From 2a082b20b8da2ffecb4f2c0e179a961c0fd26bd0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Jun 2017 11:23:45 -0400 Subject: [PATCH 01/82] main: DevelBuild=yes to `ostree --version` for devel builds And use it in `test-symbols.sh`, to fix the `distcheck` case; the previous change stopped distributing `libostree-devel.sym` in release builds. Closes: #944 Approved by: jlebon --- configure.ac | 5 ++++- src/ostree/ot-main.c | 3 +++ tests/test-symbols.sh | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 7946f4fd..58795a78 100644 --- a/configure.ac +++ b/configure.ac @@ -460,7 +460,10 @@ AS_IF([test x$enable_experimental_api = xyes], ) AM_CONDITIONAL([ENABLE_EXPERIMENTAL_API],[test x$enable_experimental_api = xyes]) AM_CONDITIONAL([BUILDOPT_IS_DEVEL_BUILD],[test x$is_release_build != xyes]) -AM_COND_IF([BUILDOPT_IS_DEVEL_BUILD], release_build_type=devel, release_build_type=release) +AM_COND_IF([BUILDOPT_IS_DEVEL_BUILD], + AC_DEFINE([BUILDOPT_IS_DEVEL_BUILD], [1], [Define if doing a development build]) + release_build_type=devel, + release_build_type=release) AC_CONFIG_FILES([ Makefile diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 9aca8287..40d77f5f 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -242,6 +242,9 @@ ostree_option_context_parse (GOptionContext *context, g_print (" Version: %s\n", PACKAGE_VERSION); if (strlen (OSTREE_GITREV) > 0) g_print (" Git: %s\n", OSTREE_GITREV); +#ifdef BUILDOPT_IS_DEVEL_BUILD + g_print (" DevelBuild: yes\n"); +#endif g_print (" Features:\n"); for (char **iter = features; iter && *iter; iter++) g_print (" - %s\n", *iter); diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index d22231d0..3f627304 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -22,7 +22,11 @@ set -xeuo pipefail echo '1..3' released_syms=${G_TEST_SRCDIR}/src/libostree/libostree-released.sym -devel_syms=${G_TEST_SRCDIR}/src/libostree/libostree-devel.sym +if echo "$OSTREE_FEATURES" | grep --quiet --no-messages "DevelBuild"; then + devel_syms=${G_TEST_SRCDIR}/src/libostree/libostree-devel.sym +else + devel_syms= +fi if echo "$OSTREE_FEATURES" | grep --quiet --no-messages "experimental"; then experimental_sym="${G_TEST_SRCDIR}/src/libostree/libostree-experimental.sym" experimental_sections="${G_TEST_SRCDIR}/apidoc/ostree-experimental-sections.txt" From b69c6ba9928f4b1b452c0d60b7bcf1867d9a1760 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Jun 2017 10:37:19 -0400 Subject: [PATCH 02/82] build-sys: Post-release version bump Closes: #944 Approved by: jlebon --- configure.ac | 4 ++-- src/libostree/libostree-devel.sym | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 58795a78..c10d2100 100644 --- a/configure.ac +++ b/configure.ac @@ -4,10 +4,10 @@ dnl update libostree-released.sym from libostree-devel.sym, and update the check dnl in test-symbols.sh, and also set is_release_build=yes below. Then make dnl another post-release commit to bump the version, and set is_release_build=no. m4_define([year_version], [2017]) -m4_define([release_version], [7]) +m4_define([release_version], [8]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 01f182f6..a9c3f610 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,6 +18,8 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ +LIBOSTREE_2017.8 { +}; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste From 82410f0e59ab830db86350adce43cc67e466ce7f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 19 Jun 2017 11:25:25 -0400 Subject: [PATCH 03/82] ci: add ci-release-build.sh Add a check that verifies that `is_release_build` is `yes` only for release commits. And also verify that the commit message has the correct version. Closes: #945 Approved by: cgwalters --- .papr.yml | 1 + ci/ci-release-build.sh | 48 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100755 ci/ci-release-build.sh diff --git a/.papr.yml b/.papr.yml index 74c16900..6946b936 100644 --- a/.papr.yml +++ b/.papr.yml @@ -19,6 +19,7 @@ env: tests: - ci/ci-commitmessage-submodules.sh - ci/build-check.sh + - ci/ci-release-build.sh timeout: 30m diff --git a/ci/ci-release-build.sh b/ci/ci-release-build.sh new file mode 100755 index 00000000..157eb030 --- /dev/null +++ b/ci/ci-release-build.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -euo pipefail + +# Makes sure that is_release_build is only set to yes in a release commit. A +# release commit must be titled: "Release $MAJOR.$MINOR". Also checks that the +# release version in the build system matches the commit msg. + +# if running under PAPR, use the branch/PR HEAD actually +# being tested rather than the merge sha +HEAD=${PAPR_COMMIT:-HEAD} + +git log --format=%B -n 1 $HEAD > log.txt + +if grep -q ^is_release_build=yes configure.ac; then + echo "*** is_release_build is set to yes ***" + + V=$(grep -Po '^#define PACKAGE_VERSION "\K[0-9]+\.[0-9]+(?=")' config.h) + if [ -z "$V" ]; then + echo "ERROR: couldn't read PACKAGE_VERSION" + exit 1 + fi + echo "OK: release version is $V" + + # check if the commit title indicates a release and has the correct version + if ! grep -q "^Release $V" log.txt; then + echo "ERROR: release commit doesn't match version" + echo "Commit message:" + cat log.txt + echo "Build version: $V" + exit 1 + fi + echo "OK: release commit matches version" + + if grep -q "^LIBOSTREE_$V" src/libostree/libostree-devel.sym; then + echo "ERROR: devel syms still references release version" + exit 1 + fi + echo "OK: devel syms no longer reference release version" + +else + echo "*** is_release_build is set to no ***" + + if grep -qE "^Release [0-9]+\.[0-9]+" log.txt; then + echo "ERROR: release commit does not have is_release_build=yes" + exit 1 + fi + echo "OK: commit is not a release" +fi From fd50c9732231895b18cd8158726e7f77bc8dcae6 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 20 Jun 2017 17:38:15 +0100 Subject: [PATCH 04/82] =?UTF-8?q?build:=20Add=20=E2=80=98devel=E2=80=99=20?= =?UTF-8?q?or=20=E2=80=98release=E2=80=99=20to=20OSTREE=5FFEATURES=20for?= =?UTF-8?q?=20test-symbols.sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test-symbols.sh was looking for the DevelBuild string, which is actually part of the output from `ostree --version`, not $OSTREE_FEATURES. Signed-off-by: Philip Withnall Closes: #948 Approved by: cgwalters --- configure.ac | 1 + tests/test-symbols.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c10d2100..9973c80a 100644 --- a/configure.ac +++ b/configure.ac @@ -464,6 +464,7 @@ AM_COND_IF([BUILDOPT_IS_DEVEL_BUILD], AC_DEFINE([BUILDOPT_IS_DEVEL_BUILD], [1], [Define if doing a development build]) release_build_type=devel, release_build_type=release) +OSTREE_FEATURES="$OSTREE_FEATURES $release_build_type" AC_CONFIG_FILES([ Makefile diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 3f627304..4a11183e 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -22,7 +22,7 @@ set -xeuo pipefail echo '1..3' released_syms=${G_TEST_SRCDIR}/src/libostree/libostree-released.sym -if echo "$OSTREE_FEATURES" | grep --quiet --no-messages "DevelBuild"; then +if echo "$OSTREE_FEATURES" | grep --quiet --no-messages "devel"; then devel_syms=${G_TEST_SRCDIR}/src/libostree/libostree-devel.sym else devel_syms= From 20dc9454b3ba257778a74b42699c8506996372db Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 14 Jun 2017 12:28:52 +0100 Subject: [PATCH 05/82] lib/core: Add ostree_validate_remote_name() for remote names There are a few places in the code where ad-hoc validation was being performed. Might as well formalise it a bit more. Signed-off-by: Philip Withnall Closes: #948 Approved by: cgwalters --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 4 +++- src/libostree/ostree-core.c | 32 ++++++++++++++++++++++++++++++- src/libostree/ostree-core.h | 3 +++ src/libostree/ostree-repo.c | 8 ++++---- tests/test-basic-c.c | 20 +++++++++++++++++++ 6 files changed, 62 insertions(+), 6 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 116c50e8..47f351d2 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -118,6 +118,7 @@ ostree_checksum_b64_inplace_from_bytes ostree_checksum_b64_inplace_to_bytes ostree_cmp_checksum_bytes ostree_validate_rev +ostree_validate_remote_name ostree_parse_refspec ostree_object_type_to_string ostree_object_type_from_string diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index a9c3f610..d2bc8399 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,7 +19,9 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2017.8 { -}; +global: + ostree_validate_remote_name; +} LIBOSTREE_2017.7; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 54e01bcb..3eb35c3f 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -101,6 +101,7 @@ ostree_validate_checksum_string (const char *sha256, #define OSTREE_REF_FRAGMENT_REGEXP "[-._\\w\\d]+" #define OSTREE_REF_REGEXP "(?:" OSTREE_REF_FRAGMENT_REGEXP "/)*" OSTREE_REF_FRAGMENT_REGEXP +#define OSTREE_REMOTE_NAME_REGEXP OSTREE_REF_FRAGMENT_REGEXP /** * ostree_parse_refspec: @@ -125,7 +126,7 @@ ostree_parse_refspec (const char *refspec, static gsize regex_initialized; if (g_once_init_enter (®ex_initialized)) { - regex = g_regex_new ("^(" OSTREE_REF_FRAGMENT_REGEXP ":)?(" OSTREE_REF_REGEXP ")$", 0, 0, NULL); + regex = g_regex_new ("^(" OSTREE_REMOTE_NAME_REGEXP ":)?(" OSTREE_REF_REGEXP ")$", 0, 0, NULL); g_assert (regex); g_once_init_leave (®ex_initialized, 1); } @@ -180,6 +181,35 @@ ostree_validate_rev (const char *rev, return TRUE; } +/** + * ostree_validate_remote_name: + * @remote_name: A remote name + * @error: Error + * + * Returns: %TRUE if @remote_name is a valid remote name + * Since: 2017.7 + */ +gboolean +ostree_validate_remote_name (const char *remote_name, + GError **error) +{ + g_autoptr(GMatchInfo) match = NULL; + + static gsize regex_initialized; + static GRegex *regex; + if (g_once_init_enter (®ex_initialized)) + { + regex = g_regex_new ("^" OSTREE_REMOTE_NAME_REGEXP "$", 0, 0, NULL); + g_assert (regex); + g_once_init_leave (®ex_initialized, 1); + } + + if (!g_regex_match (regex, remote_name, 0, &match)) + return glnx_throw (error, "Invalid remote name %s", remote_name); + + return TRUE; +} + GVariant * _ostree_file_header_new (GFileInfo *file_info, GVariant *xattrs) diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index c1e014e2..dc64d89b 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -242,6 +242,9 @@ int ostree_cmp_checksum_bytes (const guchar *a, const guchar *b); _OSTREE_PUBLIC gboolean ostree_validate_rev (const char *rev, GError **error); +_OSTREE_PUBLIC +gboolean ostree_validate_remote_name (const char *remote_name, GError **error); + _OSTREE_PUBLIC gboolean ostree_parse_refspec (const char *refspec, char **out_remote, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index e556e464..6bcc8e0d 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -876,8 +876,8 @@ impl_repo_remote_add (OstreeRepo *self, g_return_val_if_fail (url != NULL, FALSE); g_return_val_if_fail (options == NULL || g_variant_is_of_type (options, G_VARIANT_TYPE ("a{sv}")), FALSE); - if (strchr (name, '/') != NULL) - return glnx_throw (error, "Invalid character '/' in remote name: %s", name); + if (!ostree_validate_remote_name (name, error)) + return FALSE; g_autoptr(OstreeRemote) remote = _ostree_repo_get_remote (self, name, NULL); if (remote != NULL && if_not_exists) @@ -1005,8 +1005,8 @@ impl_repo_remote_delete (OstreeRepo *self, { g_return_val_if_fail (name != NULL, FALSE); - if (strchr (name, '/') != NULL) - return glnx_throw (error, "Invalid character '/' in remote name: %s", name); + if (!ostree_validate_remote_name (name, error)) + return FALSE; g_autoptr(OstreeRemote) remote = NULL; if (if_exists) diff --git a/tests/test-basic-c.c b/tests/test-basic-c.c index dbab087e..dc6d33f2 100644 --- a/tests/test-basic-c.c +++ b/tests/test-basic-c.c @@ -185,6 +185,25 @@ static gboolean hi_content_stream_new (GInputStream **out_stream, return ostree_raw_file_to_content_stream ((GInputStream*)hi_memstream, finfo, NULL, out_stream, out_length, NULL, error); } +static void +test_validate_remotename (void) +{ + const char *valid[] = {"foo", "hello-world"}; + const char *invalid[] = {"foo/bar", ""}; + for (guint i = 0; i < G_N_ELEMENTS(valid); i++) + { + g_autoptr(GError) error = NULL; + g_assert (ostree_validate_remote_name (valid[i], &error)); + g_assert_no_error (error); + } + for (guint i = 0; i < G_N_ELEMENTS(invalid); i++) + { + g_autoptr(GError) error = NULL; + g_assert (!ostree_validate_remote_name (invalid[i], &error)); + g_assert (error != NULL); + } +} + static void test_object_writes (gconstpointer data) { @@ -232,6 +251,7 @@ int main (int argc, char **argv) g_test_add_data_func ("/repo-not-system", repo, test_repo_is_not_system); g_test_add_data_func ("/raw-file-to-archive-z2-stream", repo, test_raw_file_to_archive_z2_stream); g_test_add_data_func ("/objectwrites", repo, test_object_writes); + g_test_add_func ("/remotename", test_validate_remotename); return g_test_run(); out: From 46e7f08daa924ecdf255922adff31b389a1bf8a1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Jun 2017 10:54:05 -0400 Subject: [PATCH 06/82] repo: Squash a gcc `-Wmaybe-uninitialized` warning It's spurious, but unfortunately GCC doesn't currently understand that it will always be set. Closes: #943 Approved by: jlebon --- src/libostree/ostree-repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 6bcc8e0d..d1253da5 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2683,7 +2683,7 @@ ostree_repo_load_file (OstreeRepo *self, } else { - int objdir_fd; /* referenced */ + int objdir_fd = -1; /* referenced */ if (!stat_bare_content_object (self, loose_path_buf, &objdir_fd, &ret_file_info, From 027c77c621bf8557fa1fa45ac77b247a52376131 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 20 Jun 2017 17:19:01 -0400 Subject: [PATCH 07/82] ci: Enable -Werror=maybe-uninitialized We don't want to inject this warning by default for every build like the other ones in `configure.ac`, since it can be spruriously wrong. But there's no reason not to have a more extended set of warnings for well-known toolchains (e.g. f25). Closes: #943 Approved by: jlebon --- .papr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.papr.yml b/.papr.yml index 6946b936..1addf061 100644 --- a/.papr.yml +++ b/.papr.yml @@ -12,7 +12,9 @@ packages: - git env: - CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' + # Enable all the sanitizers for this primary build. + # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain + CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2 -Werror=maybe-uninitialized' ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc # TODO when we're doing leak checks: G_SLICE: "always-malloc" From 333d264c5b35c01035372b85aa0e689627afc480 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 21 Jun 2017 12:37:12 +0100 Subject: [PATCH 08/82] =?UTF-8?q?lib/core:=20Fix=20=E2=80=98Since=E2=80=99?= =?UTF-8?q?=20line=20for=20ostree=5Fvalidate=5Fremote=5Fname()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was missed when cherry-picking it out of https://github.com/ostreedev/ostree/pull/924#discussion_r123097919. Signed-off-by: Philip Withnall Closes: #950 Approved by: cgwalters --- src/libostree/ostree-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 3eb35c3f..abc204e9 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -187,7 +187,7 @@ ostree_validate_rev (const char *rev, * @error: Error * * Returns: %TRUE if @remote_name is a valid remote name - * Since: 2017.7 + * Since: 2017.8 */ gboolean ostree_validate_remote_name (const char *remote_name, From 20829a058253cb9fbbc102d8fa347c33ce7c3e8b Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 21 Jun 2017 12:53:48 +0100 Subject: [PATCH 09/82] lib/sym: Fix symbol versions for 2017.7 experimental symbols There was a typo in the group name. It should be OK to change the version since this is all hidden behind the --enable-experimental-api configure option. Signed-off-by: Philip Withnall Closes: #950 Approved by: cgwalters --- src/libostree/libostree-experimental.sym | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index 1d4feeda..9ccca958 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -30,7 +30,7 @@ global: ostree_remote_unref; } LIBOSTREE_2017.6; -LIBOSTREE_2016.7_EXPERIMENTAL { +LIBOSTREE_2017.7_EXPERIMENTAL { global: ostree_remote_get_name; } LIBOSTREE_2017.6_EXPERIMENTAL; From 9a79d13ce307d8b5b2fe0c373c5d778ad7128b5a Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 21 Jun 2017 12:54:28 +0100 Subject: [PATCH 10/82] =?UTF-8?q?lib/remote:=20Fix=20=E2=80=98Since?= =?UTF-8?q?=E2=80=99=20line=20for=20OstreeRemote?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was a typo in it when it was first introduced. Let’s not live in the past. Signed-off-by: Philip Withnall Closes: #950 Approved by: cgwalters --- src/libostree/ostree-remote.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-remote.h b/src/libostree/ostree-remote.h index 96fd4556..df76f67e 100644 --- a/src/libostree/ostree-remote.h +++ b/src/libostree/ostree-remote.h @@ -41,7 +41,7 @@ G_BEGIN_DECLS * remotes can only be passed around as (reference counted) opaque handles. In * future, more API may be added to create and interrogate them. * - * Since: 2016.7 + * Since: 2017.6 */ #ifndef OSTREE_ENABLE_EXPERIMENTAL_API /* This is in ostree-types.h otherwise */ From 90b587e2c8c5593797b201dc1a63b59cd5fd5022 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Wed, 21 Jun 2017 10:26:03 +0100 Subject: [PATCH 11/82] build: Always include ostree-trivial-httpd.xml in tarballs Signed-off-by: Simon McVittie Closes: #949 Approved by: cgwalters --- Makefile-man.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile-man.am b/Makefile-man.am index 7996d2d5..93779509 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -32,6 +32,9 @@ ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 \ ostree-summary.1 ostree-static-delta.1 if BUILDOPT_TRIVIAL_HTTPD man1_files += ostree-trivial-httpd.1 +else +# We still want to distribute the source, even if we are not building it +EXTRA_DIST += man/ostree-trivial-httpd.xml endif if BUILDOPT_FUSE From aafda9073ad581f9b8051d58947bbab9379badcc Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Jun 2017 13:01:50 -0400 Subject: [PATCH 12/82] lib/core: Avoid NULL deref in content_file_parse() if out variable unset Prep for a change in `ostree_repo_load_file()`. We would crash if a caller had `out_file_info = NULL`, because we deref `ret_file_info` below it. Closes: #951 Approved by: jlebon --- src/libostree/ostree-core.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index abc204e9..32516b7c 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -644,7 +644,7 @@ ostree_content_stream_parse (gboolean compressed, if (compressed) { if (!zlib_file_header_parse (file_header, - out_file_info ? &ret_file_info : NULL, + &ret_file_info, out_xattrs ? &ret_xattrs : NULL, error)) return FALSE; @@ -652,12 +652,11 @@ ostree_content_stream_parse (gboolean compressed, else { if (!file_header_parse (file_header, - out_file_info ? &ret_file_info : NULL, + &ret_file_info, out_xattrs ? &ret_xattrs : NULL, error)) return FALSE; - if (ret_file_info) - g_file_info_set_size (ret_file_info, input_length - archive_header_size - 8); + g_file_info_set_size (ret_file_info, input_length - archive_header_size - 8); } g_autoptr(GInputStream) ret_input = NULL; From 63ad289a9c5ec76c071469fe8db7bb61d3b114c9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Jun 2017 13:03:14 -0400 Subject: [PATCH 13/82] lib/repo: Split archive/bare file parsing Prep for future cleanup patches (in particular I want an internal-only version at first that uses a fd+`struct stat`) to avoid allocations. The new version avoids lots of deep nesting of conditionals as well by hoisting the "not found" handling to an early return. There's a bit of code duplication between the two cases but it's quite worth the result. Closes: #951 Approved by: jlebon --- src/libostree/ostree-repo.c | 405 ++++++++++++++++++------------------ 1 file changed, 203 insertions(+), 202 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index d1253da5..ec52dc09 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2614,6 +2614,204 @@ _ostree_repo_read_bare_fd (OstreeRepo *self, return TRUE; } +static gboolean +repo_load_file_archive (OstreeRepo *self, + const char *checksum, + GInputStream **out_input, + GFileInfo **out_file_info, + GVariant **out_xattrs, + GCancellable *cancellable, + GError **error) +{ + struct stat stbuf; + char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; + _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode); + + glnx_fd_close int fd = -1; + if (!ot_openat_ignore_enoent (self->objects_dir_fd, loose_path_buf, &fd, + error)) + return FALSE; + + if (fd < 0 && self->commit_stagedir_fd != -1) + { + if (!ot_openat_ignore_enoent (self->commit_stagedir_fd, loose_path_buf, &fd, + error)) + return FALSE; + } + + if (fd != -1) + { + if (!glnx_fstat (fd, &stbuf, error)) + return FALSE; + + g_autoptr(GInputStream) tmp_stream = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE); + /* Note return here */ + return ostree_content_stream_parse (TRUE, tmp_stream, stbuf.st_size, TRUE, + out_input, out_file_info, out_xattrs, + cancellable, error); + } + else if (self->parent_repo) + { + return ostree_repo_load_file (self->parent_repo, checksum, + out_input, out_file_info, out_xattrs, + cancellable, error); + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Couldn't find file object '%s'", checksum); + return FALSE; + } +} + +static gboolean +repo_load_file_bare (OstreeRepo *self, + const char *checksum, + GInputStream **out_input, + GFileInfo **out_file_info, + GVariant **out_xattrs, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GInputStream) ret_input = NULL; + g_autoptr(GFileInfo) ret_file_info = NULL; + g_autoptr(GVariant) ret_xattrs = NULL; + char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; + _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode); + + int objdir_fd = -1; /* referenced */ + if (!stat_bare_content_object (self, loose_path_buf, + &objdir_fd, + &ret_file_info, + cancellable, error)) + return FALSE; + + if (!ret_file_info) + { + if (self->parent_repo) + { + return ostree_repo_load_file (self->parent_repo, checksum, + out_input, out_file_info, out_xattrs, + cancellable, error); + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Couldn't find file object '%s'", checksum); + return FALSE; + } + } + + if (self->mode == OSTREE_REPO_MODE_BARE_USER) + { + /* In bare-user, symlinks are stored as regular files, so we just + * always do an open, then query the user.ostreemeta xattr for + * more information. + */ + glnx_fd_close int fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return glnx_throw_errno (error); + + g_autoptr(GBytes) bytes = glnx_fgetxattr_bytes (fd, "user.ostreemeta", error); + if (bytes == NULL) + return FALSE; + + g_autoptr(GVariant) metadata = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_FILEMETA_GVARIANT_FORMAT, + bytes, FALSE)); + ret_xattrs = set_info_from_filemeta (ret_file_info, metadata); + + guint32 mode = g_file_info_get_attribute_uint32 (ret_file_info, "unix::mode"); + if (S_ISREG (mode) && out_input) + { + g_assert (fd != -1); + ret_input = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE); + } + else if (S_ISLNK (mode)) + { + g_file_info_set_file_type (ret_file_info, G_FILE_TYPE_SYMBOLIC_LINK); + g_file_info_set_size (ret_file_info, 0); + + char targetbuf[PATH_MAX+1]; + gsize target_size; + g_autoptr(GInputStream) target_input = g_unix_input_stream_new (fd, FALSE); + if (!g_input_stream_read_all (target_input, targetbuf, sizeof (targetbuf), + &target_size, cancellable, error)) + return FALSE; + + g_file_info_set_symlink_target (ret_file_info, targetbuf); + } + } + else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + { + + /* Canonical info is: uid/gid is 0 and no xattrs, which + might be wrong and thus not validate correctly, but + at least we report something consistent. */ + g_file_info_set_attribute_uint32 (ret_file_info, "unix::uid", 0); + g_file_info_set_attribute_uint32 (ret_file_info, "unix::gid", 0); + + if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR && + out_input) + { + glnx_fd_close int fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return glnx_throw_errno (error); + + ret_input = g_unix_input_stream_new (fd, TRUE); + fd = -1; /* Transfer ownership */ + } + + if (out_xattrs) + { + GVariantBuilder builder; + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a(ayay)")); + ret_xattrs = g_variant_ref_sink (g_variant_builder_end (&builder)); + } + } + else + { + g_assert (self->mode == OSTREE_REPO_MODE_BARE); + + if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR + && (out_input || out_xattrs)) + { + glnx_fd_close int fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return glnx_throw_errno (error); + + if (out_xattrs) + { + if (self->disable_xattrs) + ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); + else if (!glnx_fd_get_all_xattrs (fd, &ret_xattrs, + cancellable, error)) + return FALSE; + } + + if (out_input) + { + ret_input = g_unix_input_stream_new (fd, TRUE); + fd = -1; /* Transfer ownership */ + } + } + else if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_SYMBOLIC_LINK + && out_xattrs) + { + if (self->disable_xattrs) + ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); + else if (!glnx_dfd_name_get_all_xattrs (objdir_fd, loose_path_buf, + &ret_xattrs, + cancellable, error)) + return FALSE; + } + } + + ot_transfer_out_value (out_input, &ret_input); + ot_transfer_out_value (out_file_info, &ret_file_info); + ot_transfer_out_value (out_xattrs, &ret_xattrs); + return TRUE; +} + /** * ostree_repo_load_file: * @self: Repo @@ -2636,209 +2834,12 @@ ostree_repo_load_file (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean found = FALSE; - g_autoptr(GInputStream) ret_input = NULL; - g_autoptr(GFileInfo) ret_file_info = NULL; - g_autoptr(GVariant) ret_xattrs = NULL; - - OstreeRepoMode repo_mode = ostree_repo_get_mode (self); - - char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; - _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, repo_mode); - - if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2) - { - int fd = -1; - struct stat stbuf; - g_autoptr(GInputStream) tmp_stream = NULL; - - if (!ot_openat_ignore_enoent (self->objects_dir_fd, loose_path_buf, &fd, - error)) - return FALSE; - - if (fd < 0 && self->commit_stagedir_fd != -1) - { - if (!ot_openat_ignore_enoent (self->commit_stagedir_fd, loose_path_buf, &fd, - error)) - return FALSE; - } - - if (fd != -1) - { - tmp_stream = g_unix_input_stream_new (fd, TRUE); - fd = -1; /* Transfer ownership */ - - if (!glnx_stream_fstat ((GFileDescriptorBased*) tmp_stream, &stbuf, - error)) - return FALSE; - - if (!ostree_content_stream_parse (TRUE, tmp_stream, stbuf.st_size, TRUE, - out_input ? &ret_input : NULL, - &ret_file_info, &ret_xattrs, - cancellable, error)) - return FALSE; - - found = TRUE; - } - } + if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2) + return repo_load_file_archive (self, checksum, out_input, out_file_info, out_xattrs, + cancellable, error); else - { - int objdir_fd = -1; /* referenced */ - if (!stat_bare_content_object (self, loose_path_buf, - &objdir_fd, - &ret_file_info, - cancellable, error)) - return FALSE; - - if (ret_file_info) - { - found = TRUE; - - if (repo_mode == OSTREE_REPO_MODE_BARE_USER) - { - guint32 mode; - g_autoptr(GVariant) metadata = NULL; - g_autoptr(GBytes) bytes = NULL; - glnx_fd_close int fd = -1; - - /* In bare-user, symlinks are stored as regular files, so we just - * always do an open, then query the user.ostreemeta xattr for - * more information. - */ - fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); - if (fd < 0) - return glnx_throw_errno (error); - - bytes = glnx_fgetxattr_bytes (fd, "user.ostreemeta", error); - if (bytes == NULL) - return FALSE; - - metadata = g_variant_new_from_bytes (OSTREE_FILEMETA_GVARIANT_FORMAT, - bytes, FALSE); - g_variant_ref_sink (metadata); - - ret_xattrs = set_info_from_filemeta (ret_file_info, metadata); - - mode = g_file_info_get_attribute_uint32 (ret_file_info, "unix::mode"); - - if (S_ISREG (mode) && out_input) - { - g_assert (fd != -1); - ret_input = g_unix_input_stream_new (fd, TRUE); - fd = -1; /* Transfer ownership */ - } - else if (S_ISLNK (mode)) - { - g_autoptr(GInputStream) target_input = NULL; - char targetbuf[PATH_MAX+1]; - gsize target_size; - - g_file_info_set_file_type (ret_file_info, G_FILE_TYPE_SYMBOLIC_LINK); - g_file_info_set_size (ret_file_info, 0); - - target_input = g_unix_input_stream_new (fd, TRUE); - fd = -1; /* Transfer ownership */ - - if (!g_input_stream_read_all (target_input, targetbuf, sizeof (targetbuf), - &target_size, cancellable, error)) - return FALSE; - - g_file_info_set_symlink_target (ret_file_info, targetbuf); - } - } - else if (repo_mode == OSTREE_REPO_MODE_BARE_USER_ONLY) - { - glnx_fd_close int fd = -1; - - /* Canonical info is: uid/gid is 0 and no xattrs, which - might be wrong and thus not validate correctly, but - at least we report something consistent. */ - g_file_info_set_attribute_uint32 (ret_file_info, "unix::uid", 0); - g_file_info_set_attribute_uint32 (ret_file_info, "unix::gid", 0); - - if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR && - out_input) - { - fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); - if (fd < 0) - return glnx_throw_errno (error); - - ret_input = g_unix_input_stream_new (fd, TRUE); - fd = -1; /* Transfer ownership */ - } - - if (out_xattrs) - { - GVariantBuilder builder; - g_variant_builder_init (&builder, G_VARIANT_TYPE ("a(ayay)")); - ret_xattrs = g_variant_ref_sink (g_variant_builder_end (&builder)); - } - } - else - { - g_assert (repo_mode == OSTREE_REPO_MODE_BARE); - - if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR - && (out_input || out_xattrs)) - { - glnx_fd_close int fd = -1; - - fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); - if (fd < 0) - return glnx_throw_errno (error); - - if (out_xattrs) - { - if (self->disable_xattrs) - ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); - else if (!glnx_fd_get_all_xattrs (fd, &ret_xattrs, - cancellable, error)) - return FALSE; - } - - if (out_input) - { - ret_input = g_unix_input_stream_new (fd, TRUE); - fd = -1; /* Transfer ownership */ - } - } - else if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_SYMBOLIC_LINK - && out_xattrs) - { - if (self->disable_xattrs) - ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); - else if (!glnx_dfd_name_get_all_xattrs (objdir_fd, loose_path_buf, - &ret_xattrs, - cancellable, error)) - return FALSE; - } - } - } - } - - if (!found) - { - if (self->parent_repo) - { - if (!ostree_repo_load_file (self->parent_repo, checksum, - out_input ? &ret_input : NULL, - out_file_info ? &ret_file_info : NULL, - out_xattrs ? &ret_xattrs : NULL, - cancellable, error)) - return FALSE; - } - else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Couldn't find file object '%s'", checksum); - return FALSE; - } - } - - ot_transfer_out_value (out_input, &ret_input); - ot_transfer_out_value (out_file_info, &ret_file_info); - ot_transfer_out_value (out_xattrs, &ret_xattrs); - return TRUE; + return repo_load_file_bare (self, checksum, out_input, out_file_info, out_xattrs, + cancellable, error); } /** From 612c8a5fa8c2c6bccda2987bd21343d2837989f7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Jun 2017 12:14:07 -0400 Subject: [PATCH 14/82] lib/repo: More cleanup of load_file() internals This is followon work from previous cleanups. Basically `stat_bare_content_object()` was the `fstatat()` logic and `ostree_repo_read_bare_fd()` was the `openat()` implementation; they duplicated some bits to find the object in staging, recurse into parent etc. Further, I wanted an internal-only version of this API which didn't allocate `GFileInfo`/`GInputStream` but used a plain `fd` and `struct stat` to avoid mallocs. The end version here I think looks a lot nicer, since we deduplicate the various `open()` calls in the different cases for example. Closes: #952 Approved by: jlebon --- src/libostree/ostree-repo-private.h | 13 +- .../ostree-repo-static-delta-processing.c | 10 +- src/libostree/ostree-repo.c | 332 +++++++----------- 3 files changed, 148 insertions(+), 207 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 6cbf9ebe..8f87b103 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -321,11 +321,14 @@ _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, GError **error); gboolean -_ostree_repo_read_bare_fd (OstreeRepo *self, - const char *checksum, - int *out_fd, - GCancellable *cancellable, - GError **error); +_ostree_repo_load_file_bare (OstreeRepo *self, + const char *checksum, + int *out_fd, + struct stat *out_stbuf, + char **out_symlink, + GVariant **out_xattrs, + GCancellable *cancellable, + GError **error); gboolean _ostree_repo_update_mtime (OstreeRepo *self, diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index ea157e77..a0f51262 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -830,11 +830,13 @@ dispatch_set_read_source (OstreeRepo *repo, g_free (state->read_source_object); state->read_source_object = ostree_checksum_from_bytes (state->payload_data + source_offset); - - if (!_ostree_repo_read_bare_fd (repo, state->read_source_object, &state->read_source_fd, - cancellable, error)) + + if (!_ostree_repo_load_file_bare (repo, state->read_source_object, + &state->read_source_fd, + NULL, NULL, NULL, + cancellable, error)) goto out; - + ret = TRUE; out: if (!ret) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index ec52dc09..1b30e913 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2502,118 +2502,22 @@ load_metadata_internal (OstreeRepo *self, return TRUE; } -/* Basically fstatat(), but also looks in both the committed and staging - * directories, and returns *out_dfd for where we found the object. - */ -static gboolean -stat_bare_content_object (OstreeRepo *self, - const char *loose_path_buf, - int *out_dfd, - GFileInfo **out_info, - GCancellable *cancellable, - GError **error) -{ - struct stat stbuf; - int res; - int dirfd; - - dirfd = self->objects_dir_fd; - res = TEMP_FAILURE_RETRY (fstatat (dirfd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW)); - if (res < 0 && errno == ENOENT && self->commit_stagedir_fd != -1) - { - dirfd = self->commit_stagedir_fd; - res = TEMP_FAILURE_RETRY (fstatat (dirfd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW)); - } - if (res < 0) - { - if (errno == ENOENT) - { - *out_dfd = -1; - *out_info = NULL; - return TRUE; - } - return glnx_throw_errno (error); - } - - g_autoptr(GFileInfo) ret_info = _ostree_header_gfile_info_new (stbuf.st_mode, stbuf.st_uid, stbuf.st_gid); - - if (S_ISREG (stbuf.st_mode)) - { - g_file_info_set_size (ret_info, stbuf.st_size); - } - else if (S_ISLNK (stbuf.st_mode)) - { - if (!ot_readlinkat_gfile_info (dirfd, loose_path_buf, - ret_info, cancellable, error)) - return FALSE; - } - else - return glnx_throw (error, "Not a regular file or symlink: %s", loose_path_buf); - - *out_dfd = dirfd; - ot_transfer_out_value (out_info, &ret_info); - return TRUE; -} - static GVariant * -set_info_from_filemeta (GFileInfo *info, - GVariant *metadata) +filemeta_to_stat (struct stat *stbuf, + GVariant *metadata) { guint32 uid, gid, mode; GVariant *xattrs; g_variant_get (metadata, "(uuu@a(ayay))", &uid, &gid, &mode, &xattrs); - uid = GUINT32_FROM_BE (uid); - gid = GUINT32_FROM_BE (gid); - mode = GUINT32_FROM_BE (mode); - - g_file_info_set_attribute_uint32 (info, "unix::uid", uid); - g_file_info_set_attribute_uint32 (info, "unix::gid", gid); - g_file_info_set_attribute_uint32 (info, "unix::mode", mode); + stbuf->st_uid = GUINT32_FROM_BE (uid); + stbuf->st_gid = GUINT32_FROM_BE (gid); + stbuf->st_mode = GUINT32_FROM_BE (mode); return xattrs; } -gboolean -_ostree_repo_read_bare_fd (OstreeRepo *self, - const char *checksum, - int *out_fd, - GCancellable *cancellable, - GError **error) -{ - char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; - - g_assert (_ostree_repo_mode_is_bare (self->mode)); - - _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode); - - if (!ot_openat_ignore_enoent (self->objects_dir_fd, loose_path_buf, out_fd, error)) - return FALSE; - - if (*out_fd == -1 && self->commit_stagedir_fd != -1) - { - if (!ot_openat_ignore_enoent (self->commit_stagedir_fd, loose_path_buf, out_fd, error)) - return FALSE; - } - - if (*out_fd == -1) - { - if (self->parent_repo) - return _ostree_repo_read_bare_fd (self->parent_repo, - checksum, - out_fd, - cancellable, - error); - - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "No such file object %s", checksum); - return FALSE; - } - - return TRUE; -} - static gboolean repo_load_file_archive (OstreeRepo *self, const char *checksum, @@ -2664,102 +2568,108 @@ repo_load_file_archive (OstreeRepo *self, } } -static gboolean -repo_load_file_bare (OstreeRepo *self, - const char *checksum, - GInputStream **out_input, - GFileInfo **out_file_info, - GVariant **out_xattrs, - GCancellable *cancellable, - GError **error) +gboolean +_ostree_repo_load_file_bare (OstreeRepo *self, + const char *checksum, + int *out_fd, + struct stat *out_stbuf, + char **out_symlink, + GVariant **out_xattrs, + GCancellable *cancellable, + GError **error) { - g_autoptr(GInputStream) ret_input = NULL; - g_autoptr(GFileInfo) ret_file_info = NULL; + /* The bottom case recursing on the parent repo */ + if (self == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Couldn't find file object '%s'", checksum); + return FALSE; + } + + struct stat stbuf; + glnx_fd_close int fd = -1; + g_autofree char *ret_symlink = NULL; g_autoptr(GVariant) ret_xattrs = NULL; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode); - int objdir_fd = -1; /* referenced */ - if (!stat_bare_content_object (self, loose_path_buf, - &objdir_fd, - &ret_file_info, - cancellable, error)) - return FALSE; - - if (!ret_file_info) + /* Do a fstatat() and find the object directory that contains this object */ + int objdir_fd = self->objects_dir_fd; + int res; + if ((res = TEMP_FAILURE_RETRY (fstatat (objdir_fd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW))) < 0 + && errno == ENOENT && self->commit_stagedir_fd != -1) { - if (self->parent_repo) - { - return ostree_repo_load_file (self->parent_repo, checksum, - out_input, out_file_info, out_xattrs, - cancellable, error); - } - else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Couldn't find file object '%s'", checksum); - return FALSE; - } + objdir_fd = self->commit_stagedir_fd; + res = TEMP_FAILURE_RETRY (fstatat (objdir_fd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW)); + } + if (res < 0 && errno != ENOENT) + return glnx_throw_errno_prefix (error, "fstat"); + else if (res < 0) + { + g_assert (errno == ENOENT); + return _ostree_repo_load_file_bare (self->parent_repo, checksum, out_fd, + out_stbuf, out_symlink, out_xattrs, + cancellable, error); + } + + const gboolean need_open = + (out_fd || out_xattrs || self->mode == OSTREE_REPO_MODE_BARE_USER); + /* If it's a regular file and we're requested to return the fd, do it now. As + * a special case in bare-user, we always do an open, since the stat() metadata + * lives there. + */ + if (need_open && S_ISREG (stbuf.st_mode)) + { + fd = openat (objdir_fd, loose_path_buf, O_CLOEXEC | O_RDONLY); + if (fd < 0) + return glnx_throw_errno_prefix (error, "openat"); + } + + if (!(S_ISREG (stbuf.st_mode) || S_ISLNK (stbuf.st_mode))) + return glnx_throw (error, "Not a regular file or symlink: %s", loose_path_buf); + + /* In the non-bare-user case, gather symlink info if requested */ + if (self->mode != OSTREE_REPO_MODE_BARE_USER + && S_ISLNK (stbuf.st_mode) && out_symlink) + { + ret_symlink = glnx_readlinkat_malloc (objdir_fd, loose_path_buf, + cancellable, error); + if (!ret_symlink) + return FALSE; } if (self->mode == OSTREE_REPO_MODE_BARE_USER) { - /* In bare-user, symlinks are stored as regular files, so we just - * always do an open, then query the user.ostreemeta xattr for - * more information. - */ - glnx_fd_close int fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); - if (fd < 0) - return glnx_throw_errno (error); - g_autoptr(GBytes) bytes = glnx_fgetxattr_bytes (fd, "user.ostreemeta", error); if (bytes == NULL) return FALSE; g_autoptr(GVariant) metadata = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_FILEMETA_GVARIANT_FORMAT, bytes, FALSE)); - ret_xattrs = set_info_from_filemeta (ret_file_info, metadata); - - guint32 mode = g_file_info_get_attribute_uint32 (ret_file_info, "unix::mode"); - if (S_ISREG (mode) && out_input) + ret_xattrs = filemeta_to_stat (&stbuf, metadata); + if (S_ISLNK (stbuf.st_mode)) { - g_assert (fd != -1); - ret_input = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE); - } - else if (S_ISLNK (mode)) - { - g_file_info_set_file_type (ret_file_info, G_FILE_TYPE_SYMBOLIC_LINK); - g_file_info_set_size (ret_file_info, 0); + if (out_symlink) + { + char targetbuf[PATH_MAX+1]; + gsize target_size; + g_autoptr(GInputStream) target_input = g_unix_input_stream_new (fd, FALSE); + if (!g_input_stream_read_all (target_input, targetbuf, sizeof (targetbuf), + &target_size, cancellable, error)) + return FALSE; - char targetbuf[PATH_MAX+1]; - gsize target_size; - g_autoptr(GInputStream) target_input = g_unix_input_stream_new (fd, FALSE); - if (!g_input_stream_read_all (target_input, targetbuf, sizeof (targetbuf), - &target_size, cancellable, error)) - return FALSE; - - g_file_info_set_symlink_target (ret_file_info, targetbuf); + ret_symlink = g_strndup (targetbuf, target_size); + } + /* In the symlink case, we don't want to return the bare-user fd */ + (void) close (glnx_steal_fd (&fd)); } } else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) { - /* Canonical info is: uid/gid is 0 and no xattrs, which might be wrong and thus not validate correctly, but at least we report something consistent. */ - g_file_info_set_attribute_uint32 (ret_file_info, "unix::uid", 0); - g_file_info_set_attribute_uint32 (ret_file_info, "unix::gid", 0); - - if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR && - out_input) - { - glnx_fd_close int fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); - if (fd < 0) - return glnx_throw_errno (error); - - ret_input = g_unix_input_stream_new (fd, TRUE); - fd = -1; /* Transfer ownership */ - } + stbuf.st_uid = stbuf.st_gid = 0; if (out_xattrs) { @@ -2772,30 +2682,15 @@ repo_load_file_bare (OstreeRepo *self, { g_assert (self->mode == OSTREE_REPO_MODE_BARE); - if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR - && (out_input || out_xattrs)) + if (S_ISREG (stbuf.st_mode) && out_xattrs) { - glnx_fd_close int fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); - if (fd < 0) - return glnx_throw_errno (error); - - if (out_xattrs) - { - if (self->disable_xattrs) - ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); - else if (!glnx_fd_get_all_xattrs (fd, &ret_xattrs, - cancellable, error)) - return FALSE; - } - - if (out_input) - { - ret_input = g_unix_input_stream_new (fd, TRUE); - fd = -1; /* Transfer ownership */ - } + if (self->disable_xattrs) + ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); + else if (!glnx_fd_get_all_xattrs (fd, &ret_xattrs, + cancellable, error)) + return FALSE; } - else if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_SYMBOLIC_LINK - && out_xattrs) + else if (S_ISLNK (stbuf.st_mode) && out_xattrs) { if (self->disable_xattrs) ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); @@ -2806,8 +2701,11 @@ repo_load_file_bare (OstreeRepo *self, } } - ot_transfer_out_value (out_input, &ret_input); - ot_transfer_out_value (out_file_info, &ret_file_info); + if (out_fd) + *out_fd = glnx_steal_fd (&fd); + if (out_stbuf) + *out_stbuf = stbuf; + ot_transfer_out_value (out_symlink, &ret_symlink); ot_transfer_out_value (out_xattrs, &ret_xattrs); return TRUE; } @@ -2838,8 +2736,46 @@ ostree_repo_load_file (OstreeRepo *self, return repo_load_file_archive (self, checksum, out_input, out_file_info, out_xattrs, cancellable, error); else - return repo_load_file_bare (self, checksum, out_input, out_file_info, out_xattrs, - cancellable, error); + { + glnx_fd_close int fd = -1; + struct stat stbuf; + g_autofree char *symlink_target = NULL; + g_autoptr(GVariant) ret_xattrs = NULL; + if (!_ostree_repo_load_file_bare (self, checksum, + out_input ? &fd : NULL, + out_file_info ? &stbuf : NULL, + out_file_info ? &symlink_target : NULL, + out_xattrs ? &ret_xattrs : NULL, + cancellable, error)) + return FALSE; + + /* Convert fd → GInputStream and struct stat → GFileInfo */ + if (out_input) + { + if (fd != -1) + *out_input = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE); + else + *out_input = NULL; + } + if (out_file_info) + { + *out_file_info = _ostree_header_gfile_info_new (stbuf.st_mode, stbuf.st_uid, stbuf.st_gid); + if (S_ISREG (stbuf.st_mode)) + { + g_file_info_set_size (*out_file_info, stbuf.st_size); + } + else if (S_ISLNK (stbuf.st_mode)) + { + g_file_info_set_size (*out_file_info, 0); + g_file_info_set_symlink_target (*out_file_info, symlink_target); + } + else + g_assert_not_reached (); + } + + ot_transfer_out_value (out_xattrs, &ret_xattrs); + return TRUE; + } } /** From e23071dc396937dc91e6b3b9b610fa9263a571bb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Jun 2017 15:29:45 -0400 Subject: [PATCH 15/82] lib/deltas: Port to more to new code style Looking at the uses of `ostree_repo_load_file()` here. Closes: #953 Approved by: jlebon --- .../ostree-repo-static-delta-compilation.c | 288 +++++++----------- src/libostree/ostree-rollsum.h | 2 + 2 files changed, 116 insertions(+), 174 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 47fae9f5..7bcf5a92 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -138,11 +138,11 @@ xattr_chunk_hash (const void *vp) g_variant_get_child (v, i, "(^&ay@ay)", &name, &value); value_data = g_variant_get_fixed_array (value, &value_len, 1); - + h += g_str_hash (name); h += bufhash (value_data, value_len); } - + return h; } @@ -235,7 +235,7 @@ objtype_checksum_array_new (GPtrArray *objects) const char *checksum; guint8 csum[OSTREE_SHA256_DIGEST_LEN]; guint8 objtype_v; - + ostree_object_name_deserialize (serialized_key, &checksum, &objtype); objtype_v = (guint8) objtype; @@ -253,29 +253,25 @@ splice_stream_to_payload (OstreeStaticDeltaPartBuilder *current_part, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - const guint readlen = 4096; - gsize bytes_read; - while (TRUE) { + const guint readlen = 4096; allocate_part_buffer_space (current_part, readlen); + gsize bytes_read; if (!g_input_stream_read_all (istream, current_part->payload->str + current_part->payload->len, readlen, &bytes_read, cancellable, error)) - goto out; + return FALSE; if (bytes_read == 0) break; - + current_part->payload->len += bytes_read; } - ret = TRUE; - out: - return ret; + return TRUE; } static void @@ -293,7 +289,7 @@ write_content_mode_xattrs (OstreeRepo *repo, guint32 mode = g_file_info_get_attribute_uint32 (content_finfo, "unix::mode"); g_autoptr(GVariant) modev - = g_variant_ref_sink (g_variant_new ("(uuu)", + = g_variant_ref_sink (g_variant_new ("(uuu)", GUINT32_TO_BE (uid), GUINT32_TO_BE (gid), GUINT32_TO_BE (mode))); @@ -317,41 +313,40 @@ process_one_object (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - guint64 content_size; - g_autoptr(GInputStream) content_stream = NULL; + OstreeStaticDeltaPartBuilder *current_part = *current_part_val; g_autoptr(GFileInfo) content_finfo = NULL; g_autoptr(GVariant) content_xattrs = NULL; - guint64 compressed_size; - OstreeStaticDeltaPartBuilder *current_part = *current_part_val; + guint64 content_size; + g_autoptr(GInputStream) content_stream = NULL; if (OSTREE_OBJECT_TYPE_IS_META (objtype)) { if (!ostree_repo_load_object_stream (repo, objtype, checksum, &content_stream, &content_size, cancellable, error)) - goto out; + return FALSE; } else { if (!ostree_repo_load_file (repo, checksum, &content_stream, &content_finfo, &content_xattrs, cancellable, error)) - goto out; + return FALSE; content_size = g_file_info_get_size (content_finfo); } - + /* Check to see if this delta is maximum size */ if (current_part->objects->len > 0 && current_part->payload->len + content_size > builder->max_chunk_size_bytes) { *current_part_val = current_part = allocate_part (builder); - } + } + guint64 compressed_size; if (!ostree_repo_query_object_storage_size (repo, objtype, checksum, &compressed_size, cancellable, error)) - goto out; + return FALSE; builder->loose_compressed_size += compressed_size; current_part->uncompressed_size += content_size; @@ -366,7 +361,7 @@ process_one_object (OstreeRepo *repo, if (!splice_stream_to_payload (current_part, content_stream, cancellable, error)) - goto out; + return FALSE; g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE); _ostree_write_varuint64 (current_part->operations, content_size); @@ -375,21 +370,16 @@ process_one_object (OstreeRepo *repo, else { gsize mode_offset, xattr_offset, content_offset; - guint32 mode; - - mode = g_file_info_get_attribute_uint32 (content_finfo, "unix::mode"); + guint32 mode = g_file_info_get_attribute_uint32 (content_finfo, "unix::mode"); write_content_mode_xattrs (repo, current_part, content_finfo, content_xattrs, &mode_offset, &xattr_offset); if (S_ISLNK (mode)) { - const char *target; - g_assert (content_stream == NULL); - - target = g_file_info_get_symlink_target (content_finfo); - content_stream = + const char *target = g_file_info_get_symlink_target (content_finfo); + content_stream = g_memory_input_stream_new_from_data (target, strlen (target), NULL); content_size = strlen (target); } @@ -401,7 +391,7 @@ process_one_object (OstreeRepo *repo, content_offset = current_part->payload->len; if (!splice_stream_to_payload (current_part, content_stream, cancellable, error)) - goto out; + return FALSE; g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE); _ostree_write_varuint64 (current_part->operations, mode_offset); @@ -410,9 +400,7 @@ process_one_object (OstreeRepo *repo, _ostree_write_varuint64 (current_part->operations, content_offset); } - ret = TRUE; - out: - return ret; + return TRUE; } typedef struct { @@ -491,34 +479,28 @@ try_content_bsdiff (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; + + g_autoptr(GFileInfo) from_finfo = NULL; - g_autoptr(GFileInfo) to_finfo = NULL; - ContentBsdiff *ret_bsdiff = NULL; - - *out_bsdiff = NULL; - if (!ostree_repo_load_file (repo, from, NULL, &from_finfo, NULL, cancellable, error)) return FALSE; + g_autoptr(GFileInfo) to_finfo = NULL; if (!ostree_repo_load_file (repo, to, NULL, &to_finfo, NULL, cancellable, error)) return FALSE; - if (g_file_info_get_size (to_finfo) + g_file_info_get_size (from_finfo) > max_bsdiff_size_bytes) - { - ret = TRUE; - goto out; - } + *out_bsdiff = NULL; - ret_bsdiff = g_new0 (ContentBsdiff, 1); + /* Ignore this if it's too large */ + if (g_file_info_get_size (to_finfo) + g_file_info_get_size (from_finfo) > max_bsdiff_size_bytes) + return TRUE; + + ContentBsdiff *ret_bsdiff = g_new0 (ContentBsdiff, 1); ret_bsdiff->from_checksum = g_strdup (from); - ret = TRUE; - if (out_bsdiff) - *out_bsdiff = g_steal_pointer (&ret_bsdiff); - out: - return ret; + ot_transfer_out_value (out_bsdiff, &ret_bsdiff); + return TRUE; } static gboolean @@ -530,35 +512,27 @@ try_content_rollsum (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GBytes) tmp_from = NULL; - g_autoptr(GBytes) tmp_to = NULL; - OstreeRollsumMatches *matches = NULL; - ContentRollsum *ret_rollsum = NULL; - *out_rollsum = NULL; /* Load the content objects, splice them to uncompressed temporary files that * we can just mmap() and seek around in conveniently. */ + g_autoptr(GBytes) tmp_from = NULL; if (!get_unpacked_unlinked_content (repo, from, &tmp_from, cancellable, error)) - goto out; + return FALSE; + g_autoptr(GBytes) tmp_to = NULL; if (!get_unpacked_unlinked_content (repo, to, &tmp_to, cancellable, error)) - goto out; + return FALSE; - matches = _ostree_compute_rollsum_matches (tmp_from, tmp_to); + OstreeRollsumMatches *matches = _ostree_compute_rollsum_matches (tmp_from, tmp_to); - { guint match_ratio = (matches->bufmatches*100)/matches->total; + const guint match_ratio = (matches->bufmatches*100)/matches->total; - /* Only proceed if the file contains (arbitrary) more than 50% of - * the previous chunks. - */ - if (match_ratio < 50) - { - ret = TRUE; - goto out; - } - } + /* Only proceed if the file contains (arbitrary) more than 50% of + * the previous chunks. + */ + if (match_ratio < 50) + return TRUE; if (opts & DELTAOPT_FLAG_VERBOSE) { @@ -568,17 +542,11 @@ try_content_rollsum (OstreeRepo *repo, matches->total, (unsigned long long)matches->match_size); } - ret_rollsum = g_new0 (ContentRollsum, 1); + ContentRollsum *ret_rollsum = g_new0 (ContentRollsum, 1); ret_rollsum->from_checksum = g_strdup (from); ret_rollsum->matches = g_steal_pointer (&matches); - - ret = TRUE; - if (out_rollsum) - *out_rollsum = g_steal_pointer (&ret_rollsum); - out: - if (matches) - _ostree_rollsum_matches_free (matches); - return ret; + ot_transfer_out_value (out_rollsum, &ret_rollsum); + return TRUE; } struct bzdiff_opaque_s @@ -625,14 +593,7 @@ process_one_rollsum (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - guint64 content_size; - g_autoptr(GBytes) tmp_to = NULL; - g_autoptr(GFileInfo) content_finfo = NULL; - g_autoptr(GVariant) content_xattrs = NULL; OstreeStaticDeltaPartBuilder *current_part = *current_part_val; - const guint8 *tmp_to_buf; - gsize tmp_to_len; /* Check to see if this delta has gone over maximum size */ if (current_part->objects->len > 0 && @@ -641,17 +602,21 @@ process_one_rollsum (OstreeRepo *repo, *current_part_val = current_part = allocate_part (builder); } + g_autoptr(GBytes) tmp_to = NULL; if (!get_unpacked_unlinked_content (repo, to_checksum, &tmp_to, cancellable, error)) - goto out; + return FALSE; - tmp_to_buf = g_bytes_get_data (tmp_to, &tmp_to_len); + gsize tmp_to_len; + const guint8 *tmp_to_buf = g_bytes_get_data (tmp_to, &tmp_to_len); + g_autoptr(GFileInfo) content_finfo = NULL; + g_autoptr(GVariant) content_xattrs = NULL; if (!ostree_repo_load_file (repo, to_checksum, NULL, &content_finfo, &content_xattrs, cancellable, error)) - goto out; - content_size = g_file_info_get_size (content_finfo); + return FALSE; + guint64 content_size = g_file_info_get_size (content_finfo); g_assert_cmpint (tmp_to_len, ==, content_size); current_part->uncompressed_size += content_size; @@ -685,11 +650,10 @@ process_one_rollsum (OstreeRepo *repo, { GVariant *match = matchlist->pdata[i]; guint32 crc; - guint64 prefix; g_variant_get (match, "(uttt)", &crc, &offset, &to_start, &from_start); - prefix = to_start - writing_offset; + const guint64 prefix = to_start - writing_offset; if (prefix > 0) { @@ -720,13 +684,11 @@ process_one_rollsum (OstreeRepo *repo, if (!reading_payload) g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_UNSET_READ_SOURCE); - { guint64 remainder = tmp_to_len - writing_offset; - if (remainder > 0) - append_payload_chunk_and_write (current_part, tmp_to_buf + writing_offset, remainder); - writing_offset += remainder; - g_assert_cmpint (writing_offset, ==, tmp_to_len); - } - + const guint64 remainder = tmp_to_len - writing_offset; + if (remainder > 0) + append_payload_chunk_and_write (current_part, tmp_to_buf + writing_offset, remainder); + writing_offset += remainder; + g_assert_cmpint (writing_offset, ==, tmp_to_len); g_assert_cmpint (writing_offset, ==, content_size); } @@ -734,9 +696,7 @@ process_one_rollsum (OstreeRepo *repo, g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_CLOSE); } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -748,17 +708,7 @@ process_one_bsdiff (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - guint64 content_size; - g_autoptr(GFileInfo) content_finfo = NULL; - g_autoptr(GVariant) content_xattrs = NULL; OstreeStaticDeltaPartBuilder *current_part = *current_part_val; - g_autoptr(GBytes) tmp_from = NULL; - g_autoptr(GBytes) tmp_to = NULL; - const guint8 *tmp_to_buf; - gsize tmp_to_len; - const guint8 *tmp_from_buf; - gsize tmp_from_len; /* Check to see if this delta has gone over maximum size */ if (current_part->objects->len > 0 && @@ -767,21 +717,27 @@ process_one_bsdiff (OstreeRepo *repo, *current_part_val = current_part = allocate_part (builder); } + g_autoptr(GBytes) tmp_from = NULL; if (!get_unpacked_unlinked_content (repo, bsdiff_content->from_checksum, &tmp_from, cancellable, error)) - goto out; + return FALSE; + g_autoptr(GBytes) tmp_to = NULL; if (!get_unpacked_unlinked_content (repo, to_checksum, &tmp_to, cancellable, error)) - goto out; + return FALSE; - tmp_to_buf = g_bytes_get_data (tmp_to, &tmp_to_len); - tmp_from_buf = g_bytes_get_data (tmp_from, &tmp_from_len); + gsize tmp_to_len; + const guint8 *tmp_to_buf = g_bytes_get_data (tmp_to, &tmp_to_len); + gsize tmp_from_len; + const guint8 *tmp_from_buf = g_bytes_get_data (tmp_from, &tmp_from_len); + g_autoptr(GFileInfo) content_finfo = NULL; + g_autoptr(GVariant) content_xattrs = NULL; if (!ostree_repo_load_file (repo, to_checksum, NULL, &content_finfo, &content_xattrs, cancellable, error)) - goto out; - content_size = g_file_info_get_size (content_finfo); + return FALSE; + const guint64 content_size = g_file_info_get_size (content_finfo); g_assert_cmpint (tmp_to_len, ==, content_size); current_part->uncompressed_size += content_size; @@ -819,10 +775,8 @@ process_one_bsdiff (OstreeRepo *repo, op.cancellable = cancellable; op.error = error; stream.opaque = &op; - if (bsdiff (tmp_from_buf, tmp_from_len, tmp_to_buf, tmp_to_len, &stream) < 0) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "bsdiff generation failed"); - goto out; - } + if (bsdiff (tmp_from_buf, tmp_from_len, tmp_to_buf, tmp_to_len, &stream) < 0) + return glnx_throw (error, "bsdiff generation failed"); payload = g_memory_output_stream_get_data (G_MEMORY_OUTPUT_STREAM (out)); payload_size = g_memory_output_stream_get_data_size (G_MEMORY_OUTPUT_STREAM (out)); @@ -838,9 +792,7 @@ process_one_bsdiff (OstreeRepo *repo, g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_UNSET_READ_SOURCE); - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -862,7 +814,7 @@ check_object_world_readable (OstreeRepo *repo, return TRUE; } -static gboolean +static gboolean generate_delta_lowlatency (OstreeRepo *repo, const char *from, const char *to, @@ -871,7 +823,6 @@ generate_delta_lowlatency (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; GHashTableIter hashiter; gpointer key, value; OstreeStaticDeltaPartBuilder *current_part = NULL; @@ -892,27 +843,27 @@ generate_delta_lowlatency (OstreeRepo *repo, { if (!ostree_repo_read_commit (repo, from, &root_from, NULL, cancellable, error)) - goto out; + return FALSE; if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, from, &from_commit, error)) - goto out; + return FALSE; if (!ostree_repo_traverse_commit (repo, from, 0, &from_reachable_objects, cancellable, error)) - goto out; + return FALSE; } if (!ostree_repo_read_commit (repo, to, &root_to, NULL, cancellable, error)) - goto out; + return FALSE; if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, to, &to_commit, error)) - goto out; + return FALSE; if (!ostree_repo_traverse_commit (repo, to, 0, &to_reachable_objects, cancellable, error)) - goto out; + return FALSE; new_reachable_metadata = ostree_repo_traverse_new_reachable (); new_reachable_regfile_content = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, g_free); @@ -939,7 +890,7 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!ostree_repo_load_file (repo, checksum, NULL, &finfo, NULL, cancellable, error)) - goto out; + return FALSE; ftype = g_file_info_get_file_type (finfo); if (ftype == G_FILE_TYPE_REGULAR) @@ -958,7 +909,7 @@ generate_delta_lowlatency (OstreeRepo *repo, CONTENT_SIZE_SIMILARITY_THRESHOLD_PERCENT, &modified_regfile_content, cancellable, error)) - goto out; + return FALSE; } else modified_regfile_content = g_hash_table_new (g_str_hash, g_str_equal); @@ -1001,13 +952,13 @@ generate_delta_lowlatency (OstreeRepo *repo, * bare repository defined as its parent. */ if (!check_object_world_readable (repo, from_checksum, &from_world_readable, cancellable, error)) - goto out; + return FALSE; if (!from_world_readable) continue; if (!try_content_rollsum (repo, opts, from_checksum, to_checksum, &rollsum, cancellable, error)) - goto out; + return FALSE; if (rollsum) { @@ -1021,7 +972,7 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!try_content_bsdiff (repo, from_checksum, to_checksum, &bsdiff, builder->max_bsdiff_size_bytes, cancellable, error)) - goto out; + return FALSE; if (bsdiff) g_hash_table_insert (bsdiff_optimized_content_objects, g_strdup (to_checksum), bsdiff); @@ -1050,7 +1001,7 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!process_one_object (repo, builder, ¤t_part, checksum, objtype, cancellable, error)) - goto out; + return FALSE; } /* Now do rollsummed objects */ @@ -1064,7 +1015,7 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!process_one_rollsum (repo, builder, ¤t_part, checksum, rollsum, cancellable, error)) - goto out; + return FALSE; builder->n_rollsum++; } @@ -1080,7 +1031,7 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!process_one_bsdiff (repo, builder, ¤t_part, checksum, bsdiff, cancellable, error)) - goto out; + return FALSE; builder->n_bsdiff++; } @@ -1103,11 +1054,11 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!ostree_repo_load_object_stream (repo, OSTREE_OBJECT_TYPE_FILE, checksum, NULL, &uncompressed_size, cancellable, error)) - goto out; + return FALSE; if (builder->min_fallback_size_bytes > 0 && uncompressed_size > builder->min_fallback_size_bytes) fallback = TRUE; - + if (fallback) { g_autofree char *size = g_format_size (uncompressed_size); @@ -1115,7 +1066,7 @@ generate_delta_lowlatency (OstreeRepo *repo, if (opts & DELTAOPT_FLAG_VERBOSE) g_printerr ("fallback for %s (%s)\n", checksum, size); - g_ptr_array_add (builder->fallback_objects, + g_ptr_array_add (builder->fallback_objects, ostree_object_name_serialize (checksum, OSTREE_OBJECT_TYPE_FILE)); g_hash_table_iter_remove (&hashiter); builder->n_fallback++; @@ -1136,7 +1087,7 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!process_one_object (repo, builder, ¤t_part, checksum, OSTREE_OBJECT_TYPE_FILE, cancellable, error)) - goto out; + return FALSE; } /* Now symlinks */ @@ -1148,12 +1099,10 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!process_one_object (repo, builder, ¤t_part, checksum, OSTREE_OBJECT_TYPE_FILE, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -1163,14 +1112,10 @@ get_fallback_headers (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - guint i; - g_autoptr(GVariant) ret_headers = NULL; - g_autoptr(GVariantBuilder) fallback_builder = NULL; + g_autoptr(GVariantBuilder) fallback_builder = + g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT)); - fallback_builder = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT)); - - for (i = 0; i < builder->fallback_objects->len; i++) + for (guint i = 0; i < builder->fallback_objects->len; i++) { GVariant *serialized = builder->fallback_objects->pdata[i]; const char *checksum; @@ -1185,23 +1130,22 @@ get_fallback_headers (OstreeRepo *self, if (!ostree_repo_load_object_stream (self, objtype, checksum, NULL, &uncompressed_size, cancellable, error)) - goto out; + return FALSE; compressed_size = uncompressed_size; } else { - g_autoptr(GFileInfo) file_info = NULL; - if (!ostree_repo_query_object_storage_size (self, OSTREE_OBJECT_TYPE_FILE, checksum, &compressed_size, cancellable, error)) - goto out; + return FALSE; + g_autoptr(GFileInfo) file_info = NULL; if (!ostree_repo_load_file (self, checksum, NULL, &file_info, NULL, cancellable, error)) - goto out; + return FALSE; uncompressed_size = g_file_info_get_size (file_info); } @@ -1214,13 +1158,9 @@ get_fallback_headers (OstreeRepo *self, maybe_swap_endian_u64 (builder->swap_endian, uncompressed_size))); } - ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder)); - - ret = TRUE; - if (out_headers) - *out_headers = g_steal_pointer (&ret_headers); - out: - return ret; + g_autoptr(GVariant) ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder)); + ot_transfer_out_value (out_headers, &ret_headers); + return TRUE; } /** @@ -1352,7 +1292,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } { guint8 endianness_char; - + switch (endianness) { case G_LITTLE_ENDIAN: @@ -1413,14 +1353,14 @@ ostree_repo_static_delta_generate (OstreeRepo *self, { guint j; for (j = 0; j < part_builder->modes->len; j++) g_variant_builder_add_value (&mode_builder, part_builder->modes->pdata[j]); - + for (j = 0; j < part_builder->xattrs->len; j++) g_variant_builder_add_value (&xattr_builder, part_builder->xattrs->pdata[j]); } - + payload_b = g_string_free_to_bytes (part_builder->payload); part_builder->payload = NULL; - + operations_b = g_string_free_to_bytes (part_builder->operations); part_builder->operations = NULL; /* FIXME - avoid duplicating memory here */ diff --git a/src/libostree/ostree-rollsum.h b/src/libostree/ostree-rollsum.h index 3a96ea59..ed832c6d 100644 --- a/src/libostree/ostree-rollsum.h +++ b/src/libostree/ostree-rollsum.h @@ -21,6 +21,7 @@ #pragma once #include +#include "libglnx.h" G_BEGIN_DECLS @@ -39,5 +40,6 @@ _ostree_compute_rollsum_matches (GBytes *from, GBytes *to); void _ostree_rollsum_matches_free (OstreeRollsumMatches *rollsum); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeRollsumMatches, _ostree_rollsum_matches_free) G_END_DECLS From 553b99642c10e1085323e31723fc9e56870a0d2b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Jun 2017 16:09:32 -0400 Subject: [PATCH 16/82] cmd/fsck: Port to new style Happened to look at this code too, it's a straightforward port. Closes: #955 Approved by: jlebon --- src/ostree/ot-builtin-fsck.c | 126 ++++++++++++----------------------- 1 file changed, 44 insertions(+), 82 deletions(-) diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 7519d6db..936bdce4 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -47,13 +47,12 @@ load_and_fsck_one_object (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; gboolean missing = FALSE; g_autoptr(GVariant) metadata = NULL; g_autoptr(GInputStream) input = NULL; g_autoptr(GFileInfo) file_info = NULL; g_autoptr(GVariant) xattrs = NULL; - GError *temp_error = NULL; + g_autoptr(GError) temp_error = NULL; if (OSTREE_OBJECT_TYPE_IS_META (objtype)) { @@ -69,8 +68,8 @@ load_and_fsck_one_object (OstreeRepo *repo, } else { - g_prefix_error (error, "Loading metadata object %s: ", checksum); - goto out; + g_propagate_error (error, g_steal_pointer (&temp_error)); + return glnx_prefix_error (error, "Loading metadata object %s", checksum); } } else @@ -78,28 +77,19 @@ load_and_fsck_one_object (OstreeRepo *repo, if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!ostree_validate_structureof_commit (metadata, error)) - { - g_prefix_error (error, "While validating commit metadata '%s': ", checksum); - goto out; - } + return glnx_prefix_error (error, "While validating commit metadata '%s'", checksum); } else if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE) { if (!ostree_validate_structureof_dirtree (metadata, error)) - { - g_prefix_error (error, "While validating directory tree '%s': ", checksum); - goto out; - } + return glnx_prefix_error (error, "While validating directory tree '%s'", checksum); } else if (objtype == OSTREE_OBJECT_TYPE_DIR_META) { if (!ostree_validate_structureof_dirmeta (metadata, error)) - { - g_prefix_error (error, "While validating directory metadata '%s': ", checksum); - goto out; - } + return glnx_prefix_error (error, "While validating directory metadata '%s'", checksum); } - + input = g_memory_input_stream_new_from_data (g_variant_get_data (metadata), g_variant_get_size (metadata), NULL); @@ -122,19 +112,15 @@ load_and_fsck_one_object (OstreeRepo *repo, } else { - *error = temp_error; - g_prefix_error (error, "Loading file object %s: ", checksum); - goto out; + g_propagate_error (error, g_steal_pointer (&temp_error)); + return glnx_prefix_error (error, "Loading file object %s", checksum); } } else { mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); if (!ostree_validate_structureof_file_mode (mode, error)) - { - g_prefix_error (error, "While validating file '%s': ", checksum); - goto out; - } + return glnx_prefix_error (error, "While validating file '%s'", checksum); } } @@ -150,8 +136,8 @@ load_and_fsck_one_object (OstreeRepo *repo, if (!ostree_checksum_file_from_input (file_info, xattrs, input, objtype, &computed_csum, cancellable, error)) - goto out; - + return FALSE; + tmp_checksum = ostree_checksum_from_bytes (computed_csum); if (strcmp (checksum, tmp_checksum) != 0) { @@ -165,16 +151,11 @@ load_and_fsck_one_object (OstreeRepo *repo, *out_found_corruption = TRUE; } else - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, msg); - goto out; - } + return glnx_throw (error, "%s", msg); } } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -184,16 +165,10 @@ fsck_reachable_objects_from_commits (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; + g_autoptr(GHashTable) reachable_objects = ostree_repo_traverse_new_reachable (); + GHashTableIter hash_iter; gpointer key, value; - g_autoptr(GHashTable) reachable_objects = NULL; - guint i; - guint mod; - guint count; - - reachable_objects = ostree_repo_traverse_new_reachable (); - g_hash_table_iter_init (&hash_iter, commits); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { @@ -207,12 +182,12 @@ fsck_reachable_objects_from_commits (OstreeRepo *repo, if (!ostree_repo_traverse_commit_union (repo, checksum, 0, reachable_objects, cancellable, error)) - goto out; + return FALSE; } - count = g_hash_table_size (reachable_objects); - mod = count / 10; - i = 0; + const guint count = g_hash_table_size (reachable_objects); + const guint mod = count / 10; + guint i = 0; g_hash_table_iter_init (&hash_iter, reachable_objects); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { @@ -224,44 +199,37 @@ fsck_reachable_objects_from_commits (OstreeRepo *repo, if (!load_and_fsck_one_object (repo, checksum, objtype, out_found_corruption, cancellable, error)) - goto out; + return FALSE; if (mod == 0 || (i % mod == 0)) g_print ("%u/%u objects\n", i + 1, count); i++; } - ret = TRUE; - out: - return ret; + return TRUE; } gboolean ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GOptionContext) context = NULL; glnx_unref_object OstreeRepo *repo = NULL; - GHashTableIter hash_iter; - gpointer key, value; gboolean found_corruption = FALSE; - guint n_partial = 0; - g_autoptr(GHashTable) all_refs = NULL; - g_autoptr(GHashTable) objects = NULL; - g_autoptr(GHashTable) commits = NULL; - g_autoptr(GPtrArray) tombstones = NULL; - context = g_option_context_new ("- Check the repository for consistency"); + g_autoptr(GOptionContext) context = g_option_context_new ("- Check the repository for consistency"); if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) - goto out; + return FALSE; if (!opt_quiet) g_print ("Validating refs...\n"); /* Validate that the commit for each ref is available */ + g_autoptr(GHashTable) all_refs = NULL; if (!ostree_repo_list_refs (repo, NULL, &all_refs, cancellable, error)) return FALSE; + + GHashTableIter hash_iter; + gpointer key, value; g_hash_table_iter_init (&hash_iter, all_refs); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { @@ -270,27 +238,27 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** g_autoptr(GVariant) commit = NULL; if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum, &commit, error)) - { - g_prefix_error (error, "Loading commit for ref %s: ", refname); - goto out; - } + return glnx_prefix_error (error, "Loading commit for ref %s", refname); } if (!opt_quiet) g_print ("Enumerating objects...\n"); + g_autoptr(GHashTable) objects = NULL; if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error)) - goto out; + return FALSE; - commits = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, - (GDestroyNotify)g_variant_unref, NULL); - - g_hash_table_iter_init (&hash_iter, objects); + g_autoptr(GHashTable) commits = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, + (GDestroyNotify)g_variant_unref, NULL); + + g_autoptr(GPtrArray) tombstones = NULL; if (opt_add_tombstones) tombstones = g_ptr_array_new_with_free_func (g_free); + guint n_partial = 0; + g_hash_table_iter_init (&hash_iter, objects); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { GVariant *serialized_key = key; @@ -304,7 +272,7 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!ostree_repo_load_commit (repo, checksum, &commit, &commitstate, error)) - goto out; + return FALSE; if (opt_add_tombstones) { @@ -324,7 +292,7 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** else { g_propagate_error (error, local_error); - goto out; + return FALSE; } } } @@ -345,7 +313,7 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** if (!fsck_reachable_objects_from_commits (repo, commits, &found_corruption, cancellable, error)) - goto out; + return FALSE; if (opt_add_tombstones) { @@ -353,14 +321,14 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** if (tombstones->len) { if (!ot_enable_tombstone_commits (repo, error)) - goto out; + return FALSE; } for (i = 0; i < tombstones->len; i++) { const char *checksum = tombstones->pdata[i]; g_print ("Adding tombstone for commit %s\n", checksum); if (!ostree_repo_delete_object (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum, cancellable, error)) - goto out; + return FALSE; } } else if (n_partial > 0) @@ -369,13 +337,7 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** } if (found_corruption) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Repository corruption encountered"); - goto out; - } + return glnx_throw (error, "Repository corruption encountered"); - ret = TRUE; - out: - return ret; + return TRUE; } From da0791f484ff6c06eed1d7ba83bddbc931cfdd2d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Jun 2017 16:26:44 -0400 Subject: [PATCH 17/82] tests: add a syntax-check rule for glnx_prefix_error() Same as the errno variant; the colon-space `: ` thing got me in a different patch. Closes: #956 Approved by: jlebon --- cfg.mk | 4 ++++ src/libostree/ostree-repo-commit.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cfg.mk b/cfg.mk index b4ffaf5f..f6f1ce25 100644 --- a/cfg.mk +++ b/cfg.mk @@ -23,6 +23,10 @@ local-checks-to-skip = \ sc_prohibit_path_max_allocation \ sc_trailing_blank \ +sc_glnx_prefix_error_colon: + @prohibit='\ Date: Fri, 9 Jun 2017 09:50:38 +0100 Subject: [PATCH 18/82] lib/repo: Split out ref handling from regenerate_summary() This will make some future additions to regenerate_summary() easier. This commit introduces no functional changes. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- src/libostree/ostree-repo.c | 75 +++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1b30e913..f2b7c21b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4477,6 +4477,51 @@ ostree_repo_verify_summary (OstreeRepo *self, error); } +/* Add an entry for a @ref ↦ @checksum mapping to an `a(s(t@ay@a{sv}))` + * @refs_builder to go into a `summary` file. This includes building the + * standard additional metadata keys for the ref. */ +static gboolean +summary_add_ref_entry (OstreeRepo *self, + const char *ref, + const char *checksum, + GVariantBuilder *refs_builder, + GError **error) +{ + g_auto(GVariantDict) commit_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; + + g_assert (ref); g_assert (checksum); + + g_autofree char *remotename = NULL; + if (!ostree_parse_refspec (ref, &remotename, NULL, NULL)) + g_assert_not_reached (); + + /* Don't put remote refs in the summary */ + if (remotename != NULL) + return TRUE; + + g_autoptr(GVariant) commit_obj = NULL; + if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, checksum, &commit_obj, error)) + return FALSE; + + g_variant_dict_init (&commit_metadata_builder, NULL); + + /* Forward the commit’s timestamp if it’s valid. */ + guint64 commit_timestamp = ostree_commit_get_timestamp (commit_obj); + g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (commit_timestamp); + + if (dt != NULL) + g_variant_dict_insert_value (&commit_metadata_builder, OSTREE_COMMIT_TIMESTAMP, + g_variant_new_uint64 (GUINT64_TO_BE (commit_timestamp))); + + g_variant_builder_add_value (refs_builder, + g_variant_new ("(s(t@ay@a{sv}))", ref, + (guint64) g_variant_get_size (commit_obj), + ostree_checksum_to_bytes_v (checksum), + g_variant_dict_end (&commit_metadata_builder))); + + return TRUE; +} + /** * ostree_repo_regenerate_summary: * @self: Repo @@ -4516,37 +4561,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self, { const char *ref = iter->data; const char *commit = g_hash_table_lookup (refs, ref); - g_auto(GVariantDict) commit_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; - g_assert (commit); - - g_autofree char *remotename = NULL; - if (!ostree_parse_refspec (ref, &remotename, NULL, NULL)) - g_assert_not_reached (); - - /* Don't put remote refs in the summary */ - if (remotename != NULL) - continue; - - g_autoptr(GVariant) commit_obj = NULL; - if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_obj, error)) + if (!summary_add_ref_entry (self, ref, commit, refs_builder, error)) return FALSE; - - g_variant_dict_init (&commit_metadata_builder, NULL); - - /* Forward the commit’s timestamp if it’s valid. */ - guint64 commit_timestamp = ostree_commit_get_timestamp (commit_obj); - g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (commit_timestamp); - - if (dt != NULL) - g_variant_dict_insert_value (&commit_metadata_builder, OSTREE_COMMIT_TIMESTAMP, - g_variant_new_uint64 (GUINT64_TO_BE (commit_timestamp))); - - g_variant_builder_add_value (refs_builder, - g_variant_new ("(s(t@ay@a{sv}))", ref, - (guint64) g_variant_get_size (commit_obj), - ostree_checksum_to_bytes_v (commit), - g_variant_dict_end (&commit_metadata_builder))); } } From 0a20e7d43c43c9d5793f0fdb5bf8422affa5d841 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 15:27:10 +0100 Subject: [PATCH 19/82] lib/ref: Add OstreeCollectionRef type for globally unique refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a type representing the tuple (collection ID, ref name), which is guaranteed to be globally unique. It will be used in upcoming commits. It introduces the concept of a ‘collection’ which is a unique, curated set of refs which lie in the same trust domain (i.e. all signed by the same key and validated by the same developer). Flathub might be a collection, for example; or the set of OS refs coming from a particular OS vendor. It includes a function for validating collection IDs. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- Makefile-libostree-defines.am | 1 + Makefile-libostree.am | 2 + apidoc/ostree-experimental-sections.txt | 20 +++ src/libostree/libostree-experimental.sym | 13 ++ src/libostree/ostree-autocleanups.h | 2 + src/libostree/ostree-core-private.h | 9 ++ src/libostree/ostree-core.c | 32 ++++ src/libostree/ostree-core.h | 5 + src/libostree/ostree-ref.c | 193 +++++++++++++++++++++++ src/libostree/ostree-ref.h | 90 +++++++++++ src/libostree/ostree-repo.h | 3 + src/libostree/ostree.h | 4 + 12 files changed, 374 insertions(+) create mode 100644 src/libostree/ostree-ref.c create mode 100644 src/libostree/ostree-ref.h diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index aff7e52b..44b6cd0a 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -40,6 +40,7 @@ libostree_public_headers = \ if ENABLE_EXPERIMENTAL_API libostree_public_headers += \ + src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ $(NULL) endif diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 61ad1f4a..a9c392c1 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -94,6 +94,7 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-linuxfsutil.c \ src/libostree/ostree-diff.c \ src/libostree/ostree-mutable-tree.c \ + src/libostree/ostree-ref.c \ src/libostree/ostree-remote.c \ src/libostree/ostree-remote-private.h \ src/libostree/ostree-repo.c \ @@ -151,6 +152,7 @@ libostree_1_la_SOURCES += \ endif if !ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += \ + src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ $(NULL) endif diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 998a4f9c..30355967 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -1,3 +1,18 @@ +
+ostree-ref +OstreeCollectionRef +ostree_collection_ref_new +ostree_collection_ref_dup +ostree_collection_ref_free +ostree_collection_ref_hash +ostree_collection_ref_equal +OstreeCollectionRefv +ostree_collection_ref_dupv +ostree_collection_ref_freev + +ostree_collection_ref_get_type +
+
ostree-remote OstreeRemote @@ -5,3 +20,8 @@ ostree_remote_ref ostree_remote_unref ostree_remote_get_name
+ +
+ostree-misc-experimental +ostree_validate_collection_id +
diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index 9ccca958..0dabd591 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -34,3 +34,16 @@ LIBOSTREE_2017.7_EXPERIMENTAL { global: ostree_remote_get_name; } LIBOSTREE_2017.6_EXPERIMENTAL; + +LIBOSTREE_2017.8_EXPERIMENTAL { +global: + ostree_collection_ref_dup; + ostree_collection_ref_dupv; + ostree_collection_ref_equal; + ostree_collection_ref_free; + ostree_collection_ref_freev; + ostree_collection_ref_get_type; + ostree_collection_ref_hash; + ostree_collection_ref_new; + ostree_validate_collection_id; +} LIBOSTREE_2017.7_EXPERIMENTAL; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index 1fdb50ad..f683c2e3 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -60,6 +60,8 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSysrootUpgrader, g_object_unref) G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_commit_traverse_iter_clear) #ifdef OSTREE_ENABLE_EXPERIMENTAL_API +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) +G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 72b88aba..a4a31034 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -173,6 +173,15 @@ _ostree_raw_file_to_archive_stream (GInputStream *input, GCancellable *cancellable, GError **error); +#ifndef OSTREE_ENABLE_EXPERIMENTAL_API +gboolean ostree_validate_collection_id (const char *collection_id, GError **error); +#endif /* !OSTREE_ENABLE_EXPERIMENTAL_API */ +#if (defined(OSTREE_COMPILATION) || GLIB_CHECK_VERSION(2, 44, 0)) && !defined(OSTREE_ENABLE_EXPERIMENTAL_API) +#include +#include "ostree-ref.h" +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) +G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) +#endif G_END_DECLS diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 32516b7c..c0b7cbe3 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -210,6 +210,38 @@ ostree_validate_remote_name (const char *remote_name, return TRUE; } +/** + * ostree_validate_collection_id: + * @rev: (nullable): A collection ID + * @error: Error + * + * Check whether the given @collection_id is valid. Return an error if it is + * invalid or %NULL. + * + * Valid collection IDs are reverse DNS names: + * * They are composed of 1 or more elements separated by a period (`.`) character. + * All elements must contain at least one character. + * * Each element must only contain the ASCII characters `[A-Z][a-z][0-9]_` and must not + * begin with a digit. + * * They must contain at least one `.` (period) character (and thus at least two elements). + * * They must not begin with a `.` (period) character. + * * They must not exceed 255 characters in length. + * + * (This makes their format identical to D-Bus interface names, for consistency.) + * + * Returns: %TRUE if @collection_id is a valid collection ID, %FALSE if it is invalid + * or %NULL + */ +gboolean +ostree_validate_collection_id (const char *collection_id, GError **error) +{ + /* Abuse g_dbus_is_interface_name(), since collection IDs have the same format. */ + if (collection_id == NULL || !g_dbus_is_interface_name (collection_id)) + return glnx_throw (error, "Invalid collection ID %s", collection_id); + + return TRUE; +} + GVariant * _ostree_file_header_new (GFileInfo *file_info, GVariant *xattrs) diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index dc64d89b..fa9e5e86 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -242,6 +242,11 @@ int ostree_cmp_checksum_bytes (const guchar *a, const guchar *b); _OSTREE_PUBLIC gboolean ostree_validate_rev (const char *rev, GError **error); +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +_OSTREE_PUBLIC +gboolean ostree_validate_collection_id (const char *collection_id, GError **error); +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + _OSTREE_PUBLIC gboolean ostree_validate_remote_name (const char *remote_name, GError **error); diff --git a/src/libostree/ostree-ref.c b/src/libostree/ostree-ref.c new file mode 100644 index 00000000..8c84946c --- /dev/null +++ b/src/libostree/ostree-ref.c @@ -0,0 +1,193 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "ostree-autocleanups.h" +#include "ostree-core.h" +#include "ostree-core-private.h" +#include "ostree-ref.h" + +G_DEFINE_BOXED_TYPE (OstreeCollectionRef, ostree_collection_ref, + ostree_collection_ref_dup, ostree_collection_ref_free) + +/** + * ostree_collection_ref_new: + * @collection_id: (nullable): a collection ID, or %NULL for a plain ref + * @ref_name: a ref name + * + * Create a new #OstreeCollectionRef containing (@collection_id, @ref_name). If + * @collection_id is %NULL, this is equivalent to a plain ref name string (not a + * refspec; no remote name is included), which can be used for non-P2P + * operations. + * + * Returns: (transfer full): a new #OstreeCollectionRef + * Since: 2017.8 + */ +OstreeCollectionRef * +ostree_collection_ref_new (const gchar *collection_id, + const gchar *ref_name) +{ + g_autoptr(OstreeCollectionRef) collection_ref = NULL; + + g_return_val_if_fail (collection_id == NULL || + ostree_validate_collection_id (collection_id, NULL), NULL); + g_return_val_if_fail (ostree_validate_rev (ref_name, NULL), NULL); + + collection_ref = g_new0 (OstreeCollectionRef, 1); + collection_ref->collection_id = g_strdup (collection_id); + collection_ref->ref_name = g_strdup (ref_name); + + return g_steal_pointer (&collection_ref); +} + +/** + * ostree_collection_ref_dup: + * @ref: an #OstreeCollectionRef + * + * Create a copy of the given @ref. + * + * Returns: (transfer full): a newly allocated copy of @ref + * Since: 2017.8 + */ +OstreeCollectionRef * +ostree_collection_ref_dup (const OstreeCollectionRef *ref) +{ + g_return_val_if_fail (ref != NULL, NULL); + + return ostree_collection_ref_new (ref->collection_id, ref->ref_name); +} + +/** + * ostree_collection_ref_free: + * @ref: (transfer full): an #OstreeCollectionRef + * + * Free the given @ref. + * + * Since: 2017.8 + */ +void +ostree_collection_ref_free (OstreeCollectionRef *ref) +{ + g_return_if_fail (ref != NULL); + + g_free (ref->collection_id); + g_free (ref->ref_name); + g_free (ref); +} + +/** + * ostree_collection_ref_hash: + * @ref: an #OstreeCollectionRef + * + * Hash the given @ref. This function is suitable for use with #GHashTable. + * @ref must be non-%NULL. + * + * Returns: hash value for @ref + * Since: 2017.8 + */ +guint +ostree_collection_ref_hash (gconstpointer ref) +{ + const OstreeCollectionRef *_ref = ref; + + if (_ref->collection_id != NULL) + return g_str_hash (_ref->collection_id) ^ g_str_hash (_ref->ref_name); + else + return g_str_hash (_ref->ref_name); +} + +/** + * ostree_collection_ref_equal: + * @ref1: an #OstreeCollectionRef + * @ref2: another #OstreeCollectionRef + * + * Compare @ref1 and @ref2 and return %TRUE if they have the same collection ID and + * ref name, and %FALSE otherwise. Both @ref1 and @ref2 must be non-%NULL. + * + * Returns: %TRUE if @ref1 and @ref2 are equal, %FALSE otherwise + * Since: 2017.8 + */ +gboolean +ostree_collection_ref_equal (gconstpointer ref1, + gconstpointer ref2) +{ + const OstreeCollectionRef *_ref1 = ref1, *_ref2 = ref2; + + return (g_strcmp0 (_ref1->collection_id, _ref2->collection_id) == 0 && + g_strcmp0 (_ref1->ref_name, _ref2->ref_name) == 0); +} + +/** + * ostree_collection_ref_dupv: + * @refs: (array zero-terminated=1): %NULL-terminated array of #OstreeCollectionRefs + * + * Copy an array of #OstreeCollectionRefs, including deep copies of all its + * elements. @refs must be %NULL-terminated; it may be empty, but must not be + * %NULL. + * + * Returns: (transfer full) (array zero-terminated=1): a newly allocated copy of @refs + * Since: 2017.8 + */ +OstreeCollectionRef ** +ostree_collection_ref_dupv (const OstreeCollectionRef * const *refs) +{ + gsize i, n_refs = g_strv_length ((gchar **) refs); /* hack */ + g_auto(OstreeCollectionRefv) new_refs = NULL; + + g_return_val_if_fail (refs != NULL, NULL); + + new_refs = g_new0 (OstreeCollectionRef*, n_refs + 1); + + for (i = 0; i < n_refs; i++) + new_refs[i] = ostree_collection_ref_dup (refs[i]); + new_refs[i] = NULL; + + return g_steal_pointer (&new_refs); +} + +/** + * ostree_collection_ref_freev: + * @refs: (transfer full) (array zero-terminated=1): an array of #OstreeCollectionRefs + * + * Free the given array of @refs, including freeing all its elements. @refs + * must be %NULL-terminated; it may be empty, but must not be %NULL. + * + * Since: 2017.8 + */ +void +ostree_collection_ref_freev (OstreeCollectionRef **refs) +{ + gsize i; + + g_return_if_fail (refs != NULL); + + for (i = 0; refs[i] != NULL; i++) + ostree_collection_ref_free (refs[i]); + g_free (refs); +} diff --git a/src/libostree/ostree-ref.h b/src/libostree/ostree-ref.h new file mode 100644 index 00000000..4d43153c --- /dev/null +++ b/src/libostree/ostree-ref.h @@ -0,0 +1,90 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-types.h" + +G_BEGIN_DECLS + +/** + * OstreeCollectionRef: + * @collection_id: (nullable): collection ID which provided the ref, or %NULL if there + * is no associated collection + * @ref_name: ref name + * + * A structure which globally uniquely identifies a ref as the tuple + * (@collection_id, @ref_name). For backwards compatibility, @collection_id may be %NULL, + * indicating a ref name which is not globally unique. + * + * Since: 2017.8 + */ +typedef struct +{ + gchar *collection_id; /* (nullable) */ + gchar *ref_name; /* (not nullable) */ +} OstreeCollectionRef; + +#ifndef __GI_SCANNER__ +_OSTREE_PUBLIC +GType ostree_collection_ref_get_type (void); +#endif + +_OSTREE_PUBLIC +OstreeCollectionRef *ostree_collection_ref_new (const gchar *collection_id, + const gchar *ref_name); +_OSTREE_PUBLIC +OstreeCollectionRef *ostree_collection_ref_dup (const OstreeCollectionRef *ref); +_OSTREE_PUBLIC +void ostree_collection_ref_free (OstreeCollectionRef *ref); + +_OSTREE_PUBLIC +guint ostree_collection_ref_hash (gconstpointer ref); +_OSTREE_PUBLIC +gboolean ostree_collection_ref_equal (gconstpointer ref1, + gconstpointer ref2); + +_OSTREE_PUBLIC +OstreeCollectionRef **ostree_collection_ref_dupv (const OstreeCollectionRef * const *refs); +_OSTREE_PUBLIC +void ostree_collection_ref_freev (OstreeCollectionRef **refs); + +/** + * OstreeCollectionRefv: + * + * A %NULL-terminated array of #OstreeCollectionRef instances, designed to + * be used with g_auto(): + * + * |[ + * g_auto(OstreeCollectionRefv) refs = NULL; + * ]| + * + * Since: 2017.8 + */ +typedef OstreeCollectionRef** OstreeCollectionRefv; + +G_END_DECLS diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index ed73d4a2..04294f33 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -25,6 +25,9 @@ #include "ostree-core.h" #include "ostree-types.h" #include "ostree-async-progress.h" +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +#include "ostree-ref.h" +#endif #include "ostree-sepolicy.h" #include "ostree-gpg-verify-result.h" diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index 5d1ac1e1..c9c3bb75 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -35,5 +35,9 @@ #include #include +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +#include +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + #include #include From 4de736fdfa6751139e7773eca09812d3b7ba5286 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 12:38:59 +0100 Subject: [PATCH 20/82] lib/repo: Add collection ID support to OstreeRepo Add {get,set}_collection_id() methods to OstreeRepo and some documentation about the concept of a collection ID which globally identifies an upstream repository. See the documentation for more details. This will be used in future commits. For now, the new API is marked as experimental (--enable-experimental-api). Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- apidoc/ostree-experimental-sections.txt | 2 + src/libostree/libostree-experimental.sym | 2 + src/libostree/ostree-repo-private.h | 11 ++++ src/libostree/ostree-repo.c | 83 ++++++++++++++++++++++++ src/libostree/ostree-repo.h | 11 ++++ 5 files changed, 109 insertions(+) diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 30355967..16983ae8 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -23,5 +23,7 @@ ostree_remote_get_name
ostree-misc-experimental +ostree_repo_get_collection_id +ostree_repo_set_collection_id ostree_validate_collection_id
diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index 0dabd591..dad788b7 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -45,5 +45,7 @@ global: ostree_collection_ref_get_type; ostree_collection_ref_hash; ostree_collection_ref_new; + ostree_repo_get_collection_id; + ostree_repo_set_collection_id; ostree_validate_collection_id; } LIBOSTREE_2017.7_EXPERIMENTAL; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 8f87b103..825c1ffc 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -20,6 +20,7 @@ #pragma once +#include "ostree-ref.h" #include "ostree-repo.h" #include "ostree-remote-private.h" #include "libglnx.h" @@ -129,6 +130,7 @@ struct OstreeRepo { gboolean enable_uncompressed_cache; gboolean generate_sizes; guint64 tmp_expiry_seconds; + gchar *collection_id; OstreeRepo *parent_repo; }; @@ -349,4 +351,13 @@ _ostree_repo_get_remote_inherited (OstreeRepo *self, const char *name, GError **error); +#ifndef OSTREE_ENABLE_EXPERIMENTAL_API + +const gchar * ostree_repo_get_collection_id (OstreeRepo *self); +gboolean ostree_repo_set_collection_id (OstreeRepo *self, + const gchar *collection_id, + GError **error); + +#endif /* !OSTREE_ENABLE_EXPERIMENTAL_API */ + G_END_DECLS diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index f2b7c21b..24e94e03 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -77,6 +77,25 @@ * Once the #OstreeMutableTree is complete, write all of its metadata * with ostree_repo_write_mtree(), and finally create a commit with * ostree_repo_write_commit(). + * + * ## Collection IDs + * + * A collection ID is a globally unique identifier which, if set, is used to + * identify refs from a repository which are mirrored elsewhere, such as in + * mirror repositories or peer to peer networks. + * + * This is separate from the `collection-id` configuration key for a remote, which + * is used to store the collection ID of the repository that remote points to. + * + * The collection ID should only be set on an #OstreeRepo if it is the canonical + * collection for some refs. + * + * A collection ID must be a reverse DNS name, where the domain name is under the + * control of the curator of the collection, so they can demonstrate ownership + * of the collection. The later elements in the reverse DNS name can be used to + * disambiguate between multiple collections from the same curator. For example, + * `org.exampleos.Main` and `org.exampleos.Apps`. For the complete format of + * collection IDs, see ostree_validate_collection_id(). */ typedef struct { GObjectClass parent_class; @@ -461,6 +480,7 @@ ostree_repo_finalize (GObject *object) g_clear_pointer (&self->dirmeta_cache, (GDestroyNotify) g_hash_table_unref); g_mutex_clear (&self->cache_lock); g_mutex_clear (&self->txn_stats_lock); + g_free (self->collection_id); g_clear_pointer (&self->remotes, g_hash_table_destroy); g_mutex_clear (&self->remotes_lock); @@ -1707,6 +1727,9 @@ ostree_repo_create (OstreeRepo *self, g_string_append_printf (config_data, "mode=%s\n", mode_str); + if (self->collection_id != NULL) + g_string_append_printf (config_data, "collection-id=%s\n", self->collection_id); + if (!glnx_file_replace_contents_at (dfd, "config", (guint8*)config_data->str, config_data->len, 0, cancellable, error)) @@ -1939,6 +1962,13 @@ reload_core_config (OstreeRepo *self, self->zlib_compression_level = OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL; } + { + g_clear_pointer (&self->collection_id, g_free); + if (!ot_keyfile_get_value_with_default (self->config, "core", "collection-id", + NULL, &self->collection_id, NULL)) + return FALSE; + } + if (!ot_keyfile_get_value_with_default (self->config, "core", "parent", NULL, &parent_repo_path, error)) return FALSE; @@ -4827,3 +4857,56 @@ _ostree_repo_memory_cache_ref_destroy (OstreeRepoMemoryCacheRef *state) g_mutex_unlock (lock); g_object_unref (repo); } + +/** + * ostree_repo_get_collection_id: + * @self: an #OstreeRepo + * + * Get the collection ID of this repository. See [collection IDs][collection-ids]. + * + * Returns: (nullable): collection ID for the repository + * Since: 2017.8 + */ +const gchar * +ostree_repo_get_collection_id (OstreeRepo *self) +{ + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + + return self->collection_id; +} + +/** + * ostree_repo_set_collection_id: + * @self: an #OstreeRepo + * @collection_id: (nullable): new collection ID, or %NULL to unset it + * @error: return location for a #GError, or %NULL + * + * Set or clear the collection ID of this repository. See [collection IDs][collection-ids]. + * The update will be made in memory, but must be written out to the repository + * configuration on disk using ostree_repo_write_config(). + * + * Returns: %TRUE on success, %FALSE otherwise + * Since: 2017.8 + */ +gboolean +ostree_repo_set_collection_id (OstreeRepo *self, + const gchar *collection_id, + GError **error) +{ + if (collection_id != NULL && !ostree_validate_collection_id (collection_id, error)) + return FALSE; + + g_autofree gchar *new_collection_id = g_strdup (collection_id); + g_free (self->collection_id); + self->collection_id = g_steal_pointer (&new_collection_id); + + if (self->config != NULL) + { + if (collection_id != NULL) + g_key_file_set_string (self->config, "core", "collection-id", collection_id); + else + return g_key_file_remove_key (self->config, "core", "collection-id", error); + } + + return TRUE; +} diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 04294f33..566754d4 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -89,6 +89,17 @@ gboolean ostree_repo_create (OstreeRepo *self, GCancellable *cancellable, GError **error); +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + +_OSTREE_PUBLIC +const gchar * ostree_repo_get_collection_id (OstreeRepo *self); +_OSTREE_PUBLIC +gboolean ostree_repo_set_collection_id (OstreeRepo *self, + const gchar *collection_id, + GError **error); + +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + _OSTREE_PUBLIC GFile * ostree_repo_get_path (OstreeRepo *self); From fbf8df882964572e3735ef2b6fc9204d78f9502a Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:25:59 +0100 Subject: [PATCH 21/82] =?UTF-8?q?lib/refs:=20Add=20methods=20for=20setting?= =?UTF-8?q?/listing=20collection=E2=80=93refs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These are tuples of (collection ID, ref name) which are a globally-unique form of local ref. They use OstreeCollectionRef as an identifier, and hence need to be accessed using new API, as the existing API uses string identifiers and sometimes accepts refspecs. Remote names are not supported as part an OstreeCollectionRef. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- apidoc/ostree-experimental-sections.txt | 3 + src/libostree/libostree-experimental.sym | 3 + src/libostree/ostree-repo-commit.c | 81 ++++++++- src/libostree/ostree-repo-private.h | 43 ++++- src/libostree/ostree-repo-prune.c | 23 ++- src/libostree/ostree-repo-refs.c | 213 ++++++++++++++++++++--- src/libostree/ostree-repo.c | 122 +++++++++++-- src/libostree/ostree-repo.h | 31 ++++ tests/basic-test.sh | 2 +- 9 files changed, 471 insertions(+), 50 deletions(-) diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 16983ae8..78a50100 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -26,4 +26,7 @@ ostree_remote_get_name ostree_repo_get_collection_id ostree_repo_set_collection_id ostree_validate_collection_id +ostree_repo_list_collection_refs +ostree_repo_set_collection_ref_immediate +ostree_repo_transaction_set_collection_ref diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index dad788b7..9d2024f3 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -46,6 +46,9 @@ global: ostree_collection_ref_hash; ostree_collection_ref_new; ostree_repo_get_collection_id; + ostree_repo_list_collection_refs; ostree_repo_set_collection_id; + ostree_repo_set_collection_ref_immediate; + ostree_repo_transaction_set_collection_ref; ostree_validate_collection_id; } LIBOSTREE_2017.7_EXPERIMENTAL; diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index cf6d90e7..8d474d63 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1347,6 +1347,11 @@ ensure_txn_refs (OstreeRepo *self) { if (self->txn_refs == NULL) self->txn_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + if (self->txn_collection_refs == NULL) + self->txn_collection_refs = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + (GDestroyNotify) ostree_collection_ref_free, + g_free); } /** @@ -1410,6 +1415,41 @@ ostree_repo_transaction_set_ref (OstreeRepo *self, g_hash_table_replace (self->txn_refs, refspec, g_strdup (checksum)); } +/** + * ostree_repo_transaction_set_collection_ref: + * @self: An #OstreeRepo + * @ref: The collection–ref to write + * @checksum: (nullable): The checksum to point it to + * + * If @checksum is not %NULL, then record it as the target of local ref named + * @ref. + * + * Otherwise, if @checksum is %NULL, then record that the ref should + * be deleted. + * + * The change will not be written out immediately, but when the transaction + * is completed with ostree_repo_commit_transaction(). If the transaction + * is instead aborted with ostree_repo_abort_transaction(), no changes will + * be made to the repository. + * + * Since: 2017.8 + */ +void +ostree_repo_transaction_set_collection_ref (OstreeRepo *self, + const OstreeCollectionRef *ref, + const char *checksum) +{ + g_return_if_fail (OSTREE_IS_REPO (self)); + g_return_if_fail (self->in_transaction == TRUE); + g_return_if_fail (ref != NULL); + g_return_if_fail (checksum == NULL || ostree_validate_checksum_string (checksum, NULL)); + + ensure_txn_refs (self); + + g_hash_table_replace (self->txn_collection_refs, + ostree_collection_ref_dup (ref), g_strdup (checksum)); +} + /** * ostree_repo_set_ref_immediate: * @self: An #OstreeRepo @@ -1431,7 +1471,40 @@ ostree_repo_set_ref_immediate (OstreeRepo *self, GCancellable *cancellable, GError **error) { - return _ostree_repo_write_ref (self, remote, ref, checksum, + const OstreeCollectionRef _ref = { NULL, (gchar *) ref }; + return _ostree_repo_write_ref (self, remote, &_ref, checksum, + cancellable, error); +} + +/** + * ostree_repo_set_collection_ref_immediate: + * @self: An #OstreeRepo + * @ref: The collection–ref to write + * @checksum: (nullable): The checksum to point it to, or %NULL to unset + * @cancellable: GCancellable + * @error: GError + * + * This is like ostree_repo_transaction_set_collection_ref(), except it may be + * invoked outside of a transaction. This is presently safe for the + * case where we're creating or overwriting an existing ref. + * + * Returns: %TRUE on success, %FALSE otherwise + * Since: 2017.8 + */ +gboolean +ostree_repo_set_collection_ref_immediate (OstreeRepo *self, + const OstreeCollectionRef *ref, + const char *checksum, + GCancellable *cancellable, + GError **error) +{ + g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); + g_return_val_if_fail (ref != NULL, FALSE); + g_return_val_if_fail (checksum == NULL || ostree_validate_checksum_string (checksum, NULL), FALSE); + g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), FALSE); + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + + return _ostree_repo_write_ref (self, NULL, ref, checksum, cancellable, error); } @@ -1481,6 +1554,11 @@ ostree_repo_commit_transaction (OstreeRepo *self, return FALSE; g_clear_pointer (&self->txn_refs, g_hash_table_destroy); + if (self->txn_collection_refs) + if (!_ostree_repo_update_collection_refs (self, self->txn_collection_refs, cancellable, error)) + return FALSE; + g_clear_pointer (&self->txn_collection_refs, g_hash_table_destroy); + if (self->commit_stagedir_fd != -1) { (void) close (self->commit_stagedir_fd); @@ -1518,6 +1596,7 @@ ostree_repo_abort_transaction (OstreeRepo *self, g_hash_table_remove_all (self->loose_object_devino_hash); g_clear_pointer (&self->txn_refs, g_hash_table_destroy); + g_clear_pointer (&self->txn_collection_refs, g_hash_table_destroy); if (self->commit_stagedir_fd != -1) { diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 825c1ffc..0081eb31 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -48,6 +48,8 @@ G_BEGIN_DECLS /* Well-known keys for the additional metadata field in a summary file. */ #define OSTREE_SUMMARY_LAST_MODIFIED "ostree.summary.last-modified" #define OSTREE_SUMMARY_EXPIRES "ostree.summary.expires" +#define OSTREE_SUMMARY_COLLECTION_ID "ostree.summary.collection-id" +#define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map" /* Well-known keys for the additional metadata field in a commit in a ref entry * in a summary file. */ @@ -96,7 +98,8 @@ struct OstreeRepo { GFile *sysroot_dir; char *remotes_config_dir; - GHashTable *txn_refs; + GHashTable *txn_refs; /* (element-type utf8 utf8) */ + GHashTable *txn_collection_refs; /* (element-type OstreeCollectionRef utf8) */ GMutex txn_stats_lock; OstreeRepoTransactionStats txn_stats; @@ -221,7 +224,13 @@ _ostree_repo_update_refs (OstreeRepo *self, GCancellable *cancellable, GError **error); -gboolean +gboolean +_ostree_repo_update_collection_refs (OstreeRepo *self, + GHashTable *refs, + GCancellable *cancellable, + GError **error); + +gboolean _ostree_repo_file_replace_contents (OstreeRepo *self, int dfd, const char *path, @@ -230,13 +239,13 @@ _ostree_repo_file_replace_contents (OstreeRepo *self, GCancellable *cancellable, GError **error); -gboolean -_ostree_repo_write_ref (OstreeRepo *self, - const char *remote, - const char *ref, - const char *rev, - GCancellable *cancellable, - GError **error); +gboolean +_ostree_repo_write_ref (OstreeRepo *self, + const char *remote, + const OstreeCollectionRef *ref, + const char *rev, + GCancellable *cancellable, + GError **error); OstreeRepoFile * _ostree_repo_file_new_for_commit (OstreeRepo *repo, @@ -358,6 +367,22 @@ gboolean ostree_repo_set_collection_id (OstreeRepo *self, const gchar *collection_id, GError **error); +gboolean ostree_repo_list_collection_refs (OstreeRepo *self, + const char *match_collection_id, + GHashTable **out_all_refs, + GCancellable *cancellable, + GError **error); + +void ostree_repo_transaction_set_collection_ref (OstreeRepo *self, + const OstreeCollectionRef *ref, + const char *checksum); + +gboolean ostree_repo_set_collection_ref_immediate (OstreeRepo *self, + const OstreeCollectionRef *ref, + const char *checksum, + GCancellable *cancellable, + GError **error); + #endif /* !OSTREE_ENABLE_EXPERIMENTAL_API */ G_END_DECLS diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index bf0a2530..c0da7121 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -312,7 +312,6 @@ ostree_repo_prune (OstreeRepo *self, GHashTableIter hash_iter; gpointer key, value; g_autoptr(GHashTable) objects = NULL; - g_autoptr(GHashTable) all_refs = NULL; g_autoptr(GHashTable) reachable = NULL; gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY; @@ -325,12 +324,34 @@ ostree_repo_prune (OstreeRepo *self, if (refs_only) { + /* Ignoring collections. */ + g_autoptr(GHashTable) all_refs = NULL; /* (element-type utf8 utf8) */ + if (!ostree_repo_list_refs (self, NULL, &all_refs, cancellable, error)) return FALSE; g_hash_table_iter_init (&hash_iter, all_refs); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *checksum = value; + + g_debug ("Finding objects to keep for commit %s", checksum); + if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, + cancellable, error)) + return FALSE; + } + + /* Using collections. */ + g_autoptr(GHashTable) all_collection_refs = NULL; /* (element-type OstreeChecksumRef utf8) */ + + if (!ostree_repo_list_collection_refs (self, NULL, &all_collection_refs, + cancellable, error)) + return FALSE; + + g_hash_table_iter_init (&hash_iter, all_collection_refs); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *checksum = value; diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index adab50fe..849d5d8f 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -20,18 +20,25 @@ #include "config.h" +#include "ostree-core-private.h" #include "ostree-repo-private.h" #include "otutil.h" #include "ot-fs-utils.h" +/* This is polymorphic in @collection_id: if non-%NULL, @refs will be treated as of + * type OstreeCollectionRef ↦ checksum. Otherwise, it will be treated as of type + * refspec ↦ checksum. */ static gboolean add_ref_to_set (const char *remote, + const char *collection_id, int base_fd, const char *path, GHashTable *refs, GCancellable *cancellable, GError **error) { + g_return_val_if_fail (remote == NULL || collection_id == NULL, FALSE); + gsize len; char *contents = glnx_file_get_contents_utf8_at (base_fd, path, &len, cancellable, error); if (!contents) @@ -39,14 +46,21 @@ add_ref_to_set (const char *remote, g_strchomp (contents); - g_autoptr(GString) refname = g_string_new (""); - if (remote) + if (collection_id == NULL) { - g_string_append (refname, remote); - g_string_append_c (refname, ':'); + g_autoptr(GString) refname = g_string_new (""); + if (remote) + { + g_string_append (refname, remote); + g_string_append_c (refname, ':'); + } + g_string_append (refname, path); + g_hash_table_insert (refs, g_string_free (g_steal_pointer (&refname), FALSE), contents); + } + else + { + g_hash_table_insert (refs, ostree_collection_ref_new (collection_id, path), contents); } - g_string_append (refname, path); - g_hash_table_insert (refs, g_string_free (g_steal_pointer (&refname), FALSE), contents); return TRUE; } @@ -99,6 +113,8 @@ write_checksum_file_at (OstreeRepo *self, g_clear_error (&temp_error); + /* FIXME: Conflict detection needs to be extended to collection–refs + * using ostree_repo_list_collection_refs(). */ if (!ostree_repo_list_refs (self, name, &refs, cancellable, error)) return FALSE; @@ -456,6 +472,7 @@ ostree_repo_resolve_rev_ext (OstreeRepo *self, static gboolean enumerate_refs_recurse (OstreeRepo *repo, const char *remote, + const char *collection_id, int base_dfd, GString *base_path, int child_dfd, @@ -485,14 +502,14 @@ enumerate_refs_recurse (OstreeRepo *repo, { g_string_append_c (base_path, '/'); - if (!enumerate_refs_recurse (repo, remote, base_dfd, base_path, + if (!enumerate_refs_recurse (repo, remote, collection_id, base_dfd, base_path, dfd_iter.fd, dent->d_name, refs, cancellable, error)) return FALSE; } else if (dent->d_type == DT_REG) { - if (!add_ref_to_set (remote, base_dfd, base_path->str, refs, + if (!add_ref_to_set (remote, collection_id, base_dfd, base_path->str, refs, cancellable, error)) return FALSE; } @@ -554,7 +571,7 @@ _ostree_repo_list_refs_internal (OstreeRepo *self, if (!glnx_opendirat (self->repo_dir_fd, cut_prefix ? path : prefix_path, TRUE, &base_fd, error)) return FALSE; - if (!enumerate_refs_recurse (self, remote, base_fd, base_path, + if (!enumerate_refs_recurse (self, remote, NULL, base_fd, base_path, base_fd, cut_prefix ? "." : ref_prefix, ret_all_refs, cancellable, error)) return FALSE; @@ -566,7 +583,7 @@ _ostree_repo_list_refs_internal (OstreeRepo *self, if (!glnx_opendirat (self->repo_dir_fd, prefix_path, TRUE, &prefix_dfd, error)) return FALSE; - if (!add_ref_to_set (remote, prefix_dfd, ref_prefix, ret_all_refs, + if (!add_ref_to_set (remote, NULL, prefix_dfd, ref_prefix, ret_all_refs, cancellable, error)) return FALSE; } @@ -581,7 +598,7 @@ _ostree_repo_list_refs_internal (OstreeRepo *self, if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &refs_heads_dfd, error)) return FALSE; - if (!enumerate_refs_recurse (self, NULL, refs_heads_dfd, base_path, + if (!enumerate_refs_recurse (self, NULL, NULL, refs_heads_dfd, base_path, refs_heads_dfd, ".", ret_all_refs, cancellable, error)) return FALSE; @@ -607,7 +624,7 @@ _ostree_repo_list_refs_internal (OstreeRepo *self, if (!glnx_opendirat (dfd_iter.fd, dent->d_name, TRUE, &remote_dfd, error)) return FALSE; - if (!enumerate_refs_recurse (self, dent->d_name, remote_dfd, base_path, + if (!enumerate_refs_recurse (self, dent->d_name, NULL, remote_dfd, base_path, remote_dfd, ".", ret_all_refs, cancellable, error)) @@ -741,16 +758,19 @@ ostree_repo_remote_list_refs (OstreeRepo *self, } gboolean -_ostree_repo_write_ref (OstreeRepo *self, - const char *remote, - const char *ref, - const char *rev, - GCancellable *cancellable, - GError **error) +_ostree_repo_write_ref (OstreeRepo *self, + const char *remote, + const OstreeCollectionRef *ref, + const char *rev, + GCancellable *cancellable, + GError **error) { glnx_fd_close int dfd = -1; - if (remote == NULL) + g_return_val_if_fail (remote == NULL || ref->collection_id == NULL, FALSE); + + if (remote == NULL && + (ref->collection_id == NULL || g_strcmp0 (ref->collection_id, ostree_repo_get_collection_id (self)) == 0)) { if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &dfd, error)) @@ -759,6 +779,29 @@ _ostree_repo_write_ref (OstreeRepo *self, return FALSE; } } + else if (remote == NULL && ref->collection_id != NULL) + { + glnx_fd_close int refs_mirrors_dfd = -1; + + /* refs/mirrors might not exist in older repositories, so create it. */ + if (!glnx_shutil_mkdir_p_at_open (self->repo_dir_fd, "refs/mirrors", 0777, + &refs_mirrors_dfd, cancellable, error)) + { + g_prefix_error (error, "Opening %s: ", "refs/mirrors"); + return FALSE; + } + + if (rev != NULL) + { + /* Ensure we have a dir for the collection */ + if (!glnx_shutil_mkdir_p_at (refs_mirrors_dfd, ref->collection_id, 0777, cancellable, error)) + return FALSE; + } + + dfd = glnx_opendirat_with_errno (refs_mirrors_dfd, ref->collection_id, TRUE); + if (dfd < 0 && (errno != ENOENT || rev != NULL)) + return glnx_throw_errno_prefix (error, "Opening mirrors/ dir %s", ref->collection_id); + } else { glnx_fd_close int refs_remotes_dfd = -1; @@ -786,7 +829,7 @@ _ostree_repo_write_ref (OstreeRepo *self, { if (dfd >= 0) { - if (unlinkat (dfd, ref, 0) != 0) + if (unlinkat (dfd, ref->ref_name, 0) != 0) { if (errno != ENOENT) return glnx_throw_errno (error); @@ -795,7 +838,7 @@ _ostree_repo_write_ref (OstreeRepo *self, } else { - if (!write_checksum_file_at (self, dfd, ref, rev, cancellable, error)) + if (!write_checksum_file_at (self, dfd, ref->ref_name, rev, cancellable, error)) return FALSE; } @@ -807,7 +850,7 @@ _ostree_repo_write_ref (OstreeRepo *self, gboolean _ostree_repo_update_refs (OstreeRepo *self, - GHashTable *refs, + GHashTable *refs, /* (element-type utf8 utf8) */ GCancellable *cancellable, GError **error) { @@ -820,15 +863,135 @@ _ostree_repo_update_refs (OstreeRepo *self, const char *refspec = key; const char *rev = value; g_autofree char *remote = NULL; - g_autofree char *ref = NULL; + g_autofree char *ref_name = NULL; - if (!ostree_parse_refspec (refspec, &remote, &ref, error)) + if (!ostree_parse_refspec (refspec, &remote, &ref_name, error)) return FALSE; - if (!_ostree_repo_write_ref (self, remote, ref, rev, + const OstreeCollectionRef ref = { NULL, ref_name }; + if (!_ostree_repo_write_ref (self, remote, &ref, rev, cancellable, error)) return FALSE; } return TRUE; } + +gboolean +_ostree_repo_update_collection_refs (OstreeRepo *self, + GHashTable *refs, /* (element-type OstreeCollectionRef utf8) */ + GCancellable *cancellable, + GError **error) +{ + GHashTableIter hash_iter; + gpointer key, value; + + g_hash_table_iter_init (&hash_iter, refs); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const OstreeCollectionRef *ref = key; + const char *rev = value; + + if (!_ostree_repo_write_ref (self, NULL, ref, rev, + cancellable, error)) + return FALSE; + } + + return TRUE; +} + +/** + * ostree_repo_list_collection_refs: + * @self: Repo + * @match_collection_id: (nullable): If non-%NULL, only list refs from this collection + * @out_all_refs: (out) (element-type OstreeCollectionRef utf8): Mapping from collection–ref to checksum + * @cancellable: Cancellable + * @error: Error + * + * List all local and mirrored refs, mapping them to the commit checksums they + * currently point to in @out_all_refs. If @match_collection_id is specified, + * the results will be limited to those with an equal collection ID. + * + * #OstreeCollectionRefs are guaranteed to be returned with their collection ID + * set to a non-%NULL value; so no refs from `refs/heads` will be listed if no + * collection ID is configured for the repository + * (ostree_repo_get_collection_id()). + * + * Returns: %TRUE on success, %FALSE otherwise + * Since: 2017.8 + */ +gboolean +ostree_repo_list_collection_refs (OstreeRepo *self, + const char *match_collection_id, + GHashTable **out_all_refs, + GCancellable *cancellable, + GError **error) +{ + g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); + g_return_val_if_fail (match_collection_id == NULL || + ostree_validate_collection_id (match_collection_id, NULL), FALSE); + g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), FALSE); + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + + g_autoptr(GHashTable) ret_all_refs = NULL; + + ret_all_refs = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + (GDestroyNotify) ostree_collection_ref_free, + g_free); + + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + g_autoptr(GString) base_path = g_string_new (""); + + const gchar *main_collection_id = ostree_repo_get_collection_id (self); + + if (main_collection_id != NULL && + (match_collection_id == NULL || g_strcmp0 (match_collection_id, main_collection_id) == 0)) + { + glnx_fd_close int refs_heads_dfd = -1; + + if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &refs_heads_dfd, error)) + return FALSE; + + if (!enumerate_refs_recurse (self, NULL, main_collection_id, refs_heads_dfd, base_path, + refs_heads_dfd, ".", + ret_all_refs, cancellable, error)) + return FALSE; + } + + g_string_truncate (base_path, 0); + + gboolean refs_mirrors_exists = FALSE; + if (!ot_dfd_iter_init_allow_noent (self->repo_dir_fd, "refs/mirrors", + &dfd_iter, &refs_mirrors_exists, error)) + return FALSE; + + while (refs_mirrors_exists) + { + struct dirent *dent; + glnx_fd_close int collection_dfd = -1; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (!dent) + break; + + if (dent->d_type != DT_DIR) + continue; + + if (match_collection_id != NULL && g_strcmp0 (match_collection_id, dent->d_name) != 0) + continue; + + if (!glnx_opendirat (dfd_iter.fd, dent->d_name, TRUE, &collection_dfd, error)) + return FALSE; + + if (!enumerate_refs_recurse (self, NULL, dent->d_name, collection_dfd, base_path, + collection_dfd, ".", + ret_all_refs, + cancellable, error)) + return FALSE; + } + + ot_transfer_out_value (out_all_refs, &ret_all_refs); + return TRUE; +} diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 24e94e03..1db748cb 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -475,6 +475,7 @@ ostree_repo_finalize (GObject *object) if (self->config) g_key_file_free (self->config); g_clear_pointer (&self->txn_refs, g_hash_table_destroy); + g_clear_pointer (&self->txn_collection_refs, g_hash_table_destroy); g_clear_error (&self->writable_error); g_clear_pointer (&self->object_sizes, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&self->dirmeta_cache, (GDestroyNotify) g_hash_table_unref); @@ -1703,7 +1704,8 @@ ostree_repo_create (OstreeRepo *self, glnx_fd_close int dfd = -1; struct stat stbuf; const char *state_dirs[] = { "objects", "tmp", "extensions", "state", - "refs", "refs/heads", "refs/remotes" }; + "refs", "refs/heads", "refs/mirrors", + "refs/remotes" }; if (mkdir (repopath, 0755) != 0) { @@ -4568,6 +4570,13 @@ summary_add_ref_entry (OstreeRepo *self, * * It is regenerated automatically after a commit if * `core/commit-update-summary` is set. + * + * If the `core/collection-id` key is set in the configuration, it will be + * included as %OSTREE_SUMMARY_COLLECTION_ID in the summary file. Refs from the + * `refs/mirrors` directory will be included in the generated summary file, + * listed under the %OSTREE_SUMMARY_COLLECTION_MAP key. Collection IDs and refs + * in %OSTREE_SUMMARY_COLLECTION_MAP are guaranteed to be in lexicographic + * order. */ gboolean ostree_repo_regenerate_summary (OstreeRepo *self, @@ -4579,21 +4588,26 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_dict_init (&additional_metadata_builder, additional_metadata); g_autoptr(GVariantBuilder) refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))")); + const gchar *main_collection_id = ostree_repo_get_collection_id (self); + { - g_autoptr(GHashTable) refs = NULL; - if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error)) - return FALSE; - - g_autoptr(GList) ordered_keys = g_hash_table_get_keys (refs); - ordered_keys = g_list_sort (ordered_keys, (GCompareFunc)strcmp); - - for (GList *iter = ordered_keys; iter; iter = iter->next) + if (main_collection_id == NULL) { - const char *ref = iter->data; - const char *commit = g_hash_table_lookup (refs, ref); - - if (!summary_add_ref_entry (self, ref, commit, refs_builder, error)) + g_autoptr(GHashTable) refs = NULL; + if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error)) return FALSE; + + g_autoptr(GList) ordered_keys = g_hash_table_get_keys (refs); + ordered_keys = g_list_sort (ordered_keys, (GCompareFunc)strcmp); + + for (GList *iter = ordered_keys; iter; iter = iter->next) + { + const char *ref = iter->data; + const char *commit = g_hash_table_lookup (refs, ref); + + if (!summary_add_ref_entry (self, ref, commit, refs_builder, error)) + return FALSE; + } } } @@ -4640,6 +4654,88 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_new_uint64 (GUINT64_TO_BE (g_get_real_time () / G_USEC_PER_SEC))); } + /* Add refs which have a collection specified. ostree_repo_list_collection_refs() + * is guaranteed to only return refs which are in refs/mirrors, or those which + * are in refs/heads if the repository configuration specifies a collection ID + * (which we put in the main refs map, rather than the collection map, for + * backwards compatibility). */ + { + g_autoptr(GHashTable) collection_refs = NULL; + if (!ostree_repo_list_collection_refs (self, NULL, &collection_refs, cancellable, error)) + return FALSE; + + gsize collection_map_size = 0; + GHashTableIter iter; + g_autoptr(GHashTable) collection_map = NULL; /* (element-type utf8 GHashTable) */ + g_hash_table_iter_init (&iter, collection_refs); + collection_map = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, + (GDestroyNotify) g_hash_table_unref); + + const OstreeCollectionRef *ref; + const char *checksum; + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum)) + { + GHashTable *ref_map = g_hash_table_lookup (collection_map, ref->collection_id); + + if (ref_map == NULL) + { + ref_map = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); + g_hash_table_insert (collection_map, ref->collection_id, ref_map); + } + + g_hash_table_insert (ref_map, ref->ref_name, (gpointer) checksum); + } + + g_autoptr(GVariantBuilder) collection_refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); + + g_autoptr(GList) ordered_collection_ids = g_hash_table_get_keys (collection_map); + ordered_collection_ids = g_list_sort (ordered_collection_ids, (GCompareFunc) strcmp); + + for (GList *collection_iter = ordered_collection_ids; collection_iter; collection_iter = collection_iter->next) + { + const char *collection_id = collection_iter->data; + GHashTable *ref_map = g_hash_table_lookup (collection_map, collection_id); + + gboolean is_main_collection_id = (main_collection_id != NULL && g_str_equal (collection_id, main_collection_id)); + + if (!is_main_collection_id) + { + g_variant_builder_open (collection_refs_builder, G_VARIANT_TYPE ("{sa(s(taya{sv}))}")); + g_variant_builder_add (collection_refs_builder, "s", collection_id); + g_variant_builder_open (collection_refs_builder, G_VARIANT_TYPE ("a(s(taya{sv}))")); + } + + g_autoptr(GList) ordered_refs = g_hash_table_get_keys (ref_map); + ordered_refs = g_list_sort (ordered_refs, (GCompareFunc) strcmp); + + for (GList *ref_iter = ordered_refs; ref_iter != NULL; ref_iter = ref_iter->next) + { + const char *ref = ref_iter->data; + const char *commit = g_hash_table_lookup (ref_map, ref); + GVariantBuilder *builder = is_main_collection_id ? refs_builder : collection_refs_builder; + + if (!summary_add_ref_entry (self, ref, commit, builder, error)) + return FALSE; + + if (!is_main_collection_id) + collection_map_size++; + } + + if (!is_main_collection_id) + { + g_variant_builder_close (collection_refs_builder); /* array */ + g_variant_builder_close (collection_refs_builder); /* dict entry */ + } + } + + if (main_collection_id != NULL) + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_COLLECTION_ID, + g_variant_new_string (main_collection_id)); + if (collection_map_size > 0) + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_COLLECTION_MAP, + g_variant_builder_end (collection_refs_builder)); + } + g_autoptr(GVariant) summary = NULL; { g_autoptr(GVariantBuilder) summary_builder = diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 566754d4..388d73c4 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -309,6 +309,15 @@ void ostree_repo_transaction_set_ref (OstreeRepo *self, const char *ref, const char *checksum); +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + +_OSTREE_PUBLIC +void ostree_repo_transaction_set_collection_ref (OstreeRepo *self, + const OstreeCollectionRef *ref, + const char *checksum); + +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + _OSTREE_PUBLIC gboolean ostree_repo_set_ref_immediate (OstreeRepo *self, const char *remote, @@ -317,6 +326,17 @@ gboolean ostree_repo_set_ref_immediate (OstreeRepo *self, GCancellable *cancellable, GError **error); +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + +_OSTREE_PUBLIC +gboolean ostree_repo_set_collection_ref_immediate (OstreeRepo *self, + const OstreeCollectionRef *ref, + const char *checksum, + GCancellable *cancellable, + GError **error); + +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + _OSTREE_PUBLIC gboolean ostree_repo_has_object (OstreeRepo *self, OstreeObjectType objtype, @@ -1068,6 +1088,17 @@ gboolean ostree_repo_pull_with_options (OstreeRepo *self, GCancellable *cancellable, GError **error); +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + +_OSTREE_PUBLIC +gboolean ostree_repo_list_collection_refs (OstreeRepo *self, + const char *match_collection_id, + GHashTable **out_all_refs, + GCancellable *cancellable, + GError **error); + +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + _OSTREE_PUBLIC void ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress, gpointer user_data); diff --git a/tests/basic-test.sh b/tests/basic-test.sh index d9b20938..6f237696 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -492,7 +492,7 @@ echo "ok pull-local with --remote arg" cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=repo3 prune find repo3/objects -name '*.commit' > objlist-before-prune -rm repo3/refs/heads/* repo3/refs/remotes/* -rf +rm repo3/refs/heads/* repo3/refs/mirrors/* repo3/refs/remotes/* -rf ${CMD_PREFIX} ostree --repo=repo3 prune --refs-only find repo3/objects -name '*.commit' > objlist-after-prune if cmp -s objlist-before-prune objlist-after-prune; then From 7607d94713539e748a89656c9a85cbe04186b281 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:42:22 +0100 Subject: [PATCH 22/82] lib/pull: Add collection support to ostree_repo_pull_with_options() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new collection-refs option of type a(sss), giving a list of (collection ID, ref name, checksum) tuples to pull from the given remote. This option is intended to supersede the refs and override-commit-ids options, so is mutually exclusive with them. This includes support for resolving the refs from the remote’s summary file, or from its refs/heads and refs/mirrors directories. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 233 +++++++++++++++++++++++++------ 1 file changed, 187 insertions(+), 46 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 2c87fd60..03be117a 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -738,13 +738,20 @@ scan_dirtree_object (OtPullData *pull_data, } static gboolean -fetch_ref_contents (OtPullData *pull_data, - const char *ref, - char **out_contents, - GCancellable *cancellable, - GError **error) +fetch_ref_contents (OtPullData *pull_data, + const char *main_collection_id, + const OstreeCollectionRef *ref, + char **out_contents, + GCancellable *cancellable, + GError **error) { - g_autofree char *filename = g_build_filename ("refs", "heads", ref, NULL); + g_autofree char *filename = NULL; + + if (ref->collection_id == NULL || g_strcmp0 (ref->collection_id, main_collection_id) == 0) + filename = g_build_filename ("refs", "heads", ref->ref_name, NULL); + else + filename = g_build_filename ("refs", "mirrors", ref->collection_id, ref->ref_name, NULL); + g_autofree char *ret_contents = NULL; if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher, pull_data->meta_mirrorlist, @@ -762,16 +769,46 @@ fetch_ref_contents (OtPullData *pull_data, } static gboolean -lookup_commit_checksum_from_summary (OtPullData *pull_data, - const char *ref, - char **out_checksum, - gsize *out_size, - GError **error) +lookup_commit_checksum_and_collection_from_summary (OtPullData *pull_data, + const OstreeCollectionRef *ref, + char **out_checksum, + gsize *out_size, + char **out_collection_id, + GError **error) { - g_autoptr(GVariant) refs = g_variant_get_child_value (pull_data->summary, 0); + g_autoptr(GVariant) additional_metadata = g_variant_get_child_value (pull_data->summary, 1); + const gchar *main_collection_id; + + if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id)) + main_collection_id = NULL; + + g_autoptr(GVariant) refs = NULL; + const gchar *resolved_collection_id = NULL; + + if (ref->collection_id == NULL || g_strcmp0 (ref->collection_id, main_collection_id) == 0) + { + refs = g_variant_get_child_value (pull_data->summary, 0); + resolved_collection_id = main_collection_id; + } + else if (ref->collection_id != NULL) + { + g_autoptr(GVariant) collection_map = NULL; + + collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, + G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); + if (collection_map != NULL) + refs = g_variant_lookup_value (collection_map, ref->collection_id, G_VARIANT_TYPE ("a(s(taya{sv}))")); + resolved_collection_id = ref->collection_id; + } + int i; - if (!ot_variant_bsearch_str (refs, ref, &i)) - return glnx_throw (error, "No such branch '%s' in repository summary", ref); + if (refs == NULL || !ot_variant_bsearch_str (refs, ref->ref_name, &i)) + { + if (ref->collection_id != NULL) + return glnx_throw (error, "No such branch (%s, %s) in repository summary", ref->collection_id, ref->ref_name); + else + return glnx_throw (error, "No such branch '%s' in repository summary", ref->ref_name); + } g_autoptr(GVariant) refdata = g_variant_get_child_value (refs, i); g_autoptr(GVariant) reftargetdata = g_variant_get_child_value (refdata, 1); @@ -779,11 +816,15 @@ lookup_commit_checksum_from_summary (OtPullData *pull_data, g_autoptr(GVariant) commit_csum_v = NULL; g_variant_get (reftargetdata, "(t@ay@a{sv})", &commit_size, &commit_csum_v, NULL); + if (resolved_collection_id != NULL && + !ostree_validate_collection_id (resolved_collection_id, error)) + return FALSE; if (!ostree_validate_structureof_csum_v (commit_csum_v, error)) return FALSE; *out_checksum = ostree_checksum_from_bytes_v (commit_csum_v); *out_size = commit_size; + *out_collection_id = g_strdup (resolved_collection_id); return TRUE; } @@ -2675,7 +2716,9 @@ initiate_delta_request (OtPullData *pull_data, pull_data->n_requested_metadata++; } -/* @ref - Optional ref name +/* + * initiate_request: + * @ref: Optional ref name and collection ID * @to_revision: Target commit revision we want to fetch * * Start a request for either a ref or a commit. In the @@ -2685,10 +2728,10 @@ initiate_delta_request (OtPullData *pull_data, * `disable_static_deltas` and `require_static_deltas`. */ static gboolean -initiate_request (OtPullData *pull_data, - const char *ref, - const char *to_revision, - GError **error) +initiate_request (OtPullData *pull_data, + const OstreeCollectionRef *ref, + const char *to_revision, + GError **error) { g_autofree char *delta_from_revision = NULL; @@ -2716,7 +2759,7 @@ initiate_request (OtPullData *pull_data, initiate_delta_request (pull_data, NULL, to_revision); else if (pull_data->require_static_deltas) /* No deltas found; are they required? */ { - set_required_deltas_error (error, ref, to_revision); + set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision); return FALSE; } else /* No deltas, fall back to object fetches. */ @@ -2726,7 +2769,11 @@ initiate_request (OtPullData *pull_data, { /* Are we doing a delta via a ref? In that case we can fall back to the older * logic of just using the current tip of the ref as a delta FROM source. */ - if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE, + g_autofree char *refspec = NULL; + if (pull_data->remote_name != NULL) + refspec = g_strdup_printf ("%s:%s", pull_data->remote_name, ref->ref_name); + if (!ostree_repo_resolve_rev (pull_data->repo, + (refspec != NULL) ? refspec : ref->ref_name, TRUE, &delta_from_revision, error)) return FALSE; @@ -2782,6 +2829,9 @@ initiate_request (OtPullData *pull_data, * The following are currently defined: * * * refs (as): Array of string refs + * * collection-refs (a(sss)): Array of (collection ID, ref name, checksum) tuples to pull; + * mutually exclusive with `refs` and `override-commit-ids`. Checksums may be the empty + * string to pull the latest commit for that ref * * flags (i): An instance of #OstreeRepoPullFlags * * subdir (s): Pull just this subdirectory * * subdirs (as): Pull just these subdirectories @@ -2811,7 +2861,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, gpointer key, value; g_autoptr(GBytes) bytes_summary = NULL; g_autofree char *metalink_url_str = NULL; - g_autoptr(GHashTable) requested_refs_to_fetch = NULL; + g_autoptr(GHashTable) requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ g_autoptr(GHashTable) commits_to_fetch = NULL; g_autofree char *remote_mode_str = NULL; glnx_unref_object OstreeMetalink *metalink = NULL; @@ -2826,18 +2876,24 @@ ostree_repo_pull_with_options (OstreeRepo *self, const char *dir_to_pull = NULL; g_autofree char **dirs_to_pull = NULL; g_autofree char **refs_to_fetch = NULL; + g_autoptr(GVariantIter) collection_refs_iter = NULL; g_autofree char **override_commit_ids = NULL; GSource *update_timeout = NULL; gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; + gboolean opt_collection_refs_set = FALSE; + const char *main_collection_id = NULL; const char *url_override = NULL; gboolean inherit_transaction = FALSE; + g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ int i; if (options) { int flags_i = OSTREE_REPO_PULL_FLAGS_NONE; (void) g_variant_lookup (options, "refs", "^a&s", &refs_to_fetch); + opt_collection_refs_set = + g_variant_lookup (options, "collection-refs", "a(sss)", &collection_refs_iter); (void) g_variant_lookup (options, "flags", "i", &flags_i); /* Reduce risk of issues if enum happens to be 64 bit for some reason */ flags = flags_i; @@ -2860,6 +2916,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); + g_return_val_if_fail (!opt_collection_refs_set || + (refs_to_fetch == NULL && override_commit_ids == NULL), FALSE); if (refs_to_fetch && override_commit_ids) g_return_val_if_fail (g_strv_length (refs_to_fetch) == g_strv_length (override_commit_ids), FALSE); @@ -2990,7 +3048,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; pull_data->tmpdir_dfd = pull_data->repo->tmp_dir_fd; - requested_refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + requested_refs_to_fetch = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + (GDestroyNotify) ostree_collection_ref_free, + g_free); commits_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); if (!ostree_repo_get_remote_option (self, @@ -3102,6 +3163,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, } } + /* FIXME: Do we want an analogue of this which supports collection IDs? */ if (!ostree_repo_get_remote_list_option (self, remote_name_or_baseurl, "branches", &configured_branches, error)) @@ -3282,6 +3344,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->summary) { + additional_metadata = g_variant_get_child_value (pull_data->summary, 1); + + if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id)) + main_collection_id = NULL; + else if (!ostree_validate_collection_id (main_collection_id, error)) + goto out; + refs = g_variant_get_child_value (pull_data->summary, 0); for (i = 0, n = g_variant_n_children (refs); i < n; i++) { @@ -3293,11 +3362,47 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!ostree_validate_rev (refname, error)) goto out; - if (pull_data->is_mirror && !refs_to_fetch) - g_hash_table_insert (requested_refs_to_fetch, g_strdup (refname), NULL); + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) + { + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (main_collection_id, refname), NULL); + } + } + + g_autoptr(GVariant) collection_map = NULL; + collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); + if (collection_map != NULL) + { + GVariantIter collection_map_iter; + const char *collection_id; + g_autoptr(GVariant) collection_refs = NULL; + + g_variant_iter_init (&collection_map_iter, collection_map); + + while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs)) + { + if (!ostree_validate_collection_id (collection_id, error)) + goto out; + + for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++) + { + const char *refname; + g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i); + + g_variant_get_child (ref, 0, "&s", &refname); + + if (!ostree_validate_rev (refname, error)) + goto out; + + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) + { + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (collection_id, refname), NULL); + } + } + } } - additional_metadata = g_variant_get_child_value (pull_data->summary, 1); deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); n = deltas ? g_variant_n_children (deltas) : 0; for (i = 0; i < n; i++) @@ -3321,7 +3426,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, } } - if (pull_data->is_mirror && !refs_to_fetch && !configured_branches) + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches) { if (!bytes_summary) { @@ -3330,7 +3435,18 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; } - } + } + else if (opt_collection_refs_set) + { + const gchar *collection_id, *ref_name, *checksum; + + while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum)) + { + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (collection_id, ref_name), + (*checksum != '\0') ? g_strdup (checksum) : NULL); + } + } else if (refs_to_fetch != NULL) { char **strviter = refs_to_fetch; @@ -3348,7 +3464,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, else { char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; - g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), commitid); + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (NULL, branch), commitid); } strviter++; @@ -3371,47 +3488,67 @@ ostree_repo_pull_with_options (OstreeRepo *self, for (;branches_iter && *branches_iter; branches_iter++) { const char *branch = *branches_iter; - - g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL); + + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (NULL, branch), NULL); } } + /* Resolve the checksum for each ref. This has to be done into a new hash table, + * since we can’t modify the keys of @requested_refs_to_fetch while iterating + * over it, and we need to ensure the collection IDs are resolved too. */ g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); + updated_requested_refs_to_fetch = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + (GDestroyNotify) ostree_collection_ref_free, + g_free); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { - const char *branch = key; + const OstreeCollectionRef *ref = key; const char *override_commitid = value; - char *contents = NULL; + g_autofree char *contents = NULL; /* Support specifying "" for an override commitid */ if (override_commitid && *override_commitid) { - g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), g_strdup (override_commitid)); + g_hash_table_replace (updated_requested_refs_to_fetch, ostree_collection_ref_dup (ref), g_strdup (override_commitid)); } - else + else { + g_autoptr(OstreeCollectionRef) ref_with_collection = NULL; + if (pull_data->summary) { gsize commit_size = 0; guint64 *malloced_size; + g_autofree gchar *collection_id = NULL; - if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error)) + if (!lookup_commit_checksum_and_collection_from_summary (pull_data, ref, &contents, &commit_size, &collection_id, error)) goto out; + ref_with_collection = ostree_collection_ref_new (collection_id, ref->ref_name); + malloced_size = g_new0 (guint64, 1); *malloced_size = commit_size; g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size); } else { - if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) + if (!fetch_ref_contents (pull_data, main_collection_id, ref, &contents, cancellable, error)) goto out; + + ref_with_collection = ostree_collection_ref_dup (ref); } - /* Transfer ownership of contents */ - g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents); + + g_hash_table_replace (updated_requested_refs_to_fetch, + g_steal_pointer (&ref_with_collection), + g_steal_pointer (&contents)); } } + g_hash_table_unref (requested_refs_to_fetch); + requested_refs_to_fetch = g_steal_pointer (&updated_requested_refs_to_fetch); + /* Create the state directory here - it's new with the commitpartial code, * and may not exist in older repositories. */ @@ -3454,7 +3591,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { - const char *ref = key; + const OstreeCollectionRef *ref = key; const char *to_revision = value; if (!initiate_request (pull_data, ref, to_revision, error)) goto out; @@ -3495,26 +3632,30 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { - const char *ref = key; + const OstreeCollectionRef *ref = key; const char *checksum = value; g_autofree char *remote_ref = NULL; g_autofree char *original_rev = NULL; if (pull_data->remote_name) - remote_ref = g_strdup_printf ("%s:%s", pull_data->remote_name, ref); + remote_ref = g_strdup_printf ("%s:%s", pull_data->remote_name, ref->ref_name); else - remote_ref = g_strdup (ref); + remote_ref = g_strdup (ref->ref_name); if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error)) goto out; - + if (original_rev && strcmp (checksum, original_rev) == 0) { } else { - ostree_repo_transaction_set_ref (pull_data->repo, pull_data->is_mirror ? NULL : pull_data->remote_name, - ref, checksum); + if (pull_data->is_mirror) + ostree_repo_transaction_set_collection_ref (pull_data->repo, + ref, checksum); + else + ostree_repo_transaction_set_ref (pull_data->repo, pull_data->remote_name, + ref->ref_name, checksum); } } From 292230301dde3c774325d50b5ed95d37b1c1d217 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 18 Apr 2017 23:59:33 +0100 Subject: [PATCH 23/82] lib/repo-finder: Add basic support for finding remote URIs by ref name Add an initial OstreeRepoFinder interface (but no implementations), which will find remote URIs by ref names and collection IDs, the combination of which is globally unique. The new API is used in a new ostree_repo_find_updates() function, which resolves a list of ref names to update into a set of remote URIs to pull them from, which can be treated as mirrors. It is an attempt to generalise resolution of the URIs to pull from, and to generalise determination of the order and parallelisation which they should be downloaded from in. Includes fixes by Krzesimir Nowak . Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- Makefile-libostree-defines.am | 1 + Makefile-libostree.am | 7 +- apidoc/ostree-experimental-sections.txt | 28 + src/libostree/libostree-experimental.sym | 16 + src/libostree/ostree-autocleanups.h | 3 + src/libostree/ostree-core-private.h | 5 + src/libostree/ostree-repo-finder.c | 576 ++++++++++ src/libostree/ostree-repo-finder.h | 172 +++ src/libostree/ostree-repo-pull.c | 1311 +++++++++++++++++++++- src/libostree/ostree-repo.h | 61 + src/libostree/ostree.h | 1 + 11 files changed, 2179 insertions(+), 2 deletions(-) create mode 100644 src/libostree/ostree-repo-finder.c create mode 100644 src/libostree/ostree-repo-finder.h diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index 44b6cd0a..6214633d 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -42,6 +42,7 @@ if ENABLE_EXPERIMENTAL_API libostree_public_headers += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ + src/libostree/ostree-repo-finder.h \ $(NULL) endif diff --git a/Makefile-libostree.am b/Makefile-libostree.am index a9c392c1..a9331dd4 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -154,6 +154,11 @@ if !ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ + src/libostree/ostree-repo-finder.h \ + $(NULL) +else # if ENABLE_EXPERIMENTAL_API +libostree_1_la_SOURCES += \ + src/libostree/ostree-repo-finder.c \ $(NULL) endif @@ -230,7 +235,7 @@ OSTree_1_0_gir_INCLUDES = Gio-2.0 OSTree_1_0_gir_CFLAGS = $(libostree_1_la_CFLAGS) OSTree_1_0_gir_LIBS = libostree-1.la OSTree_1_0_gir_SCANNERFLAGS = --warn-all --identifier-prefix=Ostree --symbol-prefix=ostree -OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h,$(libostree_1_la_SOURCES)) +OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h,$(libostree_1_la_SOURCES)) INTROSPECTION_GIRS += OSTree-1.0.gir gir_DATA += OSTree-1.0.gir typelib_DATA += OSTree-1.0.typelib diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 78a50100..7e9ed084 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -21,6 +21,34 @@ ostree_remote_unref ostree_remote_get_name +
+ostree-repo-experimental +ostree_repo_find_remotes_async +ostree_repo_find_remotes_finish +ostree_repo_pull_from_remotes_async +ostree_repo_pull_from_remotes_finish +ostree_repo_resolve_keyring_for_collection +
+ +
+ostree-repo-finder +OstreeRepoFinder +ostree_repo_finder_resolve_async +ostree_repo_finder_resolve_finish +ostree_repo_finder_resolve_all_async +ostree_repo_finder_resolve_all_finish +OstreeRepoFinderResult +ostree_repo_finder_result_new +ostree_repo_finder_result_dup +ostree_repo_finder_result_free +ostree_repo_finder_result_compare +OstreeRepoFinderResultv +ostree_repo_finder_result_freev + +ostree_repo_finder_get_type +ostree_repo_finder_result_get_type +
+
ostree-misc-experimental ostree_repo_get_collection_id diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index 9d2024f3..cda34322 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -45,8 +45,24 @@ global: ostree_collection_ref_get_type; ostree_collection_ref_hash; ostree_collection_ref_new; + ostree_repo_find_remotes_async; + ostree_repo_find_remotes_finish; + ostree_repo_finder_get_type; + ostree_repo_finder_resolve_async; + ostree_repo_finder_resolve_all_async; + ostree_repo_finder_resolve_all_finish; + ostree_repo_finder_resolve_finish; + ostree_repo_finder_result_compare; + ostree_repo_finder_result_dup; + ostree_repo_finder_result_free; + ostree_repo_finder_result_freev; + ostree_repo_finder_result_get_type; + ostree_repo_finder_result_new; ostree_repo_get_collection_id; ostree_repo_list_collection_refs; + ostree_repo_pull_from_remotes_async; + ostree_repo_pull_from_remotes_finish; + ostree_repo_resolve_keyring_for_collection; ostree_repo_set_collection_id; ostree_repo_set_collection_ref_immediate; ostree_repo_transaction_set_collection_ref; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index f683c2e3..1f7716b2 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -63,6 +63,9 @@ G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_comm G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) +G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #endif diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index a4a31034..a8fbb8e1 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -182,6 +182,11 @@ gboolean ostree_validate_collection_id (const char *collection_id, GError **erro #include "ostree-ref.h" G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) + +#include "ostree-repo-finder.h" +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) +G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) #endif G_END_DECLS diff --git a/src/libostree/ostree-repo-finder.c b/src/libostree/ostree-repo-finder.c new file mode 100644 index 00000000..7893978d --- /dev/null +++ b/src/libostree/ostree-repo-finder.c @@ -0,0 +1,576 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include + +#include "ostree-autocleanups.h" +#include "ostree-core.h" +#include "ostree-remote-private.h" +#include "ostree-repo-finder.h" +#include "ostree-repo.h" + +static void ostree_repo_finder_default_init (OstreeRepoFinderInterface *iface); + +G_DEFINE_INTERFACE (OstreeRepoFinder, ostree_repo_finder, G_TYPE_OBJECT) + +static void +ostree_repo_finder_default_init (OstreeRepoFinderInterface *iface) +{ + /* Nothing to see here. */ +} + +/* Validate the given struct contains a valid collection ID and ref name, and that + * the collection ID is non-%NULL. */ +static gboolean +is_valid_collection_ref (const OstreeCollectionRef *ref) +{ + return (ref != NULL && + ostree_validate_rev (ref->ref_name, NULL) && + ostree_validate_collection_id (ref->collection_id, NULL)); +} + +/* Validate @refs is non-%NULL, non-empty, and contains only valid collection + * and ref names. */ +static gboolean +is_valid_collection_ref_array (const OstreeCollectionRef * const *refs) +{ + gsize i; + + if (refs == NULL || *refs == NULL) + return FALSE; + + for (i = 0; refs[i] != NULL; i++) + { + if (!is_valid_collection_ref (refs[i])) + return FALSE; + } + + return TRUE; +} + +/* Validate @ref_to_checksum is non-%NULL, non-empty, and contains only valid + * OstreeCollectionRefs as keys and only valid commit checksums as values. */ +static gboolean +is_valid_collection_ref_map (GHashTable *ref_to_checksum) +{ + GHashTableIter iter; + const OstreeCollectionRef *ref; + const gchar *checksum; + + if (ref_to_checksum == NULL || g_hash_table_size (ref_to_checksum) == 0) + return FALSE; + + g_hash_table_iter_init (&iter, ref_to_checksum); + + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum)) + { + g_assert (ref != NULL); + g_assert (checksum != NULL); + + if (!is_valid_collection_ref (ref)) + return FALSE; + if (!ostree_validate_checksum_string (checksum, NULL)) + return FALSE; + } + + return TRUE; +} + +static void resolve_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data); + +/** + * ostree_repo_finder_resolve_async: + * @self: an #OstreeRepoFinder + * @refs: (array zero-terminated=1): non-empty array of collection–ref pairs to find remotes for + * @parent_repo: (transfer none): the local repository which the refs are being resolved for, + * which provides configuration information and GPG keys + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: asynchronous completion callback + * @user_data: data to pass to @callback + * + * Find reachable remote URIs which claim to provide any of the given @refs. The + * specific method for finding the remotes depends on the #OstreeRepoFinder + * implementation. + * + * Any remote which is found and which claims to support any of the given @refs + * will be returned in the results. It is possible that a remote claims to + * support a given ref, but turns out not to — it is not possible to verify this + * until ostree_repo_pull_from_remotes_async() is called. + * + * The returned results will be sorted with the most useful first — this is + * typically the remote which claims to provide the most @refs, at the lowest + * latency. + * + * Each result contains a mapping of @refs to the checksums of the commits + * which the result provides. If the result provides the latest commit for a ref + * across all of the results, the checksum will be set. Otherwise, if the + * result provides an outdated commit, or doesn’t provide a given ref at all, + * the ref will not be set. Results which provide none of the requested @refs + * may be listed with an empty refs map. + * + * Pass the results to ostree_repo_pull_from_remotes_async() to pull the given + * @refs from those remotes. + * + * Since: 2017.8 + */ +void +ostree_repo_finder_resolve_async (OstreeRepoFinder *self, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + OstreeRepoFinder *finders[2] = { NULL, }; + + g_return_if_fail (OSTREE_IS_REPO_FINDER (self)); + g_return_if_fail (is_valid_collection_ref_array (refs)); + g_return_if_fail (OSTREE_IS_REPO (parent_repo)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_resolve_async); + + finders[0] = self; + + ostree_repo_finder_resolve_all_async (finders, refs, parent_repo, cancellable, + resolve_cb, g_steal_pointer (&task)); +} + +static void +resolve_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autoptr(GPtrArray) results = NULL; + g_autoptr(GError) local_error = NULL; + + task = G_TASK (user_data); + + results = ostree_repo_finder_resolve_all_finish (result, &local_error); + + g_assert ((local_error == NULL) != (results == NULL)); + + if (local_error != NULL) + g_task_return_error (task, g_steal_pointer (&local_error)); + else + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); +} + +/** + * ostree_repo_finder_resolve_finish: + * @self: an #OstreeRepoFinder + * @result: #GAsyncResult from the callback + * @error: return location for a #GError + * + * Get the results from a ostree_repo_finder_resolve_async() operation. + * + * Returns: (transfer full) (element-type OstreeRepoFinderResult): array of zero + * or more results + * Since: 2017.8 + */ +GPtrArray * +ostree_repo_finder_resolve_finish (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (OSTREE_IS_REPO_FINDER (self), NULL); + g_return_val_if_fail (g_task_is_valid (result, self), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + return g_task_propagate_pointer (G_TASK (result), error); +} + +static gint +sort_results_cb (gconstpointer a, + gconstpointer b) +{ + const OstreeRepoFinderResult *result_a = *((const OstreeRepoFinderResult **) a); + const OstreeRepoFinderResult *result_b = *((const OstreeRepoFinderResult **) b); + + return ostree_repo_finder_result_compare (result_a, result_b); +} + +typedef struct +{ + gsize n_finders_pending; + GPtrArray *results; +} ResolveAllData; + +static void +resolve_all_data_free (ResolveAllData *data) +{ + g_assert (data->n_finders_pending == 0); + g_clear_pointer (&data->results, g_ptr_array_unref); + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (ResolveAllData, resolve_all_data_free) + +static void resolve_all_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data); +static void resolve_all_finished_one (GTask *task); + +/** + * ostree_repo_finder_resolve_all_async: + * @finders: (array zero-terminated=1): non-empty array of #OstreeRepoFinders + * @refs: (array zero-terminated=1): non-empty array of collection–ref pairs to find remotes for + * @parent_repo: (transfer none): the local repository which the refs are being resolved for, + * which provides configuration information and GPG keys + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: asynchronous completion callback + * @user_data: data to pass to @callback + * + * A version of ostree_repo_finder_resolve_async() which queries one or more + * @finders in parallel and combines the results. + * + * Since: 2017.8 + */ +void +ostree_repo_finder_resolve_all_async (OstreeRepoFinder * const *finders, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autoptr(ResolveAllData) data = NULL; + gsize i; + g_autoptr(GString) refs_str = NULL; + g_autoptr(GString) finders_str = NULL; + + g_return_if_fail (finders != NULL && finders[0] != NULL); + g_return_if_fail (is_valid_collection_ref_array (refs)); + g_return_if_fail (OSTREE_IS_REPO (parent_repo)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + refs_str = g_string_new (""); + for (i = 0; refs[i] != NULL; i++) + { + if (i != 0) + g_string_append (refs_str, ", "); + g_string_append_printf (refs_str, "(%s, %s)", + refs[i]->collection_id, refs[i]->ref_name); + } + + finders_str = g_string_new (""); + for (i = 0; finders[i] != NULL; i++) + { + if (i != 0) + g_string_append (finders_str, ", "); + g_string_append (finders_str, g_type_name (G_TYPE_FROM_INSTANCE (finders[i]))); + } + + g_debug ("%s: Resolving refs [%s] with finders [%s]", G_STRFUNC, + refs_str->str, finders_str->str); + + task = g_task_new (NULL, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_resolve_all_async); + + data = g_new0 (ResolveAllData, 1); + data->n_finders_pending = 1; /* while setting up the loop */ + data->results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + g_task_set_task_data (task, data, (GDestroyNotify) resolve_all_data_free); + + /* Start all the asynchronous queries in parallel. */ + for (i = 0; finders[i] != NULL; i++) + { + OstreeRepoFinder *finder = OSTREE_REPO_FINDER (finders[i]); + OstreeRepoFinderInterface *iface; + + iface = OSTREE_REPO_FINDER_GET_IFACE (finder); + g_assert (iface->resolve_async != NULL); + iface->resolve_async (finder, refs, parent_repo, cancellable, resolve_all_cb, g_object_ref (task)); + data->n_finders_pending++; + } + + resolve_all_finished_one (task); + data = NULL; /* passed to the GTask above */ +} + +/* Modifies both arrays in place. */ +static void +array_concatenate_steal (GPtrArray *array, + GPtrArray *to_concatenate) /* (transfer full) */ +{ + g_autoptr(GPtrArray) array_to_concatenate = to_concatenate; + gsize i; + + for (i = 0; i < array_to_concatenate->len; i++) + { + /* Sanity check that the arrays do not contain any %NULL elements + * (particularly NULL terminators). */ + g_assert (g_ptr_array_index (array_to_concatenate, i) != NULL); + g_ptr_array_add (array, g_steal_pointer (&g_ptr_array_index (array_to_concatenate, i))); + } + + g_ptr_array_set_free_func (array_to_concatenate, NULL); + g_ptr_array_set_size (array_to_concatenate, 0); +} + +static void +resolve_all_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + OstreeRepoFinder *finder; + OstreeRepoFinderInterface *iface; + g_autoptr(GTask) task = NULL; + g_autoptr(GPtrArray) results = NULL; + g_autoptr(GError) local_error = NULL; + ResolveAllData *data; + + finder = OSTREE_REPO_FINDER (obj); + iface = OSTREE_REPO_FINDER_GET_IFACE (finder); + task = G_TASK (user_data); + data = g_task_get_task_data (task); + results = iface->resolve_finish (finder, result, &local_error); + + g_assert ((local_error == NULL) != (results == NULL)); + + if (local_error != NULL) + g_debug ("Error resolving refs to repository URI using %s: %s", + g_type_name (G_TYPE_FROM_INSTANCE (finder)), local_error->message); + else + array_concatenate_steal (data->results, g_steal_pointer (&results)); + + resolve_all_finished_one (task); +} + +static void +resolve_all_finished_one (GTask *task) +{ + ResolveAllData *data; + + data = g_task_get_task_data (task); + + data->n_finders_pending--; + + if (data->n_finders_pending == 0) + { + gsize i; + g_autoptr(GString) results_str = NULL; + + g_ptr_array_sort (data->results, sort_results_cb); + + results_str = g_string_new (""); + for (i = 0; i < data->results->len; i++) + { + const OstreeRepoFinderResult *result = g_ptr_array_index (data->results, i); + + if (i != 0) + g_string_append (results_str, ", "); + g_string_append (results_str, ostree_remote_get_name (result->remote)); + } + if (i == 0) + g_string_append (results_str, "(none)"); + + g_debug ("%s: Finished, results: %s", G_STRFUNC, results_str->str); + + g_task_return_pointer (task, g_steal_pointer (&data->results), (GDestroyNotify) g_ptr_array_unref); + } +} + +/** + * ostree_repo_finder_resolve_all_finish: + * @result: #GAsyncResult from the callback + * @error: return location for a #GError + * + * Get the results from a ostree_repo_finder_resolve_all_async() operation. + * + * Returns: (transfer full) (element-type OstreeRepoFinderResult): array of zero + * or more results + * Since: 2017.8 + */ +GPtrArray * +ostree_repo_finder_resolve_all_finish (GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, NULL), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + return g_task_propagate_pointer (G_TASK (result), error); +} + +G_DEFINE_BOXED_TYPE (OstreeRepoFinderResult, ostree_repo_finder_result, + ostree_repo_finder_result_dup, ostree_repo_finder_result_free) + +/** + * ostree_repo_finder_result_new: + * @remote: (transfer none): an #OstreeRemote containing the transport details + * for the result + * @finder: (transfer none): the #OstreeRepoFinder instance which produced the + * result + * @priority: static priority of the result, where higher numbers indicate lower + * priority + * @ref_to_checksum: (element-type OstreeCollectionRef utf8): map of collection–ref pairs + * to checksums provided by this result + * @summary_last_modified: Unix timestamp (seconds since the epoch, UTC) when + * the summary file for the result was last modified, or `0` if this is unknown + * + * Create a new #OstreeRepoFinderResult instance. The semantics for the arguments + * are as described in the #OstreeRepoFinderResult documentation. + * + * Returns: (transfer full): a new #OstreeRepoFinderResult + * Since: 2017.8 + */ +OstreeRepoFinderResult * +ostree_repo_finder_result_new (OstreeRemote *remote, + OstreeRepoFinder *finder, + gint priority, + GHashTable *ref_to_checksum, + guint64 summary_last_modified) +{ + g_autoptr(OstreeRepoFinderResult) result = NULL; + + g_return_val_if_fail (remote != NULL, NULL); + g_return_val_if_fail (OSTREE_IS_REPO_FINDER (finder), NULL); + g_return_val_if_fail (is_valid_collection_ref_map (ref_to_checksum), NULL); + + result = g_new0 (OstreeRepoFinderResult, 1); + result->remote = ostree_remote_ref (remote); + result->finder = g_object_ref (finder); + result->priority = priority; + result->ref_to_checksum = g_hash_table_ref (ref_to_checksum); + result->summary_last_modified = summary_last_modified; + + return g_steal_pointer (&result); +} + +/** + * ostree_repo_finder_result_dup: + * @result: (transfer none): an #OstreeRepoFinderResult to copy + * + * Copy an #OstreeRepoFinderResult. + * + * Returns: (transfer full): a newly allocated copy of @result + * Since: 2017.8 + */ +OstreeRepoFinderResult * +ostree_repo_finder_result_dup (OstreeRepoFinderResult *result) +{ + g_return_val_if_fail (result != NULL, NULL); + + return ostree_repo_finder_result_new (result->remote, result->finder, + result->priority, result->ref_to_checksum, + result->summary_last_modified); +} + +/** + * ostree_repo_finder_result_compare: + * @a: an #OstreeRepoFinderResult + * @b: an #OstreeRepoFinderResult + * + * Compare two #OstreeRepoFinderResult instances to work out which one is better + * to pull from, and hence needs to be ordered before the other. + * + * Returns: <0 if @a is ordered before @b, 0 if they are ordered equally, + * >0 if @b is ordered before @a + * Since: 2017.8 + */ +gint +ostree_repo_finder_result_compare (const OstreeRepoFinderResult *a, + const OstreeRepoFinderResult *b) +{ + guint a_n_refs, b_n_refs; + + g_return_val_if_fail (a != NULL, 0); + g_return_val_if_fail (b != NULL, 0); + + /* FIXME: Check if this is really the ordering we want. For example, we + * probably don’t want a result with 0 refs to be ordered before one with >0 + * refs, just because its priority is higher. */ + if (a->priority != b->priority) + return a->priority - b->priority; + + if (a->summary_last_modified != 0 && b->summary_last_modified != 0 && + a->summary_last_modified != b->summary_last_modified) + return a->summary_last_modified - b->summary_last_modified; + + gpointer value; + GHashTableIter iter; + a_n_refs = b_n_refs = 0; + + g_hash_table_iter_init (&iter, a->ref_to_checksum); + while (g_hash_table_iter_next (&iter, NULL, &value)) + if (value != NULL) + a_n_refs++; + + g_hash_table_iter_init (&iter, b->ref_to_checksum); + while (g_hash_table_iter_next (&iter, NULL, &value)) + if (value != NULL) + b_n_refs++; + + if (a_n_refs != b_n_refs) + return (gint) a_n_refs - (gint) b_n_refs; + + return g_strcmp0 (a->remote->name, b->remote->name); +} + +/** + * ostree_repo_finder_result_free: + * @result: (transfer full): an #OstreeRepoFinderResult + * + * Free the given @result. + * + * Since: 2017.8 + */ +void +ostree_repo_finder_result_free (OstreeRepoFinderResult *result) +{ + g_return_if_fail (result != NULL); + + g_hash_table_unref (result->ref_to_checksum); + g_object_unref (result->finder); + ostree_remote_unref (result->remote); + g_free (result); +} + +/** + * ostree_repo_finder_result_freev: + * @results: (array zero-terminated=1) (transfer full): an #OstreeRepoFinderResult + * + * Free the given @results array, freeing each element and the container. + * + * Since: 2017.8 + */ +void +ostree_repo_finder_result_freev (OstreeRepoFinderResult **results) +{ + gsize i; + + for (i = 0; results[i] != NULL; i++) + ostree_repo_finder_result_free (results[i]); + + g_free (results); +} diff --git a/src/libostree/ostree-repo-finder.h b/src/libostree/ostree-repo-finder.h new file mode 100644 index 00000000..6b0ce8ca --- /dev/null +++ b/src/libostree/ostree-repo-finder.h @@ -0,0 +1,172 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-ref.h" +#include "ostree-remote.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER (ostree_repo_finder_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +_OSTREE_PUBLIC +G_DECLARE_INTERFACE (OstreeRepoFinder, ostree_repo_finder, OSTREE, REPO_FINDER, GObject) */ + +_OSTREE_PUBLIC +GType ostree_repo_finder_get_type (void); +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeRepoFinder OstreeRepoFinder; +typedef struct _OstreeRepoFinderInterface OstreeRepoFinderInterface; + +static inline OstreeRepoFinder *OSTREE_REPO_FINDER (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_repo_finder_get_type (), OstreeRepoFinder); } +static inline gboolean OSTREE_IS_REPO_FINDER (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_repo_finder_get_type ()); } +static inline OstreeRepoFinderInterface *OSTREE_REPO_FINDER_GET_IFACE (gpointer ptr) { return G_TYPE_INSTANCE_GET_INTERFACE (ptr, ostree_repo_finder_get_type (), OstreeRepoFinderInterface); } +G_GNUC_END_IGNORE_DEPRECATIONS + +struct _OstreeRepoFinderInterface +{ + GTypeInterface g_iface; + + void (*resolve_async) (OstreeRepoFinder *self, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); + GPtrArray *(*resolve_finish) (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error); +}; + +_OSTREE_PUBLIC +void ostree_repo_finder_resolve_async (OstreeRepoFinder *self, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +GPtrArray *ostree_repo_finder_resolve_finish (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error); + +_OSTREE_PUBLIC +void ostree_repo_finder_resolve_all_async (OstreeRepoFinder * const *finders, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +GPtrArray *ostree_repo_finder_resolve_all_finish (GAsyncResult *result, + GError **error); + +/** + * OstreeRepoFinderResult: + * @remote: #OstreeRemote which contains the transport details for the result, + * such as its URI and GPG key + * @finder: the #OstreeRepoFinder instance which produced this result + * @priority: static priority of the result, where higher numbers indicate lower + * priority + * @ref_to_checksum: (element-type OstreeCollectionRef utf8): map of collection–ref + * pairs to checksums provided by this remote; values may be %NULL to + * indicate this remote doesn’t provide that ref + * @summary_last_modified: Unix timestamp (seconds since the epoch, UTC) when + * the summary file on the remote was last modified, or `0` if unknown + * + * #OstreeRepoFinderResult gives a single result from an + * ostree_repo_finder_resolve_async() or ostree_repo_finder_resolve_all_async() + * operation. This represents a single remote which provides none, some or all + * of the refs being resolved. The structure includes various bits of metadata + * which allow ostree_repo_pull_from_remotes_async() (for example) to prioritise + * how to pull the refs. + * + * The @priority is used as one input of many to ordering functions like + * ostree_repo_finder_result_compare(). + * + * @ref_to_checksum indicates which refs (out of the ones queried for as inputs + * to ostree_repo_finder_resolve_async()) are provided by this remote. The refs + * are present as keys (of type #OstreeCollectionRef), and the corresponding values + * are the checksums of the commits the remote currently has for those refs. (These + * might not be the latest commits available out of all results.) A + * checksum may be %NULL if the remote does not advertise the corresponding ref. + * After ostree_repo_finder_resolve_async() has been called, the commit metadata + * should be available locally, so the details for each checksum can be looked + * up using ostree_repo_load_commit(). + * + * Since: 2017.8 + */ +typedef struct +{ + OstreeRemote *remote; + OstreeRepoFinder *finder; + gint priority; + GHashTable *ref_to_checksum; + guint64 summary_last_modified; + + /*< private >*/ + gpointer padding[4]; +} OstreeRepoFinderResult; + +_OSTREE_PUBLIC +GType ostree_repo_finder_result_get_type (void); + +_OSTREE_PUBLIC +OstreeRepoFinderResult *ostree_repo_finder_result_new (OstreeRemote *remote, + OstreeRepoFinder *finder, + gint priority, + GHashTable *ref_to_checksum, + guint64 summary_last_modified); +_OSTREE_PUBLIC +OstreeRepoFinderResult *ostree_repo_finder_result_dup (OstreeRepoFinderResult *result); +_OSTREE_PUBLIC +gint ostree_repo_finder_result_compare (const OstreeRepoFinderResult *a, + const OstreeRepoFinderResult *b); +_OSTREE_PUBLIC +void ostree_repo_finder_result_free (OstreeRepoFinderResult *result); + +/** + * OstreeRepoFinderResultv: + * + * A %NULL-terminated array of #OstreeRepoFinderResult instances, designed to + * be used with g_auto(): + * + * |[ + * g_auto(OstreeRepoFinderResultv) results = NULL; + * ]| + * + * Since: 2017.8 + */ +typedef OstreeRepoFinderResult** OstreeRepoFinderResultv; + +_OSTREE_PUBLIC +void ostree_repo_finder_result_freev (OstreeRepoFinderResult **results); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 03be117a..b4c565a8 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1,6 +1,7 @@ /* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- * * Copyright (C) 2011,2012,2013 Colin Walters + * Copyright © 2017 Endless Mobile, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -17,7 +18,9 @@ * Free Software Foundation, Inc., 59 Temple Place - Suite 330, * Boston, MA 02111-1307, USA. * - * Author: Colin Walters + * Authors: + * - Colin Walters + * - Philip Withnall */ #include "config.h" @@ -33,8 +36,13 @@ #include "ostree-repo-static-delta-private.h" #include "ostree-metalink.h" #include "ostree-fetcher-util.h" +#include "ostree-remote-private.h" #include "ot-fs-utils.h" +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +#include "ostree-repo-finder.h" +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + #include #define OSTREE_REPO_PULL_CONTENT_PRIORITY (OSTREE_FETCHER_DEFAULT_PRIORITY) @@ -2611,6 +2619,11 @@ repo_remote_fetch_summary (OstreeRepo *self, } } + /* FIXME: Send the ETag from the cache with the request for summary.sig to + * avoid downloading summary.sig unnecessarily. This won’t normally provide + * any benefits (but won’t do any harm) since summary.sig is typically 500B + * in size. But if a repository has multiple keys, the signature file will + * grow and this optimisation may be useful. */ if (!_ostree_preload_metadata_file (self, fetcher, mirrorlist, @@ -3786,6 +3799,1302 @@ ostree_repo_pull_with_options (OstreeRepo *self, return ret; } +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + +/* Structure used in ostree_repo_find_remotes_async() which stores metadata + * about a given OSTree commit. This includes the metadata from the commit + * #GVariant, plus some working state which is used to work out which remotes + * have refs pointing to this commit. */ +typedef struct +{ + gchar *checksum; /* always set */ + guint64 commit_size; /* always set */ + guint64 timestamp; /* 0 for unknown */ + GVariant *additional_metadata; + GArray *refs; /* (element-type gsize), indexes to refs which point to this commit on at least one remote */ +} CommitMetadata; + +static void +commit_metadata_free (CommitMetadata *info) +{ + g_clear_pointer (&info->refs, g_array_unref); + g_free (info->checksum); + g_clear_pointer (&info->additional_metadata, g_variant_unref); + g_free (info); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (CommitMetadata, commit_metadata_free) + +static CommitMetadata * +commit_metadata_new (const gchar *checksum, + guint64 commit_size, + guint64 timestamp, + GVariant *additional_metadata) +{ + g_autoptr(CommitMetadata) info = NULL; + + info = g_new0 (CommitMetadata, 1); + info->checksum = g_strdup (checksum); + info->commit_size = commit_size; + info->timestamp = timestamp; + info->additional_metadata = (additional_metadata != NULL) ? g_variant_ref (additional_metadata) : NULL; + info->refs = g_array_new (FALSE, FALSE, sizeof (gsize)); + + return g_steal_pointer (&info); +} + +/* Structure used in ostree_repo_find_remotes_async() to store a grid (or table) + * of pointers, indexed by rows and columns. Basically an encapsulated 2D array. + * See the comments in ostree_repo_find_remotes_async() for its semantics + * there. */ +typedef struct +{ + gsize width; /* pointers */ + gsize height; /* pointers */ + gconstpointer pointers[]; /* n_pointers = width * height */ +} PointerTable; + +static void +pointer_table_free (PointerTable *table) +{ + g_free (table); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (PointerTable, pointer_table_free) + +/* Both dimensions are in numbers of pointers. */ +static PointerTable * +pointer_table_new (gsize width, + gsize height) +{ + g_autoptr(PointerTable) table = NULL; + + g_return_val_if_fail (width > 0, NULL); + g_return_val_if_fail (height > 0, NULL); + g_return_val_if_fail (width <= (G_MAXSIZE - sizeof (PointerTable)) / sizeof (gconstpointer) / height, NULL); + + table = g_malloc0 (sizeof (PointerTable) + sizeof (gconstpointer) * width * height); + table->width = width; + table->height = height; + + return g_steal_pointer (&table); +} + +static gconstpointer +pointer_table_get (const PointerTable *table, + gsize x, + gsize y) +{ + g_return_val_if_fail (table != NULL, FALSE); + g_return_val_if_fail (x < table->width, FALSE); + g_return_val_if_fail (y < table->height, FALSE); + + return table->pointers[table->width * y + x]; +} + +static void +pointer_table_set (PointerTable *table, + gsize x, + gsize y, + gconstpointer value) +{ + g_return_if_fail (table != NULL); + g_return_if_fail (x < table->width); + g_return_if_fail (y < table->height); + + table->pointers[table->width * y + x] = value; +} + +/* Validate the given struct contains a valid collection ID and ref name. */ +static gboolean +is_valid_collection_ref (const OstreeCollectionRef *ref) +{ + return (ref != NULL && + ostree_validate_rev (ref->ref_name, NULL) && + ostree_validate_collection_id (ref->collection_id, NULL)); +} + +/* Validate @refs is non-%NULL, non-empty, and contains only valid collection + * and ref names. */ +static gboolean +is_valid_collection_ref_array (const OstreeCollectionRef * const *refs) +{ + gsize i; + + if (refs == NULL || *refs == NULL) + return FALSE; + + for (i = 0; refs[i] != NULL; i++) + { + if (!is_valid_collection_ref (refs[i])) + return FALSE; + } + + return TRUE; +} + +/* Validate @finders is non-%NULL, non-empty, and contains only valid + * #OstreeRepoFinder instances. */ +static gboolean +is_valid_finder_array (OstreeRepoFinder **finders) +{ + gsize i; + + if (finders == NULL || *finders == NULL) + return FALSE; + + for (i = 0; finders[i] != NULL; i++) + { + if (!OSTREE_IS_REPO_FINDER (finders[i])) + return FALSE; + } + + return TRUE; +} + +/* Closure used to carry inputs from ostree_repo_find_remotes_async() to + * find_remotes_cb(). */ +typedef struct +{ + OstreeCollectionRef **refs; + GVariant *options; + OstreeAsyncProgress *progress; +} FindRemotesData; + +static void +find_remotes_data_free (FindRemotesData *data) +{ + g_clear_object (&data->progress); + g_clear_pointer (&data->options, g_variant_unref); + ostree_collection_ref_freev (data->refs); + + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (FindRemotesData, find_remotes_data_free) + +static FindRemotesData * +find_remotes_data_new (const OstreeCollectionRef * const *refs, + GVariant *options, + OstreeAsyncProgress *progress) +{ + g_autoptr(FindRemotesData) data = NULL; + + data = g_new0 (FindRemotesData, 1); + data->refs = ostree_collection_ref_dupv (refs); + data->options = (options != NULL) ? g_variant_ref (options) : NULL; + data->progress = (progress != NULL) ? g_object_ref (progress) : NULL; + + return g_steal_pointer (&data); +} + +static gchar * +uint64_secs_to_iso8601 (guint64 secs) +{ + g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (secs); + + if (dt != NULL) + return g_date_time_format (dt, "%FT%TZ"); + else + return g_strdup ("invalid"); +} + +static gint +sort_results_cb (gconstpointer a, + gconstpointer b) +{ + const OstreeRepoFinderResult **result_a = (const OstreeRepoFinderResult **) a; + const OstreeRepoFinderResult **result_b = (const OstreeRepoFinderResult **) b; + + return ostree_repo_finder_result_compare (*result_a, *result_b); +} + +static void +repo_finder_result_free0 (OstreeRepoFinderResult *result) +{ + if (result == NULL) + return; + + ostree_repo_finder_result_free (result); +} + +static void find_remotes_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data); + +/** + * ostree_repo_find_remotes_async: + * @self: an #OstreeRepo + * @refs: (array zero-terminated=1): non-empty array of collection–ref pairs to find remotes for + * @options: (nullable): a GVariant `a{sv}` with an extensible set of flags + * @finders: (array zero-terminated=1) (transfer none): non-empty array of + * #OstreeRepoFinder instances to use, or %NULL to use the system defaults + * @progress: (nullable): an #OstreeAsyncProgress to update with the operation’s + * progress, or %NULL + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: asynchronous completion callback + * @user_data: data to pass to @callback + * + * Find reachable remote URIs which claim to provide any of the given named + * @refs. This will search for configured remotes (#OstreeRepoFinderConfig), + * mounted volumes (#OstreeRepoFinderMount) and (if enabled at compile time) + * local network peers (#OstreeRepoFinderAvahi). In order to use a custom + * configuration of #OstreeRepoFinder instances, call + * ostree_repo_finder_resolve_all_async() on them individually. + * + * Any remote which is found and which claims to support any of the given @refs + * will be returned in the results. It is possible that a remote claims to + * support a given ref, but turns out not to — it is not possible to verify this + * until ostree_repo_pull_from_remotes_async() is called. + * + * The returned results will be sorted with the most useful first — this is + * typically the remote which claims to provide the most of @refs, at the lowest + * latency. + * + * Each result contains a list of the subset of @refs it claims to provide. It + * is possible for a non-empty list of results to be returned, but for some of + * @refs to not be listed in any of the results. Callers must check for this. + * + * Pass the results to ostree_repo_pull_from_remotes_async() to pull the given @refs + * from those remotes. + * + * No @options are currently supported. + * + * @finders must be a non-empty %NULL-terminated array of the #OstreeRepoFinder + * instances to use, or %NULL to use the system default set of finders, which + * will typically be all available finders using their default options (but + * this is not guaranteed). + * + * GPG verification of the summary and all commits will be used unconditionally. + * + * This will use the thread-default #GMainContext, but will not iterate it. + * + * Since: 2017.8 + */ +void +ostree_repo_find_remotes_async (OstreeRepo *self, + const OstreeCollectionRef * const *refs, + GVariant *options, + OstreeRepoFinder **finders, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autoptr(FindRemotesData) data = NULL; + GMainContext *context; + OstreeRepoFinder *default_finders[4] = { NULL, }; + + g_return_if_fail (OSTREE_IS_REPO (self)); + g_return_if_fail (is_valid_collection_ref_array (refs)); + g_return_if_fail (options == NULL || + g_variant_is_of_type (options, G_VARIANT_TYPE_VARDICT)); + g_return_if_fail (finders == NULL || is_valid_finder_array (finders)); + g_return_if_fail (progress == NULL || OSTREE_IS_ASYNC_PROGRESS (progress)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + /* Set up a task for the whole operation. */ + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_find_remotes_async); + + context = g_main_context_get_thread_default (); + + /* Are we using #OstreeRepoFinders provided by the user, or the defaults? */ + if (finders == NULL) + { + finders = default_finders; + } + + data = find_remotes_data_new (refs, options, progress); + g_task_set_task_data (task, g_steal_pointer (&data), (GDestroyNotify) find_remotes_data_free); + + /* Asynchronously resolve all possible remotes for the given refs. */ + ostree_repo_finder_resolve_all_async (finders, refs, self, cancellable, + find_remotes_cb, g_steal_pointer (&task)); +} + +/* Find the first instance of (@collection_id, @ref_name) in @refs and return + * its index; or return %FALSE if nothing’s found. */ +static gboolean +collection_refv_contains (const OstreeCollectionRef * const *refs, + const gchar *collection_id, + const gchar *ref_name, + gsize *out_index) +{ + gsize i; + + for (i = 0; refs[i] != NULL; i++) + { + if (g_str_equal (refs[i]->collection_id, collection_id) && + g_str_equal (refs[i]->ref_name, ref_name)) + { + *out_index = i; + return TRUE; + } + } + + return FALSE; +} + +/* For each ref from @refs which is listed in @summary_refs, cache its metadata + * from the summary file entry into @commit_metadatas, and add the checksum it + * points to into @refs_and_remotes_table at (@ref_index, @result_index). + * @ref_index is the ref’s index in @refs. */ +static gboolean +find_remotes_process_refs (OstreeRepo *self, + const OstreeCollectionRef * const *refs, + OstreeRepoFinderResult *result, + gsize result_index, + const gchar *summary_collection_id, + GVariant *summary_refs, + GHashTable *commit_metadatas, + PointerTable *refs_and_remotes_table) +{ + gsize j, n; + + for (j = 0, n = g_variant_n_children (summary_refs); j < n; j++) + { + const guchar *csum_bytes; + g_autoptr(GVariant) ref_v = NULL, csum_v = NULL, commit_metadata_v = NULL, stored_commit_metadata_v = NULL; + guint64 commit_size, commit_timestamp; + gchar tmp_checksum[OSTREE_SHA256_STRING_LEN + 1]; + gsize ref_index; + g_autoptr(GDateTime) dt = NULL; + g_autoptr(GError) error = NULL; + const gchar *ref_name; + CommitMetadata *commit_metadata; + + /* Check the ref name. */ + ref_v = g_variant_get_child_value (summary_refs, j); + g_variant_get_child (ref_v, 0, "&s", &ref_name); + + if (!ostree_validate_rev (ref_name, &error)) + { + g_debug ("%s: Summary for result ‘%s’ contained invalid ref name ‘%s’: %s", + G_STRFUNC, result->remote->name, ref_name, error->message); + return FALSE; + } + + /* Check the commit checksum. */ + g_variant_get_child (ref_v, 1, "(t@ay@a{sv})", &commit_size, &csum_v, &commit_metadata_v); + + csum_bytes = ostree_checksum_bytes_peek_validate (csum_v, &error); + if (csum_bytes == NULL) + { + g_debug ("%s: Summary for result ‘%s’ contained invalid ref checksum: %s", + G_STRFUNC, result->remote->name, error->message); + return FALSE; + } + + ostree_checksum_inplace_from_bytes (csum_bytes, tmp_checksum); + + /* Is this a ref we care about? */ + if (!collection_refv_contains (refs, summary_collection_id, ref_name, &ref_index)) + continue; + + /* Load the commit metadata from disk if possible, for verification. */ + if (!ostree_repo_load_commit (self, tmp_checksum, &stored_commit_metadata_v, NULL, NULL)) + stored_commit_metadata_v = NULL; + + /* Check the additional metadata. */ + if (!g_variant_lookup (commit_metadata_v, OSTREE_COMMIT_TIMESTAMP, "t", &commit_timestamp)) + commit_timestamp = 0; /* unknown */ + else + commit_timestamp = GUINT64_FROM_BE (commit_timestamp); + + dt = g_date_time_new_from_unix_utc (commit_timestamp); + + if (dt == NULL) + { + g_debug ("%s: Summary for result ‘%s’ contained commit timestamp %" G_GUINT64_FORMAT " which is too far in the future. Resetting to 0.", + G_STRFUNC, result->remote->name, commit_timestamp); + commit_timestamp = 0; + } + + /* Check and store the commit metadata. */ + commit_metadata = g_hash_table_lookup (commit_metadatas, tmp_checksum); + + if (commit_metadata == NULL) + { + commit_metadata = commit_metadata_new (tmp_checksum, commit_size, + (stored_commit_metadata_v != NULL) ? ostree_commit_get_timestamp (stored_commit_metadata_v) : 0, + NULL); + g_hash_table_insert (commit_metadatas, commit_metadata->checksum, + commit_metadata /* transfer */); + } + + /* Update the metadata if possible. */ + if (commit_metadata->timestamp == 0) + { + commit_metadata->timestamp = commit_timestamp; + } + else if (commit_timestamp != 0 && commit_metadata->timestamp != commit_timestamp) + { + g_debug ("%s: Summary for result ‘%s’ contained commit timestamp %" G_GUINT64_FORMAT " which did not match existing timestamp %" G_GUINT64_FORMAT ". Ignoring.", + G_STRFUNC, result->remote->name, commit_timestamp, commit_metadata->timestamp); + return FALSE; + } + + if (commit_size != commit_metadata->commit_size) + { + g_debug ("%s: Summary for result ‘%s’ contained commit size %" G_GUINT64_FORMAT "B which did not match existing size %" G_GUINT64_FORMAT "B. Ignoring.", + G_STRFUNC, result->remote->name, commit_size, commit_metadata->commit_size); + return FALSE; + } + + pointer_table_set (refs_and_remotes_table, ref_index, result_index, commit_metadata->checksum); + g_array_append_val (commit_metadata->refs, ref_index); + + g_debug ("%s: Remote ‘%s’ lists ref ‘%s’ mapping to commit ‘%s’.", + G_STRFUNC, result->remote->name, ref_name, commit_metadata->checksum); + } + + return TRUE; +} + +static void +find_remotes_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + OstreeRepo *self; + g_autoptr(GTask) task = NULL; + GCancellable *cancellable; + const FindRemotesData *data; + const OstreeCollectionRef * const *refs; + OstreeAsyncProgress *progress; + g_autoptr(GError) error = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + gsize i; + GHashTableIter iter; + CommitMetadata *commit_metadata; + g_autoptr(PointerTable) refs_and_remotes_table = NULL; /* (element-type commit-checksum) */ + g_autoptr(GHashTable) commit_metadatas = NULL; /* (element-type commit-checksum CommitMetadata) */ + g_autoptr(OstreeFetcher) fetcher = NULL; + g_autofree const gchar **ref_to_latest_commit = NULL; /* indexed as @refs; (element-type commit-checksum) */ + gsize n_refs; + const gchar *checksum; + g_autoptr(GPtrArray) remotes_to_remove = NULL; /* (element-type OstreeRemote) */ + g_autoptr(GPtrArray) final_results = NULL; /* (element-type OstreeRepoFinderResult) */ + + task = G_TASK (user_data); + self = OSTREE_REPO (g_task_get_source_object (task)); + cancellable = g_task_get_cancellable (task); + data = g_task_get_task_data (task); + + refs = (const OstreeCollectionRef * const *) data->refs; + progress = data->progress; + + /* Finish finding the remotes. */ + results = ostree_repo_finder_resolve_all_finish (result, &error); + + if (results == NULL) + { + g_task_return_error (task, g_steal_pointer (&error)); + return; + } + + if (results->len == 0) + { + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); + return; + } + + /* Throughout this function, we eliminate invalid results from @results by + * clearing them to %NULL. We cannot remove them from the array, as that messes + * up iteration and stored array indices. Accordingly, we need the free function + * to be %NULL-safe. */ + g_ptr_array_set_free_func (results, (GDestroyNotify) repo_finder_result_free0); + + /* FIXME: Add support for options: + * - override-commit-ids (allow downgrades) + * + * Use case: multiple pulls of separate subdirs; want them to use the same + * configuration. + * Use case: downgrading a flatpak app. + */ + + /* FIXME: In future, we also want to pull static delta superblocks in this + * phase, so that we have all the metadata we need for accurate size + * estimation for the actual pull operation. This should check the + * disable-static-deltas option first. */ + + /* FIXME: We currently do nothing with @progress. */ + + /* Each key must be a pointer to the #CommitMetadata.checksum field of its value. */ + commit_metadatas = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify) commit_metadata_free); + + /* X dimension is an index into @refs. Y dimension is an index into @results. + * Each cell stores the commit checksum which that ref resolves to on that + * remote, or %NULL if the remote doesn’t have that ref. */ + n_refs = g_strv_length ((gchar **) refs); /* it’s not a GStrv, but this works */ + refs_and_remotes_table = pointer_table_new (n_refs, results->len); + remotes_to_remove = g_ptr_array_new_with_free_func (NULL); + + /* Fetch and validate the summary file for each result. */ + /* FIXME: All these downloads could be parallelised; that requires the + * ostree_repo_remote_fetch_summary_with_options() API to be async. */ + for (i = 0; i < results->len; i++) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + g_autoptr(GBytes) summary_bytes = NULL, summary_sig_bytes = NULL; + g_autoptr(GVariant) summary_v = NULL; + guint64 summary_last_modified; + g_autoptr(GVariant) summary_refs = NULL; + g_autoptr(GVariant) additional_metadata_v = NULL; + g_autofree gchar *summary_collection_id = NULL; + g_autoptr(GVariantIter) summary_collection_map = NULL; + gboolean invalid_result = FALSE; + + /* Add the remote to our internal list of remotes, so other libostree + * API can access it. */ + if (!_ostree_repo_add_remote (self, result->remote)) + g_ptr_array_add (remotes_to_remove, result->remote); + + g_debug ("%s: Fetching summary for remote ‘%s’ with keyring ‘%s’.", + G_STRFUNC, result->remote->name, result->remote->keyring); + + /* Download the summary and signature, and validate the signature. This + * will load from the cache if possible. */ + ostree_repo_remote_fetch_summary_with_options (self, + result->remote->name, + NULL, /* no options */ + &summary_bytes, + &summary_sig_bytes, + cancellable, + &error); + + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + goto error; + else if (error != NULL) + { + g_debug ("%s: Failed to download summary for result ‘%s’. Ignoring. %s", + G_STRFUNC, result->remote->name, error->message); + g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free); + g_clear_error (&error); + continue; + } + + /* Check the metadata in the summary file, especially whether it contains + * all the @refs we are interested in. */ + summary_v = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, + summary_bytes, FALSE); + + /* Check the summary’s additional metadata and set up @commit_metadata + * and @refs_and_remotes_table with all the refs listed in the summary + * file which intersect with @refs. */ + additional_metadata_v = g_variant_get_child_value (summary_v, 1); + + if (g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_COLLECTION_ID, "s", &summary_collection_id)) + { + summary_refs = g_variant_get_child_value (summary_v, 0); + + if (!find_remotes_process_refs (self, refs, result, i, summary_collection_id, summary_refs, + commit_metadatas, refs_and_remotes_table)) + { + g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free); + continue; + } + } + + if (!g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_COLLECTION_MAP, "a{sa(s(taya{sv}))}", &summary_collection_map)) + summary_collection_map = NULL; + + while (summary_collection_map != NULL && + g_variant_iter_loop (summary_collection_map, "{s@a(s(taya{sv}))}", &summary_collection_id, &summary_refs)) + { + if (!find_remotes_process_refs (self, refs, result, i, summary_collection_id, summary_refs, + commit_metadatas, refs_and_remotes_table)) + { + g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free); + invalid_result = TRUE; + break; + } + } + + if (invalid_result) + continue; + + /* Check the summary timestamp. */ + if (!g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_LAST_MODIFIED, "t", &summary_last_modified)) + summary_last_modified = 0; + else + summary_last_modified = GUINT64_FROM_BE (summary_last_modified); + + /* Update the stored result data. Clear the @ref_to_checksum map, since + * it’s been moved to @refs_and_remotes_table and is now potentially out + * of date. */ + g_clear_pointer (&result->ref_to_checksum, g_hash_table_unref); + result->summary_last_modified = summary_last_modified; + } + + /* Fill in any gaps in the metadata for the most recent commits by pulling + * the commit metadata from the remotes. The ‘most recent commits’ are the + * set of head commits pointed to by the refs we just resolved from the + * summary files. */ + g_hash_table_iter_init (&iter, commit_metadatas); + + while (g_hash_table_iter_next (&iter, (gpointer *) &checksum, (gpointer *) &commit_metadata)) + { + char buf[_OSTREE_LOOSE_PATH_MAX]; + g_autofree gchar *commit_filename = NULL; + g_autoptr(GPtrArray) mirrorlist = NULL; /* (element-type OstreeFetcherURI) */ + g_autoptr(GBytes) commit_bytes = NULL; + g_autoptr(GVariant) commit_v = NULL; + guint64 commit_timestamp; + g_autoptr(GDateTime) dt = NULL; + + /* Already complete? */ + if (commit_metadata->timestamp != 0) + continue; + + _ostree_loose_path (buf, commit_metadata->checksum, OSTREE_OBJECT_TYPE_COMMIT, OSTREE_REPO_MODE_ARCHIVE_Z2); + commit_filename = g_build_filename ("objects", buf, NULL); + + /* For each of the remotes whose summary files contain this ref, try + * downloading the commit metadata until we succeed. Since the results are + * in priority order, the most important remotes are tried first. */ + for (i = 0; i < commit_metadata->refs->len; i++) + { + gsize ref_index = g_array_index (commit_metadata->refs, gsize, i); + gsize j; + + for (j = 0; j < results->len; j++) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, j); + + /* Previous error processing this result? */ + if (result == NULL) + continue; + + if (pointer_table_get (refs_and_remotes_table, ref_index, j) != commit_metadata->checksum) + continue; + + g_autofree gchar *uri = NULL; + g_autoptr(OstreeFetcherURI) fetcher_uri = NULL; + + if (!ostree_repo_remote_get_url (self, result->remote->name, + &uri, &error)) + goto error; + + fetcher_uri = _ostree_fetcher_uri_parse (uri, &error); + if (fetcher_uri == NULL) + goto error; + + fetcher = _ostree_repo_remote_new_fetcher (self, result->remote->name, + TRUE, &error); + if (fetcher == NULL) + goto error; + + g_debug ("%s: Fetching metadata for commit ‘%s’ from remote ‘%s’.", + G_STRFUNC, commit_metadata->checksum, result->remote->name); + + /* FIXME: Support remotes which have contenturl, mirrorlist, etc. */ + mirrorlist = g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); + g_ptr_array_add (mirrorlist, g_steal_pointer (&fetcher_uri)); + + if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, + mirrorlist, + commit_filename, + FALSE, /* don’t add trailing nul */ + TRUE, /* return NULL on ENOENT */ + &commit_bytes, + 0, /* no maximum size */ + cancellable, + &error)) + goto error; + + glnx_unref_object OstreeGpgVerifyResult *verify_result = NULL; + + verify_result = ostree_repo_verify_commit_for_remote (self, + commit_metadata->checksum, + result->remote->name, + cancellable, + &error); + if (verify_result == NULL) + { + g_prefix_error (&error, "Commit %s: ", commit_metadata->checksum); + goto error; + } + + if (!ostree_gpg_verify_result_require_valid_signature (verify_result, &error)) + { + g_prefix_error (&error, "Commit %s: ", commit_metadata->checksum); + goto error; + } + + if (commit_bytes != NULL) + break; + } + + if (commit_bytes != NULL) + break; + } + + if (commit_bytes == NULL) + { + g_set_error (&error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Metadata not found for commit ‘%s’", commit_metadata->checksum); + goto error; + } + + /* Parse the commit metadata. */ + commit_v = g_variant_new_from_bytes (OSTREE_COMMIT_GVARIANT_FORMAT, + commit_bytes, FALSE); + g_variant_get_child (commit_v, 5, "t", &commit_timestamp); + commit_timestamp = GUINT64_FROM_BE (commit_timestamp); + dt = g_date_time_new_from_unix_utc (commit_timestamp); + + if (dt == NULL) + { + g_debug ("%s: Commit ‘%s’ metadata contained timestamp %" G_GUINT64_FORMAT " which is too far in the future. Resetting to 0.", + G_STRFUNC, commit_metadata->checksum, commit_timestamp); + commit_timestamp = 0; + } + + /* Update the #CommitMetadata. */ + commit_metadata->timestamp = commit_timestamp; + } + + /* Find the latest commit for each ref. This is where we resolve the + * differences between remotes: two remotes could both contain ref R, but one + * remote could be outdated compared to the other, and point to an older + * commit. For each ref, we want to find the most recent commit any remote + * points to for it. + * + * @ref_to_latest_commit is indexed by @ref_index, and its values are the + * latest checksum for each ref. */ + ref_to_latest_commit = g_new0 (const gchar *, n_refs); + + for (i = 0; i < n_refs; i++) + { + gsize j; + const gchar *latest_checksum = NULL; + const CommitMetadata *latest_commit_metadata = NULL; + g_autofree gchar *latest_commit_timestamp_str = NULL; + + for (j = 0; j < results->len; j++) + { + const CommitMetadata *candidate_commit_metadata; + const gchar *candidate_checksum; + + candidate_checksum = pointer_table_get (refs_and_remotes_table, i, j); + + if (candidate_checksum == NULL) + continue; + + candidate_commit_metadata = g_hash_table_lookup (commit_metadatas, candidate_checksum); + g_assert (candidate_commit_metadata != NULL); + + if (latest_commit_metadata == NULL || + candidate_commit_metadata->timestamp > latest_commit_metadata->timestamp) + { + latest_checksum = candidate_checksum; + latest_commit_metadata = candidate_commit_metadata; + } + } + + /* @latest_checksum could be %NULL here if there was an error downloading + * the summary or commit metadata files above. */ + ref_to_latest_commit[i] = latest_checksum; + + if (latest_commit_metadata != NULL) + { + latest_commit_timestamp_str = uint64_secs_to_iso8601 (latest_commit_metadata->timestamp); + g_debug ("%s: Latest commit for ref (%s, %s) across all remotes is ‘%s’ with timestamp %s.", + G_STRFUNC, refs[i]->collection_id, refs[i]->ref_name, + latest_checksum, latest_commit_timestamp_str); + } + else + { + g_debug ("%s: Latest commit for ref (%s, %s) is unknown due to failure to download metadata.", + G_STRFUNC, refs[i]->collection_id, refs[i]->ref_name); + } + } + + /* Recombine @commit_metadatas and @results so that each + * #OstreeRepoFinderResult.refs lists the refs for which that remote has the + * latest commits (i.e. it’s not out of date compared to some other remote). */ + final_results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + + for (i = 0; i < results->len; i++) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + g_autoptr(GHashTable) validated_ref_to_checksum = NULL; /* (element-type utf8 utf8) */ + gsize j; + + /* Previous error processing this result? */ + if (result == NULL) + continue; + + /* Map of refs to checksums provided by this result. The checksums should + * be %NULL for each ref unless this result provides the latest checksum. */ + validated_ref_to_checksum = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + (GDestroyNotify) ostree_collection_ref_free, + g_free); + + for (j = 0; refs[j] != NULL; j++) + { + const gchar *latest_commit_for_ref = ref_to_latest_commit[j]; + + if (pointer_table_get (refs_and_remotes_table, j, i) != latest_commit_for_ref) + latest_commit_for_ref = NULL; + + g_hash_table_insert (validated_ref_to_checksum, ostree_collection_ref_dup (refs[j]), g_strdup (latest_commit_for_ref)); + } + + if (g_hash_table_size (validated_ref_to_checksum) == 0) + { + g_debug ("%s: Omitting remote ‘%s’ from results as none of its refs are new enough.", + G_STRFUNC, result->remote->name); + ostree_repo_finder_result_free (g_steal_pointer (&g_ptr_array_index (results, i))); + continue; + } + + result->ref_to_checksum = g_steal_pointer (&validated_ref_to_checksum); + g_ptr_array_add (final_results, g_steal_pointer (&g_ptr_array_index (results, i))); + } + + /* Ensure the updated results are still in priority order. */ + g_ptr_array_sort (final_results, sort_results_cb); + + /* Remove the remotes we temporarily added. + * FIXME: It would be so much better if we could pass #OstreeRemote pointers + * around internally, to avoid serialising on the global table of them. */ + for (i = 0; i < remotes_to_remove->len; i++) + { + OstreeRemote *remote = g_ptr_array_index (remotes_to_remove, i); + _ostree_repo_remove_remote (self, remote); + } + + g_task_return_pointer (task, g_steal_pointer (&final_results), (GDestroyNotify) g_ptr_array_unref); + + return; + +error: + /* Remove the remotes we temporarily added. */ + for (i = 0; i < remotes_to_remove->len; i++) + { + OstreeRemote *remote = g_ptr_array_index (remotes_to_remove, i); + _ostree_repo_remove_remote (self, remote); + } + + g_task_return_error (task, g_steal_pointer (&error)); +} + +/** + * ostree_repo_find_remotes_finish: + * @self: an #OstreeRepo + * @result: the asynchronous result + * @error: return location for a #GError, or %NULL + * + * Finish an asynchronous pull operation started with + * ostree_repo_find_remotes_async(). + * + * Returns: (transfer full) (array zero-terminated=1): a potentially empty array + * of #OstreeRepoFinderResults, followed by a %NULL terminator element; or + * %NULL on error + * Since: 2017.8 + */ +OstreeRepoFinderResult ** +ostree_repo_find_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error) +{ + g_autoptr(GPtrArray) results = NULL; + + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + g_return_val_if_fail (g_task_is_valid (result, self), NULL); + g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_find_remotes_async), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + results = g_task_propagate_pointer (G_TASK (result), error); + + if (results != NULL) + { + g_ptr_array_add (results, NULL); /* NULL terminator */ + return (OstreeRepoFinderResult **) g_ptr_array_free (g_steal_pointer (&results), FALSE); + } + else + return NULL; +} + +static void +copy_option (GVariantDict *master_options, + GVariantDict *slave_options, + const gchar *key, + const GVariantType *expected_type) +{ + g_autoptr(GVariant) option_v = g_variant_dict_lookup_value (master_options, key, expected_type); + if (option_v != NULL) + g_variant_dict_insert_value (slave_options, key, g_steal_pointer (&option_v)); +} + +/** + * ostree_repo_pull_from_remotes_async: + * @self: an #OstreeRepo + * @results: (array zero-terminated=1): %NULL-terminated array of remotes to + * pull from, including the refs to pull from each + * @options: (nullable): A GVariant `a{sv}` with an extensible set of flags + * @progress: (nullable): an #OstreeAsyncProgress to update with the operation’s + * progress, or %NULL + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: asynchronous completion callback + * @user_data: data to pass to @callback + * + * Pull refs from multiple remotes which have been found using + * ostree_repo_find_remotes_async(). + * + * @results are expected to be in priority order, with the best remotes to pull + * from listed first. ostree_repo_pull_from_remotes_async() will generally pull + * from the remotes in order, but may parallelise its downloads. + * + * If an error is encountered when pulling from a given remote, that remote will + * be ignored and another will be tried instead. If any refs have not been + * downloaded successfully after all remotes have been tried, %G_IO_ERROR_FAILED + * will be returned. The results of any successful downloads will remain cached + * in the local repository. + * + * If @cancellable is cancelled, %G_IO_ERROR_CANCELLED will be returned + * immediately. The results of any successfully completed downloads at that + * point will remain cached in the local repository. + * + * GPG verification of the summary and all commits will be used unconditionally. + * + * The following @options are currently defined: + * + * * `flags` (`i`): #OstreeRepoPullFlags to apply to the pull operation + * * `inherit-transaction` (`b`): %TRUE to inherit an ongoing transaction on + * the #OstreeRepo, rather than encapsulating the pull in a new one + * + * Since: 2017.8 + */ +void +ostree_repo_pull_from_remotes_async (OstreeRepo *self, + const OstreeRepoFinderResult * const *results, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_return_if_fail (OSTREE_IS_REPO (self)); + g_return_if_fail (results != NULL && results[0] != NULL); + g_return_if_fail (options == NULL || g_variant_is_of_type (options, G_VARIANT_TYPE ("a{sv}"))); + g_return_if_fail (progress == NULL || OSTREE_IS_ASYNC_PROGRESS (progress)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + g_autoptr(GTask) task = NULL; + g_autoptr(GHashTable) refs_pulled = NULL; /* (element-type OstreeCollectionRef gboolean) */ + gsize i, j; + g_autoptr(GString) refs_unpulled_string = NULL; + GHashTableIter iter; + const OstreeCollectionRef *ref; + gpointer is_pulled_pointer; + g_autoptr(GError) local_error = NULL; + g_auto(GVariantDict) options_dict = OT_VARIANT_BUILDER_INITIALIZER; + OstreeRepoPullFlags flags; + gboolean inherit_transaction; + + /* Set up a task for the whole operation. */ + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_pull_from_remotes_async); + + /* Keep track of the set of refs we’ve pulled already. Value is %TRUE if the + * ref has been pulled; %FALSE if it has not. */ + refs_pulled = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, NULL, NULL); + + g_variant_dict_init (&options_dict, options); + + if (!g_variant_dict_lookup (&options_dict, "flags", "i", &flags)) + flags = OSTREE_REPO_PULL_FLAGS_NONE; + if (!g_variant_dict_lookup (&options_dict, "inherit-transaction", "b", &inherit_transaction)) + inherit_transaction = FALSE; + + /* Run all the local pull operations in a single overall transaction. */ + if (!inherit_transaction && + !ostree_repo_prepare_transaction (self, NULL, cancellable, &local_error)) + { + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + /* FIXME: Rework this code to pull in parallel where possible. At the moment + * we expect the (i == 0) iteration will do all the work (all the refs) and + * subsequent iterations are only there in case of error. + * + * The code is currently all synchronous, too. Making it asynchronous requires + * the underlying pull code to be asynchronous. */ + for (i = 0; results[i] != NULL; i++) + { + const OstreeRepoFinderResult *result = results[i]; + + g_autoptr(GString) refs_to_pull_str = NULL; + g_autoptr(GPtrArray) refs_to_pull = NULL; /* (element-type OstreeCollectionRef) */ + g_auto(GVariantBuilder) refs_to_pull_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_auto(GVariantDict) local_options_dict = OT_VARIANT_BUILDER_INITIALIZER; + g_autoptr(GVariant) local_options = NULL; + const gchar *checksum; + gboolean remove_remote; + + refs_to_pull = g_ptr_array_new_with_free_func (NULL); + refs_to_pull_str = g_string_new (""); + g_variant_builder_init (&refs_to_pull_builder, G_VARIANT_TYPE ("a(sss)")); + + g_hash_table_iter_init (&iter, result->ref_to_checksum); + + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum)) + { + if (checksum != NULL && + !GPOINTER_TO_INT (g_hash_table_lookup (refs_pulled, ref))) + { + g_ptr_array_add (refs_to_pull, (gpointer) ref); + g_variant_builder_add (&refs_to_pull_builder, "(sss)", + ref->collection_id, ref->ref_name, checksum); + + if (refs_to_pull_str->len > 0) + g_string_append (refs_to_pull_str, ", "); + g_string_append_printf (refs_to_pull_str, "(%s, %s)", + ref->collection_id, ref->ref_name); + } + } + + if (refs_to_pull->len == 0) + { + g_debug ("Ignoring remote ‘%s’ as it has no relevant refs or they " + "have already been pulled.", + result->remote->name); + continue; + } + + /* NULL terminators. */ + g_ptr_array_add (refs_to_pull, NULL); + + g_debug ("Pulling from remote ‘%s’: %s", + result->remote->name, refs_to_pull_str->str); + + /* Set up the pull options. */ + g_variant_dict_init (&local_options_dict, NULL); + + g_variant_dict_insert (&local_options_dict, "flags", "i", OSTREE_REPO_PULL_FLAGS_UNTRUSTED | flags); + g_variant_dict_insert_value (&local_options_dict, "collection-refs", g_variant_builder_end (&refs_to_pull_builder)); + g_variant_dict_insert (&local_options_dict, "gpg-verify", "b", TRUE); + g_variant_dict_insert (&local_options_dict, "gpg-verify-summary", "b", TRUE); + g_variant_dict_insert (&local_options_dict, "inherit-transaction", "b", TRUE); + copy_option (&options_dict, &local_options_dict, "depth", G_VARIANT_TYPE ("i")); + copy_option (&options_dict, &local_options_dict, "disable-static-deltas", G_VARIANT_TYPE ("b")); + copy_option (&options_dict, &local_options_dict, "http-headers", G_VARIANT_TYPE ("a(ss)")); + copy_option (&options_dict, &local_options_dict, "subdirs", G_VARIANT_TYPE ("as")); + copy_option (&options_dict, &local_options_dict, "update-frequency", G_VARIANT_TYPE ("u")); + + local_options = g_variant_dict_end (&local_options_dict); + + /* FIXME: We do nothing useful with @progress at the moment. */ + remove_remote = !_ostree_repo_add_remote (self, result->remote); + ostree_repo_pull_with_options (self, result->remote->name, local_options, + progress, cancellable, &local_error); + if (remove_remote) + _ostree_repo_remove_remote (self, result->remote); + + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + { + if (!inherit_transaction) + ostree_repo_abort_transaction (self, NULL, NULL); + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + for (j = 0; refs_to_pull->pdata[j] != NULL; j++) + g_hash_table_replace (refs_pulled, refs_to_pull->pdata[j], + GINT_TO_POINTER (local_error == NULL)); + + if (local_error != NULL) + { + g_debug ("Failed to pull refs from ‘%s’: %s", + result->remote->name, local_error->message); + g_clear_error (&local_error); + continue; + } + else + { + g_debug ("Pulled refs from ‘%s’.", result->remote->name); + } + } + + /* Commit the transaction. */ + if (!inherit_transaction && + !ostree_repo_commit_transaction (self, NULL, cancellable, &local_error)) + { + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + /* Any refs left un-downloaded? If so, we’ve failed. */ + g_hash_table_iter_init (&iter, refs_pulled); + + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &is_pulled_pointer)) + { + gboolean is_pulled = GPOINTER_TO_INT (is_pulled_pointer); + + if (is_pulled) + continue; + + if (refs_unpulled_string == NULL) + refs_unpulled_string = g_string_new (""); + else + g_string_append (refs_unpulled_string, ", "); + + g_string_append_printf (refs_unpulled_string, "(%s, %s)", + ref->collection_id, ref->ref_name); + } + + if (refs_unpulled_string != NULL) + { + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to pull some refs from the remotes: %s", + refs_unpulled_string->str); + return; + } + + g_task_return_boolean (task, TRUE); +} + +/** + * ostree_repo_pull_from_remotes_finish: + * @self: an #OstreeRepo + * @result: the asynchronous result + * @error: return location for a #GError, or %NULL + * + * Finish an asynchronous pull operation started with + * ostree_repo_pull_from_remotes_async(). + * + * Returns: %TRUE on success, %FALSE otherwise + * Since: 2017.8 + */ +gboolean +ostree_repo_pull_from_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_pull_from_remotes_async), FALSE); + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + + return g_task_propagate_boolean (G_TASK (result), error); +} + +/* Check whether the given remote exists, has a `collection-id` key set, and it + * equals @collection_id. If so, return %TRUE. Otherwise, %FALSE. */ +static gboolean +check_remote_matches_collection_id (OstreeRepo *repo, + const gchar *remote_name, + const gchar *collection_id) +{ + g_autofree gchar *remote_collection_id = NULL; + + if (!ostree_repo_get_remote_option (repo, remote_name, "collection-id", NULL, + &remote_collection_id, NULL) || + remote_collection_id == NULL) + return FALSE; + + return g_str_equal (remote_collection_id, collection_id); +} + +/** + * ostree_repo_resolve_keyring_for_collection: + * @self: an #OstreeRepo + * @collection_id: the collection ID to look up a keyring for + * @cancellable: (nullable): a #GCancellable, or %NULL + * @error: return location for a #GError, or %NULL + * + * Find the GPG keyring for the given @collection_id, using the local + * configuration from the given #OstreeRepo. This will search the configured + * remotes for ones whose `collection-id` key matches @collection_id, and will + * return the GPG keyring from the first matching remote. + * + * If multiple remotes match and have different keyrings, a debug message will + * be emitted, and the first result will be returned. It is expected that the + * keyrings should match. + * + * If no match can be found, a %G_IO_ERROR_NOT_FOUND error will be returned. + * + * Returns: (transfer full): filename of the GPG keyring for @collection_id + * Since: 2017.8 + */ +gchar * +ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, + const gchar *collection_id, + GCancellable *cancellable, + GError **error) +{ + gsize i; + g_auto(GStrv) remotes = NULL; + const OstreeRemote *keyring_remote = NULL; + + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + g_return_val_if_fail (ostree_validate_collection_id (collection_id, NULL), NULL); + g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + /* Look through all the currently configured remotes for the given collection. */ + remotes = ostree_repo_remote_list (self, NULL); + + for (i = 0; remotes != NULL && remotes[i] != NULL; i++) + { + g_autoptr(GError) local_error = NULL; + + if (!check_remote_matches_collection_id (self, remotes[i], collection_id)) + continue; + + if (keyring_remote == NULL) + { + g_debug ("%s: Found match for collection ‘%s’ in remote ‘%s’.", + G_STRFUNC, collection_id, remotes[i]); + keyring_remote = _ostree_repo_get_remote_inherited (self, remotes[i], &local_error); + + if (keyring_remote == NULL) + { + g_debug ("%s: Error loading remote ‘%s’: %s", + G_STRFUNC, remotes[i], local_error->message); + continue; + } + + if (g_strcmp0 (keyring_remote->keyring, "") == 0 || + g_strcmp0 (keyring_remote->keyring, "/dev/null") == 0) + { + g_debug ("%s: Ignoring remote ‘%s’ as it has no keyring configured.", + G_STRFUNC, remotes[i]); + continue; + } + + /* continue so we can catch duplicates */ + } + else + { + g_debug ("%s: Duplicate keyring for collection ‘%s’ in remote ‘%s’." + "Keyring will be loaded from remote ‘%s’.", + G_STRFUNC, collection_id, remotes[i], + keyring_remote->name); + } + } + + if (keyring_remote != NULL) + return g_strdup (keyring_remote->keyring); + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "No keyring found configured locally for collection ‘%s’", + collection_id); + return NULL; + } +} + +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + /** * ostree_repo_remote_fetch_summary_with_options: * @self: Self diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 388d73c4..ea7a7789 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -27,6 +27,7 @@ #include "ostree-async-progress.h" #ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include "ostree-ref.h" +#include "ostree-repo-finder.h" #endif #include "ostree-sepolicy.h" #include "ostree-gpg-verify-result.h" @@ -1080,6 +1081,33 @@ ostree_repo_pull_one_dir (OstreeRepo *self, GCancellable *cancellable, GError **error); + + +#if 0 +FIXME +Called with: remote_name, refs, override-commit-ids +or: URL, refs, override-commit-ids +=> we only need refs; could use the remote_name or URL as additional results + +Summary file is downloaded first, so this would result in multiple downloads of +the summary, but we don’t care because of caching. + +Big problem preventing this from being the overall API: presenting the download +sizes in the gnome-software UI before the user chooses to download. + +_OSTREE_PUBLIC +gboolean ostree_repo_find_remotes_squashed (OstreeRepo *self, + const gchar * const *refs, -> options + GVariant *options, + OstreeRepoFinder **finders, -> options + GMainContext *context, -> nope + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error); +#endif + + + _OSTREE_PUBLIC gboolean ostree_repo_pull_with_options (OstreeRepo *self, const char *remote_name_or_baseurl, @@ -1090,6 +1118,39 @@ gboolean ostree_repo_pull_with_options (OstreeRepo *self, #ifdef OSTREE_ENABLE_EXPERIMENTAL_API +_OSTREE_PUBLIC +void ostree_repo_find_remotes_async (OstreeRepo *self, + const OstreeCollectionRef * const *refs, + GVariant *options, + OstreeRepoFinder **finders, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +OstreeRepoFinderResult **ostree_repo_find_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error); + +_OSTREE_PUBLIC +void ostree_repo_pull_from_remotes_async (OstreeRepo *self, + const OstreeRepoFinderResult * const *results, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +gboolean ostree_repo_pull_from_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error); + +_OSTREE_PUBLIC +gchar *ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, + const gchar *collection_id, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_list_collection_refs (OstreeRepo *self, const char *match_collection_id, diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index c9c3bb75..935707e3 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -37,6 +37,7 @@ #ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include +#include #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include From d15f83c9223a48454fafd47b41cf9bf2e4e4da72 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 19 Apr 2017 00:05:06 +0100 Subject: [PATCH 24/82] lib/repo-finder: Add config-file based OstreeRepoFinder implementation This is a basic implementation of OstreeRepoFinder which resolves ref names to remote URIs by looking their collection IDs up in the local configuration of remotes who have their collection-id key set. Unit tests are included. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- Makefile-libostree-defines.am | 1 + Makefile-libostree.am | 4 +- Makefile-tests.am | 10 + apidoc/ostree-experimental-sections.txt | 8 + src/libostree/libostree-experimental.sym | 2 + src/libostree/ostree-autocleanups.h | 1 + src/libostree/ostree-core-private.h | 3 + src/libostree/ostree-repo-finder-config.c | 235 ++++++++++++++++ src/libostree/ostree-repo-finder-config.h | 55 ++++ src/libostree/ostree-repo-pull.c | 6 + src/libostree/ostree.h | 1 + tests/.gitignore | 1 + tests/test-repo-finder-config.c | 327 ++++++++++++++++++++++ 13 files changed, 653 insertions(+), 1 deletion(-) create mode 100644 src/libostree/ostree-repo-finder-config.c create mode 100644 src/libostree/ostree-repo-finder-config.h create mode 100644 tests/test-repo-finder-config.c diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index 6214633d..ac5eaa00 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -43,6 +43,7 @@ libostree_public_headers += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ src/libostree/ostree-repo-finder.h \ + src/libostree/ostree-repo-finder-config.h \ $(NULL) endif diff --git a/Makefile-libostree.am b/Makefile-libostree.am index a9331dd4..4b968abe 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -155,10 +155,12 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ src/libostree/ostree-repo-finder.h \ + src/libostree/ostree-repo-finder-config.h \ $(NULL) else # if ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += \ src/libostree/ostree-repo-finder.c \ + src/libostree/ostree-repo-finder-config.c \ $(NULL) endif @@ -235,7 +237,7 @@ OSTree_1_0_gir_INCLUDES = Gio-2.0 OSTree_1_0_gir_CFLAGS = $(libostree_1_la_CFLAGS) OSTree_1_0_gir_LIBS = libostree-1.la OSTree_1_0_gir_SCANNERFLAGS = --warn-all --identifier-prefix=Ostree --symbol-prefix=ostree -OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h,$(libostree_1_la_SOURCES)) +OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h %/ostree-repo-finder-config.h,$(libostree_1_la_SOURCES)) INTROSPECTION_GIRS += OSTree-1.0.gir gir_DATA += OSTree-1.0.gir typelib_DATA += OSTree-1.0.typelib diff --git a/Makefile-tests.am b/Makefile-tests.am index 4261fa7c..1cdf8826 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -193,6 +193,12 @@ _installed_or_uninstalled_test_programs = tests/test-varint tests/test-ot-unix-u tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum \ tests/test-basic-c tests/test-sysroot-c tests/test-pull-c +if ENABLE_EXPERIMENTAL_API +test_programs += \ + tests/test-repo-finder-config \ + $(NULL) +endif + # An interactive tool noinst_PROGRAMS += tests/test-rollsum-cli @@ -219,6 +225,10 @@ tests_test_rollsum_SOURCES = src/libostree/ostree-rollsum.c tests/test-rollsum.c tests_test_rollsum_CFLAGS = $(TESTS_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) tests_test_rollsum_LDADD = $(bupsplitpath) $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS) +tests_test_repo_finder_config_SOURCES = tests/test-repo-finder-config.c +tests_test_repo_finder_config_CFLAGS = $(TESTS_CFLAGS) +tests_test_repo_finder_config_LDADD = $(TESTS_LDADD) + tests_test_mutable_tree_CFLAGS = $(TESTS_CFLAGS) tests_test_mutable_tree_LDADD = $(TESTS_LDADD) diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 7e9ed084..93210d56 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -49,6 +49,14 @@ ostree_repo_finder_get_type ostree_repo_finder_result_get_type
+
+ostree-repo-finder-config +OstreeRepoFinderConfig +ostree_repo_finder_config_new + +ostree_repo_finder_config_get_type +
+
ostree-misc-experimental ostree_repo_get_collection_id diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index cda34322..79fcdbe9 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -47,6 +47,8 @@ global: ostree_collection_ref_new; ostree_repo_find_remotes_async; ostree_repo_find_remotes_finish; + ostree_repo_finder_config_get_type; + ostree_repo_finder_config_new; ostree_repo_finder_get_type; ostree_repo_finder_resolve_async; ostree_repo_finder_resolve_all_async; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index 1f7716b2..ac0aa1fb 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -64,6 +64,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index a8fbb8e1..4c117110 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -187,6 +187,9 @@ G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_fre G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) + +#include "ostree-repo-finder-config.h" +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref) #endif G_END_DECLS diff --git a/src/libostree/ostree-repo-finder-config.c b/src/libostree/ostree-repo-finder-config.c new file mode 100644 index 00000000..79a63536 --- /dev/null +++ b/src/libostree/ostree-repo-finder-config.c @@ -0,0 +1,235 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "ostree-remote-private.h" +#include "ostree-repo.h" +#include "ostree-repo-private.h" +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-config.h" + +/** + * SECTION:ostree-repo-finder-config + * @title: OstreeRepoFinderConfig + * @short_description: Finds remote repositories from ref names using the local + * repository configuration files + * @stability: Unstable + * @include: libostree/ostree-repo-finder-config.h + * + * #OstreeRepoFinderConfig is an implementation of #OstreeRepoFinder which looks + * refs up in locally configured remotes and returns remote URIs. + * Duplicate remote URIs are combined into a single #OstreeRepoFinderResult + * which lists multiple refs. + * + * For all the locally configured remotes which have an `collection-id` specified + * (see [ostree.repo-config(5)](man:ostree.repo-config(5))), it finds the + * intersection of their refs and the set of refs to resolve. If the + * intersection is non-empty, that remote is returned as a result. Remotes which + * do not have their `collection-id` key configured are ignored. + * + * Since: 2017.8 + */ + +static void ostree_repo_finder_config_iface_init (OstreeRepoFinderInterface *iface); + +struct _OstreeRepoFinderConfig +{ + GObject parent_instance; +}; + +G_DEFINE_TYPE_WITH_CODE (OstreeRepoFinderConfig, ostree_repo_finder_config, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (OSTREE_TYPE_REPO_FINDER, ostree_repo_finder_config_iface_init)) + +static gint +results_compare_cb (gconstpointer a, + gconstpointer b) +{ + const OstreeRepoFinderResult *result_a = *((const OstreeRepoFinderResult **) a); + const OstreeRepoFinderResult *result_b = *((const OstreeRepoFinderResult **) b); + + return ostree_repo_finder_result_compare (result_a, result_b); +} + +static void +ostree_repo_finder_config_resolve_async (OstreeRepoFinder *finder, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autoptr(GPtrArray) results = NULL; + const gint priority = 100; /* arbitrarily chosen; lower than the others */ + gsize i, j; + g_autoptr(GHashTable) repo_name_to_refs = NULL; /* (element-type utf8 GHashTable) */ + GHashTable *supported_ref_to_checksum; /* (element-type OstreeCollectionRef utf8) */ + GHashTableIter iter; + const gchar *remote_name; + g_auto(GStrv) remotes = NULL; + gsize n_remotes = 0; + + task = g_task_new (finder, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_config_resolve_async); + results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + repo_name_to_refs = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, + (GDestroyNotify) g_hash_table_unref); + + /* List all remotes in this #OstreeRepo and see which of their ref lists + * intersect with @refs. */ + remotes = ostree_repo_remote_list (parent_repo, (guint *) &n_remotes); + + g_debug ("%s: Checking %" G_GSIZE_FORMAT " remotes", G_STRFUNC, n_remotes); + + for (i = 0; i < n_remotes; i++) + { + g_autoptr(GError) local_error = NULL; + g_autoptr(GHashTable) remote_refs = NULL; /* (element-type utf8 utf8) */ + const gchar *checksum; + g_autofree gchar *remote_collection_id = NULL; + + remote_name = remotes[i]; + + if (!ostree_repo_get_remote_option (parent_repo, remote_name, "collection-id", + NULL, &remote_collection_id, &local_error) || + !ostree_validate_collection_id (remote_collection_id, &local_error)) + { + g_debug ("Ignoring remote ‘%s’ due to no valid collection ID being configured for it: %s", + remote_name, local_error->message); + g_clear_error (&local_error); + continue; + } + + if (!ostree_repo_remote_list_refs (parent_repo, remote_name, &remote_refs, + cancellable, &local_error)) + { + g_debug ("Ignoring remote ‘%s’ due to error loading its refs: %s", + remote_name, local_error->message); + g_clear_error (&local_error); + continue; + } + + for (j = 0; refs[j] != NULL; j++) + { + if (g_strcmp0 (refs[j]->collection_id, remote_collection_id) == 0 && + g_hash_table_lookup_extended (remote_refs, refs[j]->ref_name, NULL, (gpointer *) &checksum)) + { + /* The requested ref is listed in the refs for this remote. Add + * the remote to the results, and the ref to its + * @supported_ref_to_checksum. */ + g_debug ("Resolved ref (%s, %s) to remote ‘%s’.", + refs[j]->collection_id, refs[j]->ref_name, remote_name); + + supported_ref_to_checksum = g_hash_table_lookup (repo_name_to_refs, remote_name); + + if (supported_ref_to_checksum == NULL) + { + supported_ref_to_checksum = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + NULL, g_free); + g_hash_table_insert (repo_name_to_refs, (gpointer) remote_name, supported_ref_to_checksum /* transfer */); + } + + g_hash_table_insert (supported_ref_to_checksum, + (gpointer) refs[j], g_strdup (checksum)); + } + } + } + + /* Aggregate the results. */ + g_hash_table_iter_init (&iter, repo_name_to_refs); + + while (g_hash_table_iter_next (&iter, (gpointer *) &remote_name, (gpointer *) &supported_ref_to_checksum)) + { + g_autoptr(GError) local_error = NULL; + OstreeRemote *remote; + + /* We don’t know what last-modified timestamp the remote has without + * making expensive HTTP queries, so leave that information blank. We + * assume that the configuration which says the refs and commits in + * @supported_ref_to_checksum are in the repository is correct; the code + * in ostree_repo_find_remotes_async() will check that. */ + remote = _ostree_repo_get_remote_inherited (parent_repo, remote_name, &local_error); + if (remote == NULL) + { + g_debug ("Configuration for remote ‘%s’ could not be found. Ignoring.", + remote_name); + continue; + } + + g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, 0)); + } + + g_ptr_array_sort (results, results_compare_cb); + + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); +} + +static GPtrArray * +ostree_repo_finder_config_resolve_finish (OstreeRepoFinder *finder, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, finder), NULL); + return g_task_propagate_pointer (G_TASK (result), error); +} + +static void +ostree_repo_finder_config_init (OstreeRepoFinderConfig *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_repo_finder_config_class_init (OstreeRepoFinderConfigClass *klass) +{ + /* Nothing to see here. */ +} + +static void +ostree_repo_finder_config_iface_init (OstreeRepoFinderInterface *iface) +{ + iface->resolve_async = ostree_repo_finder_config_resolve_async; + iface->resolve_finish = ostree_repo_finder_config_resolve_finish; +} + +/** + * ostree_repo_finder_config_new: + * + * Create a new #OstreeRepoFinderConfig. + * + * Returns: (transfer full): a new #OstreeRepoFinderConfig + * Since: 2017.8 + */ +OstreeRepoFinderConfig * +ostree_repo_finder_config_new (void) +{ + return g_object_new (OSTREE_TYPE_REPO_FINDER_CONFIG, NULL); +} diff --git a/src/libostree/ostree-repo-finder-config.h b/src/libostree/ostree-repo-finder-config.h new file mode 100644 index 00000000..28e6fc84 --- /dev/null +++ b/src/libostree/ostree-repo-finder-config.h @@ -0,0 +1,55 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER_CONFIG (ostree_repo_finder_config_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +_OSTREE_PUBLIC +G_DECLARE_FINAL_TYPE (OstreeRepoFinderConfig, ostree_repo_finder_config, OSTREE, REPO_FINDER_CONFIG, GObject) */ + +_OSTREE_PUBLIC +GType ostree_repo_finder_config_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeRepoFinderConfig OstreeRepoFinderConfig; +typedef struct { GObjectClass parent_class; } OstreeRepoFinderConfigClass; + +static inline OstreeRepoFinderConfig *OSTREE_REPO_FINDER_CONFIG (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_repo_finder_config_get_type (), OstreeRepoFinderConfig); } +static inline gboolean OSTREE_IS_REPO_FINDER_CONFIG (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_repo_finder_config_get_type ()); } +G_GNUC_END_IGNORE_DEPRECATIONS + +_OSTREE_PUBLIC +OstreeRepoFinderConfig *ostree_repo_finder_config_new (void); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index b4c565a8..56e4839e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -41,6 +41,7 @@ #ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include "ostree-repo-finder.h" +#include "ostree-repo-finder-config.h" #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include @@ -4085,6 +4086,7 @@ ostree_repo_find_remotes_async (OstreeRepo *self, g_autoptr(FindRemotesData) data = NULL; GMainContext *context; OstreeRepoFinder *default_finders[4] = { NULL, }; + g_autoptr(OstreeRepoFinder) finder_config = NULL; g_return_if_fail (OSTREE_IS_REPO (self)); g_return_if_fail (is_valid_collection_ref_array (refs)); @@ -4103,6 +4105,10 @@ ostree_repo_find_remotes_async (OstreeRepo *self, /* Are we using #OstreeRepoFinders provided by the user, or the defaults? */ if (finders == NULL) { + finder_config = OSTREE_REPO_FINDER (ostree_repo_finder_config_new ()); + + default_finders[0] = finder_config; + finders = default_finders; } diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index 935707e3..8d547041 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -38,6 +38,7 @@ #ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include #include +#include #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include diff --git a/tests/.gitignore b/tests/.gitignore index 6fc06881..bf31dd01 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -14,4 +14,5 @@ test-mutable-tree test-ot-opt-utils test-ot-tool-util test-ot-unix-utils +test-repo-finder-config test-rollsum-cli diff --git a/tests/test-repo-finder-config.c b/tests/test-repo-finder-config.c new file mode 100644 index 00000000..dc083754 --- /dev/null +++ b/tests/test-repo-finder-config.c @@ -0,0 +1,327 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "libostreetest.h" +#include "ostree-autocleanups.h" +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-config.h" + +/* Test fixture. Creates a temporary directory. */ +typedef struct +{ + OstreeRepo *parent_repo; /* owned */ + int working_dfd; /* owned */ + GFile *working_dir; /* owned */ +} Fixture; + +static void +setup (Fixture *fixture, + gconstpointer test_data) +{ + g_autofree gchar *tmp_name = NULL; + g_autoptr(GError) error = NULL; + + tmp_name = g_strdup ("test-repo-finder-config-XXXXXX"); + glnx_mkdtempat_open_in_system (tmp_name, 0700, &fixture->working_dfd, &error); + g_assert_no_error (error); + + g_test_message ("Using temporary directory: %s", tmp_name); + + glnx_shutil_mkdir_p_at (fixture->working_dfd, "repo", 0700, NULL, &error); + g_assert_no_error (error); + + g_autoptr(GFile) tmp_dir = g_file_new_for_path (g_get_tmp_dir ()); + fixture->working_dir = g_file_get_child (tmp_dir, tmp_name); + + fixture->parent_repo = ot_test_setup_repo (NULL, &error); + g_assert_no_error (error); +} + +static void +teardown (Fixture *fixture, + gconstpointer test_data) +{ + glnx_fd_close int parent_repo_dfd = -1; + g_autoptr(GError) error = NULL; + + /* Recursively remove the temporary directory. */ + glnx_shutil_rm_rf_at (fixture->working_dfd, ".", NULL, NULL); + + close (fixture->working_dfd); + fixture->working_dfd = -1; + + /* The repo also needs its source files to be removed. This is the inverse + * of setup_test_repository() in libtest.sh. */ + g_autofree gchar *parent_repo_path = g_file_get_path (ostree_repo_get_path (fixture->parent_repo)); + glnx_opendirat (-1, parent_repo_path, TRUE, &parent_repo_dfd, &error); + g_assert_no_error (error); + + glnx_shutil_rm_rf_at (parent_repo_dfd, "../files", NULL, NULL); + glnx_shutil_rm_rf_at (parent_repo_dfd, "../repo", NULL, NULL); + + g_clear_object (&fixture->working_dir); + g_clear_object (&fixture->parent_repo); +} + +/* Test the object constructor works at a basic level. */ +static void +test_repo_finder_config_init (void) +{ + g_autoptr(OstreeRepoFinderConfig) finder = NULL; + + /* Default everything. */ + finder = ostree_repo_finder_config_new (); +} + +static void +result_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data) +{ + GAsyncResult **result_out = user_data; + *result_out = g_object_ref (result); +} + +/* Test that no remotes are found if there are no config files in the refs + * directory. */ +static void +test_repo_finder_config_no_configs (Fixture *fixture, + gconstpointer test_data) +{ + g_autoptr(OstreeRepoFinderConfig) finder = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + g_autoptr(GError) error = NULL; + const OstreeCollectionRef ref1 = { "org.example.Os", "exampleos/x86_64/standard" }; + const OstreeCollectionRef ref2 = { "org.example.Os", "exampleos/x86_64/buildmaster/standard" }; + const OstreeCollectionRef * const refs[] = { &ref1, &ref2, NULL }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + finder = ostree_repo_finder_config_new (); + + ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, + fixture->parent_repo, NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (results->len, ==, 0); + + g_main_context_pop_thread_default (context); +} + +/* Add configuration for a remote named @remote_name, at @remote_uri, with a + * remote collection ID of @collection_id, to the given @repo. */ +static void +assert_create_remote_config (OstreeRepo *repo, + const gchar *remote_name, + const gchar *remote_uri, + const gchar *collection_id) +{ + g_autoptr(GError) error = NULL; + g_autoptr(GVariant) options = NULL; + + if (collection_id != NULL) + options = g_variant_new_parsed ("@a{sv} { 'collection-id': <%s> }", + collection_id); + + ostree_repo_remote_add (repo, remote_name, remote_uri, options, NULL, &error); + g_assert_no_error (error); +} + +static gchar *assert_create_remote (Fixture *fixture, + const gchar *collection_id, + ...) G_GNUC_NULL_TERMINATED; + +/* Create a new repository in a temporary directory with its collection ID set + * to @collection_id, and containing the refs given in @... (which must be + * %NULL-terminated). Return the `file://` URI of the new repository. */ +static gchar * +assert_create_remote (Fixture *fixture, + const gchar *collection_id, + ...) +{ + va_list args; + g_autoptr(GError) error = NULL; + const gchar *repo_name = (collection_id != NULL) ? collection_id : "no-collection"; + + glnx_shutil_mkdir_p_at (fixture->working_dfd, repo_name, 0700, NULL, &error); + g_assert_no_error (error); + + g_autoptr(GFile) repo_path = g_file_get_child (fixture->working_dir, repo_name); + g_autoptr(OstreeRepo) repo = ostree_repo_new (repo_path); + ostree_repo_set_collection_id (repo, collection_id, &error); + g_assert_no_error (error); + ostree_repo_create (repo, OSTREE_REPO_MODE_ARCHIVE_Z2, NULL, &error); + g_assert_no_error (error); + + /* Set up the refs from @.... */ + va_start (args, collection_id); + + for (const gchar *ref_name = va_arg (args, const gchar *); + ref_name != NULL; + ref_name = va_arg (args, const gchar *)) + { + OstreeCollectionRef collection_ref = { (gchar *) collection_id, (gchar *) ref_name }; + g_autofree gchar *checksum = NULL; + g_autoptr(OstreeMutableTree) mtree = NULL; + g_autoptr(OstreeRepoFile) repo_file = NULL; + + mtree = ostree_mutable_tree_new (); + ostree_repo_write_dfd_to_mtree (repo, AT_FDCWD, ".", mtree, NULL, NULL, &error); + g_assert_no_error (error); + ostree_repo_write_mtree (repo, mtree, (GFile **) &repo_file, NULL, &error); + g_assert_no_error (error); + + ostree_repo_write_commit (repo, NULL /* no parent */, ref_name, ref_name, + NULL /* no metadata */, repo_file, &checksum, + NULL, &error); + g_assert_no_error (error); + + if (collection_id != NULL) + ostree_repo_set_collection_ref_immediate (repo, &collection_ref, checksum, NULL, &error); + else + ostree_repo_set_ref_immediate (repo, NULL, ref_name, checksum, NULL, &error); + g_assert_no_error (error); + } + + va_end (args); + + /* Update the summary. */ + ostree_repo_regenerate_summary (repo, NULL /* no metadata */, NULL, &error); + g_assert_no_error (error); + + return g_file_get_uri (repo_path); +} + +/* Test resolving the refs against a collection of config files, which contain + * valid, invalid or duplicate repo information. */ +static void +test_repo_finder_config_mixed_configs (Fixture *fixture, + gconstpointer test_data) +{ + g_autoptr(OstreeRepoFinderConfig) finder = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + g_autoptr(GError) error = NULL; + gsize i; + const OstreeCollectionRef ref0 = { "org.example.Collection0", "exampleos/x86_64/ref0" }; + const OstreeCollectionRef ref1 = { "org.example.Collection0", "exampleos/x86_64/ref1" }; + const OstreeCollectionRef ref2 = { "org.example.Collection1", "exampleos/x86_64/ref1" }; + const OstreeCollectionRef ref3 = { "org.example.Collection1", "exampleos/x86_64/ref2" }; + const OstreeCollectionRef ref4 = { "org.example.Collection2", "exampleos/x86_64/ref3" }; + const OstreeCollectionRef * const refs[] = { &ref0, &ref1, &ref2, &ref3, &ref4, NULL }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + /* Put together various ref configuration files. */ + g_autofree gchar *collection0_uri = assert_create_remote (fixture, "org.example.Collection0", + "exampleos/x86_64/ref0", + "exampleos/x86_64/ref1", + NULL); + g_autofree gchar *collection1_uri = assert_create_remote (fixture, "org.example.Collection1", + "exampleos/x86_64/ref2", + NULL); + g_autofree gchar *no_collection_uri = assert_create_remote (fixture, NULL, + "exampleos/x86_64/ref3", + NULL); + + assert_create_remote_config (fixture->parent_repo, "remote0", collection0_uri, "org.example.Collection0"); + assert_create_remote_config (fixture->parent_repo, "remote1", collection1_uri, "org.example.Collection1"); + assert_create_remote_config (fixture->parent_repo, "remote0-copy", collection0_uri, "org.example.Collection0"); + assert_create_remote_config (fixture->parent_repo, "remote1-bad-copy", collection1_uri, "org.example.NotCollection1"); + assert_create_remote_config (fixture->parent_repo, "remote2", no_collection_uri, NULL); + + finder = ostree_repo_finder_config_new (); + + /* Resolve the refs. */ + ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, + fixture->parent_repo, NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (results->len, ==, 3); + + /* Check that the results are correct: the invalid refs should have been + * ignored, and the valid results canonicalised and deduplicated. */ + for (i = 0; i < results->len; i++) + { + const OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + + if (g_strcmp0 (ostree_remote_get_name (result->remote), "remote0") == 0 || + g_strcmp0 (ostree_remote_get_name (result->remote), "remote0-copy") == 0) + { + g_assert_cmpuint (g_hash_table_size (result->ref_to_checksum), ==, 2); + g_assert_true (g_hash_table_contains (result->ref_to_checksum, &ref0)); + g_assert_true (g_hash_table_contains (result->ref_to_checksum, &ref1)); + } + else if (g_strcmp0 (ostree_remote_get_name (result->remote), "remote1") == 0) + { + g_assert_cmpuint (g_hash_table_size (result->ref_to_checksum), ==, 1); + g_assert_true (g_hash_table_contains (result->ref_to_checksum, &ref3)); + } + else + { + g_assert_not_reached (); + } + } + + g_main_context_pop_thread_default (context); +} + +int main (int argc, char **argv) +{ + setlocale (LC_ALL, ""); + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/repo-finder-config/init", test_repo_finder_config_init); + g_test_add ("/repo-finder-config/no-configs", Fixture, NULL, setup, + test_repo_finder_config_no_configs, teardown); + g_test_add ("/repo-finder-config/mixed-configs", Fixture, NULL, setup, + test_repo_finder_config_mixed_configs, teardown); + + return g_test_run(); +} From ae335f24dca8561a8f719edec243c4f339986c4e Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 19 Apr 2017 00:07:51 +0100 Subject: [PATCH 25/82] lib/repo-finder: Add mount based OstreeRepoFinder implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a basic implementation of OstreeRepoFinder which resolves ref names to remote URIs by looking for them on any currently mounted removable storage volumes. The idea is to support OS and app updates via USB stick. Unit tests are included. This bumps libostree’s maximum GLib dependency from 2.44 to 2.50 for g_drive_is_removable(). If GLib 2.50 is not available, the call which needs it will be omitted and the OstreeRepoFinderMount implementation will scan all volumes (not just removable ones); this is a performance hit, but not a functionality hit. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- Makefile-libostree-defines.am | 1 + Makefile-libostree.am | 4 +- Makefile-tests.am | 7 +- Makefile.am | 2 +- apidoc/ostree-experimental-sections.txt | 8 + src/libostree/libostree-experimental.sym | 2 + src/libostree/ostree-autocleanups.h | 1 + src/libostree/ostree-core-private.h | 3 + src/libostree/ostree-repo-finder-mount.c | 547 +++++++++++++++++++++++ src/libostree/ostree-repo-finder-mount.h | 55 +++ src/libostree/ostree-repo-pull.c | 4 + src/libostree/ostree.h | 1 + tests/.gitignore | 1 + tests/test-mock-gio.c | 400 +++++++++++++++++ tests/test-mock-gio.h | 127 ++++++ tests/test-repo-finder-mount.c | 497 ++++++++++++++++++++ 16 files changed, 1657 insertions(+), 3 deletions(-) create mode 100644 src/libostree/ostree-repo-finder-mount.c create mode 100644 src/libostree/ostree-repo-finder-mount.h create mode 100644 tests/test-mock-gio.c create mode 100644 tests/test-mock-gio.h create mode 100644 tests/test-repo-finder-mount.c diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index ac5eaa00..7586f7b9 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -44,6 +44,7 @@ libostree_public_headers += \ src/libostree/ostree-remote.h \ src/libostree/ostree-repo-finder.h \ src/libostree/ostree-repo-finder-config.h \ + src/libostree/ostree-repo-finder-mount.h \ $(NULL) endif diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 4b968abe..01aa8663 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -156,11 +156,13 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-remote.h \ src/libostree/ostree-repo-finder.h \ src/libostree/ostree-repo-finder-config.h \ + src/libostree/ostree-repo-finder-mount.h \ $(NULL) else # if ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += \ src/libostree/ostree-repo-finder.c \ src/libostree/ostree-repo-finder-config.c \ + src/libostree/ostree-repo-finder-mount.c \ $(NULL) endif @@ -237,7 +239,7 @@ OSTree_1_0_gir_INCLUDES = Gio-2.0 OSTree_1_0_gir_CFLAGS = $(libostree_1_la_CFLAGS) OSTree_1_0_gir_LIBS = libostree-1.la OSTree_1_0_gir_SCANNERFLAGS = --warn-all --identifier-prefix=Ostree --symbol-prefix=ostree -OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h %/ostree-repo-finder-config.h,$(libostree_1_la_SOURCES)) +OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h %/ostree-repo-finder-config.h %/ostree-repo-finder-mount.h,$(libostree_1_la_SOURCES)) INTROSPECTION_GIRS += OSTree-1.0.gir gir_DATA += OSTree-1.0.gir typelib_DATA += OSTree-1.0.typelib diff --git a/Makefile-tests.am b/Makefile-tests.am index 1cdf8826..09c85818 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -196,6 +196,7 @@ _installed_or_uninstalled_test_programs = tests/test-varint tests/test-ot-unix-u if ENABLE_EXPERIMENTAL_API test_programs += \ tests/test-repo-finder-config \ + tests/test-repo-finder-mount \ $(NULL) endif @@ -210,7 +211,7 @@ common_tests_cflags = $(ostree_bin_shared_cflags) $(OT_INTERNAL_GIO_UNIX_CFLAGS) common_tests_ldadd = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_GIO_UNIX_LIBS) noinst_LTLIBRARIES += libostreetest.la -libostreetest_la_SOURCES = tests/libostreetest.c +libostreetest_la_SOURCES = tests/libostreetest.c tests/test-mock-gio.c tests/test-mock-gio.h libostreetest_la_CFLAGS = $(common_tests_cflags) -I $(srcdir)/tests libostreetest_la_LIBADD = $(common_tests_ldadd) @@ -229,6 +230,10 @@ tests_test_repo_finder_config_SOURCES = tests/test-repo-finder-config.c tests_test_repo_finder_config_CFLAGS = $(TESTS_CFLAGS) tests_test_repo_finder_config_LDADD = $(TESTS_LDADD) +tests_test_repo_finder_mount_SOURCES = tests/test-repo-finder-mount.c +tests_test_repo_finder_mount_CFLAGS = $(TESTS_CFLAGS) +tests_test_repo_finder_mount_LDADD = $(TESTS_LDADD) + tests_test_mutable_tree_CFLAGS = $(TESTS_CFLAGS) tests_test_mutable_tree_LDADD = $(TESTS_LDADD) diff --git a/Makefile.am b/Makefile.am index 53b505e3..0939d4b7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,7 +29,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DOSTREE_COMPILATION \ -DG_LOG_DOMAIN=\"OSTree\" \ -DOSTREE_GITREV='"$(OSTREE_GITREV)"' \ - -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40 \ + -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_50 \ -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 -DSOUP_VERSION_MAX_ALLOWED=SOUP_VERSION_2_48 AM_CFLAGS += -std=gnu99 $(WARN_CFLAGS) AM_DISTCHECK_CONFIGURE_FLAGS += \ diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 93210d56..c977b56c 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -57,6 +57,14 @@ ostree_repo_finder_config_new ostree_repo_finder_config_get_type
+
+ostree-repo-finder-mount +OstreeRepoFinderMount +ostree_repo_finder_mount_new + +ostree_repo_finder_mount_get_type +
+
ostree-misc-experimental ostree_repo_get_collection_id diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index 79fcdbe9..e70c80bb 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -50,6 +50,8 @@ global: ostree_repo_finder_config_get_type; ostree_repo_finder_config_new; ostree_repo_finder_get_type; + ostree_repo_finder_mount_get_type; + ostree_repo_finder_mount_new; ostree_repo_finder_resolve_async; ostree_repo_finder_resolve_all_async; ostree_repo_finder_resolve_all_finish; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index ac0aa1fb..dd3c9778 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -65,6 +65,7 @@ G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_fre G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderMount, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 4c117110..a56fdc0b 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -190,6 +190,9 @@ G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_res #include "ostree-repo-finder-config.h" G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref) + +#include "ostree-repo-finder-mount.h" +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderMount, g_object_unref) #endif G_END_DECLS diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c new file mode 100644 index 00000000..ffe31e99 --- /dev/null +++ b/src/libostree/ostree-repo-finder-mount.c @@ -0,0 +1,547 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "ostree-autocleanups.h" +#include "ostree-remote-private.h" +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-mount.h" + +/** + * SECTION:ostree-repo-finder-mount + * @title: OstreeRepoFinderMount + * @short_description: Finds remote repositories from ref names by looking at + * mounted removable volumes + * @stability: Unstable + * @include: libostree/ostree-repo-finder-mount.h + * + * #OstreeRepoFinderMount is an implementation of #OstreeRepoFinder which looks + * refs up in well-known locations on any mounted removable volumes. + * + * For an #OstreeCollectionRef, (`C`, `R`), it checks whether `.ostree/repos/C/R` + * exists and is an OSTree repository on each mounted removable volume. Collection + * IDs and ref names are not escaped when building the path, so if either + * contains `/` in its name, the repository will be checked for in a + * subdirectory of `.ostree/repos`. Non-removable volumes are ignored. + * + * For each repository which is found, a result will be returned for the + * intersection of the refs being searched for, and the refs in `refs/heads` and + * `refs/mirrors` in the repository on the removable volume. + * + * Symlinks are followed when resolving the refs, so a volume might contain a + * single OSTree at some arbitrary path, with a number of refs linking to it + * from `.ostree/repos`. Any symlink which points outside the volume’s file + * system will be ignored. Repositories are deduplicated in the results. + * + * The volume monitor used to find mounted volumes can be overridden by setting + * #OstreeRepoFinderMount:monitor. By default, g_volume_monitor_get() is used. + * + * Since: 2017.8 + */ + +typedef GList/**/ ObjectList; + +static void +object_list_free (ObjectList *list) +{ + g_list_free_full (list, g_object_unref); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (ObjectList, object_list_free) + +static void ostree_repo_finder_mount_iface_init (OstreeRepoFinderInterface *iface); + +struct _OstreeRepoFinderMount +{ + GObject parent_instance; + + GVolumeMonitor *monitor; /* owned */ +}; + +G_DEFINE_TYPE_WITH_CODE (OstreeRepoFinderMount, ostree_repo_finder_mount, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (OSTREE_TYPE_REPO_FINDER, ostree_repo_finder_mount_iface_init)) + +typedef struct +{ + gchar *uri; + gchar *keyring; +} UriAndKeyring; + +static void +uri_and_keyring_free (UriAndKeyring *data) +{ + g_free (data->uri); + g_free (data->keyring); + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (UriAndKeyring, uri_and_keyring_free) + +static UriAndKeyring * +uri_and_keyring_new (const gchar *uri, + const gchar *keyring) +{ + g_autoptr(UriAndKeyring) data = NULL; + + data = g_new0 (UriAndKeyring, 1); + data->uri = g_strdup (uri); + data->keyring = g_strdup (keyring); + + return g_steal_pointer (&data); +} + +static guint +uri_and_keyring_hash (gconstpointer key) +{ + const UriAndKeyring *_key = key; + + return g_str_hash (_key->uri) ^ g_str_hash (_key->keyring); +} + +static gboolean +uri_and_keyring_equal (gconstpointer a, + gconstpointer b) +{ + const UriAndKeyring *_a = a, *_b = b; + + return g_str_equal (_a->uri, _b->uri) && g_str_equal (_a->keyring, _b->keyring); +} + +/* This must return a valid remote name (suitable for use in a refspec). */ +static gchar * +uri_and_keyring_to_name (UriAndKeyring *data) +{ + g_autofree gchar *escaped_uri = g_uri_escape_string (data->uri, NULL, FALSE); + g_autofree gchar *escaped_keyring = g_uri_escape_string (data->keyring, NULL, FALSE); + + /* FIXME: Need a better separator than `_`, since it’s not escaped in the input. */ + g_autofree gchar *out = g_strdup_printf ("%s_%s", escaped_uri, escaped_keyring); + + for (gsize i = 0; out[i] != '\0'; i++) + { + if (out[i] == '%') + out[i] = '_'; + } + + g_return_val_if_fail (ostree_validate_remote_name (out, NULL), NULL); + + return g_steal_pointer (&out); +} + +static gint +results_compare_cb (gconstpointer a, + gconstpointer b) +{ + const OstreeRepoFinderResult *result_a = *((const OstreeRepoFinderResult **) a); + const OstreeRepoFinderResult *result_b = *((const OstreeRepoFinderResult **) b); + + return ostree_repo_finder_result_compare (result_a, result_b); +} + +static void +ostree_repo_finder_mount_resolve_async (OstreeRepoFinder *finder, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (finder); + g_autoptr(GTask) task = NULL; + g_autoptr(ObjectList) mounts = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + GList *l; + const gint priority = 50; /* arbitrarily chosen */ + + task = g_task_new (finder, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_mount_resolve_async); + + mounts = g_volume_monitor_get_mounts (self->monitor); + results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + + g_debug ("%s: Found %u mounts", G_STRFUNC, g_list_length (mounts)); + + for (l = mounts; l != NULL; l = l->next) + { + GMount *mount = G_MOUNT (l->data); + g_autofree gchar *mount_name = NULL; + g_autoptr(GFile) mount_root = NULL; + g_autofree gchar *mount_root_path = NULL; + glnx_fd_close int mount_root_dfd = -1; + struct stat mount_root_stbuf; + glnx_fd_close int repos_dfd = -1; + gsize i; + g_autoptr(GHashTable) repo_to_refs = NULL; /* (element-type UriAndKeyring GHashTable) */ + GHashTable *supported_ref_to_checksum; /* (element-type OstreeCollectionRef utf8) */ + GHashTableIter iter; + UriAndKeyring *repo; + g_autoptr(GError) local_error = NULL; + + mount_name = g_mount_get_name (mount); + + /* Check the mount’s general properties. */ + if (g_mount_is_shadowed (mount)) + { + g_debug ("Ignoring mount ‘%s’ as it’s shadowed.", mount_name); + continue; + } + + /* Check if it contains a .ostree/repos directory. */ + mount_root = g_mount_get_root (mount); + mount_root_path = g_file_get_path (mount_root); + + if (!glnx_opendirat (AT_FDCWD, mount_root_path, TRUE, &mount_root_dfd, &local_error)) + { + g_debug ("Ignoring mount ‘%s’ as ‘%s’ directory can’t be opened: %s", + mount_name, mount_root_path, local_error->message); + continue; + } + + if (!glnx_opendirat (mount_root_dfd, ".ostree/repos", TRUE, &repos_dfd, &local_error)) + { + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + g_debug ("Ignoring mount ‘%s’ as ‘%s/.ostree/repos’ directory doesn’t exist.", + mount_name, mount_root_path); + else + g_debug ("Ignoring mount ‘%s’ as ‘%s/.ostree/repos’ directory can’t be opened: %s", + mount_name, mount_root_path, local_error->message); + + continue; + } + + /* stat() the mount root so we can later check whether the resolved + * repositories for individual refs are on the same device (to avoid the + * symlinks for them pointing outside the mount root). */ + if (!glnx_fstat (mount_root_dfd, &mount_root_stbuf, &local_error)) + { + g_debug ("Ignoring mount ‘%s’ as querying info of ‘%s’ failed: %s", + mount_name, mount_root_path, local_error->message); + continue; + } + + /* Check whether a subdirectory exists for any of the @refs we’re looking + * for. If so, and it’s a symbolic link, dereference it so multiple links + * to the same repository (containing multiple refs) are coalesced. + * Otherwise, include it as a result by itself. */ + repo_to_refs = g_hash_table_new_full (uri_and_keyring_hash, uri_and_keyring_equal, + (GDestroyNotify) uri_and_keyring_free, (GDestroyNotify) g_hash_table_unref); + + for (i = 0; refs[i] != NULL; i++) + { + struct stat stbuf; + g_autofree gchar *collection_and_ref = NULL; + g_autofree gchar *repo_dir_path = NULL; + g_autofree gchar *resolved_repo_uri = NULL; + g_autofree gchar *keyring = NULL; + g_autoptr(UriAndKeyring) resolved_repo = NULL; + + collection_and_ref = g_build_filename (refs[i]->collection_id, refs[i]->ref_name, NULL); + repo_dir_path = g_build_filename (mount_root_path, ".ostree", "repos", + collection_and_ref, NULL); + + if (!glnx_fstatat (repos_dfd, collection_and_ref, &stbuf, AT_NO_AUTOMOUNT, &local_error)) + { + g_debug ("Ignoring ref (%s, %s) on mount ‘%s’ as querying info of ‘%s’ failed: %s", + refs[i]->collection_id, refs[i]->ref_name, mount_name, repo_dir_path, local_error->message); + g_clear_error (&local_error); + continue; + } + + if ((stbuf.st_mode & S_IFMT) != S_IFDIR) + { + g_debug ("Ignoring ref (%s, %s) on mount ‘%s’ as ‘%s’ is of type %u, not a directory.", + refs[i]->collection_id, refs[i]->ref_name, mount_name, repo_dir_path, (stbuf.st_mode & S_IFMT)); + g_clear_error (&local_error); + continue; + } + + /* Check the resolved repository path is below the mount point. Do not + * allow ref symlinks to point somewhere outside of the mounted + * volume. */ + if (stbuf.st_dev != mount_root_stbuf.st_dev) + { + g_debug ("Ignoring ref (%s, %s) on mount ‘%s’ as it’s on a different file system from the mount.", + refs[i]->collection_id, refs[i]->ref_name, mount_name); + g_clear_error (&local_error); + continue; + } + + /* Exclude repositories which resolve to @parent_repo. */ + g_autofree char *canonical_repo_dir_path = realpath (repo_dir_path, NULL); + g_autofree gchar *parent_repo_path = g_file_get_path (ostree_repo_get_path (parent_repo)); + g_autofree char *canonical_parent_repo_path = realpath (parent_repo_path, NULL); + + if (g_strcmp0 (canonical_repo_dir_path, canonical_parent_repo_path) == 0) + { + g_debug ("Ignoring ref (%s, %s) on mount ‘%s’ as its repository was the one we are resolving for: %s", + refs[i]->collection_id, refs[i]->ref_name, mount_name, canonical_parent_repo_path); + g_clear_error (&local_error); + continue; + } + + /* Grab the given ref and a checksum for it from the repo. + * FIXME: Ideally, there would be some ostree_repo_open_at() which we + * could use to keep the openat() chain going. See + * https://github.com/ostreedev/ostree/pull/820. */ + g_autoptr(OstreeRepo) repo = NULL; + g_autoptr(GFile) repo_dir_file = g_file_new_for_path (repo_dir_path); + repo = ostree_repo_new (repo_dir_file); + + if (!ostree_repo_open (repo, cancellable, &local_error)) + { + g_debug ("Ignoring ref (%s, %s) on mount ‘%s’ as its repository could not be opened: %s", + refs[i]->collection_id, refs[i]->ref_name, mount_name, local_error->message); + g_clear_error (&local_error); + continue; + } + + g_autoptr(GHashTable) repo_refs = NULL; /* (element-type OstreeCollectionRef utf8) */ + + if (!ostree_repo_list_collection_refs (repo, refs[i]->collection_id, &repo_refs, cancellable, &local_error)) + { + g_debug ("Ignoring ref (%s, %s) on mount ‘%s’ as its refs could not be listed: %s", + refs[i]->collection_id, refs[i]->ref_name, mount_name, local_error->message); + g_clear_error (&local_error); + continue; + } + + const gchar *checksum = g_hash_table_lookup (repo_refs, refs[i]); + + if (checksum == NULL) + { + g_debug ("Ignoring ref (%s, %s) on mount ‘%s’ as its repository doesn’t contain the ref.", + refs[i]->collection_id, refs[i]->ref_name, mount_name); + g_clear_error (&local_error); + continue; + } + + /* Finally, look up the GPG keyring for this ref. */ + keyring = ostree_repo_resolve_keyring_for_collection (parent_repo, refs[i]->collection_id, + cancellable, &local_error); + + if (keyring == NULL) + { + g_debug ("Ignoring ref (%s, %s) on mount ‘%s’ due to missing keyring: %s", + refs[i]->collection_id, refs[i]->ref_name, mount_name, local_error->message); + g_clear_error (&local_error); + continue; + } + + /* There is a valid repo at (or pointed to by) + * $mount_root/.ostree/repos/$refs[i]->collection_id/$refs[i]->ref_name. + * Add it to the results, keyed by the canonicalised repository URI + * to deduplicate the results. */ + resolved_repo_uri = g_strconcat ("file://", canonical_repo_dir_path, NULL); + g_debug ("Resolved ref (%s, %s) on mount ‘%s’ to repo URI ‘%s’ with keyring ‘%s’.", + refs[i]->collection_id, refs[i]->ref_name, mount_name, resolved_repo_uri, keyring); + + resolved_repo = uri_and_keyring_new (resolved_repo_uri, keyring); + + supported_ref_to_checksum = g_hash_table_lookup (repo_to_refs, resolved_repo); + + if (supported_ref_to_checksum == NULL) + { + supported_ref_to_checksum = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + NULL, g_free); + g_hash_table_insert (repo_to_refs, g_steal_pointer (&resolved_repo), supported_ref_to_checksum /* transfer */); + } + + g_hash_table_insert (supported_ref_to_checksum, (gpointer) refs[i], g_strdup (checksum)); + } + + /* Aggregate the results. */ + g_hash_table_iter_init (&iter, repo_to_refs); + + while (g_hash_table_iter_next (&iter, (gpointer *) &repo, (gpointer *) &supported_ref_to_checksum)) + { + g_autoptr(OstreeRemote) remote = NULL; + + /* Build an #OstreeRemote. Use the escaped URI, since remote->name + * is used in file paths, so needs to not contain special characters. */ + g_autofree gchar *name = uri_and_keyring_to_name (repo); + remote = ostree_remote_new (name); + + g_clear_pointer (&remote->keyring, g_free); + remote->keyring = g_strdup (repo->keyring); + + g_key_file_set_string (remote->options, remote->group, "url", repo->uri); + g_key_file_set_boolean (remote->options, remote->group, "gpg-verify", TRUE); + g_key_file_set_boolean (remote->options, remote->group, "gpg-verify-summary", TRUE); + + /* Set the timestamp in the #OstreeRepoFinderResult to 0 because + * the code in ostree_repo_pull_from_remotes_async() will be able to + * check it just as quickly as we can here; so don’t duplicate the + * code. */ + g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, 0)); + } + } + + g_ptr_array_sort (results, results_compare_cb); + + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); +} + +static GPtrArray * +ostree_repo_finder_mount_resolve_finish (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, self), NULL); + return g_task_propagate_pointer (G_TASK (result), error); +} + +static void +ostree_repo_finder_mount_init (OstreeRepoFinderMount *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_repo_finder_mount_constructed (GObject *object) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (object); + + G_OBJECT_CLASS (ostree_repo_finder_mount_parent_class)->constructed (object); + + if (self->monitor == NULL) + self->monitor = g_volume_monitor_get (); +} + +typedef enum +{ + PROP_MONITOR = 1, +} OstreeRepoFinderMountProperty; + +static void +ostree_repo_finder_mount_get_property (GObject *object, + guint property_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (object); + + switch ((OstreeRepoFinderMountProperty) property_id) + { + case PROP_MONITOR: + g_value_set_object (value, self->monitor); + break; + default: + g_assert_not_reached (); + } +} + +static void +ostree_repo_finder_mount_set_property (GObject *object, + guint property_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (object); + + switch ((OstreeRepoFinderMountProperty) property_id) + { + case PROP_MONITOR: + /* Construct-only. */ + g_assert (self->monitor == NULL); + self->monitor = g_value_dup_object (value); + break; + default: + g_assert_not_reached (); + } +} + +static void +ostree_repo_finder_mount_dispose (GObject *object) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (object); + + g_clear_object (&self->monitor); + + G_OBJECT_CLASS (ostree_repo_finder_mount_parent_class)->dispose (object); +} + +static void +ostree_repo_finder_mount_class_init (OstreeRepoFinderMountClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->get_property = ostree_repo_finder_mount_get_property; + object_class->set_property = ostree_repo_finder_mount_set_property; + object_class->constructed = ostree_repo_finder_mount_constructed; + object_class->dispose = ostree_repo_finder_mount_dispose; + + /** + * OstreeRepoFinderMount:monitor: + * + * Volume monitor to use to look up mounted volumes when queried. + * + * Since: 2017.8 + */ + g_object_class_install_property (object_class, PROP_MONITOR, + g_param_spec_object ("monitor", + "Volume Monitor", + "Volume monitor to use " + "to look up mounted " + "volumes when queried.", + G_TYPE_VOLUME_MONITOR, + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS)); +} + +static void +ostree_repo_finder_mount_iface_init (OstreeRepoFinderInterface *iface) +{ + iface->resolve_async = ostree_repo_finder_mount_resolve_async; + iface->resolve_finish = ostree_repo_finder_mount_resolve_finish; +} + +/** + * ostree_repo_finder_mount_new: + * @monitor: (nullable) (transfer none): volume monitor to use, or %NULL to use + * the system default + * + * Create a new #OstreeRepoFinderMount, using the given @monitor to look up + * volumes. If @monitor is %NULL, the monitor from g_volume_monitor_get() will + * be used. + * + * Returns: (transfer full): a new #OstreeRepoFinderMount + * Since: 2017.8 + */ +OstreeRepoFinderMount * +ostree_repo_finder_mount_new (GVolumeMonitor *monitor) +{ + g_return_val_if_fail (monitor == NULL || G_IS_VOLUME_MONITOR (monitor), NULL); + + return g_object_new (OSTREE_TYPE_REPO_FINDER_MOUNT, + "monitor", monitor, + NULL); +} diff --git a/src/libostree/ostree-repo-finder-mount.h b/src/libostree/ostree-repo-finder-mount.h new file mode 100644 index 00000000..d844dd44 --- /dev/null +++ b/src/libostree/ostree-repo-finder-mount.h @@ -0,0 +1,55 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER_MOUNT (ostree_repo_finder_mount_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +_OSTREE_PUBLIC +G_DECLARE_FINAL_TYPE (OstreeRepoFinderMount, ostree_repo_finder_mount, OSTREE, REPO_FINDER_MOUNT, GObject) */ + +_OSTREE_PUBLIC +GType ostree_repo_finder_mount_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeRepoFinderMount OstreeRepoFinderMount; +typedef struct { GObjectClass parent_class; } OstreeRepoFinderMountClass; + +static inline OstreeRepoFinderMount *OSTREE_REPO_FINDER_MOUNT (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_repo_finder_mount_get_type (), OstreeRepoFinderMount); } +static inline gboolean OSTREE_IS_REPO_FINDER_MOUNT (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_repo_finder_mount_get_type ()); } +G_GNUC_END_IGNORE_DEPRECATIONS + +_OSTREE_PUBLIC +OstreeRepoFinderMount *ostree_repo_finder_mount_new (GVolumeMonitor *monitor); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 56e4839e..ecc9b72c 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -42,6 +42,7 @@ #ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include "ostree-repo-finder.h" #include "ostree-repo-finder-config.h" +#include "ostree-repo-finder-mount.h" #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include @@ -4087,6 +4088,7 @@ ostree_repo_find_remotes_async (OstreeRepo *self, GMainContext *context; OstreeRepoFinder *default_finders[4] = { NULL, }; g_autoptr(OstreeRepoFinder) finder_config = NULL; + g_autoptr(OstreeRepoFinder) finder_mount = NULL; g_return_if_fail (OSTREE_IS_REPO (self)); g_return_if_fail (is_valid_collection_ref_array (refs)); @@ -4106,8 +4108,10 @@ ostree_repo_find_remotes_async (OstreeRepo *self, if (finders == NULL) { finder_config = OSTREE_REPO_FINDER (ostree_repo_finder_config_new ()); + finder_mount = OSTREE_REPO_FINDER (ostree_repo_finder_mount_new (NULL)); default_finders[0] = finder_config; + default_finders[1] = finder_mount; finders = default_finders; } diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index 8d547041..0fe2a23e 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -39,6 +39,7 @@ #include #include #include +#include #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include diff --git a/tests/.gitignore b/tests/.gitignore index bf31dd01..f3bdb177 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -15,4 +15,5 @@ test-ot-opt-utils test-ot-tool-util test-ot-unix-utils test-repo-finder-config +test-repo-finder-mount test-rollsum-cli diff --git a/tests/test-mock-gio.c b/tests/test-mock-gio.c new file mode 100644 index 00000000..f6d4f2a8 --- /dev/null +++ b/tests/test-mock-gio.c @@ -0,0 +1,400 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "test-mock-gio.h" + +/** + * SECTION:mock-gio + * @title: Mock GIO volume interfaces + * @short_description: Mock implementations of GIO volume, mount and drive + * interfaces + * @stability: Unstable + * @include: tests/test-mock-gio.h + * + * A set of classes implementing GIO interfaces for volumes, mounts, drives + * and volume monitoring, which return mock data to the caller when used. These + * are designed for use in unit tests, to mock up removable drives when testing + * code which monitors such drives being added and removed and then queries + * properties of them. + * + * By returning mock drive locations to the caller, for example, the contents of + * a removable drive may be mocked up using temporary files. + * + * Currently, all the mock data returned by these classes to callers is static, + * set at construction time. + * + * Since: 2017.8 + */ + +/* Mock volume monitor class. This returns a static set of data to the caller, + * which it was initialised with. */ +struct _OstreeMockVolumeMonitor +{ + GVolumeMonitor parent_instance; + + GList *mounts; /* (element-type OstreeMockMount) */ + GList *volumes; /* (element-type OstreeMockVolume) */ +}; + +G_DEFINE_TYPE (OstreeMockVolumeMonitor, ostree_mock_volume_monitor, G_TYPE_VOLUME_MONITOR) + +static GList * +ostree_mock_volume_monitor_get_mounts (GVolumeMonitor *monitor) +{ + OstreeMockVolumeMonitor *self = OSTREE_MOCK_VOLUME_MONITOR (monitor); + return g_list_copy_deep (self->mounts, (GCopyFunc) g_object_ref, NULL); +} + +static GList * +ostree_mock_volume_monitor_get_volumes (GVolumeMonitor *monitor) +{ + OstreeMockVolumeMonitor *self = OSTREE_MOCK_VOLUME_MONITOR (monitor); + return g_list_copy_deep (self->volumes, (GCopyFunc) g_object_ref, NULL); +} + +static void +ostree_mock_volume_monitor_init (OstreeMockVolumeMonitor *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_volume_monitor_dispose (GObject *object) +{ + OstreeMockVolumeMonitor *self = OSTREE_MOCK_VOLUME_MONITOR (object); + + g_list_free_full (self->volumes, g_object_unref); + self->volumes = NULL; + + g_list_free_full (self->mounts, g_object_unref); + self->mounts = NULL; + + G_OBJECT_CLASS (ostree_mock_volume_monitor_parent_class)->dispose (object); +} + +static void +ostree_mock_volume_monitor_class_init (OstreeMockVolumeMonitorClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + GVolumeMonitorClass *monitor_class = G_VOLUME_MONITOR_CLASS (klass); + + object_class->dispose = ostree_mock_volume_monitor_dispose; + + monitor_class->get_mounts = ostree_mock_volume_monitor_get_mounts; + monitor_class->get_volumes = ostree_mock_volume_monitor_get_volumes; +} + +/** + * ostree_mock_volume_monitor_new: + * @mounts: (element-type GMount) (transfer none): list of current #GMounts + * @volumes: (element-type GVolume) (transfer none): list of current #GVolumes + * + * Create a new mock #GVolumeMonitor which will return the given static lists of + * #GMounts and #GVolumes to any caller of g_volume_monitor_get_mounts() or + * g_volume_monitor_get_volumes(). + * + * Typically, the elements of @mounts will be #OstreeMockMount objects and the + * elements of @volumes will be #OstreeMockVolume objects; but this does not + * have to be the case. + * + * Returns: (transfer full): a new #GVolumeMonitor object + * Since: 2017.8 + */ +GVolumeMonitor * +ostree_mock_volume_monitor_new (GList *mounts, + GList *volumes) +{ + g_autoptr(OstreeMockVolumeMonitor) monitor = NULL; + + monitor = g_object_new (OSTREE_TYPE_MOCK_VOLUME_MONITOR, NULL); + monitor->mounts = g_list_copy_deep (mounts, (GCopyFunc) g_object_ref, NULL); + monitor->volumes = g_list_copy_deep (volumes, (GCopyFunc) g_object_ref, NULL); + + return g_steal_pointer (&monitor); +} + +/* Mock volume class. This returns a static set of data to the caller, which it + * was initialised with. */ +struct _OstreeMockVolume +{ + GObject parent_instance; + + gchar *name; + GDrive *drive; /* (owned) (nullable) */ + GMount *mount; /* (owned) (nullable) */ +}; + +static void ostree_mock_volume_iface_init (GVolumeIface *iface); + +G_DEFINE_TYPE_WITH_CODE (OstreeMockVolume, ostree_mock_volume, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (G_TYPE_VOLUME, ostree_mock_volume_iface_init)) + +static gchar * +ostree_mock_volume_get_name (GVolume *volume) +{ + OstreeMockVolume *self = OSTREE_MOCK_VOLUME (volume); + return g_strdup (self->name); +} + +static GDrive * +ostree_mock_volume_get_drive (GVolume *volume) +{ + OstreeMockVolume *self = OSTREE_MOCK_VOLUME (volume); + return (self->drive != NULL) ? g_object_ref (self->drive) : NULL; +} + +static GMount * +ostree_mock_volume_get_mount (GVolume *volume) +{ + OstreeMockVolume *self = OSTREE_MOCK_VOLUME (volume); + return (self->mount != NULL) ? g_object_ref (self->mount) : NULL; +} + +static void +ostree_mock_volume_init (OstreeMockVolume *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_volume_dispose (GObject *object) +{ + OstreeMockVolume *self = OSTREE_MOCK_VOLUME (object); + + g_clear_pointer (&self->name, g_free); + g_clear_object (&self->drive); + g_clear_object (&self->mount); + + G_OBJECT_CLASS (ostree_mock_volume_parent_class)->dispose (object); +} + +static void +ostree_mock_volume_class_init (OstreeMockVolumeClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->dispose = ostree_mock_volume_dispose; +} + +static void +ostree_mock_volume_iface_init (GVolumeIface *iface) +{ + iface->get_name = ostree_mock_volume_get_name; + iface->get_drive = ostree_mock_volume_get_drive; + iface->get_mount = ostree_mock_volume_get_mount; +} + +/** + * ostree_mock_volume_new: + * @name: volume name + * @drive: (transfer none) (nullable): drive for the volume, or %NULL if none + * should be associated + * @mount: (transfer none) (nullable): mount for the volume, or %NULL if it’s + * not mounted + * + * Create a new mock #GVolume which will return the given static @name, @drive + * and @mount to any caller of its getter methods. There is currently no + * provision for changing these values dynamically. There is also currently no + * provision for mocking the other getters of #GVolume. + * + * Typically, @drive will be an #OstreeMockDrive object and @mount will be an + * #OstreeMockMount object; but this does not have to be the case. + * + * Returns: (transfer full): a new #GVolume object + * Since: 2017.8 + */ +OstreeMockVolume * +ostree_mock_volume_new (const gchar *name, + GDrive *drive, + GMount *mount) +{ + g_autoptr(OstreeMockVolume) volume = NULL; + + volume = g_object_new (OSTREE_TYPE_MOCK_VOLUME, NULL); + volume->name = g_strdup (name); + volume->drive = (drive != NULL) ? g_object_ref (drive) : NULL; + volume->mount = (mount != NULL) ? g_object_ref (mount) : NULL; + + return g_steal_pointer (&volume); +} + +/* Mock drive class. This returns a static set of data to the caller, which it + * was initialised with. */ +struct _OstreeMockDrive +{ + GObject parent_instance; + + gboolean is_removable; +}; + +static void ostree_mock_drive_iface_init (GDriveIface *iface); + +G_DEFINE_TYPE_WITH_CODE (OstreeMockDrive, ostree_mock_drive, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (G_TYPE_DRIVE, ostree_mock_drive_iface_init)) + +#if GLIB_CHECK_VERSION(2, 50, 0) +static gboolean +ostree_mock_drive_is_removable (GDrive *drive) +{ + OstreeMockDrive *self = OSTREE_MOCK_DRIVE (drive); + return self->is_removable; +} +#endif + +static void +ostree_mock_drive_init (OstreeMockDrive *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_drive_class_init (OstreeMockDriveClass *klass) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_drive_iface_init (GDriveIface *iface) +{ +#if GLIB_CHECK_VERSION(2, 50, 0) + iface->is_removable = ostree_mock_drive_is_removable; +#endif +} + +/** + * ostree_mock_drive_new: + * @is_removable: %TRUE if the drive is removable; %FALSE otherwise + * + * Create a new mock #GDrive which will return the given static @is_removable to + * any caller of its getter methods. There is currently no provision for mocking + * the other getters of #GDrive. + * + * Returns: (transfer full): a new #GDrive object + * Since: 2017.8 + */ +OstreeMockDrive * +ostree_mock_drive_new (gboolean is_removable) +{ + g_autoptr(OstreeMockDrive) drive = NULL; + + drive = g_object_new (OSTREE_TYPE_MOCK_DRIVE, NULL); + drive->is_removable = is_removable; + + return g_steal_pointer (&drive); +} + +/* Mock mount class. This returns a static set of data to the caller, which it + * was initialised with. */ +struct _OstreeMockMount +{ + GObject parent_instance; + + gchar *name; /* (owned) */ + GFile *root; /* (owned) */ +}; + +static void ostree_mock_mount_iface_init (GMountIface *iface); + +G_DEFINE_TYPE_WITH_CODE (OstreeMockMount, ostree_mock_mount, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (G_TYPE_MOUNT, ostree_mock_mount_iface_init)) + +static gchar * +ostree_mock_mount_get_name (GMount *mount) +{ + OstreeMockMount *self = OSTREE_MOCK_MOUNT (mount); + return g_strdup (self->name); +} + +static GFile * +ostree_mock_mount_get_root (GMount *mount) +{ + OstreeMockMount *self = OSTREE_MOCK_MOUNT (mount); + return g_object_ref (self->root); +} + +static void +ostree_mock_mount_init (OstreeMockMount *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_mount_dispose (GObject *object) +{ + OstreeMockMount *self = OSTREE_MOCK_MOUNT (object); + + g_clear_pointer (&self->name, g_free); + g_clear_object (&self->root); + + G_OBJECT_CLASS (ostree_mock_mount_parent_class)->dispose (object); +} + +static void +ostree_mock_mount_class_init (OstreeMockMountClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->dispose = ostree_mock_mount_dispose; +} + +static void +ostree_mock_mount_iface_init (GMountIface *iface) +{ + iface->get_name = ostree_mock_mount_get_name; + iface->get_root = ostree_mock_mount_get_root; +} + +/** + * ostree_mock_mount_new: + * @name: mount name + * @root: (transfer none): root path for the mounted file system + * + * Create a new mock #GMount which will return the given static @name and @root + * to any caller of its getter methods. There is currently no provision for + * mocking the other getters of #GMount. + * + * Typically, @root will point to a temporary directory where a mocked file + * system is present; but this does not have to be the case. + * + * Returns: (transfer full): a new #GMount object + * Since: 2017.8 + */ +OstreeMockMount * +ostree_mock_mount_new (const gchar *name, + GFile *root) +{ + g_autoptr(OstreeMockMount) mount = NULL; + + mount = g_object_new (OSTREE_TYPE_MOCK_MOUNT, NULL); + mount->name = g_strdup (name); + mount->root = g_object_ref (root); + + return g_steal_pointer (&mount); +} diff --git a/tests/test-mock-gio.h b/tests/test-mock-gio.h new file mode 100644 index 00000000..9f544759 --- /dev/null +++ b/tests/test-mock-gio.h @@ -0,0 +1,127 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include +#include + +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_MOCK_VOLUME_MONITOR (ostree_mock_volume_monitor_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +G_GNUC_INTERNAL +G_DECLARE_FINAL_TYPE (OstreeMockVolumeMonitor, ostree_mock_volume_monitor, OSTREE, MOCK_VOLUME_MONITOR, GVolumeMonitor) */ + +G_GNUC_INTERNAL +GType ostree_mock_volume_monitor_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeMockVolumeMonitor OstreeMockVolumeMonitor; +typedef struct { GVolumeMonitorClass parent_class; } OstreeMockVolumeMonitorClass; + +static inline OstreeMockVolumeMonitor *OSTREE_MOCK_VOLUME_MONITOR (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_mock_volume_monitor_get_type (), OstreeMockVolumeMonitor); } +static inline gboolean OSTREE_IS_MOCK_VOLUME_MONITOR (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_mock_volume_monitor_get_type ()); } +G_GNUC_END_IGNORE_DEPRECATIONS + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeMockVolumeMonitor, g_object_unref) + +G_GNUC_INTERNAL +GVolumeMonitor *ostree_mock_volume_monitor_new (GList *mounts, + GList *volumes); + +#define OSTREE_TYPE_MOCK_VOLUME (ostree_mock_volume_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +G_GNUC_INTERNAL +G_DECLARE_FINAL_TYPE (OstreeMockVolume, ostree_mock_volume, OSTREE, MOCK_VOLUME, GObject) */ + +G_GNUC_INTERNAL +GType ostree_mock_volume_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeMockVolume OstreeMockVolume; +typedef struct { GObjectClass parent_class; } OstreeMockVolumeClass; + +static inline OstreeMockVolume *OSTREE_MOCK_VOLUME (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_mock_volume_get_type (), OstreeMockVolume); } +static inline gboolean OSTREE_IS_MOCK_VOLUME (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_mock_volume_get_type ()); } +G_GNUC_END_IGNORE_DEPRECATIONS + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeMockVolume, g_object_unref) + +G_GNUC_INTERNAL +OstreeMockVolume *ostree_mock_volume_new (const gchar *name, + GDrive *drive, + GMount *mount); + +#define OSTREE_TYPE_MOCK_DRIVE (ostree_mock_drive_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +G_GNUC_INTERNAL +G_DECLARE_FINAL_TYPE (OstreeMockDrive, ostree_mock_drive, OSTREE, MOCK_DRIVE, GObject) */ + +G_GNUC_INTERNAL +GType ostree_mock_drive_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeMockDrive OstreeMockDrive; +typedef struct { GObjectClass parent_class; } OstreeMockDriveClass; + +static inline OstreeMockDrive *OSTREE_MOCK_DRIVE (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_mock_drive_get_type (), OstreeMockDrive); } +static inline gboolean OSTREE_IS_MOCK_DRIVE (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_mock_drive_get_type ()); } +G_GNUC_END_IGNORE_DEPRECATIONS + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeMockDrive, g_object_unref) + +G_GNUC_INTERNAL +OstreeMockDrive *ostree_mock_drive_new (gboolean is_removable); + +#define OSTREE_TYPE_MOCK_MOUNT (ostree_mock_mount_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +G_GNUC_INTERNAL +G_DECLARE_FINAL_TYPE (OstreeMockMount, ostree_mock_mount, OSTREE, MOCK_MOUNT, GObject) */ + +G_GNUC_INTERNAL +GType ostree_mock_mount_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeMockMount OstreeMockMount; +typedef struct { GObjectClass parent_class; } OstreeMockMountClass; + +static inline OstreeMockMount *OSTREE_MOCK_MOUNT (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_mock_mount_get_type (), OstreeMockMount); } +static inline gboolean OSTREE_IS_MOCK_MOUNT (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_mock_mount_get_type ()); } +G_GNUC_END_IGNORE_DEPRECATIONS + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeMockMount, g_object_unref) + +G_GNUC_INTERNAL +OstreeMockMount *ostree_mock_mount_new (const gchar *name, + GFile *root); + +G_END_DECLS diff --git a/tests/test-repo-finder-mount.c b/tests/test-repo-finder-mount.c new file mode 100644 index 00000000..c84feb52 --- /dev/null +++ b/tests/test-repo-finder-mount.c @@ -0,0 +1,497 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "libostreetest.h" +#include "ostree-autocleanups.h" +#include "ostree-remote-private.h" +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-mount.h" +#include "ostree-types.h" +#include "test-mock-gio.h" + +/* Test fixture. Creates a temporary directory and repository. */ +typedef struct +{ + OstreeRepo *parent_repo; + int working_dfd; /* owned */ + GFile *working_dir; /* owned */ +} Fixture; + +static void +setup (Fixture *fixture, + gconstpointer test_data) +{ + g_autofree gchar *tmp_name = NULL; + g_autoptr(GError) error = NULL; + + tmp_name = g_strdup ("test-repo-finder-mount-XXXXXX"); + glnx_mkdtempat_open_in_system (tmp_name, 0700, &fixture->working_dfd, &error); + g_assert_no_error (error); + + g_test_message ("Using temporary directory: %s", tmp_name); + + glnx_shutil_mkdir_p_at (fixture->working_dfd, "repo", 0700, NULL, &error); + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_EXISTS)) + g_clear_error (&error); + g_assert_no_error (error); + + g_autoptr(GFile) tmp_dir = g_file_new_for_path (g_get_tmp_dir ()); + fixture->working_dir = g_file_get_child (tmp_dir, tmp_name); + + fixture->parent_repo = ot_test_setup_repo (NULL, &error); + g_assert_no_error (error); +} + +static void +teardown (Fixture *fixture, + gconstpointer test_data) +{ + glnx_fd_close int parent_repo_dfd = -1; + g_autoptr(GError) error = NULL; + + /* Recursively remove the temporary directory. */ + glnx_shutil_rm_rf_at (fixture->working_dfd, ".", NULL, NULL); + + close (fixture->working_dfd); + fixture->working_dfd = -1; + + /* The repo also needs its source files to be removed. This is the inverse + * of setup_test_repository() in libtest.sh. */ + g_autofree gchar *parent_repo_path = g_file_get_path (ostree_repo_get_path (fixture->parent_repo)); + glnx_opendirat (-1, parent_repo_path, TRUE, &parent_repo_dfd, &error); + g_assert_no_error (error); + + glnx_shutil_rm_rf_at (parent_repo_dfd, "../files", NULL, NULL); + glnx_shutil_rm_rf_at (parent_repo_dfd, "../repo", NULL, NULL); + + g_clear_object (&fixture->working_dir); + g_clear_object (&fixture->parent_repo); +} + +/* Test the object constructor works at a basic level. */ +static void +test_repo_finder_mount_init (void) +{ + g_autoptr(OstreeRepoFinderMount) finder = NULL; + g_autoptr(GVolumeMonitor) monitor = NULL; + + /* Default #GVolumeMonitor. */ + finder = ostree_repo_finder_mount_new (NULL); + g_clear_object (&finder); + + /* Explicit #GVolumeMonitor. */ + monitor = ostree_mock_volume_monitor_new (NULL, NULL); + finder = ostree_repo_finder_mount_new (monitor); + g_clear_object (&finder); +} + +static void +result_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data) +{ + GAsyncResult **result_out = user_data; + *result_out = g_object_ref (result); +} + +/* Test that no remotes are found if the #GVolumeMonitor returns no mounts. */ +static void +test_repo_finder_mount_no_mounts (Fixture *fixture, + gconstpointer test_data) +{ + g_autoptr(OstreeRepoFinderMount) finder = NULL; + g_autoptr(GVolumeMonitor) monitor = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + g_autoptr(GError) error = NULL; + const OstreeCollectionRef ref1 = { "org.example.Collection1", "exampleos/x86_64/standard" }; + const OstreeCollectionRef ref2 = { "org.example.Collection1", "exampleos/x86_64/buildmaster/standard" }; + const OstreeCollectionRef ref3 = { "org.example.Collection2", "exampleos/x86_64/standard" }; + const OstreeCollectionRef ref4 = { "org.example.Collection2", "exampleos/arm64/standard" }; + const OstreeCollectionRef * const refs[] = { &ref1, &ref2, &ref3, &ref4, NULL }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + monitor = ostree_mock_volume_monitor_new (NULL, NULL); + finder = ostree_repo_finder_mount_new (monitor); + + ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, + fixture->parent_repo, + NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (results->len, ==, 0); + + g_main_context_pop_thread_default (context); +} + +/* Create a .ostree/repos directory under the given @mount_root, or abort. */ +static gboolean +assert_create_repos_dir (Fixture *fixture, + const gchar *mount_root_name, + int *out_repos_dfd, + GMount **out_mount) +{ + glnx_fd_close int repos_dfd = -1; + g_autoptr(GError) error = NULL; + + g_autofree gchar *path = g_build_filename (mount_root_name, ".ostree", "repos", NULL); + glnx_shutil_mkdir_p_at_open (fixture->working_dfd, path, 0700, &repos_dfd, NULL, &error); + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_EXISTS)) + g_clear_error (&error); + g_assert_no_error (error); + + *out_repos_dfd = glnx_steal_fd (&repos_dfd); + g_autoptr(GFile) mount_root = g_file_get_child (fixture->working_dir, mount_root_name); + *out_mount = G_MOUNT (ostree_mock_mount_new (mount_root_name, mount_root)); + + return TRUE; +} + +/* Create a new repository in @repo_dir with its collection ID unset, and + * containing the refs given in @... (which must be %NULL-terminated). Each + * #OstreeCollectionRef in @... is followed by a gchar** return address for the + * checksum committed for that ref. Return the new repository. */ +static OstreeRepo * +assert_create_remote_va (Fixture *fixture, + GFile *repo_dir, + va_list args) +{ + g_autoptr(GError) error = NULL; + + g_autoptr(OstreeRepo) repo = ostree_repo_new (repo_dir); + ostree_repo_create (repo, OSTREE_REPO_MODE_ARCHIVE_Z2, NULL, &error); + g_assert_no_error (error); + + /* Set up the refs from @.... */ + for (const OstreeCollectionRef *ref = va_arg (args, const OstreeCollectionRef *); + ref != NULL; + ref = va_arg (args, const OstreeCollectionRef *)) + { + g_autofree gchar *checksum = NULL; + g_autoptr(OstreeMutableTree) mtree = NULL; + g_autoptr(OstreeRepoFile) repo_file = NULL; + gchar **out_checksum = va_arg (args, gchar **); + + mtree = ostree_mutable_tree_new (); + ostree_repo_write_dfd_to_mtree (repo, AT_FDCWD, ".", mtree, NULL, NULL, &error); + g_assert_no_error (error); + ostree_repo_write_mtree (repo, mtree, (GFile **) &repo_file, NULL, &error); + g_assert_no_error (error); + + ostree_repo_write_commit (repo, NULL /* no parent */, ref->ref_name, ref->ref_name, + NULL /* no metadata */, repo_file, &checksum, + NULL, &error); + g_assert_no_error (error); + + if (ref->collection_id != NULL) + ostree_repo_set_collection_ref_immediate (repo, ref, checksum, NULL, &error); + else + ostree_repo_set_ref_immediate (repo, NULL, ref->ref_name, checksum, NULL, &error); + g_assert_no_error (error); + + if (out_checksum != NULL) + *out_checksum = g_steal_pointer (&checksum); + } + + /* Update the summary. */ + ostree_repo_regenerate_summary (repo, NULL /* no metadata */, NULL, &error); + g_assert_no_error (error); + + return g_steal_pointer (&repo); +} + +static OstreeRepo * +assert_create_repo_dir (Fixture *fixture, + int repos_dfd, + GMount *repos_mount, + const OstreeCollectionRef *ref, + gchar **out_uri, + ...) G_GNUC_NULL_TERMINATED; + +/* Create a @ref directory under the given @repos_dfd, or abort. Create a new + * repository in it with the refs given in @..., as per assert_create_remote_va(). + * Return the URI of the repository. */ +static OstreeRepo * +assert_create_repo_dir (Fixture *fixture, + int repos_dfd, + GMount *repos_mount, + const OstreeCollectionRef *ref, + gchar **out_uri, + ...) +{ + glnx_fd_close int ref_dfd = -1; + g_autoptr(OstreeRepo) repo = NULL; + g_autoptr(GError) error = NULL; + va_list args; + + g_autofree gchar *path = g_build_filename (ref->collection_id, ref->ref_name, NULL); + glnx_shutil_mkdir_p_at_open (repos_dfd, path, 0700, &ref_dfd, NULL, &error); + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_EXISTS)) + g_clear_error (&error); + g_assert_no_error (error); + + g_autoptr(GFile) mount_root = g_mount_get_root (repos_mount); + g_autoptr(GFile) repos_dir = g_file_get_child (mount_root, ".ostree/repos"); + g_autoptr(GFile) repo_dir = g_file_get_child (repos_dir, path); + + va_start (args, out_uri); + repo = assert_create_remote_va (fixture, repo_dir, args); + va_end (args); + + *out_uri = g_file_get_uri (repo_dir); + + return g_steal_pointer (&repo); +} + +/* Create a @ref symlink under the given @repos_dfd, pointing to + * @symlink_target, or abort. */ +static int +assert_create_repo_symlink (int repos_dfd, + const OstreeCollectionRef *ref, + const gchar *symlink_target_path) +{ + glnx_fd_close int symlink_target_dfd = -1; + g_autoptr(GError) error = NULL; + + /* The @ref_parent_dir is not necessarily @collection_dir, since @ref may + * contain slashes. */ + g_autofree gchar *path = g_build_filename (ref->collection_id, ref->ref_name, NULL); + g_autofree gchar *path_parent = g_path_get_dirname (path); + + glnx_shutil_mkdir_p_at (repos_dfd, path_parent, 0700, NULL, &error); + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_EXISTS)) + g_clear_error (&error); + g_assert_no_error (error); + + if (TEMP_FAILURE_RETRY (symlinkat (symlink_target_path, repos_dfd, path)) != 0) + { + g_autoptr(GError) error = NULL; + glnx_throw_errno_prefix (&error, "symlinkat"); + g_assert_no_error (error); + } + + /* Return a dir FD for the symlink target. */ + glnx_opendirat (repos_dfd, path, TRUE, &symlink_target_dfd, &error); + g_assert_no_error (error); + + return glnx_steal_fd (&symlink_target_dfd); +} + +/* Add configuration for a remote named @remote_name, at @remote_uri, with a + * remote collection ID of @collection_id, to the given @repo. */ +static void +assert_create_remote_config (OstreeRepo *repo, + const gchar *remote_name, + const gchar *remote_uri, + const gchar *collection_id) +{ + g_autoptr(GError) error = NULL; + g_autoptr(GVariant) options = NULL; + + if (collection_id != NULL) + options = g_variant_new_parsed ("@a{sv} { 'collection-id': <%s> }", + collection_id); + + ostree_repo_remote_add (repo, remote_name, remote_uri, options, NULL, &error); + g_assert_no_error (error); +} + +/* Test resolving the refs against a collection of mock volumes, some of which + * are mounted, some of which are removable, some of which contain valid or + * invalid repo information on the file system, etc. */ +static void +test_repo_finder_mount_mixed_mounts (Fixture *fixture, + gconstpointer test_data) +{ + g_autoptr(OstreeRepoFinderMount) finder = NULL; + g_autoptr(GVolumeMonitor) monitor = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + g_autoptr(GError) error = NULL; + g_autoptr(GList) mounts = NULL; /* (element-type OstreeMockMount) */ + g_autoptr(GMount) non_removable_mount = NULL; + g_autoptr(GMount) no_repos_mount = NULL; + g_autoptr(GMount) repo1_mount = NULL; + g_autoptr(GMount) repo2_mount = NULL; + g_autoptr(GFile) non_removable_root = NULL; + glnx_fd_close int no_repos_repos = -1; + glnx_fd_close int repo1_repos = -1; + glnx_fd_close int repo2_repos = -1; + g_autoptr(OstreeRepo) repo1_repo_a = NULL, repo1_repo_b = NULL; + g_autoptr(OstreeRepo) repo2_repo_a = NULL; + g_autofree gchar *repo1_repo_a_uri = NULL, *repo1_repo_b_uri = NULL; + g_autofree gchar *repo2_repo_a_uri = NULL; + g_autofree gchar *repo1_ref0_checksum = NULL, *repo1_ref1_checksum = NULL, *repo1_ref2_checksum = NULL; + g_autofree gchar *repo2_ref0_checksum = NULL, *repo2_ref1_checksum = NULL, *repo2_ref2_checksum = NULL; + g_autofree gchar *repo1_ref5_checksum = NULL; + gsize i; + const OstreeCollectionRef ref0 = { "org.example.Collection1", "exampleos/x86_64/ref0" }; + const OstreeCollectionRef ref1 = { "org.example.Collection1", "exampleos/x86_64/ref1" }; + const OstreeCollectionRef ref2 = { "org.example.Collection1", "exampleos/x86_64/ref2" }; + const OstreeCollectionRef ref3 = { "org.example.Collection1", "exampleos/x86_64/ref3" }; + const OstreeCollectionRef ref4 = { "org.example.UnconfiguredCollection", "exampleos/x86_64/ref4" }; + const OstreeCollectionRef ref5 = { "org.example.Collection3", "exampleos/x86_64/ref0" }; + const OstreeCollectionRef * const refs[] = { &ref0, &ref1, &ref2, &ref3, &ref4, &ref5, NULL }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + /* Build the various mock drives/volumes/mounts, and some repositories with + * refs within them. We use "/" under the assumption that it’s on a separate + * file system from /tmp, so it’s an example of a symlink pointing outside + * its mount point. */ + non_removable_root = g_file_get_child (fixture->working_dir, "non-removable-mount"); + non_removable_mount = G_MOUNT (ostree_mock_mount_new ("non-removable", non_removable_root)); + + assert_create_repos_dir (fixture, "no-repos-mount", &no_repos_repos, &no_repos_mount); + + assert_create_repos_dir (fixture, "repo1-mount", &repo1_repos, &repo1_mount); + repo1_repo_a = assert_create_repo_dir (fixture, repo1_repos, repo1_mount, refs[0], &repo1_repo_a_uri, + refs[0], &repo1_ref0_checksum, + refs[2], &repo1_ref2_checksum, + refs[5], &repo1_ref5_checksum, + NULL); + repo1_repo_b = assert_create_repo_dir (fixture, repo1_repos, repo1_mount, refs[1], &repo1_repo_b_uri, + refs[1], &repo1_ref1_checksum, + NULL); + assert_create_repo_symlink (repo1_repos, refs[2], "ref0"); /* repo1_repo_a */ + assert_create_repo_symlink (repo1_repos, refs[5], "../../../org.example.Collection1/exampleos/x86_64/ref0"); /* repo1_repo_a */ + + assert_create_repos_dir (fixture, "repo2-mount", &repo2_repos, &repo2_mount); + repo2_repo_a = assert_create_repo_dir (fixture, repo2_repos, repo2_mount, refs[0], &repo2_repo_a_uri, + refs[0], &repo2_ref0_checksum, + refs[1], &repo2_ref1_checksum, + refs[2], &repo2_ref2_checksum, + refs[3], NULL, + NULL); + assert_create_repo_symlink (repo2_repos, refs[1], "ref0"); /* repo2_repo_a */ + assert_create_repo_symlink (repo2_repos, refs[2], "ref1"); /* repo2_repo_b */ + assert_create_repo_symlink (repo2_repos, refs[3], "/"); + + mounts = g_list_prepend (mounts, non_removable_mount); + mounts = g_list_prepend (mounts, no_repos_mount); + mounts = g_list_prepend (mounts, repo1_mount); + mounts = g_list_prepend (mounts, repo2_mount); + + monitor = ostree_mock_volume_monitor_new (mounts, NULL); + finder = ostree_repo_finder_mount_new (monitor); + + assert_create_remote_config (fixture->parent_repo, "remote1", "https://nope1", "org.example.Collection1"); + assert_create_remote_config (fixture->parent_repo, "remote2", "https://nope2", "org.example.Collection2"); + /* don’t configure org.example.UnconfiguredCollection */ + assert_create_remote_config (fixture->parent_repo, "remote3", "https://nope3", "org.example.Collection3"); + + /* Resolve the refs. */ + ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, + fixture->parent_repo, + NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (results->len, ==, 4); + + /* Check that the results are correct: the invalid refs should have been + * ignored, and the valid results canonicalised and deduplicated. */ + for (i = 0; i < results->len; i++) + { + g_autofree gchar *uri = NULL; + const gchar *keyring; + const OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + + uri = g_key_file_get_string (result->remote->options, result->remote->group, "url", &error); + g_assert_no_error (error); + keyring = result->remote->keyring; + + if (g_strcmp0 (uri, repo1_repo_a_uri) == 0 && + g_strcmp0 (keyring, "remote1.trustedkeys.gpg") == 0) + { + g_assert_cmpuint (g_hash_table_size (result->ref_to_checksum), ==, 2); + g_assert_cmpstr (g_hash_table_lookup (result->ref_to_checksum, refs[0]), ==, repo1_ref0_checksum); + g_assert_cmpstr (g_hash_table_lookup (result->ref_to_checksum, refs[2]), ==, repo1_ref2_checksum); + } + else if (g_strcmp0 (uri, repo1_repo_a_uri) == 0 && + g_strcmp0 (keyring, "remote3.trustedkeys.gpg") == 0) + { + g_assert_cmpuint (g_hash_table_size (result->ref_to_checksum), ==, 1); + g_assert_cmpstr (g_hash_table_lookup (result->ref_to_checksum, refs[5]), ==, repo1_ref5_checksum); + } + else if (g_strcmp0 (uri, repo1_repo_b_uri) == 0 && + g_strcmp0 (keyring, "remote1.trustedkeys.gpg") == 0) + { + g_assert_cmpuint (g_hash_table_size (result->ref_to_checksum), ==, 1); + g_assert_cmpstr (g_hash_table_lookup (result->ref_to_checksum, refs[1]), ==, repo1_ref1_checksum); + } + else if (g_strcmp0 (uri, repo2_repo_a_uri) == 0 && + g_strcmp0 (keyring, "remote1.trustedkeys.gpg") == 0) + { + g_assert_cmpuint (g_hash_table_size (result->ref_to_checksum), ==, 3); + g_assert_cmpstr (g_hash_table_lookup (result->ref_to_checksum, refs[0]), ==, repo2_ref0_checksum); + g_assert_cmpstr (g_hash_table_lookup (result->ref_to_checksum, refs[1]), ==, repo2_ref1_checksum); + g_assert_cmpstr (g_hash_table_lookup (result->ref_to_checksum, refs[2]), ==, repo2_ref2_checksum); + } + else + { + g_test_message ("Unknown result ‘%s’ with keyring ‘%s’.", + result->remote->name, result->remote->keyring); + g_assert_not_reached (); + } + } + + g_main_context_pop_thread_default (context); +} + +int main (int argc, char **argv) +{ + setlocale (LC_ALL, ""); + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/repo-finder-mount/init", test_repo_finder_mount_init); + g_test_add ("/repo-finder-mount/no-mounts", Fixture, NULL, setup, + test_repo_finder_mount_no_mounts, teardown); + g_test_add ("/repo-finder-mount/mixed-mounts", Fixture, NULL, setup, + test_repo_finder_mount_mixed_mounts, teardown); + + return g_test_run(); +} From 7ee4e1295ae083c33bda801d59c88699f59c049c Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 19 Apr 2017 00:11:28 +0100 Subject: [PATCH 26/82] lib/bloom: Add an internal bloom filter implementation This will be used in an upcoming commit. It adds a basic bloom filter implementation, using the SipHash family of hash functions. The implementation (including its parameter choices and hash functions) will become a protocol detail in future, so must not be changed so that its output is bitwise incompatible between OSTree versions. Unit tests are included. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- Makefile-libostree.am | 2 + Makefile-tests.am | 5 + src/libostree/ostree-bloom-private.h | 104 +++++ src/libostree/ostree-bloom.c | 603 +++++++++++++++++++++++++++ tests/.gitignore | 1 + tests/test-bloom.c | 154 +++++++ 6 files changed, 869 insertions(+) create mode 100644 src/libostree/ostree-bloom-private.h create mode 100644 src/libostree/ostree-bloom.c create mode 100644 tests/test-bloom.c diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 01aa8663..6a7c4820 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -160,6 +160,8 @@ libostree_1_la_SOURCES += \ $(NULL) else # if ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += \ + src/libostree/ostree-bloom.c \ + src/libostree/ostree-bloom-private.h \ src/libostree/ostree-repo-finder.c \ src/libostree/ostree-repo-finder-config.c \ src/libostree/ostree-repo-finder-mount.c \ diff --git a/Makefile-tests.am b/Makefile-tests.am index 09c85818..d04a1cbc 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -195,6 +195,7 @@ _installed_or_uninstalled_test_programs = tests/test-varint tests/test-ot-unix-u if ENABLE_EXPERIMENTAL_API test_programs += \ + tests/test-bloom \ tests/test-repo-finder-config \ tests/test-repo-finder-mount \ $(NULL) @@ -226,6 +227,10 @@ tests_test_rollsum_SOURCES = src/libostree/ostree-rollsum.c tests/test-rollsum.c tests_test_rollsum_CFLAGS = $(TESTS_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) tests_test_rollsum_LDADD = $(bupsplitpath) $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS) +tests_test_bloom_SOURCES = src/libostree/ostree-bloom.c tests/test-bloom.c +tests_test_bloom_CFLAGS = $(TESTS_CFLAGS) +tests_test_bloom_LDADD = $(TESTS_LDADD) + tests_test_repo_finder_config_SOURCES = tests/test-repo-finder-config.c tests_test_repo_finder_config_CFLAGS = $(TESTS_CFLAGS) tests_test_repo_finder_config_LDADD = $(TESTS_LDADD) diff --git a/src/libostree/ostree-bloom-private.h b/src/libostree/ostree-bloom-private.h new file mode 100644 index 00000000..47f828b3 --- /dev/null +++ b/src/libostree/ostree-bloom-private.h @@ -0,0 +1,104 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +G_BEGIN_DECLS + +/** + * OstreeBloom: + * + * An implementation of a [bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) + * which is suitable for building a filter and looking keys up in an existing + * filter. + * + * Since: 2017.8 + */ +typedef struct _OstreeBloom OstreeBloom; + +/** + * OstreeBloomHashFunc: + * @element: a pointer to the element to hash + * @k: hash function parameter + * + * Function prototype for a + * [universal hash function](https://en.wikipedia.org/wiki/Universal_hashing), + * parameterised on @k, which hashes @element to a #guint64 hash value. + * + * It is up to the implementer of the hash function whether %NULL is valid for + * @element. + * + * Since: 2017.8 + */ +typedef guint64 (*OstreeBloomHashFunc) (gconstpointer element, + guint8 k); + +#define OSTREE_TYPE_BLOOM (ostree_bloom_get_type ()) + +G_GNUC_INTERNAL +GType ostree_bloom_get_type (void); + +G_GNUC_INTERNAL +OstreeBloom *ostree_bloom_new (gsize n_bytes, + guint8 k, + OstreeBloomHashFunc hash_func); + +G_GNUC_INTERNAL +OstreeBloom *ostree_bloom_new_from_bytes (GBytes *bytes, + guint8 k, + OstreeBloomHashFunc hash_func); + +G_GNUC_INTERNAL +OstreeBloom *ostree_bloom_ref (OstreeBloom *bloom); +G_GNUC_INTERNAL +void ostree_bloom_unref (OstreeBloom *bloom); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeBloom, ostree_bloom_unref) + +G_GNUC_INTERNAL +gboolean ostree_bloom_maybe_contains (OstreeBloom *bloom, + gconstpointer element); + +G_GNUC_INTERNAL +GBytes *ostree_bloom_seal (OstreeBloom *bloom); + +G_GNUC_INTERNAL +void ostree_bloom_add_element (OstreeBloom *bloom, + gconstpointer element); + +G_GNUC_INTERNAL +gsize ostree_bloom_get_size (OstreeBloom *bloom); +G_GNUC_INTERNAL +guint8 ostree_bloom_get_k (OstreeBloom *bloom); +G_GNUC_INTERNAL +OstreeBloomHashFunc ostree_bloom_get_hash_func (OstreeBloom *bloom); + +G_GNUC_INTERNAL +guint64 ostree_str_bloom_hash (gconstpointer element, + guint8 k); + +G_END_DECLS diff --git a/src/libostree/ostree-bloom.c b/src/libostree/ostree-bloom.c new file mode 100644 index 00000000..7677b3ba --- /dev/null +++ b/src/libostree/ostree-bloom.c @@ -0,0 +1,603 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "ostree-bloom-private.h" + +/** + * SECTION:bloom + * @title: Bloom filter + * @short_description: Bloom filter implementation supporting building and + * reading filters + * @stability: Unstable + * @include: libostree/ostree-bloom-private.h + * + * #OstreeBloom is an implementation of a bloom filter which supports writing to + * and loading from a #GBytes bit array. The caller must store metadata about + * the bloom filter (its hash function and `k` parameter value) separately, as + * the same values must be used when reading from a serialised bit array as were + * used to build the array in the first place. + * + * This is a standard implementation of a bloom filter, and background reading + * on the theory can be + * [found on Wikipedia](https://en.wikipedia.org/wiki/Bloom_filter). In + * particular, a bloom filter is parameterised by `m` and `k` parameters: the + * size of the bit array (in bits) is `m`, and the number of hash functions + * applied to each element is `k`. Bloom filters require a universal hash + * function which can be parameterised by `k`. We have #OstreeBloomHashFunc, + * with ostree_str_bloom_hash() being an implementation for strings. + * + * The serialised output from a bloom filter is guaranteed to be stable across + * versions of libostree as long as the same values for `k` and the hash + * function are used. + * + * #OstreeBloom is mutable when constructed with ostree_bloom_new(), and elements + * can be added to it using ostree_bloom_add_element(), until ostree_bloom_seal() + * is called to serialise it and make it immutable. After then, the bloom filter + * can only be queried using ostree_bloom_maybe_contains(). + * + * If constructed with ostree_bloom_new_from_bytes(), the bloom filter is + * immutable from construction, and can only be queried. + * + * Reference: + * - https://en.wikipedia.org/wiki/Bloom_filter + * - https://llimllib.github.io/bloomfilter-tutorial/ + * + * Since: 2017.8 + */ + +struct _OstreeBloom +{ + guint ref_count; + gsize n_bytes; + gboolean is_mutable; /* determines which of [im]mutable_bytes is accessed */ + union + { + guint8 *mutable_bytes; /* owned; mutually exclusive */ + GBytes *immutable_bytes; /* owned; mutually exclusive */ + }; + guint8 k; + OstreeBloomHashFunc hash_func; +}; + +G_DEFINE_BOXED_TYPE (OstreeBloom, ostree_bloom, ostree_bloom_ref, ostree_bloom_unref) + +/** + * ostree_bloom_new: + * @n_bytes: size to make the bloom filter, in bytes + * @k: number of hash functions to use + * @hash_func: universal hash function to use + * + * Create a new mutable #OstreeBloom filter, with all its bits initialised to + * zero. Set elements in the filter using ostree_bloom_add_element(), and seal + * it to return an immutable #GBytes using ostree_bloom_seal(). + * + * To load an #OstreeBloom from an existing #GBytes, use + * ostree_bloom_new_from_bytes(). + * + * Note that @n_bytes is in bytes, so is 8 times smaller than the parameter `m` + * which is used when describing bloom filters academically. + * + * Returns: (transfer full): a new mutable bloom filter + * + * Since: 2017.8 + */ +OstreeBloom * +ostree_bloom_new (gsize n_bytes, + guint8 k, + OstreeBloomHashFunc hash_func) +{ + g_autoptr(OstreeBloom) bloom = NULL; + + g_return_val_if_fail (n_bytes > 0, NULL); + g_return_val_if_fail (k > 0, NULL); + g_return_val_if_fail (hash_func != NULL, NULL); + + bloom = g_new0 (OstreeBloom, 1); + bloom->ref_count = 1; + + bloom->is_mutable = TRUE; + bloom->mutable_bytes = g_malloc0 (n_bytes); + bloom->n_bytes = n_bytes; + bloom->k = k; + bloom->hash_func = hash_func; + + return g_steal_pointer (&bloom); +} + +/** + * ostree_bloom_new_from_bytes: + * @bytes: array of bytes containing the filter data + * @k: number of hash functions to use + * @hash_func: universal hash function to use + * + * Load an immutable #OstreeBloom filter from the given @bytes. Check whether + * elements are probably set in the filter using ostree_bloom_maybe_contains(). + * + * To create a new mutable #OstreeBloom, use ostree_bloom_new(). + * + * Note that all the bits in @bytes are loaded, so the parameter `m` for the + * filter (as commonly used in academic literature) is always a multiple of 8. + * + * Returns: (transfer full): a new immutable bloom filter + * + * Since: 2017.8 + */ +OstreeBloom * +ostree_bloom_new_from_bytes (GBytes *bytes, + guint8 k, + OstreeBloomHashFunc hash_func) +{ + g_autoptr(OstreeBloom) bloom = NULL; + + g_return_val_if_fail (bytes != NULL, NULL); + g_return_val_if_fail (g_bytes_get_size (bytes) > 0, NULL); + g_return_val_if_fail (k > 0, NULL); + g_return_val_if_fail (hash_func != NULL, NULL); + + bloom = g_new0 (OstreeBloom, 1); + bloom->ref_count = 1; + + bloom->is_mutable = FALSE; + bloom->immutable_bytes = g_bytes_ref (bytes); + bloom->n_bytes = g_bytes_get_size (bytes); + bloom->k = k; + bloom->hash_func = hash_func; + + return g_steal_pointer (&bloom); +} + +/** + * ostree_bloom_ref: + * @bloom: an #OstreeBloom + * + * Increase the reference count of @bloom. + * + * Returns: (transfer full): @bloom + * Since: 2017.8 + */ +OstreeBloom * +ostree_bloom_ref (OstreeBloom *bloom) +{ + g_return_val_if_fail (bloom != NULL, NULL); + g_return_val_if_fail (bloom->ref_count >= 1, NULL); + g_return_val_if_fail (bloom->ref_count == G_MAXUINT - 1, NULL); + + bloom->ref_count++; + + return bloom; +} + +/** + * ostree_bloom_unref: + * @bloom: (transfer full): an #OstreeBloom + * + * Decrement the reference count of @bloom. If it reaches zero, the filter + * is destroyed. + * + * Since: 2017.8 + */ +void +ostree_bloom_unref (OstreeBloom *bloom) +{ + g_return_if_fail (bloom != NULL); + g_return_if_fail (bloom->ref_count >= 1); + + bloom->ref_count--; + + if (bloom->ref_count == 0) + { + if (bloom->is_mutable) + g_clear_pointer (&bloom->mutable_bytes, g_free); + else + g_clear_pointer (&bloom->immutable_bytes, g_bytes_unref); + bloom->n_bytes = 0; + g_free (bloom); + } +} + +/* @idx is in bits, not bytes. */ +static inline gboolean +ostree_bloom_get_bit (OstreeBloom *bloom, + gsize idx) +{ + const guint8 *bytes; + + if (bloom->is_mutable) + bytes = bloom->mutable_bytes; + else + bytes = g_bytes_get_data (bloom->immutable_bytes, NULL); + + g_assert (idx / 8 < bloom->n_bytes); + return (bytes[idx / 8] & (1 << (idx % 8))); +} + +/* @idx is in bits, not bytes. */ +static inline void +ostree_bloom_set_bit (OstreeBloom *bloom, + gsize idx) +{ + g_assert (bloom->is_mutable); + g_assert (idx / 8 < bloom->n_bytes); + bloom->mutable_bytes[idx / 8] |= (1 << (idx % 8)); +} + +/** + * ostree_bloom_maybe_contains: + * @bloom: an #OstreeBloom + * @element: (nullable): element to check for membership + * + * Check whether @element is potentially in @bloom, or whether it definitely + * isn’t. @element may be %NULL only if the hash function passed to @bloom at + * construction time supports %NULL elements. + * + * Returns: %TRUE if @element is potentially in @bloom; %FALSE if it definitely + * isn’t + * Since: 2017.8 + */ +gboolean +ostree_bloom_maybe_contains (OstreeBloom *bloom, + gconstpointer element) +{ + guint8 i; + + g_return_val_if_fail (bloom != NULL, TRUE); + g_return_val_if_fail (bloom->ref_count >= 1, TRUE); + + for (i = 0; i < bloom->k; i++) + { + gsize idx; + + idx = bloom->hash_func (element, i); + + if (!ostree_bloom_get_bit (bloom, idx % (bloom->n_bytes * 8))) + return FALSE; /* definitely not in the set */ + } + + return TRUE; /* possibly in the set */ +} + +/** + * ostree_bloom_seal: + * @bloom: an #OstreeBloom + * + * Seal a constructed bloom filter, so that elements may no longer be added to + * it, and queries can now be performed against it. The serialised form of the + * bloom filter is returned as a bit array. Note that this does not include + * information about the filter hash function or parameters; the caller is + * responsible for serialising those separately if appropriate. + * + * It is safe to call this function multiple times. + * + * Returns: (transfer full): a #GBytes containing the immutable filter data + * Since: 2017.8 + */ +GBytes * +ostree_bloom_seal (OstreeBloom *bloom) +{ + g_return_val_if_fail (bloom != NULL, NULL); + g_return_val_if_fail (bloom->ref_count >= 1, NULL); + + if (bloom->is_mutable) + { + bloom->is_mutable = FALSE; + bloom->immutable_bytes = g_bytes_new_take (g_steal_pointer (&bloom->mutable_bytes), bloom->n_bytes); + } + + return g_bytes_ref (bloom->immutable_bytes); +} + +/** + * ostree_bloom_add_element: + * @bloom: an #OstreeBloom + * @element: (nullable): element to add to the filter + * + * Add the given @element to the bloom filter, which must not yet have been + * sealed (ostree_bloom_seal()). @element may be %NULL if the hash function + * passed to @bloom at construction time supports %NULL elements. + * + * Since: 2017.8 + */ +void +ostree_bloom_add_element (OstreeBloom *bloom, + gconstpointer element) +{ + guint8 i; + + g_return_if_fail (bloom != NULL); + g_return_if_fail (bloom->ref_count >= 1); + g_return_if_fail (bloom->is_mutable); + + for (i = 0; i < bloom->k; i++) + { + gsize idx = bloom->hash_func (element, i); + ostree_bloom_set_bit (bloom, idx % (bloom->n_bytes * 8)); + } +} + +/** + * ostree_bloom_get_size: + * @bloom: an #OstreeBloom + * + * Get the size of the #OstreeBloom filter, in bytes, as configured at + * construction time. + * + * Returns: the bloom filter’s size in bytes, guaranteed to be >0 + * Since: 2017.8 + */ +gsize +ostree_bloom_get_size (OstreeBloom *bloom) +{ + g_return_val_if_fail (bloom != NULL, 0); + + return bloom->n_bytes; +} + +/** + * ostree_bloom_get_k: + * @bloom: an #OstreeBloom + * + * Get the `k` value from the #OstreeBloom filter, as configured at + * construction time. + * + * Returns: the bloom filter’s `k` value, guaranteed to be >0 + * Since: 2017.8 + */ +guint8 +ostree_bloom_get_k (OstreeBloom *bloom) +{ + g_return_val_if_fail (bloom != NULL, 0); + + return bloom->k; +} + +/** + * ostree_bloom_get_hash_func: + * @bloom: an #OstreeBloom + * + * Get the #OstreeBloomHashFunc from the #OstreeBloom filter, as configured at + * construction time. + * + * Returns: the bloom filter’s universal hash function + * Since: 2017.8 + */ +OstreeBloomHashFunc +ostree_bloom_get_hash_func (OstreeBloom *bloom) +{ + g_return_val_if_fail (bloom != NULL, NULL); + + return bloom->hash_func; +} + +/* SipHash code adapted from https://github.com/veorq/SipHash/blob/master/siphash.c */ + +/* + SipHash reference C implementation + Copyright (c) 2012-2016 Jean-Philippe Aumasson + + Copyright (c) 2012-2014 Daniel J. Bernstein + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + You should have received a copy of the CC0 Public Domain Dedication along + with + this software. If not, see + . + */ + +/* default: SipHash-2-4 */ +#define cROUNDS 2 +#define dROUNDS 4 + +#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) + +#define U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v)); \ + (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); \ + (p)[3] = (uint8_t)((v) >> 24); + +#define U64TO8_LE(p, v) \ + U32TO8_LE((p), (uint32_t)((v))); \ + U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define U8TO64_LE(p) \ + (((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) + +#define SIPROUND \ + do { \ + v0 += v1; \ + v1 = ROTL(v1, 13); \ + v1 ^= v0; \ + v0 = ROTL(v0, 32); \ + v2 += v3; \ + v3 = ROTL(v3, 16); \ + v3 ^= v2; \ + v0 += v3; \ + v3 = ROTL(v3, 21); \ + v3 ^= v0; \ + v2 += v1; \ + v1 = ROTL(v1, 17); \ + v1 ^= v2; \ + v2 = ROTL(v2, 32); \ + } while (0) + +#ifdef DEBUG +#define TRACE \ + do { \ + printf("(%3d) v0 %08x %08x\n", (int)inlen, (uint32_t)(v0 >> 32), \ + (uint32_t)v0); \ + printf("(%3d) v1 %08x %08x\n", (int)inlen, (uint32_t)(v1 >> 32), \ + (uint32_t)v1); \ + printf("(%3d) v2 %08x %08x\n", (int)inlen, (uint32_t)(v2 >> 32), \ + (uint32_t)v2); \ + printf("(%3d) v3 %08x %08x\n", (int)inlen, (uint32_t)(v3 >> 32), \ + (uint32_t)v3); \ + } while (0) +#else +#define TRACE +#endif + +static int siphash(const uint8_t *in, const size_t inlen, const uint8_t *k, + uint8_t *out, const size_t outlen) { + + assert((outlen == 8) || (outlen == 16)); + uint64_t v0 = 0x736f6d6570736575ULL; + uint64_t v1 = 0x646f72616e646f6dULL; + uint64_t v2 = 0x6c7967656e657261ULL; + uint64_t v3 = 0x7465646279746573ULL; + uint64_t k0 = U8TO64_LE(k); + uint64_t k1 = U8TO64_LE(k + 8); + uint64_t m; + int i; + const uint8_t *end = in + inlen - (inlen % sizeof(uint64_t)); + const int left = inlen & 7; + uint64_t b = ((uint64_t)inlen) << 56; + v3 ^= k1; + v2 ^= k0; + v1 ^= k1; + v0 ^= k0; + + if (outlen == 16) + v1 ^= 0xee; + + for (; in != end; in += 8) { + m = U8TO64_LE(in); + v3 ^= m; + + TRACE; + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= m; + } + + switch (left) { + case 7: + b |= ((uint64_t)in[6]) << 48; + case 6: + b |= ((uint64_t)in[5]) << 40; + case 5: + b |= ((uint64_t)in[4]) << 32; + case 4: + b |= ((uint64_t)in[3]) << 24; + case 3: + b |= ((uint64_t)in[2]) << 16; + case 2: + b |= ((uint64_t)in[1]) << 8; + case 1: + b |= ((uint64_t)in[0]); + break; + case 0: + break; + } + + v3 ^= b; + + TRACE; + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= b; + + if (outlen == 16) + v2 ^= 0xee; + else + v2 ^= 0xff; + + TRACE; + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + U64TO8_LE(out, b); + + if (outlen == 8) + return 0; + + v1 ^= 0xdd; + + TRACE; + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + U64TO8_LE(out + 8, b); + + return 0; +} + +/* End SipHash copied code. */ + +/** + * ostree_str_bloom_hash: + * @element: element to calculate the hash for + * @k: hash function index + * + * A universal hash function implementation for strings. It expects @element to + * be a pointer to a string (i.e. @element has type `const gchar*`), and expects + * @k to be in the range `[0, k_max)`, where `k_max` is the `k` value used to + * construct the bloom filter. The output range from this hash function could be + * any value in #guint64, and it handles input strings of any length. + * + * This function does not allow %NULL as a valid value for @element. + * + * Reference: + * - https://www.131002.net/siphash/ + * + * Returns: hash of the string at @element using parameter @k + * Since: 2017.8 + */ +guint64 +ostree_str_bloom_hash (gconstpointer element, + guint8 k) +{ + const gchar *str = element; + gsize str_len; + union + { + guint64 u64; + guint8 u8[8]; + } out_le; + guint8 k_array[16]; + gsize i; + + str_len = strlen (str); + for (i = 0; i < G_N_ELEMENTS (k_array); i++) + k_array[i] = k; + + siphash ((const guint8 *) str, str_len, k_array, out_le.u8, sizeof (out_le)); + + return le64toh (out_le.u64); +} diff --git a/tests/.gitignore b/tests/.gitignore index f3bdb177..5ece7ea1 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -6,6 +6,7 @@ ostree-http-server run-apache tmpdir-lifecycle test-rollsum +test-bloom test-bsdiff test-checksum test-gpg-verify-result diff --git a/tests/test-bloom.c b/tests/test-bloom.c new file mode 100644 index 00000000..2e89456e --- /dev/null +++ b/tests/test-bloom.c @@ -0,0 +1,154 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include + +#include "ostree-bloom-private.h" + +/* Test the two different constructors work at a basic level. */ +static void +test_bloom_init (void) +{ + g_autoptr(OstreeBloom) bloom = NULL; + g_autoptr(GBytes) bytes = NULL; + + bloom = ostree_bloom_new (1, 1, ostree_str_bloom_hash); + g_assert_cmpuint (ostree_bloom_get_size (bloom), ==, 1); + g_assert_cmpuint (ostree_bloom_get_k (bloom), ==, 1); + g_assert (ostree_bloom_get_hash_func (bloom) == ostree_str_bloom_hash); + g_clear_pointer (&bloom, ostree_bloom_unref); + + bytes = g_bytes_new_take (g_malloc0 (4), 4); + bloom = ostree_bloom_new_from_bytes (bytes, 1, ostree_str_bloom_hash); + g_assert_cmpuint (ostree_bloom_get_size (bloom), ==, 4); + g_assert_cmpuint (ostree_bloom_get_k (bloom), ==, 1); + g_assert (ostree_bloom_get_hash_func (bloom) == ostree_str_bloom_hash); + g_clear_pointer (&bloom, ostree_bloom_unref); +} + +/* Test that building a bloom filter, marshalling it through GBytes, and loading + * it again, gives the same element membership. */ +static void +test_bloom_construction (void) +{ + g_autoptr(OstreeBloom) bloom = NULL; + g_autoptr(OstreeBloom) immutable_bloom = NULL; + g_autoptr(GBytes) bytes = NULL; + gsize i; + const gchar *members[] = + { + "hello", "there", "these", "are", "test", "strings" + }; + const gchar *non_members[] = + { + "not", "an", "element" + }; + const gsize n_bytes = 256; + const guint8 k = 8; + const OstreeBloomHashFunc hash = ostree_str_bloom_hash; + + /* Build a bloom filter. */ + bloom = ostree_bloom_new (n_bytes, k, hash); + + for (i = 0; i < G_N_ELEMENTS (members); i++) + ostree_bloom_add_element (bloom, members[i]); + + bytes = ostree_bloom_seal (bloom); + + /* Read it back from the GBytes. */ + immutable_bloom = ostree_bloom_new_from_bytes (bytes, k, hash); + + for (i = 0; i < G_N_ELEMENTS (members); i++) + g_assert_true (ostree_bloom_maybe_contains (bloom, members[i])); + + /* This should never fail in future, as we guarantee the hash function will + * never change. But given the definition of a bloom filter, it would also + * be valid for these calls to return %TRUE. */ + for (i = 0; i < G_N_ELEMENTS (non_members); i++) + g_assert_false (ostree_bloom_maybe_contains (bloom, non_members[i])); +} + +/* Test that an empty bloom filter definitely contains no elements. */ +static void +test_bloom_empty (void) +{ + g_autoptr(OstreeBloom) bloom = NULL; + const gsize n_bytes = 256; + const guint8 k = 8; + const OstreeBloomHashFunc hash = ostree_str_bloom_hash; + + /* Build an empty bloom filter. */ + bloom = ostree_bloom_new (n_bytes, k, hash); + + g_assert_false (ostree_bloom_maybe_contains (bloom, "hello")); + g_assert_false (ostree_bloom_maybe_contains (bloom, "there")); +} + +/* Build a bloom filter, and check the membership of the members as they are + * added. */ +static void +test_bloom_membership_during_construction (void) +{ + g_autoptr(OstreeBloom) bloom = NULL; + gsize i, j; + const gchar *members[] = + { + "hello", "there", "these", "are", "test", "strings" + }; + const gsize n_bytes = 256; + const guint8 k = 8; + const OstreeBloomHashFunc hash = ostree_str_bloom_hash; + + /* These membership checks should never fail in future, as we guarantee + * the hash function will never change. But given the definition of a bloom + * filter, it would also be valid for these checks to fail. */ + bloom = ostree_bloom_new (n_bytes, k, hash); + + for (i = 0; i < G_N_ELEMENTS (members); i++) + { + ostree_bloom_add_element (bloom, members[i]); + + for (j = 0; j < G_N_ELEMENTS (members); j++) + { + if (j <= i) + g_assert_true (ostree_bloom_maybe_contains (bloom, members[j])); + else + g_assert_false (ostree_bloom_maybe_contains (bloom, members[j])); + } + } +} + +int main (int argc, char **argv) +{ + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/bloom/init", test_bloom_init); + g_test_add_func ("/bloom/construction", test_bloom_construction); + g_test_add_func ("/bloom/empty", test_bloom_empty); + g_test_add_func ("/bloom/membership-during-construction", test_bloom_membership_during_construction); + + return g_test_run(); +} From e3d4eeacbca699e3a0551bd02f49eb912df09b3d Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 19 Apr 2017 00:13:28 +0100 Subject: [PATCH 27/82] lib/repo-finder: Add Avahi based OstreeRepoFinder implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a more complex implementation of OstreeRepoFinder which resolves ref names to remote URIs by looking for refs advertised by peers on the local network using DNS-SD records and mDNS (Avahi). The idea is to allow OS and app updates to be propagated over local networks, without the internet. It requires an OSTree server and code to generate the DNS-SD adverts in order to be fully functional — support for this will be added separately. Unit tests are included. Includes fixes by Krzesimir Nowak . Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- Makefile-libostree-defines.am | 1 + Makefile-libostree.am | 18 +- Makefile-tests.am | 10 + apidoc/ostree-experimental-sections.txt | 10 + configure.ac | 25 + src/libostree/libostree-experimental.sym | 4 + src/libostree/ostree-autocleanups.h | 1 + src/libostree/ostree-core-private.h | 3 + .../ostree-repo-finder-avahi-parser.c | 137 ++ .../ostree-repo-finder-avahi-private.h | 35 + src/libostree/ostree-repo-finder-avahi.c | 1506 +++++++++++++++++ src/libostree/ostree-repo-finder-avahi.h | 62 + src/libostree/ostree-repo-pull.c | 39 +- src/libostree/ostree.h | 1 + tests/.gitignore | 1 + tests/test-repo-finder-avahi.c | 228 +++ 16 files changed, 2075 insertions(+), 6 deletions(-) create mode 100644 src/libostree/ostree-repo-finder-avahi-parser.c create mode 100644 src/libostree/ostree-repo-finder-avahi-private.h create mode 100644 src/libostree/ostree-repo-finder-avahi.c create mode 100644 src/libostree/ostree-repo-finder-avahi.h create mode 100644 tests/test-repo-finder-avahi.c diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index 7586f7b9..6fc4a18a 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -43,6 +43,7 @@ libostree_public_headers += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ src/libostree/ostree-repo-finder.h \ + src/libostree/ostree-repo-finder-avahi.h \ src/libostree/ostree-repo-finder-config.h \ src/libostree/ostree-repo-finder-mount.h \ $(NULL) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 6a7c4820..8ff89e52 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -155,6 +155,7 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ src/libostree/ostree-repo-finder.h \ + src/libostree/ostree-repo-finder-avahi.h \ src/libostree/ostree-repo-finder-config.h \ src/libostree/ostree-repo-finder-mount.h \ $(NULL) @@ -163,9 +164,17 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-bloom.c \ src/libostree/ostree-bloom-private.h \ src/libostree/ostree-repo-finder.c \ + src/libostree/ostree-repo-finder-avahi.c \ src/libostree/ostree-repo-finder-config.c \ src/libostree/ostree-repo-finder-mount.c \ $(NULL) + +if USE_AVAHI +libostree_1_la_SOURCES += \ + src/libostree/ostree-repo-finder-avahi-parser.c \ + src/libostree/ostree-repo-finder-avahi-private.h \ + $(NULL) +endif # USE_AVAHI endif symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym @@ -193,6 +202,13 @@ libostree_1_la_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_LIBARCHIVE_LIBS) endif +if ENABLE_EXPERIMENTAL_API +if USE_AVAHI +libostree_1_la_CFLAGS += $(OT_DEP_AVAHI_CFLAGS) +libostree_1_la_LIBADD += $(OT_DEP_AVAHI_LIBS) +endif +endif + if BUILDOPT_LIBSYSTEMD libostree_1_la_CFLAGS += $(LIBSYSTEMD_CFLAGS) libostree_1_la_LIBADD += $(LIBSYSTEMD_LIBS) @@ -241,7 +257,7 @@ OSTree_1_0_gir_INCLUDES = Gio-2.0 OSTree_1_0_gir_CFLAGS = $(libostree_1_la_CFLAGS) OSTree_1_0_gir_LIBS = libostree-1.la OSTree_1_0_gir_SCANNERFLAGS = --warn-all --identifier-prefix=Ostree --symbol-prefix=ostree -OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h %/ostree-repo-finder-config.h %/ostree-repo-finder-mount.h,$(libostree_1_la_SOURCES)) +OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h %/ostree-repo-finder-avahi.h %/ostree-repo-finder-config.h %/ostree-repo-finder-mount.h,$(libostree_1_la_SOURCES)) INTROSPECTION_GIRS += OSTree-1.0.gir gir_DATA += OSTree-1.0.gir typelib_DATA += OSTree-1.0.typelib diff --git a/Makefile-tests.am b/Makefile-tests.am index d04a1cbc..c21e29f7 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -199,6 +199,10 @@ test_programs += \ tests/test-repo-finder-config \ tests/test-repo-finder-mount \ $(NULL) + +if USE_AVAHI +test_programs += tests/test-repo-finder-avahi +endif endif # An interactive tool @@ -231,6 +235,12 @@ tests_test_bloom_SOURCES = src/libostree/ostree-bloom.c tests/test-bloom.c tests_test_bloom_CFLAGS = $(TESTS_CFLAGS) tests_test_bloom_LDADD = $(TESTS_LDADD) +if USE_AVAHI +tests_test_repo_finder_avahi_SOURCES = src/libostree/ostree-repo-finder-avahi-parser.c tests/test-repo-finder-avahi.c +tests_test_repo_finder_avahi_CFLAGS = $(TESTS_CFLAGS) +tests_test_repo_finder_avahi_LDADD = $(TESTS_LDADD) +endif + tests_test_repo_finder_config_SOURCES = tests/test-repo-finder-config.c tests_test_repo_finder_config_CFLAGS = $(TESTS_CFLAGS) tests_test_repo_finder_config_LDADD = $(TESTS_LDADD) diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index c977b56c..4c71fad2 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -49,6 +49,16 @@ ostree_repo_finder_get_type ostree_repo_finder_result_get_type
+
+ostree-repo-finder-avahi +OstreeRepoFinderAvahi +ostree_repo_finder_avahi_new +ostree_repo_finder_avahi_start +ostree_repo_finder_avahi_stop + +ostree_repo_finder_avahi_get_type +
+
ostree-repo-finder-config OstreeRepoFinderConfig diff --git a/configure.ac b/configure.ac index 9973c80a..67e70b3e 100644 --- a/configure.ac +++ b/configure.ac @@ -320,6 +320,31 @@ if test x$with_openssl != xno; then OSTREE_FEATURES="$OSTREE_FEATURES openssl"; AM_CONDITIONAL(USE_OPENSSL, test $with_openssl != no) dnl end openssl +dnl Avahi dependency for finding repos +AVAHI_DEPENDENCY="avahi-client >= 0.6.31 avahi-glib >= 0.6.31" + +AC_ARG_WITH(avahi, + AS_HELP_STRING([--without-avahi], [Do not use Avahi]), + :, with_avahi=maybe) + +AS_IF([ test x$with_avahi != xno ], [ + AC_MSG_CHECKING([for $AVAHI_DEPENDENCY]) + PKG_CHECK_EXISTS($AVAHI_DEPENDENCY, have_avahi=yes, have_avahi=no) + AC_MSG_RESULT([$have_avahi]) + AS_IF([ test x$have_avahi = xno && test x$with_avahi != xmaybe ], [ + AC_MSG_ERROR([Avahi is enabled but could not be found]) + ]) + AS_IF([ test x$have_avahi = xyes], [ + AC_DEFINE([HAVE_AVAHI], 1, [Define if we have avahi-client.pc and avahi-glib.pc]) + PKG_CHECK_MODULES(OT_DEP_AVAHI, $AVAHI_DEPENDENCY) + with_avahi=yes + ], [ + with_avahi=no + ]) +], [ with_avahi=no ]) +if test x$with_avahi != xno; then OSTREE_FEATURES="$OSTREE_FEATURES avahi"; fi +AM_CONDITIONAL(USE_AVAHI, test $with_avahi != no) + dnl This is what is in RHEL7.2 right now, picking it arbitrarily LIBMOUNT_DEPENDENCY="mount >= 2.23.0" diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index e70c80bb..32ba0929 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -47,6 +47,10 @@ global: ostree_collection_ref_new; ostree_repo_find_remotes_async; ostree_repo_find_remotes_finish; + ostree_repo_finder_avahi_get_type; + ostree_repo_finder_avahi_new; + ostree_repo_finder_avahi_start; + ostree_repo_finder_avahi_stop; ostree_repo_finder_config_get_type; ostree_repo_finder_config_new; ostree_repo_finder_get_type; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index dd3c9778..b375413c 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -64,6 +64,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderAvahi, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderMount, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index a56fdc0b..76c76cc7 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -188,6 +188,9 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) +#include "ostree-repo-finder-avahi.h" +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderAvahi, g_object_unref) + #include "ostree-repo-finder-config.h" G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref) diff --git a/src/libostree/ostree-repo-finder-avahi-parser.c b/src/libostree/ostree-repo-finder-avahi-parser.c new file mode 100644 index 00000000..805f5dff --- /dev/null +++ b/src/libostree/ostree-repo-finder-avahi-parser.c @@ -0,0 +1,137 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2016 Kinvolk GmbH + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Krzesimir Nowak + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "ostree-autocleanups.h" +#include "ostree-repo-finder-avahi.h" +#include "ostree-repo-finder-avahi-private.h" + +/* Reference: RFC 6763, §6. */ +static gboolean +parse_txt_record (const guint8 *txt, + gsize txt_len, + const gchar **key, + gsize *key_len, + const guint8 **value, + gsize *value_len) +{ + gsize i; + + g_return_val_if_fail (key != NULL, FALSE); + g_return_val_if_fail (key_len != NULL, FALSE); + g_return_val_if_fail (value != NULL, FALSE); + g_return_val_if_fail (value_len != NULL, FALSE); + + /* RFC 6763, §6.1. */ + if (txt_len > 8900) + return FALSE; + + *key = (const gchar *) txt; + *key_len = 0; + *value = NULL; + *value_len = 0; + + for (i = 0; i < txt_len; i++) + { + if (txt[i] >= 0x20 && txt[i] <= 0x7e && txt[i] != '=') + { + /* Key character. */ + *key_len = *key_len + 1; + continue; + } + else if (*key_len > 0 && txt[i] == '=') + { + /* Separator. */ + *value = txt + (i + 1); + *value_len = txt_len - (i + 1); + return TRUE; + } + else + { + return FALSE; + } + } + + /* The entire TXT record is the key; there is no ‘=’ or value. */ + *value = NULL; + *value_len = 0; + + return (*key_len > 0); +} + +/* TODO: Docs. Return value is only valid as long as @txt is. Reference: RFC 6763, §6. */ +GHashTable * +_ostree_txt_records_parse (AvahiStringList *txt) +{ + AvahiStringList *l; + g_autoptr(GHashTable) out = NULL; + + out = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) g_bytes_unref); + + for (l = txt; l != NULL; l = avahi_string_list_get_next (l)) + { + const guint8 *txt; + gsize txt_len; + const gchar *key; + const guint8 *value; + gsize key_len, value_len; + g_autofree gchar *key_allocated = NULL; + g_autoptr(GBytes) value_allocated = NULL; + + txt = avahi_string_list_get_text (l); + txt_len = avahi_string_list_get_size (l); + + if (!parse_txt_record (txt, txt_len, &key, &key_len, &value, &value_len)) + { + g_debug ("Ignoring invalid TXT record of length %" G_GSIZE_FORMAT, + txt_len); + continue; + } + + key_allocated = g_ascii_strdown (key, key_len); + + if (g_hash_table_lookup_extended (out, key_allocated, NULL, NULL)) + { + g_debug ("Ignoring duplicate TXT record ‘%s’", key_allocated); + continue; + } + + /* Distinguish between the case where the entire record is the key + * (value == NULL) and the case where the record is the key + ‘=’ and the + * value is empty (value != NULL && value_len == 0). */ + if (value != NULL) + value_allocated = g_bytes_new_static (value, value_len); + + g_hash_table_insert (out, g_steal_pointer (&key_allocated), g_steal_pointer (&value_allocated)); + } + + return g_steal_pointer (&out); +} diff --git a/src/libostree/ostree-repo-finder-avahi-private.h b/src/libostree/ostree-repo-finder-avahi-private.h new file mode 100644 index 00000000..6429cd7f --- /dev/null +++ b/src/libostree/ostree-repo-finder-avahi-private.h @@ -0,0 +1,35 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include +#include + +G_BEGIN_DECLS + +GHashTable *_ostree_txt_records_parse (AvahiStringList *txt); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-finder-avahi.c b/src/libostree/ostree-repo-finder-avahi.c new file mode 100644 index 00000000..433914b4 --- /dev/null +++ b/src/libostree/ostree-repo-finder-avahi.c @@ -0,0 +1,1506 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2016 Kinvolk GmbH + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Krzesimir Nowak + * - Philip Withnall + */ + +#include "config.h" + +#ifdef HAVE_AVAHI +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif /* HAVE_AVAHI */ + +#include +#include +#include + +#include "ostree-autocleanups.h" +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-avahi.h" + +#ifdef HAVE_AVAHI +#include "ostree-bloom-private.h" +#include "ostree-remote-private.h" +#include "ostree-repo-private.h" +#include "ostree-repo.h" +#include "ostree-repo-finder-avahi-private.h" +#include "otutil.h" +#endif /* HAVE_AVAHI */ + +/** + * SECTION:ostree-repo-finder-avahi + * @title: OstreeRepoFinderAvahi + * @short_description: Finds remote repositories from ref names by looking at + * adverts of refs from peers on the local network + * @stability: Unstable + * @include: libostree/ostree-repo-finder-avahi.h + * + * #OstreeRepoFinderAvahi is an implementation of #OstreeRepoFinder which looks + * for refs being hosted by peers on the local network. + * + * Any ref which matches by collection ID and ref name is returned as a result, + * with no limitations on the peers which host them, as long as they are + * accessible over the local network, and their adverts reach this machine via + * DNS-SD/mDNS. + * + * For each repository which is found, a result will be returned for the + * intersection of the refs being searched for, and the refs in `refs/mirrors` + * in the remote repository. + * + * DNS-SD resolution is performed using Avahi, which will continue to scan for + * matching peers throughout the lifetime of the process. It’s recommended that + * ostree_repo_finder_avahi_start() be called early on in the process’ lifetime, + * and the #GMainContext which is passed to ostree_repo_finder_avahi_new() + * continues to be iterated until ostree_repo_finder_avahi_stop() is called. + * + * The values stored in DNS-SD TXT records are stored as big-endian whenever + * endianness is relevant. + * + * Internally, #OstreeRepoFinderAvahi has an Avahi client, browser and resolver + * which work in the background to track all available peers on the local + * network. Whenever a resolve request is made using + * ostree_repo_finder_resolve_async(), the request is blocked until the + * background tracking is in a consistent state (typically this only happens at + * startup), and is then answered using the current cache of background data. + * The Avahi client tracks the #OstreeRepoFinderAvahi’s connection with the + * Avahi D-Bus service. The browser looks for DNS-SD peers on the local network; + * and the resolver is used to retrieve information about services advertised by + * each peer, including the services’ TXT records. + * + * Since: 2017.8 + */ + +#ifdef HAVE_AVAHI +/* FIXME: Submit these upstream */ +G_DEFINE_AUTOPTR_CLEANUP_FUNC (AvahiClient, avahi_client_free) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (AvahiServiceBrowser, avahi_service_browser_free) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (AvahiServiceResolver, avahi_service_resolver_free) + +/* FIXME: Register this with IANA? https://tools.ietf.org/html/rfc6335#section-5.2 */ +const gchar * const OSTREE_AVAHI_SERVICE_TYPE = "_ostree_repo._tcp"; + +static const gchar * +ostree_avahi_client_state_to_string (AvahiClientState state) +{ + switch (state) + { + case AVAHI_CLIENT_S_REGISTERING: + return "registering"; + case AVAHI_CLIENT_S_RUNNING: + return "running"; + case AVAHI_CLIENT_S_COLLISION: + return "collision"; + case AVAHI_CLIENT_CONNECTING: + return "connecting"; + case AVAHI_CLIENT_FAILURE: + return "failure"; + default: + return "unknown"; + } +} + +static const gchar * +ostree_avahi_resolver_event_to_string (AvahiResolverEvent event) +{ + switch (event) + { + case AVAHI_RESOLVER_FOUND: + return "found"; + case AVAHI_RESOLVER_FAILURE: + return "failure"; + default: + return "unknown"; + } +} + +static const gchar * +ostree_avahi_browser_event_to_string (AvahiBrowserEvent event) +{ + switch (event) + { + case AVAHI_BROWSER_NEW: + return "new"; + case AVAHI_BROWSER_REMOVE: + return "remove"; + case AVAHI_BROWSER_CACHE_EXHAUSTED: + return "cache-exhausted"; + case AVAHI_BROWSER_ALL_FOR_NOW: + return "all-for-now"; + case AVAHI_BROWSER_FAILURE: + return "failure"; + default: + return "unknown"; + } +} + +typedef struct +{ + gchar *uri; + gchar *keyring; +} UriAndKeyring; + +static void +uri_and_keyring_free (UriAndKeyring *data) +{ + g_free (data->uri); + g_free (data->keyring); + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (UriAndKeyring, uri_and_keyring_free) + +static UriAndKeyring * +uri_and_keyring_new (const gchar *uri, + const gchar *keyring) +{ + g_autoptr(UriAndKeyring) data = NULL; + + data = g_new0 (UriAndKeyring, 1); + data->uri = g_strdup (uri); + data->keyring = g_strdup (keyring); + + return g_steal_pointer (&data); +} + +static guint +uri_and_keyring_hash (gconstpointer key) +{ + const UriAndKeyring *_key = key; + + return g_str_hash (_key->uri) ^ g_str_hash (_key->keyring); +} + +static gboolean +uri_and_keyring_equal (gconstpointer a, + gconstpointer b) +{ + const UriAndKeyring *_a = a, *_b = b; + + return g_str_equal (_a->uri, _b->uri) && g_str_equal (_a->keyring, _b->keyring); +} + +/* This must return a valid remote name (suitable for use in a refspec). */ +static gchar * +uri_and_keyring_to_name (UriAndKeyring *data) +{ + g_autofree gchar *escaped_uri = g_uri_escape_string (data->uri, NULL, FALSE); + g_autofree gchar *escaped_keyring = g_uri_escape_string (data->keyring, NULL, FALSE); + + /* FIXME: Need a better separator than `_`, since it’s not escaped in the input. */ + g_autofree gchar *out = g_strdup_printf ("%s_%s", escaped_uri, escaped_keyring); + + for (gsize i = 0; out[i] != '\0'; i++) + { + if (out[i] == '%') + out[i] = '_'; + } + + g_return_val_if_fail (ostree_validate_remote_name (out, NULL), NULL); + + return g_steal_pointer (&out); +} + +/* Internal structure representing a service found advertised by a peer on the + * local network. This includes details for connecting to the service, and the + * metadata associated with the advert (@txt). */ +typedef struct +{ + gchar *name; + gchar *domain; + gchar *address; + guint16 port; + AvahiStringList *txt; +} OstreeAvahiService; + +static void +ostree_avahi_service_free (OstreeAvahiService *service) +{ + g_free (service->name); + g_free (service->domain); + g_free (service->address); + avahi_string_list_free (service->txt); + g_free (service); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeAvahiService, ostree_avahi_service_free) + +/* Convert an AvahiAddress to a string which is suitable for use in URIs (for + * example). Take into account the scope ID, if the address is IPv6 and a + * link-local address. + * (See https://en.wikipedia.org/wiki/IPv6_address#Link-local_addresses_and_zone_indices and + * https://github.com/lathiat/avahi/issues/110.) */ +static gchar * +address_to_string (const AvahiAddress *address, + AvahiIfIndex interface) +{ + char address_string[AVAHI_ADDRESS_STR_MAX]; + + avahi_address_snprint (address_string, sizeof (address_string), address); + + switch (address->proto) + { + case AVAHI_PROTO_INET6: + if (IN6_IS_ADDR_LINKLOCAL (address->data.data) || + IN6_IS_ADDR_LOOPBACK (address->data.data)) + return g_strdup_printf ("%s%%%d", address_string, interface); + /* else fall through */ + case AVAHI_PROTO_INET: + case AVAHI_PROTO_UNSPEC: + default: + return g_strdup (address_string); + } +} + +static OstreeAvahiService * +ostree_avahi_service_new (const gchar *name, + const gchar *domain, + const AvahiAddress *address, + AvahiIfIndex interface, + guint16 port, + AvahiStringList *txt) +{ + g_autoptr(OstreeAvahiService) service = NULL; + + g_return_val_if_fail (name != NULL, NULL); + g_return_val_if_fail (domain != NULL, NULL); + g_return_val_if_fail (address != NULL, NULL); + g_return_val_if_fail (port > 0, NULL); + + service = g_new0 (OstreeAvahiService, 1); + + service->name = g_strdup (name); + service->domain = g_strdup (domain); + service->address = address_to_string (address, interface); + service->port = port; + service->txt = avahi_string_list_copy (txt); + + return g_steal_pointer (&service); +} + +/* Check whether @str is entirely lower case. */ +static gboolean +str_is_lowercase (const gchar *str) +{ + gsize i; + + for (i = 0; str[i] != '\0'; i++) + { + if (!g_ascii_islower (str[i])) + return FALSE; + } + + return TRUE; +} + +/* Look up @key in the @attributes table derived from a TXT record, and validate + * that its value is of type @value_type. If the key is not found, or its value + * is of the wrong type or is not in normal form, %NULL is returned. @key must + * be lowercase in order to match reliably. */ +static GVariant * +_ostree_txt_records_lookup_variant (GHashTable *attributes, + const gchar *key, + const GVariantType *value_type) +{ + GBytes *value; + g_autoptr(GVariant) variant = NULL; + + g_return_val_if_fail (attributes != NULL, NULL); + g_return_val_if_fail (str_is_lowercase (key), NULL); + g_return_val_if_fail (value_type != NULL, NULL); + + value = g_hash_table_lookup (attributes, key); + + if (value == NULL) + { + g_debug ("TXT attribute ‘%s’ not found.", key); + return NULL; + } + + variant = g_variant_new_from_bytes (value_type, value, FALSE); + + if (!g_variant_is_normal_form (variant)) + { + g_debug ("TXT attribute ‘%s’ value is not in normal form. Ignoring.", key); + return NULL; + } + + return g_steal_pointer (&variant); +} + +/* Bloom hash function family for #OstreeCollectionRef, parameterised by @k. */ +static guint64 +ostree_collection_ref_bloom_hash (gconstpointer element, + guint8 k) +{ + const OstreeCollectionRef *ref = element; + + return ostree_str_bloom_hash (ref->collection_id, k) ^ ostree_str_bloom_hash (ref->ref_name, k); +} + +/* Return the (possibly empty) subset of @refs which are possibly in the given + * encoded bloom filter, @bloom_encoded. The returned array is not + * %NULL-terminated. If there is an error decoding the bloom filter (invalid + * type, zero length, unknown hash function), %NULL will be returned. */ +static GPtrArray * +bloom_refs_intersection (GVariant *bloom_encoded, + const OstreeCollectionRef * const *refs) +{ + g_autoptr(OstreeBloom) bloom = NULL; + g_autoptr(GVariant) bloom_variant = NULL; + guint8 k, hash_id; + OstreeBloomHashFunc hash_func; + const guint8 *bloom_bytes; + gsize n_bloom_bytes; + g_autoptr(GBytes) bytes = NULL; + gsize i; + g_autoptr(GPtrArray) possible_refs = NULL; /* (element-type OstreeCollectionRef) */ + + g_variant_get (bloom_encoded, "(yy@ay)", &k, &hash_id, &bloom_variant); + + if (k == 0) + return NULL; + + switch (hash_id) + { + case 1: + hash_func = ostree_collection_ref_bloom_hash; + break; + default: + return NULL; + } + + bloom_bytes = g_variant_get_fixed_array (bloom_variant, &n_bloom_bytes, sizeof (guint8)); + bytes = g_bytes_new_static (bloom_bytes, n_bloom_bytes); + bloom = ostree_bloom_new_from_bytes (bytes, k, hash_func); + + possible_refs = g_ptr_array_new_with_free_func (NULL); + + for (i = 0; refs[i] != NULL; i++) + { + if (ostree_bloom_maybe_contains (bloom, refs[i])) + g_ptr_array_add (possible_refs, (gpointer) refs[i]); + } + + return g_steal_pointer (&possible_refs); +} + +/* Given a @summary_map of ref name to commit details, and the @collection_id + * for all the refs in the @summary_map (which may be %NULL if the summary does + * not specify one), add the refs to @refs_and_checksums. + * + * The @summary_map is validated as it’s iterated over; on error, @error will be + * set and @refs_and_checksums will be left in an undefined state. */ +static gboolean +fill_refs_and_checksums_from_summary_map (GVariantIter *summary_map, + const gchar *collection_id, + GHashTable *refs_and_checksums /* (element-type OstreeCollectionRef utf8) */, + GError **error) +{ + g_autofree gchar *ref_name = NULL; + g_autoptr(GVariant) checksum_variant = NULL; + + while (g_variant_iter_next (summary_map, "(s(t@aya{sv}))", + (gpointer *) &ref_name, NULL, + (gpointer *) &checksum_variant, NULL)) + { + const OstreeCollectionRef ref = { (gchar *) collection_id, ref_name }; + + if (!ostree_validate_rev (ref_name, error)) + return FALSE; + if (!ostree_validate_structureof_csum_v (checksum_variant, error)) + return FALSE; + + if (g_hash_table_contains (refs_and_checksums, &ref)) + { + g_autofree gchar *checksum_string = ostree_checksum_from_bytes_v (checksum_variant); + + g_hash_table_replace (refs_and_checksums, + ostree_collection_ref_dup (&ref), + g_steal_pointer (&checksum_string)); + } + } + + return TRUE; +} + +/* Given a @summary file, add the refs it lists to @refs_and_checksums. This + * includes the main refs list in the summary, and the map of collection IDs + * to further refs lists. + * + * The @summary is validated as it’s explored; on error, @error will be + * set and @refs_and_checksums will be left in an undefined state. */ +static gboolean +fill_refs_and_checksums_from_summary (GVariant *summary, + GHashTable *refs_and_checksums /* (element-type OstreeCollectionRef utf8) */, + GError **error) +{ + g_autoptr(GVariant) ref_map_v = NULL; + g_autoptr(GVariant) additional_metadata_v = NULL; + GVariantIter ref_map; + g_auto(GVariantDict) additional_metadata = OT_VARIANT_BUILDER_INITIALIZER; + const gchar *collection_id; + g_autoptr(GVariantIter) collection_map = NULL; + + ref_map_v = g_variant_get_child_value (summary, 0); + additional_metadata_v = g_variant_get_child_value (summary, 1); + + g_variant_iter_init (&ref_map, ref_map_v); + g_variant_dict_init (&additional_metadata, additional_metadata_v); + + /* If the summary file specifies a collection ID (to apply to all the refs in its + * ref map), use that to start matching against the queried refs. Otherwise, + * it might specify all its refs in a collection-map; or the summary format is + * old and unsuitable for P2P redistribution and we should bail. */ + if (g_variant_dict_lookup (&additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &collection_id)) + { + if (!ostree_validate_collection_id (collection_id, error)) + return FALSE; + if (!fill_refs_and_checksums_from_summary_map (&ref_map, collection_id, refs_and_checksums, error)) + return FALSE; + } + + /* Repeat for the other collections listed in the summary. */ + if (g_variant_dict_lookup (&additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, "a{sa(s(taya{sv}))}", &collection_map)) + { + while (g_variant_iter_loop (collection_map, "{sa(s(taya{sv}))}", &collection_id, &ref_map)) + { + if (!ostree_validate_collection_id (collection_id, error)) + return FALSE; + if (!fill_refs_and_checksums_from_summary_map (&ref_map, collection_id, refs_and_checksums, error)) + return FALSE; + } + } + + return TRUE; +} + +/* Given a summary file (@summary_bytes), extract the refs it lists, and use that + * to fill in the checksums in the @supported_ref_to_checksum map. This includes + * the main refs list in the summary, and the map of collection IDs to further + * refs lists. + * + * The @summary is validated as it’s explored; on error, @error will be + * set and %FALSE will be returned. If the intersection of the summary file refs + * and the keys in @supported_ref_to_checksum is empty, an error is set. */ +static gboolean +get_refs_and_checksums_from_summary (GBytes *summary_bytes, + GHashTable *supported_ref_to_checksum /* (element-type OstreeCollectionRef utf8) */, + GError **error) +{ + g_autoptr(GVariant) summary = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, summary_bytes, FALSE)); + GHashTableIter iter; + const OstreeCollectionRef *ref; + const gchar *checksum; + + if (!g_variant_is_normal_form (summary)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not normal form"); + return FALSE; + } + if (!g_variant_is_of_type (summary, OSTREE_SUMMARY_GVARIANT_FORMAT)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Doesn't match variant type '%s'", + (char *)OSTREE_SUMMARY_GVARIANT_FORMAT); + return FALSE; + } + + if (!fill_refs_and_checksums_from_summary (summary, supported_ref_to_checksum, error)) + return FALSE; + + /* Check that at least one of the refs has a non-%NULL checksum set, otherwise + * we can discard this peer. */ + g_hash_table_iter_init (&iter, supported_ref_to_checksum); + while (g_hash_table_iter_next (&iter, + (gpointer *) &ref, + (gpointer *) &checksum)) + { + if (checksum != NULL) + return TRUE; + } + + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "No matching refs were found in the summary file"); + return FALSE; +} + +/* Download the summary file from @remote, and return the bytes of the file in + * @out_summary_bytes. */ +static gboolean +fetch_summary_from_remote (OstreeRepo *repo, + OstreeRemote *remote, + GBytes **out_summary_bytes, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GBytes) summary_bytes = NULL; + gboolean remote_already_existed = _ostree_repo_add_remote (repo, remote); + gboolean success = ostree_repo_remote_fetch_summary_with_options (repo, + remote->name, + NULL /* options */, + &summary_bytes, + NULL /* signature */, + cancellable, + error); + + if (!remote_already_existed) + _ostree_repo_remove_remote (repo, remote); + + if (!success) + return FALSE; + + g_assert (out_summary_bytes != NULL); + *out_summary_bytes = g_steal_pointer (&summary_bytes); + return TRUE; +} +#endif /* HAVE_AVAHI */ + +struct _OstreeRepoFinderAvahi +{ + GObject parent_instance; + +#ifdef HAVE_AVAHI + /* All elements of this structure must only be accessed from @avahi_context + * after construction. */ + + /* Note: There is a ref-count loop here: each #GTask has a reference to the + * #OstreeRepoFinderAvahi, and we have to keep a reference to the #GTask. */ + GPtrArray *resolve_tasks; /* (element-type (owned) GTask) */ + + AvahiGLibPoll *poll; + AvahiClient *client; + AvahiServiceBrowser *browser; + + AvahiClientState client_state; + gboolean browser_failed; + gboolean browser_all_for_now; + + GCancellable *avahi_cancellable; + GMainContext *avahi_context; + + /* Map of service name (typically human readable) to a #GPtrArray of the + * #AvahiServiceResolver instances we have running against that name. We + * could end up with more than one resolver if the same name is advertised to + * us over multiple interfaces or protocols (for example, IPv4 and IPv6). + * Resolve all of them just in case one doesn’t work. */ + GHashTable *resolvers; /* (element-type (owned) utf8 (owned) GPtrArray (element-type (owned) AvahiServiceResolver)) */ + + /* Array of #OstreeAvahiService instances representing all the services which + * we currently think are valid. */ + GPtrArray *found_services; /* (element-type (owned OstreeAvahiService) */ +#endif /* HAVE_AVAHI */ +}; + +static void ostree_repo_finder_avahi_iface_init (OstreeRepoFinderInterface *iface); + +G_DEFINE_TYPE_WITH_CODE (OstreeRepoFinderAvahi, ostree_repo_finder_avahi, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (OSTREE_TYPE_REPO_FINDER, ostree_repo_finder_avahi_iface_init)) + +#ifdef HAVE_AVAHI + +/* Download the summary file from @remote and fill in the checksums in the given + * @supported_ref_to_checksum hash table, given the existing refs in it as keys. + * See get_refs_and_checksums_from_summary() for more details. */ +static gboolean +get_checksums (OstreeRepoFinderAvahi *finder, + OstreeRepo *repo, + OstreeRemote *remote, + GHashTable *supported_ref_to_checksum /* (element-type OstreeCollectionRef utf8) */, + GError **error) +{ + g_autoptr(GBytes) summary_bytes = NULL; + + if (!fetch_summary_from_remote (repo, + remote, + &summary_bytes, + finder->avahi_cancellable, + error)) + return FALSE; + + return get_refs_and_checksums_from_summary (summary_bytes, supported_ref_to_checksum, error); +} + +/* Build some #OstreeRepoFinderResults out of the given #OstreeAvahiService by + * parsing its DNS-SD TXT records and finding the intersection between the refs + * it advertises and @refs. One or more results will be added to @results, with + * multiple results being added if the intersection of refs covers refs which + * need different GPG keyrings. One result is added per (uri, keyring) pair. + * + * If any of the TXT records are malformed or missing, or if the intersection of + * refs is empty, return early without modifying @results. + * + * This recognises the following TXT records: + * - `v` (`y`): Version of the TXT record format. Only version `1` is currently + * supported. + * - `rb` (`(yyay)`): Bloom filter indicating which refs are served by the peer. + * - `st` (`t`): Timestamp (seconds since the Unix epoch, big endian) the + * summary file was last modified. + * - `ri` (`q`): Repository index, indicating which of several repositories + * hosted on the peer this is. Big endian. + */ +static void +ostree_avahi_service_build_repo_finder_result (OstreeAvahiService *service, + OstreeRepoFinderAvahi *finder, + OstreeRepo *parent_repo, + gint priority, + const OstreeCollectionRef * const *refs, + GPtrArray *results, + GCancellable *cancellable) +{ + g_autoptr(GHashTable) attributes = NULL; + g_autoptr(GVariant) version = NULL; + g_autoptr(GVariant) bloom = NULL; + g_autoptr(GVariant) summary_timestamp = NULL; + g_autoptr(GVariant) repo_index = NULL; + g_autofree gchar *repo_path = NULL; + g_autoptr(GPtrArray) possible_refs = NULL; /* (element-type OstreeCollectionRef) */ + SoupURI *_uri = NULL; + g_autofree gchar *uri = NULL; + g_autoptr(GError) error = NULL; + gsize i; + g_autoptr(GHashTable) repo_to_refs = NULL; /* (element-type UriAndKeyring GHashTable) */ + GHashTable *supported_ref_to_checksum; /* (element-type OstreeCollectionRef utf8) */ + GHashTableIter iter; + UriAndKeyring *repo; + + g_return_if_fail (service != NULL); + g_return_if_fail (refs != NULL); + + attributes = _ostree_txt_records_parse (service->txt); + + /* Check the record version. */ + version = _ostree_txt_records_lookup_variant (attributes, "v", G_VARIANT_TYPE_BYTE); + + if (g_variant_get_byte (version) != 1) + { + g_debug ("Unknown v=%02x attribute provided in TXT record. Ignoring.", + g_variant_get_byte (version)); + return; + } + + /* Refs bloom filter? */ + bloom = _ostree_txt_records_lookup_variant (attributes, "rb", G_VARIANT_TYPE ("(yyay)")); + + if (bloom == NULL) + { + g_debug ("Missing rb (refs bloom) attribute in TXT record. Ignoring."); + return; + } + + possible_refs = bloom_refs_intersection (bloom, refs); + if (possible_refs == NULL) + { + g_debug ("Wrong k parameter or hash id in rb (refs bloom) attribute in TXT record. Ignoring."); + return; + } + if (possible_refs->len == 0) + { + g_debug ("TXT record definitely has no matching refs. Ignoring."); + return; + } + + /* Summary timestamp. */ + summary_timestamp = _ostree_txt_records_lookup_variant (attributes, "st", G_VARIANT_TYPE_UINT64); + if (summary_timestamp == NULL) + { + g_debug ("Missing st (summary timestamp) attribute in TXT record. Ignoring."); + return; + } + + /* Repository index. */ + repo_index = _ostree_txt_records_lookup_variant (attributes, "ri", G_VARIANT_TYPE_UINT16); + if (repo_index == NULL) + { + g_debug ("Missing ri (repository index) attribute in TXT record. Ignoring."); + return; + } + repo_path = g_strdup_printf ("/%u", GUINT16_FROM_BE (g_variant_get_uint16 (repo_index))); + + /* Create a new result for each keyring needed by @possible_refs. Typically, + * there will be a separate keyring per collection, but some might be shared. */ + repo_to_refs = g_hash_table_new_full (uri_and_keyring_hash, uri_and_keyring_equal, + (GDestroyNotify) uri_and_keyring_free, (GDestroyNotify) g_hash_table_unref); + + _uri = soup_uri_new (NULL); + soup_uri_set_scheme (_uri, "http"); + soup_uri_set_host (_uri, service->address); + soup_uri_set_port (_uri, service->port); + soup_uri_set_path (_uri, repo_path); + uri = soup_uri_to_string (_uri, FALSE); + soup_uri_free (_uri); + + for (i = 0; i < possible_refs->len; i++) + { + const OstreeCollectionRef *ref = g_ptr_array_index (possible_refs, i); + g_autofree gchar *keyring = NULL; + g_autoptr(UriAndKeyring) resolved_repo = NULL; + + /* Look up the GPG keyring for this ref. */ + keyring = ostree_repo_resolve_keyring_for_collection (parent_repo, ref->collection_id, + cancellable, &error); + + if (keyring == NULL) + { + g_debug ("Ignoring ref (%s, %s) on host ‘%s’ due to missing keyring: %s", + ref->collection_id, refs[i]->ref_name, service->address, + error->message); + g_clear_error (&error); + continue; + } + + /* Add this repo to the results, keyed by the canonicalised repository URI + * to deduplicate the results. */ + g_debug ("Resolved ref (%s, %s) to repo URI ‘%s’ with keyring ‘%s’.", + ref->collection_id, ref->ref_name, uri, keyring); + + resolved_repo = uri_and_keyring_new (uri, keyring); + + supported_ref_to_checksum = g_hash_table_lookup (repo_to_refs, resolved_repo); + + if (supported_ref_to_checksum == NULL) + { + supported_ref_to_checksum = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + NULL, g_free); + g_hash_table_insert (repo_to_refs, g_steal_pointer (&resolved_repo), supported_ref_to_checksum /* transfer */); + } + + /* Add a placeholder to @supported_ref_to_checksum for this ref. It will + * be filled out by the get_checksums() call below. */ + g_hash_table_insert (supported_ref_to_checksum, (gpointer) ref, NULL); + } + + /* Aggregate the results. */ + g_hash_table_iter_init (&iter, repo_to_refs); + + while (g_hash_table_iter_next (&iter, (gpointer *) &repo, (gpointer *) &supported_ref_to_checksum)) + { + g_autoptr(OstreeRemote) remote = NULL; + + /* Build an #OstreeRemote. Use the escaped URI, since remote->name + * is used in file paths, so needs to not contain special characters. */ + g_autofree gchar *name = uri_and_keyring_to_name (repo); + remote = ostree_remote_new (name); + + g_clear_pointer (&remote->keyring, g_free); + remote->keyring = g_strdup (repo->keyring); + + g_key_file_set_string (remote->options, remote->group, "url", repo->uri); + g_key_file_set_boolean (remote->options, remote->group, "gpg-verify", TRUE); + g_key_file_set_boolean (remote->options, remote->group, "gpg-verify-summary", TRUE); + + get_checksums (finder, parent_repo, remote, supported_ref_to_checksum, &error); + if (error != NULL) + { + g_debug ("Failed to get checksums for possible refs; ignoring: %s", error->message); + g_clear_error (&error); + continue; + } + + g_ptr_array_add (results, ostree_repo_finder_result_new (remote, OSTREE_REPO_FINDER (finder), + priority, supported_ref_to_checksum, + (summary_timestamp != NULL) ? GUINT64_FROM_BE (g_variant_get_uint64 (summary_timestamp)) : 0)); + } +} + +typedef struct +{ + OstreeCollectionRef **refs; /* (owned) (array zero-terminated=1) */ + OstreeRepo *parent_repo; /* (owned) */ +} ResolveData; + +static void +resolve_data_free (ResolveData *data) +{ + g_object_unref (data->parent_repo); + ostree_collection_ref_freev (data->refs); + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (ResolveData, resolve_data_free) + +static ResolveData * +resolve_data_new (const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo) +{ + g_autoptr(ResolveData) data = NULL; + + data = g_new0 (ResolveData, 1); + data->refs = ostree_collection_ref_dupv (refs); + data->parent_repo = g_object_ref (parent_repo); + + return g_steal_pointer (&data); +} + +static void +fail_all_pending_tasks (OstreeRepoFinderAvahi *self, + GQuark domain, + gint code, + const gchar *format, + ...) G_GNUC_PRINTF(4, 5); + +/* Executed in @self->avahi_context. + * + * Return the given error from all the pending resolve tasks in + * self->resolve_tasks. */ +static void +fail_all_pending_tasks (OstreeRepoFinderAvahi *self, + GQuark domain, + gint code, + const gchar *format, + ...) +{ + gsize i; + va_list args; + g_autoptr(GError) error = NULL; + + g_assert (g_main_context_is_owner (self->avahi_context)); + + va_start (args, format); + error = g_error_new_valist (domain, code, format, args); + va_end (args); + + for (i = 0; i < self->resolve_tasks->len; i++) + { + GTask *task = G_TASK (g_ptr_array_index (self->resolve_tasks, i)); + g_task_return_error (task, g_error_copy (error)); + } + + g_ptr_array_set_size (self->resolve_tasks, 0); +} + +static gint +results_compare_cb (gconstpointer a, + gconstpointer b) +{ + const OstreeRepoFinderResult *result_a = *((const OstreeRepoFinderResult **) a); + const OstreeRepoFinderResult *result_b = *((const OstreeRepoFinderResult **) b); + + return ostree_repo_finder_result_compare (result_a, result_b); +} + +/* Executed in @self->avahi_context. + * + * For each of the pending resolve tasks in self->resolve_tasks, calculate and + * return the result set for its query given the currently known services from + * Avahi which are stored in self->found_services. */ +static void +complete_all_pending_tasks (OstreeRepoFinderAvahi *self) +{ + gsize i; + const gint priority = 60; /* arbitrarily chosen */ + g_autoptr(GPtrArray) results_for_tasks = g_ptr_array_new_full (self->resolve_tasks->len, (GDestroyNotify)g_ptr_array_unref); + gboolean cancelled = FALSE; + + g_assert (g_main_context_is_owner (self->avahi_context)); + g_debug ("%s: Completing %u tasks", G_STRFUNC, self->resolve_tasks->len); + + for (i = 0; i < self->resolve_tasks->len; i++) + { + g_autoptr(GPtrArray) results = NULL; + GTask *task; + ResolveData *data; + const OstreeCollectionRef * const *refs; + gsize j; + + task = G_TASK (g_ptr_array_index (self->resolve_tasks, i)); + data = g_task_get_task_data (task); + refs = (const OstreeCollectionRef * const *) data->refs; + results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + + for (j = 0; j < self->found_services->len; j++) + { + OstreeAvahiService *service = g_ptr_array_index (self->found_services, j); + + ostree_avahi_service_build_repo_finder_result (service, self, data->parent_repo, + priority, refs, results, + self->avahi_cancellable); + if (g_cancellable_is_cancelled (self->avahi_cancellable)) + { + cancelled = TRUE; + break; + } + } + if (cancelled) + break; + + g_ptr_array_add (results_for_tasks, g_steal_pointer (&results)); + } + + if (!cancelled) + { + for (i = 0; i < self->resolve_tasks->len; i++) + { + GTask *task = G_TASK (g_ptr_array_index (self->resolve_tasks, i)); + GPtrArray *results = g_ptr_array_index (results_for_tasks, i); + + g_ptr_array_sort (results, results_compare_cb); + + g_task_return_pointer (task, + g_ptr_array_ref (results), + (GDestroyNotify) g_ptr_array_unref); + } + + g_ptr_array_set_size (self->resolve_tasks, 0); + } + else + { + fail_all_pending_tasks (self, G_IO_ERROR, G_IO_ERROR_CANCELLED, + "Avahi service resolution cancelled."); + } +} + +/* Executed in @self->avahi_context. */ +static void +maybe_complete_all_pending_tasks (OstreeRepoFinderAvahi *self) +{ + g_assert (g_main_context_is_owner (self->avahi_context)); + g_debug ("%s: client_state: %s, browser_failed: %u, cancelled: %u, " + "browser_all_for_now: %u, n_resolvers: %u", + G_STRFUNC, ostree_avahi_client_state_to_string (self->client_state), + self->browser_failed, + g_cancellable_is_cancelled (self->avahi_cancellable), + self->browser_all_for_now, g_hash_table_size (self->resolvers)); + + if (self->client_state == AVAHI_CLIENT_FAILURE) + fail_all_pending_tasks (self, G_IO_ERROR, G_IO_ERROR_FAILED, + "Avahi client error: %s", + avahi_strerror (avahi_client_errno (self->client))); + else if (self->browser_failed) + fail_all_pending_tasks (self, G_IO_ERROR, G_IO_ERROR_FAILED, + "Avahi browser error: %s", + avahi_strerror (avahi_client_errno (self->client))); + else if (g_cancellable_is_cancelled (self->avahi_cancellable)) + fail_all_pending_tasks (self, G_IO_ERROR, G_IO_ERROR_CANCELLED, + "Avahi service resolution cancelled."); + else if (self->browser_all_for_now && + g_hash_table_size (self->resolvers) == 0) + complete_all_pending_tasks (self); +} + +/* Executed in @self->avahi_context. */ +static void +client_cb (AvahiClient *client, + AvahiClientState state, + void *finder_ptr) +{ + /* Completing the pending tasks might drop the final reference to @self. */ + g_autoptr(OstreeRepoFinderAvahi) self = g_object_ref (finder_ptr); + + /* self->client will be NULL if client_cb() is called from + * ostree_repo_finder_avahi_start(). */ + g_assert (self->client == NULL || g_main_context_is_owner (self->avahi_context)); + + g_debug ("%s: Entered state ‘%s’.", + G_STRFUNC, ostree_avahi_client_state_to_string (state)); + + /* We only care about entering and leaving %AVAHI_CLIENT_FAILURE. */ + self->client_state = state; + if (self->client != NULL) + maybe_complete_all_pending_tasks (self); +} + +/* Executed in @self->avahi_context. */ +static void +resolve_cb (AvahiServiceResolver *resolver, + AvahiIfIndex interface, + AvahiProtocol protocol, + AvahiResolverEvent event, + const char *name, + const char *type, + const char *domain, + const char *host_name, + const AvahiAddress *address, + uint16_t port, + AvahiStringList *txt, + AvahiLookupResultFlags flags, + void *finder_ptr) +{ + /* Completing the pending tasks might drop the final reference to @self. */ + g_autoptr(OstreeRepoFinderAvahi) self = g_object_ref (finder_ptr); + g_autoptr(OstreeAvahiService) service = NULL; + GPtrArray *resolvers; + + g_assert (g_main_context_is_owner (self->avahi_context)); + + g_debug ("%s: Resolve event ‘%s’ for name ‘%s’.", + G_STRFUNC, ostree_avahi_resolver_event_to_string (event), name); + + /* Track the resolvers active for this @name. There may be several, + * as @name might appear to us over several interfaces or protocols. Most + * commonly this happens when both hosts are connected via IPv4 and IPv6. */ + resolvers = g_hash_table_lookup (self->resolvers, name); + + if (resolvers == NULL || resolvers->len == 0) + { + /* maybe it was removed in the meantime */ + g_hash_table_remove (self->resolvers, name); + return; + } + else if (resolvers->len == 1) + { + g_hash_table_remove (self->resolvers, name); + } + else + { + g_ptr_array_remove_fast (resolvers, resolver); + } + + /* Was resolution successful? */ + switch (event) + { + case AVAHI_RESOLVER_FOUND: + service = ostree_avahi_service_new (name, domain, address, interface, + port, txt); + g_ptr_array_add (self->found_services, g_steal_pointer (&service)); + break; + case AVAHI_RESOLVER_FAILURE: + default: + g_warning ("Failed to resolve service ‘%s’: %s", name, + avahi_strerror (avahi_client_errno (self->client))); + break; + } + + maybe_complete_all_pending_tasks (self); +} + +/* Executed in @self->avahi_context. */ +static void +browse_new (OstreeRepoFinderAvahi *self, + AvahiIfIndex interface, + AvahiProtocol protocol, + const gchar *name, + const gchar *type, + const gchar *domain) +{ + g_autoptr(AvahiServiceResolver) resolver = NULL; + GPtrArray *resolvers; /* (element-type AvahiServiceResolver) */ + + g_assert (g_main_context_is_owner (self->avahi_context)); + + resolver = avahi_service_resolver_new (self->client, + interface, + protocol, + name, + type, + domain, + AVAHI_PROTO_UNSPEC, + 0, + resolve_cb, + self); + if (resolver == NULL) + { + g_warning ("Failed to resolve service ‘%s’: %s", name, + avahi_strerror (avahi_client_errno (self->client))); + return; + } + + g_debug ("Found name service %s on the network; type: %s, domain: %s, " + "protocol: %u, interface: %u", name, type, domain, protocol, + interface); + + /* Start a resolver for this (interface, protocol, name, type, domain) + * combination. */ + resolvers = g_hash_table_lookup (self->resolvers, name); + if (resolvers == NULL) + { + resolvers = g_ptr_array_new_with_free_func ((GDestroyNotify) avahi_service_resolver_free); + g_hash_table_insert (self->resolvers, g_strdup (name), resolvers); + } + + g_ptr_array_add (resolvers, g_steal_pointer (&resolver)); +} + +/* Executed in @self->avahi_context. Caller must call maybe_complete_all_pending_tasks(). */ +static void +browse_remove (OstreeRepoFinderAvahi *self, + const char *name) +{ + gsize i; + gboolean removed = FALSE; + + g_assert (g_main_context_is_owner (self->avahi_context)); + + g_hash_table_remove (self->resolvers, name); + + for (i = 0; i < self->found_services->len; i += (removed ? 0 : 1)) + { + OstreeAvahiService *service = g_ptr_array_index (self->found_services, i); + + removed = FALSE; + + if (g_strcmp0 (service->name, name) == 0) + { + g_ptr_array_remove_index_fast (self->found_services, i); + removed = TRUE; + continue; + } + } +} + +/* Executed in @self->avahi_context. */ +static void +browse_cb (AvahiServiceBrowser *browser, + AvahiIfIndex interface, + AvahiProtocol protocol, + AvahiBrowserEvent event, + const char *name, + const char *type, + const char *domain, + AvahiLookupResultFlags flags, + void *finder_ptr) +{ + /* Completing the pending tasks might drop the final reference to @self. */ + g_autoptr(OstreeRepoFinderAvahi) self = g_object_ref (finder_ptr); + + g_assert (g_main_context_is_owner (self->avahi_context)); + + g_debug ("%s: Browse event ‘%s’ for name ‘%s’.", + G_STRFUNC, ostree_avahi_browser_event_to_string (event), name); + + self->browser_failed = FALSE; + + switch (event) + { + case AVAHI_BROWSER_NEW: + browse_new (self, interface, protocol, name, type, domain); + break; + + case AVAHI_BROWSER_REMOVE: + browse_remove (self, name); + break; + + case AVAHI_BROWSER_CACHE_EXHAUSTED: + /* don’t care about this. */ + break; + + case AVAHI_BROWSER_ALL_FOR_NOW: + self->browser_all_for_now = TRUE; + break; + + case AVAHI_BROWSER_FAILURE: + self->browser_failed = TRUE; + break; + + default: + g_assert_not_reached (); + } + + /* Check all the tasks for any event, since the @browser_failed state + * may have changed. */ + maybe_complete_all_pending_tasks (self); +} + +static gboolean add_resolve_task_cb (gpointer user_data); +#endif /* HAVE_AVAHI */ + +static void +ostree_repo_finder_avahi_resolve_async (OstreeRepoFinder *finder, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + OstreeRepoFinderAvahi *self = OSTREE_REPO_FINDER_AVAHI (finder); + g_autoptr(GTask) task = NULL; + + g_debug ("%s: Starting resolving", G_STRFUNC); + + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_avahi_resolve_async); + +#ifdef HAVE_AVAHI + g_task_set_task_data (task, resolve_data_new (refs, parent_repo), (GDestroyNotify) resolve_data_free); + + /* Move @task to the @avahi_context where it can be processed. */ + g_main_context_invoke (self->avahi_context, add_resolve_task_cb, g_steal_pointer (&task)); +#else /* if !HAVE_AVAHI */ + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "Avahi support was not compiled in to libostree"); +#endif /* !HAVE_AVAHI */ +} + +#ifdef HAVE_AVAHI +/* Executed in @self->avahi_context. */ +static gboolean +add_resolve_task_cb (gpointer user_data) +{ + g_autoptr(GTask) task = G_TASK (user_data); + OstreeRepoFinderAvahi *self = g_task_get_source_object (task); + + g_assert (g_main_context_is_owner (self->avahi_context)); + g_debug ("%s", G_STRFUNC); + + /* Track the task and check to see if the browser and resolvers are in a + * quiescent state suitable for returning a result immediately. */ + g_ptr_array_add (self->resolve_tasks, g_object_ref (task)); + maybe_complete_all_pending_tasks (self); + + return G_SOURCE_REMOVE; +} +#endif /* HAVE_AVAHI */ + +static GPtrArray * +ostree_repo_finder_avahi_resolve_finish (OstreeRepoFinder *finder, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, finder), NULL); + return g_task_propagate_pointer (G_TASK (result), error); +} + +static void +ostree_repo_finder_avahi_dispose (GObject *obj) +{ +#ifdef HAVE_AVAHI + OstreeRepoFinderAvahi *self = OSTREE_REPO_FINDER_AVAHI (obj); + + ostree_repo_finder_avahi_stop (self); + + g_assert (self->resolve_tasks == NULL || self->resolve_tasks->len == 0); + + g_clear_pointer (&self->resolve_tasks, g_ptr_array_unref); + g_clear_pointer (&self->browser, avahi_service_browser_free); + g_clear_pointer (&self->client, avahi_client_free); + g_clear_pointer (&self->poll, avahi_glib_poll_free); + g_clear_pointer (&self->avahi_context, g_main_context_unref); + g_clear_pointer (&self->found_services, g_ptr_array_unref); + g_clear_pointer (&self->resolvers, g_hash_table_unref); + g_clear_object (&self->avahi_cancellable); +#endif /* HAVE_AVAHI */ + + /* Chain up. */ + G_OBJECT_CLASS (ostree_repo_finder_avahi_parent_class)->dispose (obj); +} + +static void +ostree_repo_finder_avahi_class_init (OstreeRepoFinderAvahiClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->dispose = ostree_repo_finder_avahi_dispose; +} + +static void +ostree_repo_finder_avahi_iface_init (OstreeRepoFinderInterface *iface) +{ + iface->resolve_async = ostree_repo_finder_avahi_resolve_async; + iface->resolve_finish = ostree_repo_finder_avahi_resolve_finish; +} + +static void +ostree_repo_finder_avahi_init (OstreeRepoFinderAvahi *self) +{ +#ifdef HAVE_AVAHI + self->resolve_tasks = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref); + self->avahi_cancellable = g_cancellable_new (); + self->client_state = AVAHI_CLIENT_S_REGISTERING; + self->resolvers = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) g_ptr_array_unref); + self->found_services = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_avahi_service_free); +#endif /* HAVE_AVAHI */ +} + +/** + * ostree_repo-finder_avahi_new: + * @context: (transfer none) (nullable): a #GMainContext for processing Avahi + * events in, or %NULL to use the current thread-default + * + * Create a new #OstreeRepoFinderAvahi instance. It is intended that one such + * instance be created per process, and it be used to answer all resolution + * requests from #OstreeRepos. + * + * The calling code is responsible for ensuring that @context is iterated while + * the #OstreeRepoFinderAvahi is running (after ostree_repo_finder_avahi_start() + * is called). This may be done from any thread. + * + * If @context is %NULL, the current thread-default #GMainContext is used. + * + * Returns: (transfer full): a new #OstreeRepoFinderAvahi + * Since: 2017.8 + */ +OstreeRepoFinderAvahi * +ostree_repo_finder_avahi_new (GMainContext *context) +{ + g_autoptr(OstreeRepoFinderAvahi) finder = NULL; + + finder = g_object_new (OSTREE_TYPE_REPO_FINDER_AVAHI, NULL); + +#ifdef HAVE_AVAHI + /* FIXME: Make this a property */ + if (context != NULL) + finder->avahi_context = g_main_context_ref (context); + else + finder->avahi_context = g_main_context_ref_thread_default (); + + /* Avahi setup. Note: Technically the allocator is per-process state which we + * shouldn’t set here, but it’s probably fine. It’s unlikely that code which + * is using libostree is going to use an allocator which is not GLib, and + * *also* use Avahi API itself. */ + avahi_set_allocator (avahi_glib_allocator ()); + finder->poll = avahi_glib_poll_new (finder->avahi_context, G_PRIORITY_DEFAULT); +#endif /* HAVE_AVAHI */ + + return g_steal_pointer (&finder); +} + +/** + * ostree_repo_finder_avahi_start: + * @self: an #OstreeRepoFinderAvahi + * @error: return location for a #GError + * + * Start monitoring the local network for peers who are advertising OSTree + * repositories, using Avahi. In order for this to work, the #GMainContext + * passed to @self at construction time must be iterated (so it will typically + * be the global #GMainContext, or be a separate #GMainContext in a worker + * thread). + * + * This will return an error (%G_IO_ERROR_FAILED) if initialisation fails, or if + * Avahi support is not available (%G_IO_ERROR_NOT_SUPPORTED). In either case, + * the #OstreeRepoFinderAvahi instance is useless afterwards and should be + * destroyed. + * + * Call ostree_repo_finder_avahi_stop() to stop the repo finder. + * + * It is an error to call this function multiple times on the same + * #OstreeRepoFinderAvahi instance, or to call it after + * ostree_repo_finder_avahi_stop(). + * + * Since: 2017.8 + */ +void +ostree_repo_finder_avahi_start (OstreeRepoFinderAvahi *self, + GError **error) +{ + g_return_if_fail (OSTREE_IS_REPO_FINDER_AVAHI (self)); + g_return_if_fail (error == NULL || *error == NULL); + +#ifdef HAVE_AVAHI + g_autoptr(AvahiClient) client = NULL; + g_autoptr(AvahiServiceBrowser) browser = NULL; + int failure = 0; + + if (g_cancellable_set_error_if_cancelled (self->avahi_cancellable, error)) + return; + + g_assert (self->client == NULL); + + client = avahi_client_new (avahi_glib_poll_get (self->poll), + AVAHI_CLIENT_NO_FAIL, + client_cb, self, &failure); + + if (client == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to create finder client: %s", + avahi_strerror (failure)); + return; + } + + /* Query for the OSTree DNS-SD service on the local network. */ + browser = avahi_service_browser_new (client, + AVAHI_IF_UNSPEC, + AVAHI_PROTO_UNSPEC, + OSTREE_AVAHI_SERVICE_TYPE, + NULL, + 0, + browse_cb, + self); + + if (browser == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to create service browser: %s", + avahi_strerror (avahi_client_errno (client))); + return; + } + + /* Success. */ + self->client = g_steal_pointer (&client); + self->browser = g_steal_pointer (&browser); +#else /* if !HAVE_AVAHI */ + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "Avahi support was not compiled in to libostree"); +#endif /* !HAVE_AVAHI */ +} + +#ifdef HAVE_AVAHI +static gboolean stop_cb (gpointer user_data); +#endif /* HAVE_AVAHI */ + +/** + * ostree_repo_finder_avahi_stop: + * @self: an #OstreeRepoFinderAvahi + * + * Stop monitoring the local network for peers who are advertising OSTree + * repositories. If any resolve tasks (from ostree_repo_finder_resolve_async()) + * are in progress, they will be cancelled and will return %G_IO_ERROR_CANCELLED. + * + * Call ostree_repo_finder_avahi_start() to start the repo finder. + * + * It is an error to call this function multiple times on the same + * #OstreeRepoFinderAvahi instance, or to call it before + * ostree_repo_finder_avahi_start(). + * + * Since: 2017.8 + */ +void +ostree_repo_finder_avahi_stop (OstreeRepoFinderAvahi *self) +{ + g_return_if_fail (OSTREE_IS_REPO_FINDER_AVAHI (self)); + +#ifdef HAVE_AVAHI + if (self->browser == NULL) + return; + + g_main_context_invoke (self->avahi_context, stop_cb, g_object_ref (self)); +#endif /* HAVE_AVAHI */ +} + +#ifdef HAVE_AVAHI +static gboolean +stop_cb (gpointer user_data) +{ + g_autoptr(OstreeRepoFinderAvahi) self = OSTREE_REPO_FINDER_AVAHI (user_data); + + g_cancellable_cancel (self->avahi_cancellable); + maybe_complete_all_pending_tasks (self); + + g_clear_pointer (&self->browser, avahi_service_browser_free); + g_clear_pointer (&self->client, avahi_client_free); + g_hash_table_remove_all (self->resolvers); + + return G_SOURCE_REMOVE; +} +#endif /* HAVE_AVAHI */ diff --git a/src/libostree/ostree-repo-finder-avahi.h b/src/libostree/ostree-repo-finder-avahi.h new file mode 100644 index 00000000..98d37723 --- /dev/null +++ b/src/libostree/ostree-repo-finder-avahi.h @@ -0,0 +1,62 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER_AVAHI (ostree_repo_finder_avahi_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +_OSTREE_PUBLIC +G_DECLARE_FINAL_TYPE (OstreeRepoFinderAvahi, ostree_repo_finder_avahi, OSTREE, REPO_FINDER_AVAHI, GObject) */ + +_OSTREE_PUBLIC +GType ostree_repo_finder_avahi_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeRepoFinderAvahi OstreeRepoFinderAvahi; +typedef struct { GObjectClass parent_class; } OstreeRepoFinderAvahiClass; + +static inline OstreeRepoFinderAvahi *OSTREE_REPO_FINDER_AVAHI (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_repo_finder_avahi_get_type (), OstreeRepoFinderAvahi); } +static inline gboolean OSTREE_IS_REPO_FINDER_AVAHI (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_repo_finder_avahi_get_type ()); } +G_GNUC_END_IGNORE_DEPRECATIONS + +_OSTREE_PUBLIC +OstreeRepoFinderAvahi *ostree_repo_finder_avahi_new (GMainContext *context); + +_OSTREE_PUBLIC +void ostree_repo_finder_avahi_start (OstreeRepoFinderAvahi *self, + GError **error); + +_OSTREE_PUBLIC +void ostree_repo_finder_avahi_stop (OstreeRepoFinderAvahi *self); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index ecc9b72c..4c87199a 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -43,6 +43,9 @@ #include "ostree-repo-finder.h" #include "ostree-repo-finder-config.h" #include "ostree-repo-finder-mount.h" +#ifdef HAVE_AVAHI +#include "ostree-repo-finder-avahi.h" +#endif /* HAVE_AVAHI */ #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include @@ -3961,11 +3964,13 @@ typedef struct OstreeCollectionRef **refs; GVariant *options; OstreeAsyncProgress *progress; + OstreeRepoFinder *default_finder_avahi; } FindRemotesData; static void find_remotes_data_free (FindRemotesData *data) { + g_clear_object (&data->default_finder_avahi); g_clear_object (&data->progress); g_clear_pointer (&data->options, g_variant_unref); ostree_collection_ref_freev (data->refs); @@ -3978,7 +3983,8 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (FindRemotesData, find_remotes_data_free) static FindRemotesData * find_remotes_data_new (const OstreeCollectionRef * const *refs, GVariant *options, - OstreeAsyncProgress *progress) + OstreeAsyncProgress *progress, + OstreeRepoFinder *default_finder_avahi) { g_autoptr(FindRemotesData) data = NULL; @@ -3986,6 +3992,7 @@ find_remotes_data_new (const OstreeCollectionRef * const *refs, data->refs = ostree_collection_ref_dupv (refs); data->options = (options != NULL) ? g_variant_ref (options) : NULL; data->progress = (progress != NULL) ? g_object_ref (progress) : NULL; + data->default_finder_avahi = (default_finder_avahi != NULL) ? g_object_ref (default_finder_avahi) : NULL; return g_steal_pointer (&data); } @@ -4085,10 +4092,10 @@ ostree_repo_find_remotes_async (OstreeRepo *self, { g_autoptr(GTask) task = NULL; g_autoptr(FindRemotesData) data = NULL; - GMainContext *context; OstreeRepoFinder *default_finders[4] = { NULL, }; g_autoptr(OstreeRepoFinder) finder_config = NULL; g_autoptr(OstreeRepoFinder) finder_mount = NULL; + g_autoptr(OstreeRepoFinder) finder_avahi = NULL; g_return_if_fail (OSTREE_IS_REPO (self)); g_return_if_fail (is_valid_collection_ref_array (refs)); @@ -4102,21 +4109,43 @@ ostree_repo_find_remotes_async (OstreeRepo *self, task = g_task_new (self, cancellable, callback, user_data); g_task_set_source_tag (task, ostree_repo_find_remotes_async); - context = g_main_context_get_thread_default (); - /* Are we using #OstreeRepoFinders provided by the user, or the defaults? */ if (finders == NULL) { +#ifdef HAVE_AVAHI + GMainContext *context = g_main_context_get_thread_default (); + g_autoptr(GError) local_error = NULL; +#endif /* HAVE_AVAHI */ + finder_config = OSTREE_REPO_FINDER (ostree_repo_finder_config_new ()); finder_mount = OSTREE_REPO_FINDER (ostree_repo_finder_mount_new (NULL)); +#ifdef HAVE_AVAHI + finder_avahi = OSTREE_REPO_FINDER (ostree_repo_finder_avahi_new (context)); +#endif /* HAVE_AVAHI */ default_finders[0] = finder_config; default_finders[1] = finder_mount; + default_finders[2] = finder_avahi; finders = default_finders; + +#ifdef HAVE_AVAHI + ostree_repo_finder_avahi_start (OSTREE_REPO_FINDER_AVAHI (finder_avahi), + &local_error); + + if (local_error != NULL) + { + g_warning ("Avahi finder failed; removing it: %s", local_error->message); + default_finders[2] = NULL; + g_clear_object (&finder_avahi); + } +#endif /* HAVE_AVAHI */ } - data = find_remotes_data_new (refs, options, progress); + /* We need to keep a pointer to the default Avahi finder so we can stop it + * again after the operation, which happens implicitly by dropping the final + * ref. */ + data = find_remotes_data_new (refs, options, progress, finder_avahi); g_task_set_task_data (task, g_steal_pointer (&data), (GDestroyNotify) find_remotes_data_free); /* Asynchronously resolve all possible remotes for the given refs. */ diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index 0fe2a23e..0f727384 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -38,6 +38,7 @@ #ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include #include +#include #include #include #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ diff --git a/tests/.gitignore b/tests/.gitignore index 5ece7ea1..9bec67a3 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -15,6 +15,7 @@ test-mutable-tree test-ot-opt-utils test-ot-tool-util test-ot-unix-utils +test-repo-finder-avahi test-repo-finder-config test-repo-finder-mount test-rollsum-cli diff --git a/tests/test-repo-finder-avahi.c b/tests/test-repo-finder-avahi.c new file mode 100644 index 00000000..b2fddf70 --- /dev/null +++ b/tests/test-repo-finder-avahi.c @@ -0,0 +1,228 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "ostree-autocleanups.h" +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-avahi.h" +#include "ostree-repo-finder-avahi-private.h" + +/* FIXME: Upstream this */ +G_DEFINE_AUTOPTR_CLEANUP_FUNC (AvahiStringList, avahi_string_list_free) + +/* Test the object constructor works at a basic level. */ +static void +test_repo_finder_avahi_init (void) +{ + g_autoptr(OstreeRepoFinderAvahi) finder = NULL; + g_autoptr(GMainContext) context = NULL; + + /* Default main context. */ + finder = ostree_repo_finder_avahi_new (NULL); + g_clear_object (&finder); + + /* Explicit main context. */ + context = g_main_context_new (); + finder = ostree_repo_finder_avahi_new (context); + g_clear_object (&finder); +} + +/* Test parsing valid and invalid TXT records. */ +static void +test_repo_finder_avahi_txt_records_parse (void) +{ + struct + { + const guint8 *txt; + gsize txt_len; + const gchar *expected_key; /* (nullable) to indicate parse failure */ + const guint8 *expected_value; /* (nullable) to allow for valueless keys */ + gsize expected_value_len; + } + vectors[] = + { + { (const guint8 *) "", 0, NULL, NULL, 0 }, + { (const guint8 *) "\x00", 1, NULL, NULL, 0 }, + { (const guint8 *) "\xff", 1, NULL, NULL, 0 }, + { (const guint8 *) "k\x00", 2, NULL, NULL, 0 }, + { (const guint8 *) "k\xff", 2, NULL, NULL, 0 }, + { (const guint8 *) "=", 1, NULL, NULL, 0 }, + { (const guint8 *) "=value", 6, NULL, NULL, 0 }, + { (const guint8 *) "k=v", 3, "k", (const guint8 *) "v", 1 }, + { (const guint8 *) "key=value", 9, "key", (const guint8 *) "value", 5 }, + { (const guint8 *) "k=v=", 4, "k", (const guint8 *) "v=", 2 }, + { (const guint8 *) "k=", 2, "k", (const guint8 *) "", 0 }, + { (const guint8 *) "k", 1, "k", NULL, 0 }, + { (const guint8 *) "k==", 3, "k", (const guint8 *) "=", 1 }, + { (const guint8 *) "k=\x00\x01\x02", 5, "k", (const guint8 *) "\x00\x01\x02", 3 }, + }; + gsize i; + + for (i = 0; i < G_N_ELEMENTS (vectors); i++) + { + g_autoptr(AvahiStringList) string_list = NULL; + g_autoptr(GHashTable) attributes = NULL; + + g_test_message ("Vector %" G_GSIZE_FORMAT, i); + + string_list = avahi_string_list_add_arbitrary (NULL, vectors[i].txt, vectors[i].txt_len); + + attributes = _ostree_txt_records_parse (string_list); + + if (vectors[i].expected_key != NULL) + { + GBytes *value; + g_autoptr(GBytes) expected_value = NULL; + + g_assert_true (g_hash_table_lookup_extended (attributes, + vectors[i].expected_key, + NULL, + (gpointer *) &value)); + g_assert_cmpuint (g_hash_table_size (attributes), ==, 1); + + if (vectors[i].expected_value != NULL) + { + g_assert_nonnull (value); + expected_value = g_bytes_new_static (vectors[i].expected_value, vectors[i].expected_value_len); + g_assert_true (g_bytes_equal (value, expected_value)); + } + else + { + g_assert_null (value); + } + } + else + { + g_assert_cmpuint (g_hash_table_size (attributes), ==, 0); + } + } +} + +/* Test that the first value for a set of duplicate records is returned. + * See RFC 6763, §6.4. */ +static void +test_repo_finder_avahi_txt_records_duplicates (void) +{ + g_autoptr(AvahiStringList) string_list = NULL; + g_autoptr(GHashTable) attributes = NULL; + GBytes *value; + g_autoptr(GBytes) expected_value = NULL; + + /* Reverse the list before using it, as they are built in reverse order. + * (See the #AvahiStringList documentation.) */ + string_list = avahi_string_list_new ("k=value1", "k=value2", "k=value3", NULL); + string_list = avahi_string_list_reverse (string_list); + attributes = _ostree_txt_records_parse (string_list); + + g_assert_cmpuint (g_hash_table_size (attributes), ==, 1); + value = g_hash_table_lookup (attributes, "k"); + g_assert_nonnull (value); + + expected_value = g_bytes_new_static ("value1", strlen ("value1")); + g_assert_true (g_bytes_equal (value, expected_value)); +} + +/* Test that keys are parsed and looked up case insensitively. + * See RFC 6763, §6.4. */ +static void +test_repo_finder_avahi_txt_records_case_sensitivity (void) +{ + g_autoptr(AvahiStringList) string_list = NULL; + g_autoptr(GHashTable) attributes = NULL; + GBytes *value1, *value2; + g_autoptr(GBytes) expected_value1 = NULL, expected_value2 = NULL; + + /* Reverse the list before using it, as they are built in reverse order. + * (See the #AvahiStringList documentation.) */ + string_list = avahi_string_list_new ("k=value1", + "K=value2", + "KeY2=v", + NULL); + string_list = avahi_string_list_reverse (string_list); + attributes = _ostree_txt_records_parse (string_list); + + g_assert_cmpuint (g_hash_table_size (attributes), ==, 2); + + value1 = g_hash_table_lookup (attributes, "k"); + g_assert_nonnull (value1); + expected_value1 = g_bytes_new_static ("value1", strlen ("value1")); + g_assert_true (g_bytes_equal (value1, expected_value1)); + + g_assert_null (g_hash_table_lookup (attributes, "K")); + + value2 = g_hash_table_lookup (attributes, "key2"); + g_assert_nonnull (value2); + expected_value2 = g_bytes_new_static ("v", 1); + g_assert_true (g_bytes_equal (value2, expected_value2)); + + g_assert_null (g_hash_table_lookup (attributes, "KeY2")); +} + +/* Test that keys which have an empty value can be distinguished from those + * which have no value. See RFC 6763, §6.4. */ +static void +test_repo_finder_avahi_txt_records_empty_and_missing (void) +{ + g_autoptr(AvahiStringList) string_list = NULL; + g_autoptr(GHashTable) attributes = NULL; + GBytes *value1, *value2; + g_autoptr(GBytes) expected_value1 = NULL; + + string_list = avahi_string_list_new ("empty=", + "missing", + NULL); + attributes = _ostree_txt_records_parse (string_list); + + g_assert_cmpuint (g_hash_table_size (attributes), ==, 2); + + value1 = g_hash_table_lookup (attributes, "empty"); + g_assert_nonnull (value1); + expected_value1 = g_bytes_new_static ("", 0); + g_assert_true (g_bytes_equal (value1, expected_value1)); + + g_assert_true (g_hash_table_lookup_extended (attributes, "missing", NULL, (gpointer *) &value2)); + g_assert_null (value2); +} + +int main (int argc, char **argv) +{ + setlocale (LC_ALL, ""); + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/repo-finder-avahi/init", test_repo_finder_avahi_init); + g_test_add_func ("/repo-finder-avahi/txt-records/parse", test_repo_finder_avahi_txt_records_parse); + g_test_add_func ("/repo-finder-avahi/txt-records/duplicates", test_repo_finder_avahi_txt_records_duplicates); + g_test_add_func ("/repo-finder-avahi/txt-records/case-sensitivity", test_repo_finder_avahi_txt_records_case_sensitivity); + g_test_add_func ("/repo-finder-avahi/txt-records/empty-and-missing", test_repo_finder_avahi_txt_records_empty_and_missing); + /* FIXME: Add tests for service processing, probably by splitting the + * code in OstreeRepoFinderAvahi around found_services. */ + + return g_test_run(); +} From 6453203f54c429baa963cf5662313a3be030af9e Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 25 Apr 2017 19:10:04 +0100 Subject: [PATCH 28/82] find-remotes: Add a find-remotes built-in command This is a wrapper around the new ostree_repo_find_remotes() method; it tries to find available remotes which can serve updates for the user-provided refs. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- Makefile-ostree.am | 4 + src/ostree/main.c | 3 + src/ostree/ot-builtin-find-remotes.c | 255 +++++++++++++++++++++++++++ src/ostree/ot-builtins.h | 5 + 4 files changed, 267 insertions(+) create mode 100644 src/ostree/ot-builtin-find-remotes.c diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 4bd623d7..6414233f 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -53,6 +53,10 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/parse-datetime.c \ $(NULL) +if ENABLE_EXPERIMENTAL_API +ostree_SOURCES += src/ostree/ot-builtin-find-remotes.c +endif + # Admin subcommand ostree_SOURCES += \ src/ostree/ot-admin-builtin-init-fs.c \ diff --git a/src/ostree/main.c b/src/ostree/main.c index 5a2ed661..113edc66 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -41,6 +41,9 @@ static OstreeCommand commands[] = { { "config", ostree_builtin_config }, { "diff", ostree_builtin_diff }, { "export", ostree_builtin_export }, +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + { "find-remotes", ostree_builtin_find_remotes }, +#endif { "fsck", ostree_builtin_fsck }, { "gpg-sign", ostree_builtin_gpg_sign }, { "init", ostree_builtin_init }, diff --git a/src/ostree/ot-builtin-find-remotes.c b/src/ostree/ot-builtin-find-remotes.c new file mode 100644 index 00000000..8130ec35 --- /dev/null +++ b/src/ostree/ot-builtin-find-remotes.c @@ -0,0 +1,255 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include "ot-main.h" +#include "ot-builtins.h" +#include "ostree.h" +#include "otutil.h" + +#include "ostree-remote-private.h" + +static gchar *opt_cache_dir = NULL; +static gboolean opt_disable_fsync = FALSE; + +static GOptionEntry options[] = + { + { "cache-dir", 0, 0, G_OPTION_ARG_FILENAME, &opt_cache_dir, "Use custom cache dir", NULL }, + { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, + { NULL } + }; + +static gchar * +uint64_secs_to_iso8601 (guint64 secs) +{ + g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (secs); + + if (dt != NULL) + return g_date_time_format (dt, "%FT%TZ"); + else + return g_strdup ("invalid"); +} + +static gchar * +format_ref_to_checksum (GHashTable *ref_to_checksum /* (element-type OstreeCollectionRef utf8) */, + const gchar *line_prefix) +{ + GHashTableIter iter; + const OstreeCollectionRef *ref; + const gchar *checksum; + g_autoptr(GString) out = NULL; + + g_hash_table_iter_init (&iter, ref_to_checksum); + out = g_string_new (""); + + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum)) + g_string_append_printf (out, "%s - (%s, %s) = %s\n", + line_prefix, ref->collection_id, ref->ref_name, + (checksum != NULL) ? checksum : "(not found)"); + + return g_string_free (g_steal_pointer (&out), FALSE); +} + +static gchar * +remote_get_uri (OstreeRemote *remote) +{ + g_autoptr(GError) error = NULL; + g_autofree gchar *uri = NULL; + + uri = g_key_file_get_string (remote->options, remote->group, "url", &error); + g_assert_no_error (error); + + return g_steal_pointer (&uri); +} + +/* Add each key from @keys_input to @set iff its value is non-%NULL. */ +static void +add_keys_to_set_if_non_null (GHashTable *set, + GHashTable *keys_input) +{ + GHashTableIter iter; + gpointer key, value; + + g_hash_table_iter_init (&iter, keys_input); + + while (g_hash_table_iter_next (&iter, &key, &value)) + if (value != NULL) + g_hash_table_add (set, key); +} + +static void +get_result_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + GAsyncResult **result_out = user_data; + *result_out = g_object_ref (result); +} + +static void +collection_ref_free0 (OstreeCollectionRef *ref) +{ + if (ref == NULL) + return; + ostree_collection_ref_free (ref); +} + +/* TODO: Add a man page. */ +gboolean +ostree_builtin_find_remotes (int argc, + char **argv, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GOptionContext) context = NULL; + glnx_unref_object OstreeRepo *repo = NULL; + g_autoptr(GPtrArray) refs = NULL; /* (element-type OstreeCollectionRef) */ + glnx_unref_object OstreeAsyncProgress *progress = NULL; + gsize i; + g_autoptr(GAsyncResult) find_result = NULL; + g_auto(OstreeRepoFinderResultv) results = NULL; + g_auto(GLnxConsoleRef) console = { 0, }; + g_autoptr(GHashTable) refs_found = NULL; /* set (element-type OstreeCollectionRef) */ + + context = g_option_context_new ("COLLECTION-ID REF [COLLECTION-ID REF...] - Find remotes to serve the given refs"); + + /* Parse options. */ + if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) + return FALSE; + + if (!ostree_ensure_repo_writable (repo, error)) + return FALSE; + + if (argc < 3) + { + ot_util_usage_error (context, "At least one COLLECTION-ID REF pair must be specified", error); + return FALSE; + } + + if (argc % 2 == 0) + { + ot_util_usage_error (context, "Only complete COLLECTION-ID REF pairs may be specified", error); + return FALSE; + } + + if (opt_disable_fsync) + ostree_repo_set_disable_fsync (repo, TRUE); + + if (opt_cache_dir && + !ostree_repo_set_cache_dir (repo, AT_FDCWD, opt_cache_dir, cancellable, error)) + return FALSE; + + /* Read in the refs to search for remotes for. */ + refs = g_ptr_array_new_full (argc, (GDestroyNotify) collection_ref_free0); + + for (i = 1; i < argc; i += 2) + { + if (!ostree_validate_collection_id (argv[i], error) || + !ostree_validate_rev (argv[i + 1], error)) + return FALSE; + + g_ptr_array_add (refs, ostree_collection_ref_new (argv[i], argv[i + 1])); + } + + g_ptr_array_add (refs, NULL); + + /* Run the operation. */ + glnx_console_lock (&console); + + if (console.is_tty) + progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); + + /* FIXME: Eventually some command line options for customising the finders + * list would be good. */ + ostree_repo_find_remotes_async (repo, + (const OstreeCollectionRef * const *) refs->pdata, + NULL /* no options */, + NULL /* default finders */, + progress, cancellable, + get_result_cb, &find_result); + + while (find_result == NULL) + g_main_context_iteration (NULL, TRUE); + + results = ostree_repo_find_remotes_finish (repo, find_result, error); + + if (results == NULL) + return FALSE; + + if (progress) + ostree_async_progress_finish (progress); + + /* Print results and work out which refs were not found. */ + refs_found = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, NULL, NULL); + + for (i = 0; results[i] != NULL; i++) + { + g_autofree gchar *uri = NULL; + g_autofree gchar *refs_string = NULL; + g_autofree gchar *last_modified_string = NULL; + + uri = remote_get_uri (results[i]->remote); + refs_string = format_ref_to_checksum (results[i]->ref_to_checksum, " "); + add_keys_to_set_if_non_null (refs_found, results[i]->ref_to_checksum); + + if (results[i]->summary_last_modified > 0) + last_modified_string = uint64_secs_to_iso8601 (results[i]->summary_last_modified); + else + last_modified_string = g_strdup ("unknown"); + + g_print ("Result %" G_GSIZE_FORMAT ": %s\n" + " - Finder: %s\n" + " - Keyring: %s\n" + " - Priority: %d\n" + " - Summary last modified: %s\n" + " - Refs:\n" + "%s\n", + i, uri, G_OBJECT_TYPE_NAME (results[i]->finder), results[i]->remote->keyring, + results[i]->priority, last_modified_string, refs_string); + } + + if (results[0] == NULL) + { + g_print ("No results.\n"); + return TRUE; + } + + g_print ("%u/%u refs were found.\n", g_hash_table_size (refs_found), refs->len - 1); + + /* Print out the refs which weren’t found. */ + if (g_hash_table_size (refs_found) != refs->len - 1 /* NULL terminator */) + { + g_print ("Refs not found in any remote:\n"); + + for (i = 0; i < refs->len && refs->pdata[i] != NULL; i++) + { + const OstreeCollectionRef *ref = g_ptr_array_index (refs, i); + if (!g_hash_table_contains (refs_found, ref)) + g_print (" - (%s, %s)\n", ref->collection_id, ref->ref_name); + } + } + + return TRUE; +} diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index 1c862925..9c648458 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -22,6 +22,8 @@ #pragma once +#include "config.h" + #include "ostree.h" G_BEGIN_DECLS @@ -36,6 +38,9 @@ BUILTINPROTO(checksum); BUILTINPROTO(commit); BUILTINPROTO(diff); BUILTINPROTO(export); +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +BUILTINPROTO(find_remotes); +#endif BUILTINPROTO(gpg_sign); BUILTINPROTO(init); BUILTINPROTO(log); From 37fc49f36da045348c12281878c296b2d44835f5 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 4 May 2017 10:47:23 +0100 Subject: [PATCH 29/82] find-remotes: Add pull support to the find-remotes built-in command This will pull the remotes after finding them. This potentially needs to go in its own pull-from-remotes built-in command, but it will be fine here for now. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- src/ostree/ot-builtin-find-remotes.c | 30 +++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-find-remotes.c b/src/ostree/ot-builtin-find-remotes.c index 8130ec35..344febb5 100644 --- a/src/ostree/ot-builtin-find-remotes.c +++ b/src/ostree/ot-builtin-find-remotes.c @@ -32,11 +32,13 @@ static gchar *opt_cache_dir = NULL; static gboolean opt_disable_fsync = FALSE; +static gboolean opt_pull = FALSE; static GOptionEntry options[] = { { "cache-dir", 0, 0, G_OPTION_ARG_FILENAME, &opt_cache_dir, "Use custom cache dir", NULL }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, + { "pull", 0, 0, G_OPTION_ARG_NONE, &opt_pull, "Pull the updates after finding them", NULL }, { NULL } }; @@ -127,7 +129,7 @@ ostree_builtin_find_remotes (int argc, g_autoptr(GPtrArray) refs = NULL; /* (element-type OstreeCollectionRef) */ glnx_unref_object OstreeAsyncProgress *progress = NULL; gsize i; - g_autoptr(GAsyncResult) find_result = NULL; + g_autoptr(GAsyncResult) find_result = NULL, pull_result = NULL; g_auto(OstreeRepoFinderResultv) results = NULL; g_auto(GLnxConsoleRef) console = { 0, }; g_autoptr(GHashTable) refs_found = NULL; /* set (element-type OstreeCollectionRef) */ @@ -251,5 +253,31 @@ ostree_builtin_find_remotes (int argc, } } + /* Does the user want us to pull the updates? */ + if (!opt_pull) + return TRUE; + + /* Run the pull operation. */ + if (console.is_tty) + progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); + + ostree_repo_pull_from_remotes_async (repo, + (const OstreeRepoFinderResult * const *) results, + NULL, /* no options */ + progress, cancellable, + get_result_cb, &pull_result); + + while (pull_result == NULL) + g_main_context_iteration (NULL, TRUE); + + if (!ostree_repo_pull_from_remotes_finish (repo, pull_result, error)) + return FALSE; + + if (progress) + ostree_async_progress_finish (progress); + + /* The pull operation fails if any of the refs can’t be pulled. */ + g_print ("Pulled %u/%u refs successfully.\n", refs->len - 1, refs->len - 1); + return TRUE; } From 144e325579769c1af36cd135d8fa568327418538 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:48:35 +0100 Subject: [PATCH 30/82] init: Add a --collection-id argument to the built-in init command This allows new repositories to be configured with a collection ID which can be used to uniquely identify refs which originated from this repository. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- man/ostree-init.xml | 26 ++++++++++++++++++++++++++ src/ostree/ot-builtin-init.c | 11 +++++++++++ 2 files changed, 37 insertions(+) diff --git a/man/ostree-init.xml b/man/ostree-init.xml index 95f0b162..fb35c0da 100644 --- a/man/ostree-init.xml +++ b/man/ostree-init.xml @@ -71,6 +71,32 @@ Boston, MA 02111-1307, USA. Initialize repository in given mode (bare, bare-user, archive-z2). Default is "bare". + + + =COLLECTION-ID + + Set the collection ID of the repository. Remotes in clones + of this repository must configure the same value in order to + pull refs which originated in this repository over peer to + peer. + + This collection ID must be persistent and globally + unique. It is formatted as a reverse DNS name (like a D-Bus + interface). It must be set to a reverse DNS domain under your + control. + + This option may be omitted (the default) to leave + peer to peer distribution unsupported for the repository. A + collection ID may be added to an existing repository in + future to enable peer to peer distribution from that point + onwards. + + If the collection ID is changed for the repository + in future, peer to peer distribution of refs from the + repository will break for all peers who do not update their + remote configuration to the new collection ID. + + diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c index 8180a8ae..0dabd458 100644 --- a/src/ostree/ot-builtin-init.c +++ b/src/ostree/ot-builtin-init.c @@ -27,9 +27,16 @@ #include "ostree.h" static char *opt_mode = "bare"; +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +static char *opt_collection_id = NULL; +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ static GOptionEntry options[] = { { "mode", 0, 0, G_OPTION_ARG_STRING, &opt_mode, "Initialize repository in given mode (bare, archive-z2)", NULL }, +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + { "collection-id", 0, 0, G_OPTION_ARG_STRING, &opt_collection_id, + "Globally unique ID for this repository as an collection of refs for redistribution to other repositories", "COLLECTION-ID" }, +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ { NULL } }; @@ -48,6 +55,10 @@ ostree_builtin_init (int argc, char **argv, GCancellable *cancellable, GError ** if (!ostree_repo_mode_from_string (opt_mode, &mode, error)) goto out; +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + if (!ostree_repo_set_collection_id (repo, opt_collection_id, error)) + goto out; +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ if (!ostree_repo_create (repo, mode, NULL, error)) goto out; From 205a84b36e44808b26ccecbf490e58843548d40c Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:50:51 +0100 Subject: [PATCH 31/82] remote-add: Add a --collection-id argument to the built-in add command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows newly configured remotes to have their collection ID specified, so that refs from them can be downloaded from peers as well as the upstream collection, using the remote’s configuration. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- man/ostree-remote.xml | 10 ++++++++++ src/ostree/ot-remote-builtin-add.c | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/man/ostree-remote.xml b/man/ostree-remote.xml index 88e61ac0..2bb8aa71 100644 --- a/man/ostree-remote.xml +++ b/man/ostree-remote.xml @@ -137,6 +137,16 @@ Boston, MA 02111-1307, USA. ostree remote gpg-import --keyring=FILE. + + + =COLLECTION-ID + + + Set the collection ID for the remote to a value provided by + the repository owner, which allows refs from this remote to be + shared peer to peer. + + diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index 3e3aeda9..db115efd 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -31,6 +31,9 @@ static gboolean opt_no_gpg_verify; static gboolean opt_if_not_exists; static char *opt_gpg_import; static char *opt_contenturl; +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +static char *opt_collection_id; +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ static GOptionEntry option_entries[] = { { "set", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_set, "Set config option KEY=VALUE for remote", "KEY=VALUE" }, @@ -38,6 +41,10 @@ static GOptionEntry option_entries[] = { { "if-not-exists", 0, 0, G_OPTION_ARG_NONE, &opt_if_not_exists, "Do nothing if the provided remote exists", NULL }, { "gpg-import", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_import, "Import GPG key from FILE", "FILE" }, { "contenturl", 0, 0, G_OPTION_ARG_STRING, &opt_contenturl, "Use URL when fetching content", "URL" }, +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + { "collection-id", 0, 0, G_OPTION_ARG_STRING, &opt_collection_id, + "Globally unique ID for this repository as an collection of refs for redistribution to other repositories", "COLLECTION-ID" }, +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ { NULL } }; @@ -110,6 +117,12 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError "gpg-verify", g_variant_new_variant (g_variant_new_boolean (FALSE))); +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + if (opt_collection_id != NULL) + g_variant_builder_add (optbuilder, "{s@v}", "collection-id", + g_variant_new_variant (g_variant_new_take_string (g_steal_pointer (&opt_collection_id)))); +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + options = g_variant_ref_sink (g_variant_builder_end (optbuilder)); if (!ostree_repo_remote_change (repo, NULL, From b7b79fa78d34a34cb75aeca4d0b041586ed74224 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:51:58 +0100 Subject: [PATCH 32/82] refs: Add a --collections argument to the built-in refs command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than change the output format used by the existing refs command to output collection IDs in addition to ref names, this functionality has been hidden behind an --collections argument. If it’s not specified `ostree refs` will output the same content as before for a given repository. If it is specified, the collection ID for each ref will be included in the output as (collection ID, ref name). Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- man/ostree-refs.xml | 20 ++++++++ src/ostree/ot-builtin-refs.c | 97 ++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/man/ostree-refs.xml b/man/ostree-refs.xml index 43e934ff..726a94ac 100644 --- a/man/ostree-refs.xml +++ b/man/ostree-refs.xml @@ -83,6 +83,26 @@ Boston, MA 02111-1307, USA. you will then need to ostree prune or ostree admin cleanup. + + + + + + Enable interactions with refs using the combination of their + collection IDs and ref names. When listing refs, this changes + the output format to include collection IDs, and enables + listing remote mirrored refs. + + When creating refs, the refspec value passed to the + option is treated as + COLLECTION-ID:REF-NAME and a mirrored ref + is created. (This is an abuse of the refspec syntax.) + + When deleting refs, all refs whose collection ID equals + the value of the argument are + deleted. + + diff --git a/src/ostree/ot-builtin-refs.c b/src/ostree/ot-builtin-refs.c index c7ea9a95..19420842 100644 --- a/src/ostree/ot-builtin-refs.c +++ b/src/ostree/ot-builtin-refs.c @@ -29,14 +29,106 @@ static gboolean opt_delete; static gboolean opt_list; static char *opt_create; +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +static gboolean opt_collections; +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ static GOptionEntry options[] = { { "delete", 0, 0, G_OPTION_ARG_NONE, &opt_delete, "Delete refs which match PREFIX, rather than listing them", NULL }, { "list", 0, 0, G_OPTION_ARG_NONE, &opt_list, "Do not remove the prefix from the refs", NULL }, { "create", 0, 0, G_OPTION_ARG_STRING, &opt_create, "Create a new ref for an existing commit", "NEWREF" }, +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + { "collections", 'c', 0, G_OPTION_ARG_NONE, &opt_collections, "Enable listing collection IDs for refs", NULL }, +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ { NULL } }; +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +static gboolean +do_ref_with_collections (OstreeRepo *repo, + const char *refspec_prefix, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GHashTable) refs = NULL; /* (element-type OstreeCollectionRef utf8) */ + GHashTableIter hashiter; + gpointer hashkey, hashvalue; + gboolean ret = FALSE; + + if (!ostree_repo_list_collection_refs (repo, + (!opt_create) ? refspec_prefix : NULL, + &refs, cancellable, error)) + goto out; + + if (!opt_delete && !opt_create) + { + g_hash_table_iter_init (&hashiter, refs); + while (g_hash_table_iter_next (&hashiter, &hashkey, &hashvalue)) + { + const OstreeCollectionRef *ref = hashkey; + g_print ("(%s, %s)\n", ref->collection_id, ref->ref_name); + } + } + else if (opt_create) + { + g_autofree char *checksum = NULL; + g_autofree char *checksum_existing = NULL; + + if (!ostree_repo_resolve_rev_ext (repo, opt_create, TRUE, OSTREE_REPO_RESOLVE_REV_EXT_NONE, &checksum_existing, error)) + { + if (g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_IS_DIRECTORY)) + { + /* A folder exists with the specified ref name, + * which is handled by _ostree_repo_write_ref */ + g_clear_error (error); + } + else goto out; + } + + if (checksum_existing != NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "--create specified but ref %s already exists", opt_create); + goto out; + } + + if (!ostree_repo_resolve_rev (repo, refspec_prefix, FALSE, &checksum, error)) + goto out; + + /* This is technically an abuse of the refspec syntax: collection IDs + * should not be treated like remote names. */ + g_auto(GStrv) parts = g_strsplit (opt_create, ":", 2); + const char *collection_id = parts[0]; + const char *ref_name = parts[1]; + if (!ostree_validate_collection_id (collection_id, error)) + goto out; + if (!ostree_validate_rev (ref_name, error)) + goto out; + + const OstreeCollectionRef ref = { (gchar *) collection_id, (gchar *) ref_name }; + if (!ostree_repo_set_collection_ref_immediate (repo, &ref, checksum, + cancellable, error)) + goto out; + } + else + /* delete */ + { + g_hash_table_iter_init (&hashiter, refs); + while (g_hash_table_iter_next (&hashiter, &hashkey, &hashvalue)) + { + const OstreeCollectionRef *ref = hashkey; + + if (!ostree_repo_set_collection_ref_immediate (repo, ref, NULL, + cancellable, error)) + goto out; + } + } + ret = TRUE; + out: + return ret; +} +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellable *cancellable, GError **error) { g_autoptr(GHashTable) refs = NULL; @@ -44,6 +136,11 @@ static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellab gpointer hashkey, hashvalue; gboolean ret = FALSE; +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + if (opt_collections) + return do_ref_with_collections (repo, refspec_prefix, cancellable, error); +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + if (opt_delete || opt_list) { if (!ostree_repo_list_refs_ext (repo, refspec_prefix, &refs, OSTREE_REPO_LIST_REFS_EXT_NONE, From 18456d25fb1071909b83e203c16ad898cd651f7b Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:53:58 +0100 Subject: [PATCH 33/82] ostree/dump: Include collection IDs and mirrored refs in summary dumps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a repository’s summary file includes a collection ID, output that. If it includes refs from other collections (in the ‘collection map’), output those and include the same metadata detail as for refs in the summary file’s main refs map. If collection IDs are specified in the summary file, this changes the output format from `ostree summary -v` to use (collection ID, ref name) tuples. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- src/ostree/ot-dump.c | 86 ++++++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index e6b8859b..83eb307f 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -176,7 +176,8 @@ ot_dump_object (OstreeObjectType objtype, } static void -dump_summary_ref (const char *ref_name, +dump_summary_ref (const char *collection_id, + const char *ref_name, guint64 commit_size, GVariant *csum_v, GVariantIter *metadata) @@ -187,7 +188,10 @@ dump_summary_ref (const char *ref_name, GVariant *value; char *key; - g_print ("* %s\n", ref_name); + if (collection_id == NULL) + g_print ("* %s\n", ref_name); + else + g_print ("* (%s, %s)\n", collection_id, ref_name); size = g_format_size (commit_size); g_print (" Latest Commit (%s):\n", size); @@ -229,6 +233,39 @@ dump_summary_ref (const char *ref_name, } } +static void +dump_summary_refs (const gchar *collection_id, + GVariant *refs) +{ + GVariantIter iter; + GVariant *value; + + g_variant_iter_init (&iter, refs); + + while ((value = g_variant_iter_next_value (&iter)) != NULL) + { + const char *ref_name = NULL; + + g_variant_get_child (value, 0, "&s", &ref_name); + + if (ref_name != NULL) + { + g_autoptr(GVariant) csum_v = NULL; + g_autoptr(GVariantIter) metadata = NULL; + guint64 commit_size; + + g_variant_get_child (value, 1, "(t@aya{sv})", + &commit_size, &csum_v, &metadata); + + dump_summary_ref (collection_id, ref_name, commit_size, csum_v, metadata); + + g_print ("\n"); + } + + g_variant_unref (value); + } +} + void ot_dump_summary_bytes (GBytes *summary_bytes, OstreeDumpFlags flags) @@ -254,31 +291,26 @@ ot_dump_summary_bytes (GBytes *summary_bytes, refs = g_variant_get_child_value (summary, 0); exts = g_variant_get_child_value (summary, 1); - g_variant_iter_init (&iter, refs); + /* Print the refs, including those with a collection ID specified. */ + const gchar *main_collection_id; + g_autoptr(GVariant) collection_map = NULL; + const gchar *collection_id; - while ((value = g_variant_iter_next_value (&iter)) != NULL) + if (!g_variant_lookup (exts, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id)) + main_collection_id = NULL; + + dump_summary_refs (main_collection_id, refs); + + collection_map = g_variant_lookup_value (exts, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); + if (collection_map != NULL) { - const char *ref_name = NULL; + g_variant_iter_init (&iter, collection_map); - g_variant_get_child (value, 0, "&s", &ref_name); - - if (ref_name != NULL) - { - g_autoptr(GVariant) csum_v = NULL; - g_autoptr(GVariantIter) metadata = NULL; - guint64 commit_size; - - g_variant_get_child (value, 1, "(t@aya{sv})", - &commit_size, &csum_v, &metadata); - - dump_summary_ref (ref_name, commit_size, csum_v, metadata); - - g_print ("\n"); - } - - g_variant_unref (value); + while (g_variant_iter_loop (&iter, "{&s@a(s(taya{sv}))}", &collection_id, &refs)) + dump_summary_refs (collection_id, refs); } + /* Print out the additional metadata. */ g_variant_iter_init (&iter, exts); while (g_variant_iter_loop (&iter, "{sv}", &key, &value)) @@ -301,6 +333,16 @@ ot_dump_summary_bytes (GBytes *summary_bytes, pretty_key = "Expires"; value_str = uint64_secs_to_iso8601 (GUINT64_FROM_BE (g_variant_get_uint64 (value))); } + else if (g_strcmp0 (key, OSTREE_SUMMARY_COLLECTION_ID) == 0) + { + pretty_key = "Collection ID"; + value_str = g_variant_dup_string (value, NULL); + } + else if (g_strcmp0 (key, OSTREE_SUMMARY_COLLECTION_MAP) == 0) + { + pretty_key = "Collection Map"; + value_str = g_strdup ("(printed above)"); + } else { value_str = g_variant_print (value, FALSE); From 3dd4848c9606f3e8d5e4c974f16be5e910769a5f Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 9 Jun 2017 18:45:39 +0100 Subject: [PATCH 34/82] =?UTF-8?q?ostree/builtins:=20Add=20support=20for=20?= =?UTF-8?q?collection=E2=80=93refs=20to=20a=20few=20utilities?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These utilities were not needed for the initial port to support OstreeCollectionRef, so have been delayed a bit and, in some cases, left as FIXME comments for follow up later. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- src/ostree/ot-builtin-fsck.c | 22 ++++++++++++++++++++++ src/ostree/ot-builtin-prune.c | 27 ++++++++++++++++++++++++++- src/ostree/ot-builtin-pull-local.c | 2 ++ src/ostree/ot-builtin-reset.c | 1 + 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 936bdce4..66f5536d 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -241,6 +241,28 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** return glnx_prefix_error (error, "Loading commit for ref %s", refname); } +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + if (!opt_quiet) + g_print ("Validating refs in collections...\n"); + + g_autoptr(GHashTable) all_collection_refs = NULL; /* (element-type OstreeCollectionRef utf8) */ + if (!ostree_repo_list_collection_refs (repo, NULL, &all_collection_refs, + cancellable, error)) + return FALSE; + + g_hash_table_iter_init (&hash_iter, all_collection_refs); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const OstreeCollectionRef *ref = key; + const char *checksum = value; + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, + checksum, &commit, error)) + return glnx_prefix_error (error, "Loading commit for ref (%s, %s)", + ref->collection_id, ref->ref_name); + } +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + if (!opt_quiet) g_print ("Enumerating objects...\n"); diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c index 853c051f..8c595b73 100644 --- a/src/ostree/ot-builtin-prune.c +++ b/src/ostree/ot-builtin-prune.c @@ -50,11 +50,15 @@ static GOptionEntry options[] = { static gboolean delete_commit (OstreeRepo *repo, const char *commit_to_delete, GCancellable *cancellable, GError **error) { - g_autoptr(GHashTable) refs = NULL; + g_autoptr(GHashTable) refs = NULL; /* (element-type utf8 utf8) */ +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + g_autoptr(GHashTable) collection_refs = NULL; /* (element-type OstreeCollectionRef utf8) */ +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ GHashTableIter hashiter; gpointer hashkey, hashvalue; gboolean ret = FALSE; + /* Check refs which are not in a collection. */ if (!ostree_repo_list_refs (repo, NULL, &refs, cancellable, error)) goto out; @@ -71,6 +75,26 @@ delete_commit (OstreeRepo *repo, const char *commit_to_delete, GCancellable *can } } +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + /* And check refs which *are* in a collection. */ + if (!ostree_repo_list_collection_refs (repo, NULL, &collection_refs, cancellable, error)) + goto out; + + g_hash_table_iter_init (&hashiter, collection_refs); + while (g_hash_table_iter_next (&hashiter, &hashkey, &hashvalue)) + { + const OstreeCollectionRef *ref = hashkey; + const char *commit = hashvalue; + if (g_strcmp0 (commit_to_delete, commit) == 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Commit '%s' is referenced by (%s, %s)", + commit_to_delete, ref->collection_id, ref->ref_name); + goto out; + } + } +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + if (!ot_enable_tombstone_commits (repo, error)) goto out; @@ -249,6 +273,7 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError * } /* We start from the refs */ + /* FIXME: Do we also want to look at ostree_repo_list_collection_refs()? */ if (!ostree_repo_list_refs (repo, NULL, &all_refs, cancellable, error)) return FALSE; diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index b19a4c6a..66e189bc 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -111,6 +111,8 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr if (!ostree_repo_open (src_repo, cancellable, error)) goto out; + /* FIXME: This should grow support for pulling refs from refs/mirrors on + * a local repository, using ostree_repo_list_collection_refs(). */ if (!ostree_repo_list_refs (src_repo, NULL, &refs_to_clone, cancellable, error)) goto out; diff --git a/src/ostree/ot-builtin-reset.c b/src/ostree/ot-builtin-reset.c index cab579ae..344d692c 100644 --- a/src/ostree/ot-builtin-reset.c +++ b/src/ostree/ot-builtin-reset.c @@ -45,6 +45,7 @@ ostree_builtin_reset (int argc, const char *target = NULL; g_autofree char *checksum = NULL; + /* FIXME: Add support for collection–refs. */ context = g_option_context_new ("REF COMMIT - Reset a REF to a previous COMMIT"); if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) From 11ad3ec5293dbcd6241cdb3220e62785b07b9290 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 13 Jun 2017 18:55:53 +0100 Subject: [PATCH 35/82] tests: Add integration tests for collections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test various pieces of core and command line utility functionality relating to the newly-introduced concept of collections. Mostly focussed around the find-remotes utility, and around handling of collection–refs. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- Makefile-tests.am | 12 ++ tests/libtest.sh | 12 ++ tests/test-find-remotes.sh | 229 +++++++++++++++++++++++++++ tests/test-fsck-collections.sh | 72 +++++++++ tests/test-init-collections.sh | 33 ++++ tests/test-prune-collections.sh | 79 +++++++++ tests/test-refs-collections.sh | 125 +++++++++++++++ tests/test-remote-add-collections.sh | 35 ++++ tests/test-summary-collections.sh | 58 +++++++ 9 files changed, 655 insertions(+) create mode 100755 tests/test-find-remotes.sh create mode 100755 tests/test-fsck-collections.sh create mode 100755 tests/test-init-collections.sh create mode 100755 tests/test-prune-collections.sh create mode 100755 tests/test-refs-collections.sh create mode 100755 tests/test-remote-add-collections.sh create mode 100755 tests/test-summary-collections.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index c21e29f7..cb775320 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -111,6 +111,18 @@ _installed_or_uninstalled_test_scripts = \ tests/test-summary-view.sh \ $(NULL) +if ENABLE_EXPERIMENTAL_API +_installed_or_uninstalled_test_scripts += \ + tests/test-find-remotes.sh \ + tests/test-fsck-collections.sh \ + tests/test-init-collections.sh \ + tests/test-prune-collections.sh \ + tests/test-refs-collections.sh \ + tests/test-remote-add-collections.sh \ + tests/test-summary-collections.sh \ + $(NULL) +endif + if BUILDOPT_FUSE _installed_or_uninstalled_test_scripts += tests/test-rofiles-fuse.sh else diff --git a/tests/libtest.sh b/tests/libtest.sh index 15802dfd..ee3f4af8 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -537,3 +537,15 @@ ostree_file_path_to_object_path() { relpath=$(ostree_file_path_to_relative_object_path $repo $ref $path) echo ${repo}/${relpath} } + +# Assert ref $2 in repo $1 has checksum $3. +assert_ref () { + assert_streq $(${CMD_PREFIX} ostree rev-parse --repo=$1 $2) $3 +} + +# Assert no ref named $2 is present in repo $1. +assert_not_ref () { + if ${CMD_PREFIX} ostree rev-parse --repo=$1 $2 2>/dev/null; then + fatal "rev-parse $2 unexpectedly succeeded!" + fi +} diff --git a/tests/test-find-remotes.sh b/tests/test-find-remotes.sh new file mode 100755 index 00000000..0b887664 --- /dev/null +++ b/tests/test-find-remotes.sh @@ -0,0 +1,229 @@ +#!/bin/bash +# +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..1' + +# Create two upstream collection repositories with some example commits +cd ${test_tmpdir} + +mkdir apps-collection +ostree_repo_init apps-collection --collection-id org.example.AppsCollection +mkdir -p files +pushd files +${CMD_PREFIX} ostree --repo=../apps-collection commit -s "Test apps-collection commit 1" -b app1 --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1} > ../app1-checksum +${CMD_PREFIX} ostree --repo=../apps-collection commit -s "Test apps-collection commit 2" -b app2 --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1} > ../app2-checksum +popd +${CMD_PREFIX} ostree --repo=apps-collection summary --update --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1} + +mkdir os-collection +ostree_repo_init os-collection --collection-id org.example.OsCollection +mkdir -p files +pushd files +${CMD_PREFIX} ostree --repo=../os-collection commit -s "Test os-collection commit 1" -b os/amd64/master --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_2} > ../os-checksum +popd +${CMD_PREFIX} ostree --repo=os-collection summary --update --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_2} + +# Create a local repository where we pull the branches from the two remotes as normal, using GPG. +mkdir local +ostree_repo_init local +${CMD_PREFIX} ostree --repo=local remote add apps-remote file://$(pwd)/apps-collection --collection-id org.example.AppsCollection --gpg-import=${test_tmpdir}/gpghome/key1.asc +${CMD_PREFIX} ostree --repo=local remote add os-remote file://$(pwd)/os-collection --collection-id org.example.OsCollection --gpg-import=${test_tmpdir}/gpghome/key2.asc + +${CMD_PREFIX} ostree --repo=local pull apps-remote app1 +${CMD_PREFIX} ostree --repo=local pull os-remote os/amd64/master + +${CMD_PREFIX} ostree --repo=local refs > refs +assert_file_has_content refs "^apps-remote:app1$" +assert_file_has_content refs "^os-remote:os/amd64/master$" + +${CMD_PREFIX} ostree --repo=local refs --collections | wc -l > refscount +assert_file_has_content refscount "^0$" + +# Create a local mirror repository where we pull the branches *in mirror mode* from the two remotes. +# This should pull them into refs/mirrors, since the remotes advertise a collection ID. +mkdir local-mirror +ostree_repo_init local-mirror +${CMD_PREFIX} ostree --repo=local-mirror remote add apps-remote file://$(pwd)/apps-collection --collection-id org.example.AppsCollection --gpg-import=${test_tmpdir}/gpghome/key1.asc +${CMD_PREFIX} ostree --repo=local-mirror remote add os-remote file://$(pwd)/os-collection --collection-id org.example.OsCollection --gpg-import=${test_tmpdir}/gpghome/key2.asc + +${CMD_PREFIX} ostree --repo=local-mirror pull --mirror apps-remote app1 +${CMD_PREFIX} ostree --repo=local-mirror pull --mirror os-remote os/amd64/master + +${CMD_PREFIX} ostree --repo=local-mirror refs | wc -l > refscount +assert_file_has_content refscount "^0$" + +${CMD_PREFIX} ostree --repo=local-mirror refs --collections > refs +assert_file_has_content refs "^(org.example.AppsCollection, app1)$" +assert_file_has_content refs "^(org.example.OsCollection, os/amd64/master)$" + +for repo in local local-mirror; do + # Try finding an update for an existing branch. + ${CMD_PREFIX} ostree --repo=$repo find-remotes org.example.AppsCollection app1 > find + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/apps-collection$" + assert_file_has_content find "^ - Keyring: apps-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.AppsCollection, app1) = $(cat app1-checksum)$" + assert_file_has_content find "^1/1 refs were found.$" + assert_not_file_has_content find "^No results.$" + + # Find several updates for several existing branches. + ${CMD_PREFIX} ostree --repo=$repo find-remotes org.example.AppsCollection app1 org.example.OsCollection os/amd64/master > find + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/apps-collection$" + assert_file_has_content find "^ - Keyring: apps-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.AppsCollection, app1) = $(cat app1-checksum)$" + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/os-collection$" + assert_file_has_content find "^ - Keyring: os-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.OsCollection, os/amd64/master) = $(cat os-checksum)$" + assert_file_has_content find "^2/2 refs were found.$" + assert_not_file_has_content find "^No results.$" + + # Find some updates and a new branch. + ${CMD_PREFIX} ostree --repo=$repo find-remotes org.example.AppsCollection app1 org.example.AppsCollection app2 org.example.OsCollection os/amd64/master > find + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/apps-collection$" + assert_file_has_content find "^ - Keyring: apps-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.AppsCollection, app1) = $(cat app1-checksum)$" + assert_file_has_content find "^ - (org.example.AppsCollection, app2) = $(cat app2-checksum)$" + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/os-collection$" + assert_file_has_content find "^ - Keyring: os-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.OsCollection, os/amd64/master) = $(cat os-checksum)$" + assert_file_has_content find "^3/3 refs were found.$" + assert_not_file_has_content find "^No results.$" + + # Find an update and a non-existent branch. + ${CMD_PREFIX} ostree --repo=$repo find-remotes org.example.AppsCollection app1 org.example.AppsCollection not-an-app > find + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/apps-collection$" + assert_file_has_content find "^ - Keyring: apps-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.AppsCollection, not-an-app) = (not found)$" + assert_file_has_content find "^ - (org.example.AppsCollection, app1) = $(cat app1-checksum)$" + assert_file_has_content find "^Refs not found in any remote:$" + assert_file_has_content find "^ - (org.example.AppsCollection, not-an-app)$" + assert_file_has_content find "^1/2 refs were found.$" + assert_not_file_has_content find "^No results.$" + + # Do all the above, but pull this time. + ${CMD_PREFIX} ostree --repo=$repo find-remotes --pull org.example.AppsCollection app1 > pull || true + assert_file_has_content pull "^1/1 refs were found.$" + assert_file_has_content pull "^Pulled 1/1 refs successfully.$" + assert_not_file_has_content pull "Failed to pull some refs from the remotes" + assert_ref $repo app1 $(cat app1-checksum) + + ${CMD_PREFIX} ostree --repo=$repo find-remotes --pull org.example.AppsCollection app1 org.example.OsCollection os/amd64/master > pull + assert_file_has_content pull "^2/2 refs were found.$" + assert_file_has_content pull "^Pulled 2/2 refs successfully.$" + assert_not_file_has_content pull "Failed to pull some refs from the remotes" + assert_ref $repo app1 $(cat app1-checksum) + assert_ref $repo os/amd64/master $(cat os-checksum) + + ${CMD_PREFIX} ostree --repo=$repo find-remotes --pull org.example.AppsCollection app1 org.example.AppsCollection app2 org.example.OsCollection os/amd64/master > pull + assert_file_has_content pull "^3/3 refs were found.$" + assert_file_has_content pull "^Pulled 3/3 refs successfully.$" + assert_not_file_has_content pull "Failed to pull some refs from the remotes" + assert_ref $repo app1 $(cat app1-checksum) + assert_ref $repo app2 $(cat app2-checksum) + assert_ref $repo os/amd64/master $(cat os-checksum) + + ${CMD_PREFIX} ostree --repo=$repo find-remotes --pull org.example.AppsCollection app1 org.example.AppsCollection not-an-app > pull + assert_file_has_content pull "^1/2 refs were found.$" + assert_not_file_has_content pull "Failed to pull some refs from the remotes" + assert_ref $repo app1 $(cat app1-checksum) + assert_not_ref $repo not-an-app +done + +# Test pulling a new commit into the local mirror from one of the repositories. +pushd files +${CMD_PREFIX} ostree --repo=../os-collection commit -s "Test os-collection commit 2" -b os/amd64/master --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_2} > ../os-checksum-2 +popd +${CMD_PREFIX} ostree --repo=os-collection summary --update --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_2} + +for repo in local-mirror; do + # Try finding an update for that branch. + ${CMD_PREFIX} ostree --repo=$repo find-remotes org.example.OsCollection os/amd64/master > find + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/os-collection$" + assert_file_has_content find "^ - Keyring: os-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.OsCollection, os/amd64/master) = $(cat os-checksum-2)$" + assert_file_has_content find "^1/1 refs were found.$" + assert_not_file_has_content find "^No results.$" + + # Pull it. + ${CMD_PREFIX} ostree --repo=$repo find-remotes --pull org.example.OsCollection os/amd64/master > pull || true + assert_file_has_content pull "^1/1 refs were found.$" + assert_file_has_content pull "^Pulled 1/1 refs successfully.$" + assert_not_file_has_content pull "Failed to pull some refs from the remotes" + assert_ref $repo os/amd64/master $(cat os-checksum-2) +done + +# Add the local mirror to the local repository as a remote, so that the local repo +# has two configured remotes for the os-collection. Ensure its summary is up to date first. +#${CMD_PREFIX} ostree --repo=local-mirror summary --update +# FIXME: This `cp` can be changed to `ostree summary --update` once PR #946 lands. +# Prior to that, we need to preserve the signatures. +cp os-collection/summary{,.sig} local-mirror/ +${CMD_PREFIX} ostree --repo=local remote add os-remote-local-mirror file://$(pwd)/local-mirror --collection-id org.example.OsCollection --gpg-import=${test_tmpdir}/gpghome/key2.asc + +for repo in local; do + # Try finding an update for that branch. + ${CMD_PREFIX} ostree --repo=$repo find-remotes org.example.OsCollection os/amd64/master > find + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/os-collection$" + assert_file_has_content find "^ - Keyring: os-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.OsCollection, os/amd64/master) = $(cat os-checksum-2)$" + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/local-mirror$" + assert_file_has_content find "^ - Keyring: os-remote-local-mirror.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.OsCollection, os/amd64/master) = $(cat os-checksum-2)$" + assert_file_has_content find "^1/1 refs were found.$" + assert_not_file_has_content find "^No results.$" + + # Pull it. + ${CMD_PREFIX} ostree --repo=$repo find-remotes --pull org.example.OsCollection os/amd64/master > pull || true + assert_file_has_content pull "^1/1 refs were found.$" + assert_file_has_content pull "^Pulled 1/1 refs successfully.$" + assert_not_file_has_content pull "Failed to pull some refs from the remotes" + assert_ref $repo os/amd64/master $(cat os-checksum-2) +done + +# Add another commit to the OS collection, but don’t update the mirror. Then try pulling +# into the local repository again, and check that the outdated ref in the mirror is ignored. +pushd files +${CMD_PREFIX} ostree --repo=../os-collection commit -s "Test os-collection commit 3" -b os/amd64/master --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_2} > ../os-checksum-3 +popd +${CMD_PREFIX} ostree --repo=os-collection summary --update --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_2} + +for repo in local; do + # Try finding an update for that branch. + ${CMD_PREFIX} ostree --repo=$repo find-remotes org.example.OsCollection os/amd64/master > find + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/os-collection$" + assert_file_has_content find "^ - Keyring: os-remote.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.OsCollection, os/amd64/master) = $(cat os-checksum-3)$" + assert_file_has_content find "^Result [0-9]\+: file://$(pwd)/local-mirror$" + assert_file_has_content find "^ - Keyring: os-remote-local-mirror.trustedkeys.gpg$" + assert_file_has_content find "^ - (org.example.OsCollection, os/amd64/master) = $(cat os-checksum-3)$" + assert_file_has_content find "^1/1 refs were found.$" + assert_not_file_has_content find "^No results.$" + + # Pull it. + ${CMD_PREFIX} ostree --repo=$repo find-remotes --pull org.example.OsCollection os/amd64/master > pull || true + assert_file_has_content pull "^1/1 refs were found.$" + assert_file_has_content pull "^Pulled 1/1 refs successfully.$" + assert_not_file_has_content pull "Failed to pull some refs from the remotes" + assert_ref $repo os/amd64/master $(cat os-checksum-3) +done + +echo "ok find-remotes" diff --git a/tests/test-fsck-collections.sh b/tests/test-fsck-collections.sh new file mode 100755 index 00000000..5de17bcf --- /dev/null +++ b/tests/test-fsck-collections.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..2' + +cd ${test_tmpdir} + +# Check that fsck detects errors with refs which have collection IDs (i.e. refs in refs/mirrors). +set_up_repo() { + rm -rf repo files + + mkdir repo + ostree_repo_init repo + + mkdir files + pushd files + ${CMD_PREFIX} ostree --repo=../repo commit -s "Commit 1" -b original-ref > ../original-ref-checksum + popd + ${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.Collection:some-ref $(cat original-ref-checksum) +} + +set_up_repo + +# fsck at this point should succeed +${CMD_PREFIX} ostree fsck --repo=repo > fsck +assert_file_has_content fsck "^Validating refs in collections...$" + +# Drop the commit the ref points to, and drop the original ref so that fsck doesn’t prematurely fail on that. +find repo/objects -name '*.commit' -delete -print | wc -l > commitcount +assert_file_has_content commitcount "^1$" + +rm repo/refs/heads/original-ref + +# fsck should now fail +if ${CMD_PREFIX} ostree fsck --repo=repo > fsck; then + assert_not_reached "fsck unexpectedly succeeded after deleting commit!" +fi +assert_file_has_content fsck "^Validating refs...$" +assert_file_has_content fsck "^Validating refs in collections...$" + +echo "ok 1 fsck-collections" + +# Try fsck in an old repository where refs/mirrors doesn’t exist to begin with. +# It should succeed. +set_up_repo +rm -rf repo/refs/mirrors + +${CMD_PREFIX} ostree fsck --repo=repo > fsck +assert_file_has_content fsck "^Validating refs...$" +assert_file_has_content fsck "^Validating refs in collections...$" + +echo "ok 2 fsck-collections in old repository" diff --git a/tests/test-init-collections.sh b/tests/test-init-collections.sh new file mode 100755 index 00000000..fe78fc97 --- /dev/null +++ b/tests/test-init-collections.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..1' + +cd ${test_tmpdir} + +# Check that initialising a repository with a collection ID results in the ID being in the config. +mkdir repo +ostree_repo_init repo --collection-id org.example.Collection +assert_file_has_content repo/config "^collection-id=org.example.Collection$" + +echo "ok init-collections" diff --git a/tests/test-prune-collections.sh b/tests/test-prune-collections.sh new file mode 100755 index 00000000..823d2805 --- /dev/null +++ b/tests/test-prune-collections.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..2' + +cd ${test_tmpdir} + +# Check that refs with collection IDs (i.e. refs in refs/mirrors) are taken into account +# when computing reachability of objects while pruning. +set_up_repo() { + rm -rf repo files + + mkdir repo + ostree_repo_init repo + + mkdir files + pushd files + ${CMD_PREFIX} ostree --repo=../repo commit -s "Commit 1" -b original-ref > ../original-ref-checksum + popd + ${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.Collection:some-ref $(cat original-ref-checksum) +} + +set_up_repo + +# Try deleting a specific commit which is still pointed to by both refs. +if ${CMD_PREFIX} ostree --repo=repo prune --delete-commit=$(cat original-ref-checksum) 2>/dev/null; then + assert_not_reached "prune unexpectedly succeeded in deleting a referenced commit!" +fi + +# Pruning normally should do nothing. +${CMD_PREFIX} ostree --repo=repo prune --refs-only > prune +assert_file_has_content prune "^Total objects: 3$" +assert_file_has_content prune "^No unreachable objects$" + +# Remove the original-ref so that only the some-ref with a collection ID points to the commit. +${CMD_PREFIX} ostree --repo=repo refs --delete original-ref + +${CMD_PREFIX} ostree --repo=repo prune --refs-only > prune +assert_file_has_content prune "^Total objects: 3$" +assert_file_has_content prune "^No unreachable objects$" + +# Remove the second ref so that the commit is now orphaned. +${CMD_PREFIX} ostree --repo=repo refs --collections --delete org.example.Collection + +${CMD_PREFIX} ostree --repo=repo prune --refs-only > prune +assert_file_has_content prune "^Total objects: 3$" +assert_file_has_content prune "^Deleted 3 objects, [0-9]\+ bytes freed$" + +echo "ok 1 prune-collections" + +# Try again, but in an old repository where refs/mirrors doesn’t exist to begin with. +set_up_repo +rm -rf repo/refs/mirrors + +${CMD_PREFIX} ostree --repo=repo prune --refs-only > prune +assert_file_has_content prune "^Total objects: 3$" +assert_file_has_content prune "^No unreachable objects$" + +echo "ok 2 prune-collections in old repository" diff --git a/tests/test-refs-collections.sh b/tests/test-refs-collections.sh new file mode 100755 index 00000000..4d49247d --- /dev/null +++ b/tests/test-refs-collections.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# +# Copyright © 2016 Red Hat, Inc. +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..2' + +cd ${test_tmpdir} +mkdir repo +ostree_repo_init repo --collection-id org.example.Collection + +mkdir -p tree/root +touch tree/root/a + +# Add a few commits +seq 5 | while read i; do + echo a >> tree/root/a + ${CMD_PREFIX} ostree --repo=repo commit --branch=test-$i -m test -s test tree +done + +# The collection IDs should only be listed if --collections is passed. +${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount +assert_file_has_content refscount "^5$" + +${CMD_PREFIX} ostree --repo=repo refs > refs +assert_file_has_content refs "^test\-1$" +assert_file_has_content refs "^test\-5$" +assert_not_file_has_content refs "org.example.Collection" + +${CMD_PREFIX} ostree --repo=repo refs --collections > refs +assert_file_has_content refs "^(org.example.Collection, test-1)$" +assert_file_has_content refs "^(org.example.Collection, test-5)$" + +# Similarly, the collection IDs should only be listed when filtering if --collections is passed. +${CMD_PREFIX} ostree --repo=repo refs --list org.example.Collection | wc -l > refscount +assert_file_has_content refscount "^0$" + +${CMD_PREFIX} ostree --repo=repo refs --collections --list org.example.Collection | wc -l > refscount +assert_file_has_content refscount "^5$" + +# --delete by itself should fail. +${CMD_PREFIX} ostree --repo=repo refs --delete 2>/dev/null || true +${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount.delete1 +assert_file_has_content refscount.delete1 "^5$" + +# Deleting specific refs should work. +${CMD_PREFIX} ostree refs --delete 2>/dev/null && (echo 1>&2 "refs --delete (without prefix) unexpectedly succeeded!"; exit 1) +${CMD_PREFIX} ostree --repo=repo refs --delete test-1 test-2 +${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount.delete2 +assert_file_has_content refscount.delete2 "^3$" +${CMD_PREFIX} ostree --repo=repo refs > refs.delete2 +assert_not_file_has_content refs.delete2 '^test-1$' +assert_not_file_has_content refs.delete2 '^test-2$' +assert_file_has_content refs.delete2 '^test-3$' + +# Deleting by collection ID should only work if --collections is passed. +${CMD_PREFIX} ostree refs --repo=repo --delete org.example.Collection +${CMD_PREFIX} ostree refs --repo=repo | wc -l > refscount.delete3 +assert_file_has_content refscount.delete3 "^3$" + +${CMD_PREFIX} ostree refs --repo=repo --collections --delete org.example.Collection +${CMD_PREFIX} ostree refs --repo=repo | wc -l > refscount.delete4 +assert_file_has_content refscount.delete4 "^0$" + +# Add a few more commits, to test --create +${CMD_PREFIX} ostree --repo=repo commit --branch=ctest -m ctest -s ctest tree + +${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount +assert_file_has_content refscount "^1$" + +# and test mirrored branches +${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.NewCollection:ctest-mirror ctest + +${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount +assert_file_has_content refscount "^1$" +${CMD_PREFIX} ostree --repo=repo refs --collections | wc -l > refscount +assert_file_has_content refscount "^2$" + +${CMD_PREFIX} ostree --repo=repo refs > refs +assert_file_has_content refs "^ctest$" +assert_not_file_has_content refs "^ctest-mirror$" + +${CMD_PREFIX} ostree --repo=repo refs --collections > refs +assert_file_has_content refs "^(org.example.Collection, ctest)$" +assert_file_has_content refs "^(org.example.NewCollection, ctest-mirror)$" + +echo "ok 1 refs collections" + +# Test that listing, creating and deleting refs works from an old repository +# where refs/mirrors doesn’t exist to begin with. +rm -rf repo/refs/mirrors +${CMD_PREFIX} ostree --repo=repo refs + +rm -rf repo/refs/mirrors +${CMD_PREFIX} ostree --repo=repo refs --collections + +rm -rf repo/refs/mirrors +${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.NewCollection:ctest-mirror ctest +${CMD_PREFIX} ostree --repo=repo refs --collections > refs +assert_file_has_content refs "^(org.example.Collection, ctest)$" +assert_file_has_content refs "^(org.example.NewCollection, ctest-mirror)$" + +rm -rf repo/refs/mirrors +${CMD_PREFIX} ostree refs --repo=repo --collections --delete org.example.NonexistentCollection + +echo "ok 2 refs collections in old repository" diff --git a/tests/test-remote-add-collections.sh b/tests/test-remote-add-collections.sh new file mode 100755 index 00000000..23deaa8e --- /dev/null +++ b/tests/test-remote-add-collections.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..1' + +cd ${test_tmpdir} + +# Check that adding a remote with a collection ID results in the ID being in the config. +mkdir repo +ostree_repo_init repo +${CMD_PREFIX} ostree --repo=repo remote add some-remote https://example.com/ --collection-id example-id --gpg-import=${test_tmpdir}/gpghome/key1.asc + +assert_file_has_content repo/config "^collection-id=example-id$" + +echo "ok remote-add-collections" diff --git a/tests/test-summary-collections.sh b/tests/test-summary-collections.sh new file mode 100755 index 00000000..989e63e8 --- /dev/null +++ b/tests/test-summary-collections.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright © 2016 Red Hat, Inc. +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..1' + +cd ${test_tmpdir} +mkdir repo +ostree_repo_init repo --collection-id org.example.Collection + +mkdir -p tree/root +touch tree/root/a + +# Add a few commits +seq 5 | while read i; do + echo a >> tree/root/a + ${CMD_PREFIX} ostree --repo=repo commit --branch=test-$i -m test -s test tree +done + +# Check that they are all listed, with the repository’s collection ID, in the summary. +${CMD_PREFIX} ostree --repo=repo summary --update + +${CMD_PREFIX} ostree --repo=repo summary --view > summary +assert_file_has_content summary "(org.example.Collection, test-1)$" +assert_file_has_content summary "(org.example.Collection, test-2)$" +assert_file_has_content summary "(org.example.Collection, test-3)$" +assert_file_has_content summary "(org.example.Collection, test-4)$" +assert_file_has_content summary "(org.example.Collection, test-5)$" +assert_file_has_content summary "^Collection ID (ostree\.summary\.collection-id): org.example.Collection$" + +# Test that mirrored branches are listed too. +${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.OtherCollection:test-1-mirror test-1 +${CMD_PREFIX} ostree --repo=repo summary --update + +${CMD_PREFIX} ostree --repo=repo summary --view > summary +assert_file_has_content summary "(org.example.OtherCollection, test-1-mirror)$" + +echo "ok summary collections" From d0d5f54ce2b6c47ec80bfb0fce8e31049d39361d Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 23 Jun 2017 15:55:19 +0100 Subject: [PATCH 36/82] lib/refs: Add runtime error checking for collection ID validity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of treating it as a programming error — given that it’s user input, that’s not really appropriate. This modifies write_ref() and list_collection_refs() to implement validation. Signed-off-by: Philip Withnall Closes: #924 Approved by: cgwalters --- src/libostree/ostree-repo-refs.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index 849d5d8f..491f22bd 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -769,6 +769,13 @@ _ostree_repo_write_ref (OstreeRepo *self, g_return_val_if_fail (remote == NULL || ref->collection_id == NULL, FALSE); + if (remote != NULL && !ostree_validate_remote_name (remote, error)) + return FALSE; + if (ref->collection_id != NULL && !ostree_validate_collection_id (ref->collection_id, error)) + return FALSE; + if (!ostree_validate_rev (ref->ref_name, error)) + return FALSE; + if (remote == NULL && (ref->collection_id == NULL || g_strcmp0 (ref->collection_id, ostree_repo_get_collection_id (self)) == 0)) { @@ -928,11 +935,12 @@ ostree_repo_list_collection_refs (OstreeRepo *self, GError **error) { g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); - g_return_val_if_fail (match_collection_id == NULL || - ostree_validate_collection_id (match_collection_id, NULL), FALSE); g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), FALSE); g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + if (match_collection_id != NULL && !ostree_validate_collection_id (match_collection_id, error)) + return FALSE; + g_autoptr(GHashTable) ret_all_refs = NULL; ret_all_refs = g_hash_table_new_full (ostree_collection_ref_hash, From 1147267e4d849d99dbdc4b8635100b08dc089dae Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Jun 2017 17:13:15 -0400 Subject: [PATCH 37/82] lib/commit: Clean up commit file type handling variables The variables here were duplicative; we don't need two booleans to distinguish between symlinks and regular files. What we do need to handle is the "physical" state versus the "object" state. Symlinks objects are stored as regular files in `bare-user` and `archive`. Prep for more cleanup. Closes: #957 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 37 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8d474d63..37e3d7e5 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -614,30 +614,32 @@ write_content_object (OstreeRepo *self, cancellable, error)) return FALSE; - gboolean temp_file_is_regular = g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR; - gboolean temp_file_is_symlink = g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK; - gboolean object_is_symlink = temp_file_is_symlink; + gboolean phys_object_is_symlink = FALSE; + const GFileType object_file_type = g_file_info_get_file_type (file_info); + switch (object_file_type) + { + case G_FILE_TYPE_REGULAR: + break; + case G_FILE_TYPE_SYMBOLIC_LINK: + if (self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + phys_object_is_symlink = TRUE; + break; + default: + return glnx_throw (error, "Unsupported file type %u", object_file_type); + } - if (repo_mode == OSTREE_REPO_MODE_BARE_USER && object_is_symlink) + /* For bare-user, convert the symlink target to the input stream */ + if (repo_mode == OSTREE_REPO_MODE_BARE_USER && object_file_type == G_FILE_TYPE_SYMBOLIC_LINK) { const char *target_str = g_file_info_get_symlink_target (file_info); g_autoptr(GBytes) target = g_bytes_new (target_str, strlen (target_str) + 1); - /* For bare-user we can't store symlinks as symlinks, as symlinks don't - support user xattrs to store the ownership. So, instead store them - as regular files */ - temp_file_is_regular = TRUE; - temp_file_is_symlink = FALSE; - if (file_input != NULL) g_object_unref (file_input); /* Include the terminating zero so we can e.g. mmap this file */ file_input = g_memory_input_stream_new_from_bytes (target); } - if (!(temp_file_is_regular || temp_file_is_symlink)) - return glnx_throw (error, "Unsupported file type %u", g_file_info_get_file_type (file_info)); - /* For regular files, we create them with default mode, and only * later apply any xattrs and setuid bits. The rationale here * is that an attacker on the network with the ability to MITM @@ -654,7 +656,7 @@ write_content_object (OstreeRepo *self, g_autofree char *temp_filename = NULL; gssize unpacked_size = 0; gboolean indexable = FALSE; - if ((_ostree_repo_mode_is_bare (repo_mode)) && temp_file_is_regular) + if ((_ostree_repo_mode_is_bare (repo_mode)) && !phys_object_is_symlink) { guint64 size = g_file_info_get_size (file_info); @@ -664,7 +666,7 @@ write_content_object (OstreeRepo *self, return FALSE; tmp_unlinker.path = temp_filename; } - else if (_ostree_repo_mode_is_bare (repo_mode) && temp_file_is_symlink) + else if (_ostree_repo_mode_is_bare (repo_mode) && phys_object_is_symlink) { /* Note: This will not be hit for bare-user mode because its converted to a regular file and take the branch above */ @@ -690,7 +692,6 @@ write_content_object (OstreeRepo *self, error)) return FALSE; tmp_unlinker.path = temp_filename; - temp_file_is_regular = TRUE; temp_out = g_unix_output_stream_new (temp_fd, FALSE); file_meta = _ostree_zlib_file_header_new (file_info, xattrs); @@ -758,7 +759,7 @@ write_content_object (OstreeRepo *self, const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); if (!commit_loose_content_object (self, actual_checksum, temp_filename, - object_is_symlink, + object_file_type == G_FILE_TYPE_SYMBOLIC_LINK, uid, gid, mode, xattrs, temp_fd, cancellable, error)) @@ -768,7 +769,7 @@ write_content_object (OstreeRepo *self, tmp_unlinker.path = NULL; /* Update size metadata if configured */ - if (indexable && temp_file_is_regular) + if (indexable && object_file_type == G_FILE_TYPE_REGULAR) { struct stat stbuf; From 4dee1984dccace10572192d2f63b7a6fc7d453ee Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 24 Jun 2017 09:30:02 -0400 Subject: [PATCH 38/82] lib: Hoist unlinkat() cleanup API to fsutil, use in pull The pull code also could make use of this in both the metadata and content paths. I changed it to own the tempfile malloc (just like `GLnxTmpFile`), since there's no reason to have different lifetimes for the filename and the file, and that way we only have one variable rather than two. The content path turns out to be a special case though, where at least for mirroring archives, we directly pass the file *path* down into `_ostree_repo_commit_loose_final()`. This is prep for `GLnxTmpFile` porting. Closes: #957 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 35 ++++++------------------------ src/libostree/ostree-repo-pull.c | 34 ++++++++++++++--------------- src/libotutil/ot-fs-utils.h | 28 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 46 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 37e3d7e5..32170ac7 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -567,23 +567,6 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self, return TRUE; } -/* A little helper to call unlinkat() as a cleanup - * function. Mostly only necessary to handle - * deletion of temporary symlinks. - */ -typedef struct { - int dfd; - const char *path; -} CleanupUnlinkat; - -static void -cleanup_unlinkat (CleanupUnlinkat *cleanup) -{ - if (cleanup->path) - (void) unlinkat (cleanup->dfd, cleanup->path, 0); -} -G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(CleanupUnlinkat, cleanup_unlinkat); - /* Write a content object. */ static gboolean write_content_object (OstreeRepo *self, @@ -651,9 +634,8 @@ write_content_object (OstreeRepo *self, * temp_filename might also be a symlink. Hence the CleanupUnlinkat * which handles that case. */ - g_auto(CleanupUnlinkat) tmp_unlinker = { self->tmp_dir_fd, NULL }; + g_auto(OtCleanupUnlinkat) tmp_unlinker = { self->tmp_dir_fd, NULL }; glnx_fd_close int temp_fd = -1; - g_autofree char *temp_filename = NULL; gssize unpacked_size = 0; gboolean indexable = FALSE; if ((_ostree_repo_mode_is_bare (repo_mode)) && !phys_object_is_symlink) @@ -661,10 +643,9 @@ write_content_object (OstreeRepo *self, guint64 size = g_file_info_get_size (file_info); if (!create_regular_tmpfile_linkable_with_content (self, size, file_input, - &temp_fd, &temp_filename, + &temp_fd, &tmp_unlinker.path, cancellable, error)) return FALSE; - tmp_unlinker.path = temp_filename; } else if (_ostree_repo_mode_is_bare (repo_mode) && phys_object_is_symlink) { @@ -672,10 +653,9 @@ write_content_object (OstreeRepo *self, regular file and take the branch above */ if (!_ostree_make_temporary_symlink_at (self->tmp_dir_fd, g_file_info_get_symlink_target (file_info), - &temp_filename, + &tmp_unlinker.path, cancellable, error)) return FALSE; - tmp_unlinker.path = temp_filename; } else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2) { @@ -688,10 +668,9 @@ write_content_object (OstreeRepo *self, indexable = TRUE; if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, - &temp_fd, &temp_filename, + &temp_fd, &tmp_unlinker.path, error)) return FALSE; - tmp_unlinker.path = temp_filename; temp_out = g_unix_output_stream_new (temp_fd, FALSE); file_meta = _ostree_zlib_file_header_new (file_info, xattrs); @@ -758,15 +737,15 @@ write_content_object (OstreeRepo *self, const guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid"); const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); if (!commit_loose_content_object (self, actual_checksum, - temp_filename, + tmp_unlinker.path, object_file_type == G_FILE_TYPE_SYMBOLIC_LINK, uid, gid, mode, xattrs, temp_fd, cancellable, error)) return glnx_prefix_error (error, "Writing object %s.%s", actual_checksum, ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE)); - /* Clear the unlinker path, it was consumed */ - tmp_unlinker.path = NULL; + /* Clear the unlinker, it was consumed */ + ot_cleanup_unlinkat_clear (&tmp_unlinker); /* Update size metadata if configured */ if (indexable && object_file_type == G_FILE_TYPE_REGULAR) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 4c87199a..edfd8d4f 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -910,13 +910,13 @@ content_fetch_on_complete (GObject *object, g_autoptr(GVariant) xattrs = NULL; g_autoptr(GInputStream) file_in = NULL; g_autoptr(GInputStream) object_input = NULL; - g_autofree char *temp_path = NULL; + g_auto(OtCleanupUnlinkat) tmp_unlinker = { _ostree_fetcher_get_dfd (fetcher), NULL }; const char *checksum; g_autofree char *checksum_obj = NULL; OstreeObjectType objtype; gboolean free_fetch_data = TRUE; - if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error)) + if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmp_unlinker.path, error)) goto out; ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype); @@ -941,9 +941,11 @@ content_fetch_on_complete (GObject *object, if (!have_object) { if (!_ostree_repo_commit_loose_final (pull_data->repo, checksum, OSTREE_OBJECT_TYPE_FILE, - _ostree_fetcher_get_dfd (fetcher), -1, temp_path, + tmp_unlinker.dfd, -1, tmp_unlinker.path, cancellable, error)) goto out; + /* The path was consumed */ + ot_cleanup_unlinkat_clear (&tmp_unlinker); } pull_data->n_fetched_content++; } @@ -952,20 +954,16 @@ content_fetch_on_complete (GObject *object, /* Non-mirroring path */ if (!ostree_content_file_parse_at (TRUE, _ostree_fetcher_get_dfd (fetcher), - temp_path, FALSE, + tmp_unlinker.path, FALSE, &file_in, &file_info, &xattrs, cancellable, error)) - { - /* If it appears corrupted, delete it */ - (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0); - goto out; - } + goto out; /* Also, delete it now that we've opened it, we'll hold * a reference to the fd. If we fail to validate or write, then * the temp space will be cleaned up. */ - (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0); + ot_cleanup_unlinkat (&tmp_unlinker); if (!validate_bareuseronly_mode (pull_data, checksum, @@ -1046,7 +1044,7 @@ meta_fetch_on_complete (GObject *object, FetchObjectData *fetch_data = user_data; OtPullData *pull_data = fetch_data->pull_data; g_autoptr(GVariant) metadata = NULL; - g_autofree char *temp_path = NULL; + g_auto(OtCleanupUnlinkat) tmp_unlinker = { _ostree_fetcher_get_dfd (fetcher), NULL }; const char *checksum; g_autofree char *checksum_obj = NULL; OstreeObjectType objtype; @@ -1060,7 +1058,7 @@ meta_fetch_on_complete (GObject *object, g_debug ("fetch of %s%s complete", checksum_obj, fetch_data->is_detached_meta ? " (detached)" : ""); - if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error)) + if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmp_unlinker.path, error)) { if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) { @@ -1103,22 +1101,24 @@ meta_fetch_on_complete (GObject *object, if (objtype == OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT) goto out; - fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC); + fd = openat (_ostree_fetcher_get_dfd (fetcher), tmp_unlinker.path, O_RDONLY | O_CLOEXEC); if (fd == -1) { glnx_set_error_from_errno (error); goto out; } + /* Now delete it, keeping the fd open as the last reference); see comment in + * corresponding content fetch path. + */ + ot_cleanup_unlinkat (&tmp_unlinker); + if (fetch_data->is_detached_meta) { if (!ot_util_variant_map_fd (fd, 0, G_VARIANT_TYPE ("a{sv}"), FALSE, &metadata, error)) goto out; - /* Now delete it, see comment in corresponding content fetch path */ - (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0); - if (!ostree_repo_write_commit_detached_metadata (pull_data->repo, checksum, metadata, pull_data->cancellable, error)) goto out; @@ -1136,8 +1136,6 @@ meta_fetch_on_complete (GObject *object, FALSE, &metadata, error)) goto out; - (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0); - /* Write the commitpartial file now while we're still fetching data */ if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h index 26c5a499..0f62cb7f 100644 --- a/src/libotutil/ot-fs-utils.h +++ b/src/libotutil/ot-fs-utils.h @@ -49,6 +49,34 @@ ot_link_tmpfile_at (OtTmpfile *tmpf, const char *target, GError **error); + +/* A little helper to call unlinkat() as a cleanup + * function. Mostly only necessary to handle + * deletion of temporary symlinks. + */ +typedef struct { + int dfd; + char *path; +} OtCleanupUnlinkat; + +static inline void +ot_cleanup_unlinkat_clear (OtCleanupUnlinkat *cleanup) +{ + g_clear_pointer (&cleanup->path, g_free); +} + +static inline void +ot_cleanup_unlinkat (OtCleanupUnlinkat *cleanup) +{ + if (cleanup->path) + { + (void) unlinkat (cleanup->dfd, cleanup->path, 0); + ot_cleanup_unlinkat_clear (cleanup); + } +} +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OtCleanupUnlinkat, ot_cleanup_unlinkat); + + GFile * ot_fdrel_to_gfile (int dfd, const char *path); gboolean ot_readlinkat_gfile_info (int dfd, From af3a96755ba4d897852fa8f2cf483c0c21adc40b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 24 Jun 2017 10:28:09 -0400 Subject: [PATCH 39/82] lib: Use OtTmpFile for static delta processing The `OstreeRepoContentBareCommit` struct was basically an `OtTmpFile`, so let's make it one. I moved the "convert to `GOutputStream`" logic into the callers, since that bit can't fail; it makes the implementation much simpler since we can just return the result of `ot_open_tmpfile_linkable_at()`. Prep for `GLnxTmpfile` porting. Closes: #957 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 83 +++++++------------ src/libostree/ostree-repo-private.h | 12 +-- src/libostree/ostree-repo-pull.c | 2 +- .../ostree-repo-static-delta-processing.c | 19 +++-- 4 files changed, 45 insertions(+), 71 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 32170ac7..02811c14 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -446,57 +446,39 @@ ot_fallocate (int fd, goffset size, GError **error) return TRUE; } +/* Combines a check for whether or not we already have the object with + * allocating a tempfile if we don't. Used by the static delta code. + */ gboolean _ostree_repo_open_content_bare (OstreeRepo *self, const char *checksum, guint64 content_len, - OstreeRepoContentBareCommit *out_state, - GOutputStream **out_stream, + OtTmpfile *out_tmpf, gboolean *out_have_object, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autofree char *temp_filename = NULL; - g_autoptr(GOutputStream) ret_stream = NULL; gboolean have_obj; - if (!_ostree_repo_has_loose_object (self, checksum, OSTREE_OBJECT_TYPE_FILE, &have_obj, cancellable, error)) - goto out; - - if (!have_obj) - { - int fd; - - if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, - &fd, &temp_filename, error)) - goto out; - - if (!ot_fallocate (fd, content_len, error)) - goto out; - - ret_stream = g_unix_output_stream_new (fd, TRUE); - } - - ret = TRUE; - if (!have_obj) - { - out_state->temp_filename = temp_filename; - temp_filename = NULL; - out_state->fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)ret_stream); - if (out_stream) - *out_stream = g_steal_pointer (&ret_stream); - } + return FALSE; + /* Do we already have this object? */ *out_have_object = have_obj; - out: - return ret; + if (have_obj) + { + /* Make sure the tempfile is unset */ + out_tmpf->initialized = 0; + return TRUE; + } + + return ot_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, + out_tmpf, error); } gboolean _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, const char *checksum, - OstreeRepoContentBareCommit *state, + OtTmpfile *tmpf, guint32 uid, guint32 gid, guint32 mode, @@ -504,25 +486,22 @@ _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; + /* I don't think this is necessary, but a similar check was here previously, + * keeping it for extra redundancy. + */ + if (!tmpf->initialized || tmpf->fd == -1) + return TRUE; - if (state->fd != -1) - { - if (!commit_loose_content_object (self, checksum, - state->temp_filename, - FALSE, uid, gid, mode, - xattrs, state->fd, - cancellable, error)) - { - g_prefix_error (error, "Writing object %s.%s: ", checksum, ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE)); - goto out; - } - } - - ret = TRUE; - out: - g_free (state->temp_filename); - return ret; + if (!commit_loose_content_object (self, checksum, + tmpf->path, + FALSE, uid, gid, mode, + xattrs, tmpf->fd, + cancellable, error)) + return glnx_prefix_error (error, "Writing object %s.%s", checksum, ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE)); + /* The path was consumed */ + g_clear_pointer (&tmpf->path, g_free); + tmpf->initialized = FALSE; + return TRUE; } static gboolean diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 0081eb31..1bd797fa 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -23,7 +23,7 @@ #include "ostree-ref.h" #include "ostree-repo.h" #include "ostree-remote-private.h" -#include "libglnx.h" +#include "otutil.h" G_BEGIN_DECLS @@ -305,17 +305,11 @@ _ostree_repo_commit_loose_final (OstreeRepo *self, GCancellable *cancellable, GError **error); -typedef struct { - int fd; - char *temp_filename; -} OstreeRepoContentBareCommit; - gboolean _ostree_repo_open_content_bare (OstreeRepo *self, const char *checksum, guint64 content_len, - OstreeRepoContentBareCommit *out_state, - GOutputStream **out_stream, + OtTmpfile *out_tmpf, gboolean *out_have_object, GCancellable *cancellable, GError **error); @@ -323,7 +317,7 @@ _ostree_repo_open_content_bare (OstreeRepo *self, gboolean _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, const char *checksum, - OstreeRepoContentBareCommit *state, + OtTmpfile *tmpf, guint32 uid, guint32 gid, guint32 mode, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index edfd8d4f..4607f2f1 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1108,7 +1108,7 @@ meta_fetch_on_complete (GObject *object, goto out; } - /* Now delete it, keeping the fd open as the last reference); see comment in + /* Now delete it, keeping the fd open as the last reference; see comment in * corresponding content fetch path. */ ot_cleanup_unlinkat (&tmp_unlinker); diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index a0f51262..b8bd6587 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -56,7 +56,7 @@ typedef struct { GError **async_error; OstreeObjectType output_objtype; - OstreeRepoContentBareCommit barecommitstate; + OtTmpfile tmpf; guint64 content_size; GOutputStream *content_out; GChecksum *content_checksum; @@ -281,6 +281,8 @@ _ostree_static_delta_part_execute (OstreeRepo *repo, ret = TRUE; out: + ot_tmpfile_clear (&state->tmpf); + g_clear_object (&state->content_out); g_clear_pointer (&state->content_checksum, g_checksum_free); return ret; } @@ -419,8 +421,6 @@ do_content_open_generic (OstreeRepo *repo, if (!read_varuint64 (state, &xattr_offset, error)) goto out; - state->barecommitstate.fd = -1; - modev = g_variant_get_child_value (state->mode_dict, mode_offset); g_variant_get (modev, "(uuu)", &uid, &gid, &mode); state->uid = GUINT32_FROM_BE (uid); @@ -608,14 +608,14 @@ dispatch_open_splice_and_close (OstreeRepo *repo, { if (!_ostree_repo_open_content_bare (repo, state->checksum, state->content_size, - &state->barecommitstate, - &state->content_out, + &state->tmpf, &state->have_obj, cancellable, error)) goto out; if (!state->have_obj) { + state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE); if (!handle_untrusted_content_checksum (repo, state, cancellable, error)) goto out; @@ -711,11 +711,12 @@ dispatch_open (OstreeRepo *repo, if (!_ostree_repo_open_content_bare (repo, state->checksum, state->content_size, - &state->barecommitstate, - &state->content_out, + &state->tmpf, &state->have_obj, cancellable, error)) goto out; + if (!state->have_obj) + state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE); if (!handle_untrusted_content_checksum (repo, state, cancellable, error)) goto out; @@ -899,11 +900,12 @@ dispatch_close (OstreeRepo *repo, } } - if (!_ostree_repo_commit_trusted_content_bare (repo, state->checksum, &state->barecommitstate, + if (!_ostree_repo_commit_trusted_content_bare (repo, state->checksum, &state->tmpf, state->uid, state->gid, state->mode, state->xattrs, cancellable, error)) goto out; + g_clear_object (&state->content_out); } if (!dispatch_unset_read_source (repo, state, cancellable, error)) @@ -911,7 +913,6 @@ dispatch_close (OstreeRepo *repo, g_clear_pointer (&state->xattrs, g_variant_unref); g_clear_pointer (&state->content_checksum, g_checksum_free); - g_clear_object (&state->content_out); state->checksum_index++; state->output_target = NULL; From 5effceeba8657a293216f8a611de45f05566e3f0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 26 Jun 2017 12:30:21 -0400 Subject: [PATCH 40/82] lib/commit: Fix fallocate size for bare-user symlinks We need to account for the trailing NUL. Closes: #957 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 02811c14..916b99d2 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -590,6 +590,8 @@ write_content_object (OstreeRepo *self, return glnx_throw (error, "Unsupported file type %u", object_file_type); } + guint64 size; + /* For bare-user, convert the symlink target to the input stream */ if (repo_mode == OSTREE_REPO_MODE_BARE_USER && object_file_type == G_FILE_TYPE_SYMBOLIC_LINK) { @@ -600,7 +602,12 @@ write_content_object (OstreeRepo *self, g_object_unref (file_input); /* Include the terminating zero so we can e.g. mmap this file */ file_input = g_memory_input_stream_new_from_bytes (target); + size = g_bytes_get_size (target); } + else if (!phys_object_is_symlink) + size = g_file_info_get_size (file_info); + else + size = 0; /* For regular files, we create them with default mode, and only * later apply any xattrs and setuid bits. The rationale here @@ -619,8 +626,6 @@ write_content_object (OstreeRepo *self, gboolean indexable = FALSE; if ((_ostree_repo_mode_is_bare (repo_mode)) && !phys_object_is_symlink) { - guint64 size = g_file_info_get_size (file_info); - if (!create_regular_tmpfile_linkable_with_content (self, size, file_input, &temp_fd, &tmp_unlinker.path, cancellable, error)) From 7871bc3051e02f64c334faf851f2b4878d50aff7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 26 Jun 2017 18:11:40 -0400 Subject: [PATCH 41/82] lib/ref: Suppress more collection ref methods from introspection This squashes some warnings. Closes: #965 Approved by: pwithnall --- src/libostree/ostree-ref.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-ref.h b/src/libostree/ostree-ref.h index 4d43153c..b23af171 100644 --- a/src/libostree/ostree-ref.h +++ b/src/libostree/ostree-ref.h @@ -52,7 +52,6 @@ typedef struct #ifndef __GI_SCANNER__ _OSTREE_PUBLIC GType ostree_collection_ref_get_type (void); -#endif _OSTREE_PUBLIC OstreeCollectionRef *ostree_collection_ref_new (const gchar *collection_id, @@ -61,6 +60,7 @@ _OSTREE_PUBLIC OstreeCollectionRef *ostree_collection_ref_dup (const OstreeCollectionRef *ref); _OSTREE_PUBLIC void ostree_collection_ref_free (OstreeCollectionRef *ref); +#endif _OSTREE_PUBLIC guint ostree_collection_ref_hash (gconstpointer ref); From 21cb4d17155bab066855f1c4b60e3bdefdebfd90 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 26 Jun 2017 18:12:33 -0400 Subject: [PATCH 42/82] ci: Make introspection warnings fatal Closes: #965 Approved by: pwithnall --- .papr.yml | 2 ++ Makefile-libostree.am | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.papr.yml b/.papr.yml index 1addf061..b2eeb972 100644 --- a/.papr.yml +++ b/.papr.yml @@ -15,6 +15,8 @@ env: # Enable all the sanitizers for this primary build. # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2 -Werror=maybe-uninitialized' + # Only for CI with a known g-ir-scanner + GI_SCANNERFLAGS: '--warn-error' ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc # TODO when we're doing leak checks: G_SLICE: "always-malloc" diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 8ff89e52..86c9693f 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -256,7 +256,7 @@ OSTree_1_0_gir_EXPORT_PACKAGES = ostree-1 OSTree_1_0_gir_INCLUDES = Gio-2.0 OSTree_1_0_gir_CFLAGS = $(libostree_1_la_CFLAGS) OSTree_1_0_gir_LIBS = libostree-1.la -OSTree_1_0_gir_SCANNERFLAGS = --warn-all --identifier-prefix=Ostree --symbol-prefix=ostree +OSTree_1_0_gir_SCANNERFLAGS = --warn-all --identifier-prefix=Ostree --symbol-prefix=ostree $(GI_SCANNERFLAGS) OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h %/ostree-repo-finder-avahi.h %/ostree-repo-finder-config.h %/ostree-repo-finder-mount.h,$(libostree_1_la_SOURCES)) INTROSPECTION_GIRS += OSTree-1.0.gir gir_DATA += OSTree-1.0.gir From c2b6afe5b995de7ca776e3694496820200ecd987 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 26 Jun 2017 18:07:44 -0400 Subject: [PATCH 43/82] lib/deltas: Some style porting Just a few functions to keep up momentum. Closes: #964 Approved by: jlebon --- src/libostree/ostree-repo-static-delta-core.c | 255 +++++++----------- 1 file changed, 102 insertions(+), 153 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index d83b7f0f..878d607d 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -165,31 +165,30 @@ _ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; guint8 *checksums_data; - guint i,n_checksums; + guint n_checksums; gboolean have_object = TRUE; if (!_ostree_static_delta_parse_checksum_array (checksum_array, &checksums_data, &n_checksums, error)) - goto out; + return FALSE; - for (i = 0; i < n_checksums; i++) + for (guint i = 0; i < n_checksums; i++) { guint8 objtype = *checksums_data; const guint8 *csum = checksums_data + 1; char tmp_checksum[OSTREE_SHA256_STRING_LEN+1]; if (G_UNLIKELY(!ostree_validate_structureof_objtype (objtype, error))) - goto out; + return FALSE; ostree_checksum_inplace_from_bytes (csum, tmp_checksum); if (!ostree_repo_has_object (repo, (OstreeObjectType) objtype, tmp_checksum, &have_object, cancellable, error)) - goto out; + return FALSE; if (!have_object) break; @@ -197,10 +196,8 @@ _ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo, checksums_data += OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN; } - ret = TRUE; *out_have_all = have_object; - out: - return ret; + return TRUE; } /** @@ -223,57 +220,43 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - guint i, n; - const char *dir_or_file_path = NULL; - glnx_fd_close int meta_fd = -1; - glnx_fd_close int dfd = -1; - g_autoptr(GVariant) meta = NULL; - g_autoptr(GVariant) headers = NULL; - g_autoptr(GVariant) metadata = NULL; - g_autoptr(GVariant) fallback = NULL; - g_autofree char *to_checksum = NULL; - g_autofree char *from_checksum = NULL; g_autofree char *basename = NULL; - dir_or_file_path = gs_file_get_path_cached (dir_or_file); + const char *dir_or_file_path = gs_file_get_path_cached (dir_or_file); /* First, try opening it as a directory */ - dfd = glnx_opendirat_with_errno (AT_FDCWD, dir_or_file_path, TRUE); + glnx_fd_close int dfd = glnx_opendirat_with_errno (AT_FDCWD, dir_or_file_path, TRUE); if (dfd < 0) { if (errno != ENOTDIR) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "openat(O_DIRECTORY)"); else { g_autofree char *dir = dirname (g_strdup (dir_or_file_path)); basename = g_path_get_basename (dir_or_file_path); if (!glnx_opendirat (AT_FDCWD, dir, TRUE, &dfd, error)) - goto out; + return FALSE; } } else basename = g_strdup ("superblock"); - meta_fd = openat (dfd, basename, O_RDONLY | O_CLOEXEC); + glnx_fd_close int meta_fd = openat (dfd, basename, O_RDONLY | O_CLOEXEC); if (meta_fd < 0) - { - glnx_set_error_from_errno (error); - goto out; - } - + return glnx_throw_errno_prefix (error, "openat(%s)", basename); + + g_autoptr(GVariant) meta = NULL; if (!ot_util_variant_map_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), FALSE, &meta, error)) - goto out; + return FALSE; /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */ - metadata = g_variant_get_child_value (meta, 0); + g_autoptr(GVariant) metadata = g_variant_get_child_value (meta, 0); + g_autofree char *to_checksum = NULL; + g_autofree char *from_checksum = NULL; /* Write the to-commit object */ { g_autoptr(GVariant) to_csum_v = NULL; @@ -284,32 +267,28 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, to_csum_v = g_variant_get_child_value (meta, 3); if (!ostree_validate_structureof_csum_v (to_csum_v, error)) - goto out; + return FALSE; to_checksum = ostree_checksum_from_bytes_v (to_csum_v); from_csum_v = g_variant_get_child_value (meta, 2); if (g_variant_n_children (from_csum_v) > 0) { if (!ostree_validate_structureof_csum_v (from_csum_v, error)) - goto out; + return FALSE; from_checksum = ostree_checksum_from_bytes_v (from_csum_v); if (!ostree_repo_has_object (self, OSTREE_OBJECT_TYPE_COMMIT, from_checksum, &have_from_commit, cancellable, error)) - goto out; + return FALSE; if (!have_from_commit) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Commit %s, which is the delta source, is not in repository", from_checksum); - goto out; - } + return glnx_throw (error, "Commit %s, which is the delta source, is not in repository", from_checksum); } if (!ostree_repo_has_object (self, OSTREE_OBJECT_TYPE_COMMIT, to_checksum, &have_to_commit, cancellable, error)) - goto out; - + return FALSE; + if (!have_to_commit) { g_autofree char *detached_path = _ostree_get_relative_static_delta_path (from_checksum, to_checksum, "commitmeta"); @@ -321,27 +300,23 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, detached_data, cancellable, error)) - goto out; + return FALSE; to_commit = g_variant_get_child_value (meta, 4); if (!ostree_repo_write_metadata (self, OSTREE_OBJECT_TYPE_COMMIT, to_checksum, to_commit, NULL, cancellable, error)) - goto out; + return FALSE; } } - fallback = g_variant_get_child_value (meta, 7); + g_autoptr(GVariant) fallback = g_variant_get_child_value (meta, 7); if (g_variant_n_children (fallback) > 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Cannot execute delta offline: contains nonempty http fallback entries"); - goto out; - } + return glnx_throw (error, "Cannot execute delta offline: contains nonempty http fallback entries"); - headers = g_variant_get_child_value (meta, 6); - n = g_variant_n_children (headers); - for (i = 0; i < n; i++) + g_autoptr(GVariant) headers = g_variant_get_child_value (meta, 6); + const guint n = g_variant_n_children (headers); + for (guint i = 0; i < n; i++) { guint32 version; guint64 size; @@ -349,29 +324,21 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, const guchar *csum; char checksum[OSTREE_SHA256_STRING_LEN+1]; gboolean have_all; - g_autoptr(GInputStream) part_in = NULL; - g_autoptr(GVariant) inline_part_data = NULL; - g_autoptr(GVariant) header = NULL; g_autoptr(GVariant) csum_v = NULL; g_autoptr(GVariant) objects = NULL; g_autoptr(GVariant) part = NULL; g_autofree char *deltapart_path = NULL; - OstreeStaticDeltaOpenFlags delta_open_flags = + OstreeStaticDeltaOpenFlags delta_open_flags = skip_validation ? OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM : 0; - - header = g_variant_get_child_value (headers, i); + g_autoptr(GVariant) header = g_variant_get_child_value (headers, i); g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects); if (version > OSTREE_DELTAPART_VERSION) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Delta part has too new version %u", version); - goto out; - } + return glnx_throw (error, "Delta part has too new version %u", version); if (!_ostree_repo_static_delta_part_have_all_objects (self, objects, &have_all, cancellable, error)) - goto out; + return FALSE; /* If we already have these objects, don't bother executing the * static delta. @@ -381,13 +348,14 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, csum = ostree_checksum_bytes_peek_validate (csum_v, error); if (!csum) - goto out; + return FALSE; ostree_checksum_inplace_from_bytes (csum, checksum); deltapart_path = _ostree_get_relative_static_delta_part_path (from_checksum, to_checksum, i); - inline_part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)")); + g_autoptr(GInputStream) part_in = NULL; + g_autoptr(GVariant) inline_part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)")); if (inline_part_data) { g_autoptr(GBytes) inline_part_bytes = g_variant_get_data_as_bytes (inline_part_data); @@ -405,40 +373,31 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, NULL, &part, cancellable, error)) - goto out; + return FALSE; } else { g_autofree char *relpath = g_strdup_printf ("%u", i); /* TODO avoid malloc here */ glnx_fd_close int part_fd = openat (dfd, relpath, O_RDONLY | O_CLOEXEC); if (part_fd < 0) - { - glnx_set_error_from_errno (error); - g_prefix_error (error, "Opening deltapart '%s': ", deltapart_path); - goto out; - } + return glnx_throw_errno_prefix (error, "Opening deltapart '%s'", relpath); part_in = g_unix_input_stream_new (part_fd, FALSE); - if (!_ostree_static_delta_part_open (part_in, NULL, + if (!_ostree_static_delta_part_open (part_in, NULL, delta_open_flags, checksum, &part, cancellable, error)) - goto out; + return FALSE; } if (!_ostree_static_delta_part_execute (self, objects, part, skip_validation, NULL, cancellable, error)) - { - g_prefix_error (error, "Executing delta part %i: ", i); - goto out; - } + return glnx_prefix_error (error, "Executing delta part %i", i); } - ret = TRUE; - out: - return ret; + return TRUE; } gboolean @@ -781,35 +740,30 @@ _ostree_repo_static_delta_delete (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autofree char *from = NULL; g_autofree char *to = NULL; - g_autofree char *deltadir = NULL; - struct stat buf; - if (!_ostree_parse_delta_name (delta_id, &from, &to, error)) - goto out; - - deltadir = _ostree_get_relative_static_delta_path (from, to, NULL); + return FALSE; + g_autofree char *deltadir = _ostree_get_relative_static_delta_path (from, to, NULL); + struct stat buf; if (fstatat (self->repo_dir_fd, deltadir, &buf, 0) != 0) { if (errno == ENOENT) - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Can't find delta %s", delta_id); + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Can't find delta %s", delta_id); + return FALSE; + } else - glnx_set_error_from_errno (error); - - goto out; + return glnx_throw_errno_prefix (error, "fstatat(%s)", deltadir); } if (!glnx_shutil_rm_rf_at (self->repo_dir_fd, deltadir, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } gboolean @@ -852,28 +806,27 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autofree char *from = NULL; + g_autofree char *from = NULL; g_autofree char *to = NULL; g_autofree char *superblock_path = NULL; g_autoptr(GVariant) delta_superblock = NULL; guint64 total_size = 0, total_usize = 0; guint64 total_fallback_size = 0, total_fallback_usize = 0; - guint i; OstreeDeltaEndianness endianness; gboolean swap_endian = FALSE; if (!_ostree_parse_delta_name (delta_id, &from, &to, error)) - goto out; + return FALSE; superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); if (!ot_util_variant_map_at (self->repo_dir_fd, superblock_path, (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, OT_VARIANT_MAP_TRUSTED, &delta_superblock, error)) - goto out; + return FALSE; g_print ("Delta: %s\n", delta_id); + { const char *endianness_description; gboolean was_heuristic; @@ -903,65 +856,63 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, default: g_assert_not_reached (); } - + g_print ("Endianness: %s\n", endianness_description); } - { guint64 ts; - g_variant_get_child (delta_superblock, 1, "t", &ts); - g_print ("Timestamp: %" G_GUINT64_FORMAT "\n", GUINT64_FROM_BE (ts)); - } - { g_autoptr(GVariant) recurse = NULL; - g_variant_get_child (delta_superblock, 5, "@ay", &recurse); - g_print ("Number of parents: %u\n", (guint)(g_variant_get_size (recurse) / (OSTREE_SHA256_DIGEST_LEN * 2))); - } - { g_autoptr(GVariant) fallback = NULL; - guint n_fallback; + guint64 ts; + g_variant_get_child (delta_superblock, 1, "t", &ts); + g_print ("Timestamp: %" G_GUINT64_FORMAT "\n", GUINT64_FROM_BE (ts)); - g_variant_get_child (delta_superblock, 7, "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT, &fallback); - n_fallback = g_variant_n_children (fallback); + g_autoptr(GVariant) recurse = NULL; + g_variant_get_child (delta_superblock, 5, "@ay", &recurse); + g_print ("Number of parents: %u\n", (guint)(g_variant_get_size (recurse) / (OSTREE_SHA256_DIGEST_LEN * 2))); - g_print ("Number of fallback entries: %u\n", n_fallback); + g_autoptr(GVariant) fallback = NULL; + g_variant_get_child (delta_superblock, 7, "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT, &fallback); + const guint n_fallback = g_variant_n_children (fallback); - for (i = 0; i < n_fallback; i++) - { - guint64 size, usize; - g_autoptr(GVariant) checksum_v = NULL; - char checksum[OSTREE_SHA256_STRING_LEN+1]; - g_variant_get_child (fallback, i, "(y@aytt)", NULL, &checksum_v, &size, &usize); - ostree_checksum_inplace_from_bytes (ostree_checksum_bytes_peek (checksum_v), checksum); - size = maybe_swap_endian_u64 (swap_endian, size); - usize = maybe_swap_endian_u64 (swap_endian, usize); - g_print (" %s\n", checksum); - total_fallback_size += size; - total_fallback_usize += usize; - } - { g_autofree char *sizestr = g_format_size (total_fallback_size); - g_autofree char *usizestr = g_format_size (total_fallback_usize); - g_print ("Total Fallback Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_size, sizestr); - g_print ("Total Fallback Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_usize, usizestr); + g_print ("Number of fallback entries: %u\n", n_fallback); + + for (guint i = 0; i < n_fallback; i++) + { + guint64 size, usize; + g_autoptr(GVariant) checksum_v = NULL; + char checksum[OSTREE_SHA256_STRING_LEN+1]; + g_variant_get_child (fallback, i, "(y@aytt)", NULL, &checksum_v, &size, &usize); + ostree_checksum_inplace_from_bytes (ostree_checksum_bytes_peek (checksum_v), checksum); + size = maybe_swap_endian_u64 (swap_endian, size); + usize = maybe_swap_endian_u64 (swap_endian, usize); + g_print (" %s\n", checksum); + total_fallback_size += size; + total_fallback_usize += usize; } + { g_autofree char *sizestr = g_format_size (total_fallback_size); + g_autofree char *usizestr = g_format_size (total_fallback_usize); + g_print ("Total Fallback Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_size, sizestr); + g_print ("Total Fallback Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_usize, usizestr); } - { g_autoptr(GVariant) meta_entries = NULL; - guint n_parts; - g_variant_get_child (delta_superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries); - n_parts = g_variant_n_children (meta_entries); - g_print ("Number of parts: %u\n", n_parts); + g_autoptr(GVariant) meta_entries = NULL; + guint n_parts; - for (i = 0; i < n_parts; i++) - { - if (!show_one_part (self, swap_endian, from, to, meta_entries, i, - &total_size, &total_usize, - cancellable, error)) - goto out; - } - } + g_variant_get_child (delta_superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries); + n_parts = g_variant_n_children (meta_entries); + g_print ("Number of parts: %u\n", n_parts); + + for (guint i = 0; i < n_parts; i++) + { + if (!show_one_part (self, swap_endian, from, to, meta_entries, i, + &total_size, &total_usize, + cancellable, error)) + return FALSE; + } { g_autofree char *sizestr = g_format_size (total_size); g_autofree char *usizestr = g_format_size (total_usize); g_print ("Total Part Size: %" G_GUINT64_FORMAT " (%s)\n", total_size, sizestr); g_print ("Total Part Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_usize, usizestr); } + { guint64 overall_size = total_size + total_fallback_size; guint64 overall_usize = total_usize + total_fallback_usize; g_autofree char *sizestr = g_format_size (overall_size); @@ -970,7 +921,5 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, g_print ("Total Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", overall_usize, usizestr); } - ret = TRUE; - out: - return ret; + return TRUE; } From 54db9ecab36858bbf5afb506d7362577c157ab43 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 27 Jun 2017 12:57:47 -0400 Subject: [PATCH 44/82] build: Don't scan ostree-remote.h for introspection if !experimental-api It's a bit unfortunate that the scanner doesn't error on this, but anyways Closes: https://github.com/ostreedev/ostree/issues/966 Closes: #967 Approved by: jlebon --- Makefile-libostree.am | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 86c9693f..b589f1b2 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -150,8 +150,7 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-tls-cert-interaction.h \ $(NULL) endif -if !ENABLE_EXPERIMENTAL_API -libostree_1_la_SOURCES += \ +libostree_experimental_headers = \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ src/libostree/ostree-repo-finder.h \ @@ -159,6 +158,8 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-repo-finder-config.h \ src/libostree/ostree-repo-finder-mount.h \ $(NULL) +if !ENABLE_EXPERIMENTAL_API +libostree_1_la_SOURCES += $(libostree_experimental_headers) else # if ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += \ src/libostree/ostree-bloom.c \ @@ -257,7 +258,7 @@ OSTree_1_0_gir_INCLUDES = Gio-2.0 OSTree_1_0_gir_CFLAGS = $(libostree_1_la_CFLAGS) OSTree_1_0_gir_LIBS = libostree-1.la OSTree_1_0_gir_SCANNERFLAGS = --warn-all --identifier-prefix=Ostree --symbol-prefix=ostree $(GI_SCANNERFLAGS) -OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h %/ostree-repo-finder-avahi.h %/ostree-repo-finder-config.h %/ostree-repo-finder-mount.h,$(libostree_1_la_SOURCES)) +OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h $(libostree_experimental_headers),$(libostree_1_la_SOURCES)) INTROSPECTION_GIRS += OSTree-1.0.gir gir_DATA += OSTree-1.0.gir typelib_DATA += OSTree-1.0.typelib From 4796cd04540027a753e7f3decea3e045cc8dec82 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 26 Jun 2017 15:10:57 -0400 Subject: [PATCH 45/82] pull: Check free space when pulling deltas Computing download/storage size for `archive` pulls is hard; there's `OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES` which was from a pre-static-deltas effort by Endless, but we aren't currently making use of this much. Static deltas were designed to solve this problem; we have the total uncompressed size. Let's check free space before doing a delta pull. Related: https://github.com/ostreedev/ostree/issues/962 Closes: #963 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 4607f2f1..acc5098a 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -49,6 +49,7 @@ #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include +#include #define OSTREE_REPO_PULL_CONTENT_PRIORITY (OSTREE_FETCHER_DEFAULT_PRIORITY) #define OSTREE_REPO_PULL_METADATA_PRIORITY (OSTREE_REPO_PULL_CONTENT_PRIORITY - 100) @@ -1814,6 +1815,11 @@ process_one_static_delta (OtPullData *pull_data, headers = g_variant_get_child_value (delta_superblock, 6); fallback_objects = g_variant_get_child_value (delta_superblock, 7); + /* Gather free space so we can do a check below */ + struct statvfs stvfsbuf; + if (TEMP_FAILURE_RETRY (fstatvfs (pull_data->repo->repo_dir_fd, &stvfsbuf)) < 0) + return glnx_throw_errno_prefix (error, "fstatvfs"); + /* First process the fallbacks */ n = g_variant_n_children (fallback_objects); for (i = 0; i < n; i++) @@ -1982,6 +1988,21 @@ process_one_static_delta (OtPullData *pull_data, } } + /* The free space check is here since at this point we've parsed the delta not + * only the total size of the parts, but also whether or not we already have + * them. TODO: Ideally this free space check would be above, but we'd have to + * walk everything twice and keep track of state. + */ + const guint64 delta_required_blocks = (pull_data->total_deltapart_usize / stvfsbuf.f_bsize); + if (delta_required_blocks > stvfsbuf.f_bfree) + { + g_autofree char *formatted_required = g_format_size (pull_data->total_deltapart_usize); + g_autofree char *formatted_avail = g_format_size (((guint64)stvfsbuf.f_bsize) * stvfsbuf.f_bfree); + glnx_throw (error, "Delta requires %s free space, but only %s available", + formatted_required, formatted_avail); + goto out; + } + ret = TRUE; out: return ret; From 2f297ba4d30384a84ef0fcc5f2505aa1153f9155 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 19 Jun 2017 23:29:55 +0100 Subject: [PATCH 46/82] lib/repo: Fix a typo in a documentation comment Looks like a copy-paste error. Signed-off-by: Philip Withnall Closes: #961 Approved by: cgwalters --- src/libostree/ostree-repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1db748cb..6dcb70de 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4064,7 +4064,7 @@ ostree_repo_sign_delta (OstreeRepo *self, * @cancellable: A #GCancellable * @error: a #GError * - * Add a GPG signature to a static delta. + * Add a GPG signature to a summary file. */ gboolean ostree_repo_add_gpg_signature_summary (OstreeRepo *self, From a432a2b42029b7bc014b61375579994d13f22413 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 23 Jun 2017 21:44:29 +0100 Subject: [PATCH 47/82] tests: Fix incorrect `summary --update` usage in test-local-pull.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the spurious ‘update’ atom was ignored; now that `ostree summary --update` accepts extra arguments as additional metadata, it’s causing an error. Drop it. Signed-off-by: Philip Withnall Closes: #961 Approved by: cgwalters --- tests/test-local-pull.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index 43b96010..1c9767f6 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -84,13 +84,13 @@ if ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-sum assert_not_reached "GPG summary verification with no summary unexpectedly succeeded" fi -${OSTREE} summary -u update +${OSTREE} summary --update if ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1; then assert_not_reached "GPG summary verification with signed no summary unexpectedly succeeded" fi -${OSTREE} summary -u update --gpg-sign=${TEST_GPG_KEYID_1} --gpg-homedir=${TEST_GPG_KEYHOME} +${OSTREE} summary --update --gpg-sign=${TEST_GPG_KEYID_1} --gpg-homedir=${TEST_GPG_KEYHOME} ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1 From e0ad9b226604c08ec0fd25db68b7747e06e470d2 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 22 Jun 2017 15:16:53 +0100 Subject: [PATCH 48/82] ostree/summary: Add support for adding additional metadata When updating a summary file, parse additional arguments to the `ostree summary` command as additional metadata to be put into the summary. Add some tests for this. Signed-off-by: Philip Withnall Closes: #961 Approved by: cgwalters --- Makefile-tests.am | 1 + man/ostree-summary.xml | 11 +++- src/ostree/ot-builtin-summary.c | 47 ++++++++++++++++- tests/test-summary-update.sh | 94 +++++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 2 deletions(-) create mode 100755 tests/test-summary-update.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index cb775320..2fe0a235 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -108,6 +108,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-switchroot.sh \ tests/test-pull-contenturl.sh \ tests/test-pull-mirrorlist.sh \ + tests/test-summary-update.sh \ tests/test-summary-view.sh \ $(NULL) diff --git a/man/ostree-summary.xml b/man/ostree-summary.xml index 3fa287eb..4e897592 100644 --- a/man/ostree-summary.xml +++ b/man/ostree-summary.xml @@ -73,7 +73,16 @@ Boston, MA 02111-1307, USA. - Update the summary file. + Update the summary file. + + Any additional arguments to the command + are treated as additional key–value pairs to be added to the + summary file as additional metadata. They must be in the format + KEY=VALUE + or as two separate arguments. The keys must be namespaced for + your organisation or repository using a dot prefix. The values + must be in GVariant text format. For example, + exampleos.end-of-life "@t 1445385600". diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c index 9055d972..f2e687ec 100644 --- a/src/ostree/ot-builtin-summary.c +++ b/src/ostree/ot-builtin-summary.c @@ -30,6 +30,7 @@ static gboolean opt_update, opt_view, opt_raw; static char **opt_key_ids; static char *opt_gpg_homedir; +static char **opt_metadata; static GOptionEntry options[] = { { "update", 'u', 0, G_OPTION_ARG_NONE, &opt_update, "Update the summary", NULL }, @@ -37,9 +38,44 @@ static GOptionEntry options[] = { { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "View the raw bytes of the summary file", NULL }, { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the summary with", "KEY-ID"}, { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, + { "add-metadata", 'm', 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata, "Additional metadata field to add to the summary", "KEY=VALUE" }, { NULL } }; +/* Take arguments of the form KEY=VALUE and put them into an a{sv} variant. The + * value arguments must be parsable using g_variant_parse(). */ +static GVariant * +build_additional_metadata (const char * const *args, + GError **error) +{ + g_autoptr(GVariantBuilder) builder = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE_VARDICT); + + for (gsize i = 0; args[i] != NULL; i++) + { + const gchar *equals = strchr (args[i], '='); + g_autofree gchar *key = NULL; + const gchar *value_str; + g_autoptr(GVariant) value = NULL; + + if (equals == NULL) + return glnx_null_throw (error, + "Missing '=' in KEY=VALUE metadata '%s'", args[i]); + + key = g_strndup (args[i], equals - args[i]); + value_str = equals + 1; + + value = g_variant_parse (NULL, value_str, NULL, NULL, error); + if (value == NULL) + return glnx_prefix_error_null (error, "Error parsing variant ‘%s’: ", value_str); + + g_variant_builder_add (builder, "{sv}", key, value); + } + + return g_variant_ref_sink (g_variant_builder_end (builder)); +} + gboolean ostree_builtin_summary (int argc, char **argv, GCancellable *cancellable, GError **error) { @@ -55,10 +91,19 @@ ostree_builtin_summary (int argc, char **argv, GCancellable *cancellable, GError if (opt_update) { + g_autoptr(GVariant) additional_metadata = NULL; + if (!ostree_ensure_repo_writable (repo, error)) goto out; - if (!ostree_repo_regenerate_summary (repo, NULL, cancellable, error)) + if (opt_metadata != NULL) + { + additional_metadata = build_additional_metadata ((const char * const *) opt_metadata, error); + if (additional_metadata == NULL) + goto out; + } + + if (!ostree_repo_regenerate_summary (repo, additional_metadata, cancellable, error)) goto out; if (opt_key_ids) diff --git a/tests/test-summary-update.sh b/tests/test-summary-update.sh new file mode 100755 index 00000000..457debbe --- /dev/null +++ b/tests/test-summary-update.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# +# Authors: +# - Philip Withnall + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo "1..2" + +cd ${test_tmpdir} +mkdir repo +ostree_repo_init repo + +mkdir -p tree/root +touch tree/root/a + +# Add a few commits +seq 5 | while read i; do + echo a >> tree/root/a + ${CMD_PREFIX} ostree --repo=repo commit --branch=test-$i -m test -s test tree +done + +# Generate a plain summary file. +${CMD_PREFIX} ostree --repo=repo summary --update + +# Generate a signed summary file. +${CMD_PREFIX} ostree --repo=repo summary --update --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1} + +# Try various ways of adding additional data. +${CMD_PREFIX} ostree --repo=repo summary --update --add-metadata key="'value'" --add-metadata=key2=true +${CMD_PREFIX} ostree --repo=repo summary --update -m some-int='@t 123' +${CMD_PREFIX} ostree --repo=repo summary --update --add-metadata=map='@a{sv} {}' + +# Check the additional metadata turns up in the output. +${CMD_PREFIX} ostree --repo=repo summary --view > summary +assert_file_has_content summary "^map: {}$" + +echo "ok 1 update summary" + +# Test again, but with collections enabled in the repository (if supported). +if ! ostree --version | grep -q -e '- experimental'; then + echo "ok 2 # skip No experimental API is compiled in" + exit 0 +fi + +cd ${test_tmpdir} +rm -rf repo +ostree_repo_init repo --collection-id org.example.Collection1 + +mkdir -p tree/root +touch tree/root/a + +# Add a few commits +seq 5 | while read i; do + echo a >> tree/root/a + ${CMD_PREFIX} ostree --repo=repo commit --branch=test-$i -m test -s test tree + ${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.Collection2:test-$i test-$i +done + +# Generate a plain summary file. +${CMD_PREFIX} ostree --repo=repo summary --update + +# Generate a signed summary file. +${CMD_PREFIX} ostree --repo=repo summary --update --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1} + +# Try various ways of adding additional data. +${CMD_PREFIX} ostree --repo=repo summary --update --add-metadata key="'value'" --add-metadata=key2=true +${CMD_PREFIX} ostree --repo=repo summary --update -m some-int='@t 123' +${CMD_PREFIX} ostree --repo=repo summary --update --add-metadata=map='@a{sv} {}' + +# Check the additional metadata turns up in the output. +${CMD_PREFIX} ostree --repo=repo summary --view > summary +assert_file_has_content summary "^map: {}$" + +echo "ok 2 update summary with collections" From eb4887d619002932b348496c5613424b0ee120e0 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 19 Jun 2017 15:26:50 +0100 Subject: [PATCH 49/82] =?UTF-8?q?lib/pull:=20Don=E2=80=99t=20cache=20summa?= =?UTF-8?q?ry=20file=20until=20its=20signature=20is=20verified?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes no difference to the validity of the code, since any summary file loaded from the cache will be verified before being read anyway; but it will make some upcoming changes a little simpler. Signed-off-by: Philip Withnall Closes: #961 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 43 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index acc5098a..cc8a51eb 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3339,28 +3339,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; } - if (bytes_summary) - { - pull_data->summary_data = g_bytes_ref (bytes_summary); - pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE); - - if (bytes_sig) - pull_data->summary_data_sig = g_bytes_ref (bytes_sig); - } - - - if (!summary_from_cache && bytes_summary && bytes_sig) - { - if (!pull_data->remote_repo_local && - !_ostree_repo_cache_summary (self, - remote_name_or_baseurl, - bytes_summary, - bytes_sig, - cancellable, - error)) - goto out; - } - if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig) { g_autoptr(GVariant) sig_variant = NULL; @@ -3379,6 +3357,27 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; } + if (bytes_summary) + { + pull_data->summary_data = g_bytes_ref (bytes_summary); + pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE); + + if (bytes_sig) + pull_data->summary_data_sig = g_bytes_ref (bytes_sig); + } + + if (!summary_from_cache && bytes_summary && bytes_sig) + { + if (!pull_data->remote_repo_local && + !_ostree_repo_cache_summary (self, + remote_name_or_baseurl, + bytes_summary, + bytes_sig, + cancellable, + error)) + goto out; + } + if (pull_data->summary) { additional_metadata = g_variant_get_child_value (pull_data->summary, 1); From 4c238c350d4b910d40b92be8f6f7b226f79da0b8 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 19 Jun 2017 15:58:08 +0100 Subject: [PATCH 50/82] lib/pull: Check whether summary is in normal form when loading it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check that it’s in normal form and has the correct type when loading it, since it could come from an untrusted source. Signed-off-by: Philip Withnall Closes: #961 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index cc8a51eb..0b7dae17 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3362,6 +3362,20 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->summary_data = g_bytes_ref (bytes_summary); pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE); + if (!g_variant_is_normal_form (pull_data->summary)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not normal form"); + goto out; + } + if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Doesn't match variant type '%s'", + (char *)OSTREE_SUMMARY_GVARIANT_FORMAT); + goto out; + } + if (bytes_sig) pull_data->summary_data_sig = g_bytes_ref (bytes_sig); } From a03f0447cb716f315f68f225efca6cc261e43e56 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 23 Jun 2017 16:10:03 +0100 Subject: [PATCH 51/82] lib/pull: Use ostree_repo_verify_summary() to verify summary on pull Rather than duplicating the code. This introduces no functional changes. Signed-off-by: Philip Withnall Closes: #961 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0b7dae17..fc29a2ec 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3341,18 +3341,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig) { - g_autoptr(GVariant) sig_variant = NULL; - glnx_unref_object OstreeGpgVerifyResult *result = NULL; + g_autoptr(OstreeGpgVerifyResult) result = NULL; - sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, bytes_sig, FALSE); - result = _ostree_repo_gpg_verify_with_metadata (self, - bytes_summary, - sig_variant, - pull_data->remote_name, - NULL, - NULL, - cancellable, - error); + result = ostree_repo_verify_summary (self, pull_data->remote_name, + bytes_summary, bytes_sig, + cancellable, error); if (!ostree_gpg_verify_result_require_valid_signature (result, error)) goto out; } From 7d0e6fb63fc19975c52bf2947246ef191cd7b202 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 27 Jun 2017 17:07:11 -0400 Subject: [PATCH 52/82] ci: Actually run installed tests again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We did it *again* 😭 Closes: #969 Approved by: jlebon --- ci/build-check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build-check.sh b/ci/build-check.sh index d15032e7..da03f3cc 100755 --- a/ci/build-check.sh +++ b/ci/build-check.sh @@ -11,7 +11,7 @@ make syntax-check # TODO: do syntax-check under check # And now run the installed tests make install if test -x /usr/bin/gnome-desktop-testing-runner; then - gnome-desktop-testing-runner -p 0 ostree + gnome-desktop-testing-runner -p 0 libostree/ fi if test -x /usr/bin/clang; then From 48e49df7f7d786a2bb71b3d1d13cde5ecbdad3c4 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 27 Jun 2017 13:39:27 -0700 Subject: [PATCH 53/82] papr: build and test on c7 Start testing on CentOS 7 as well to cover kernel differences (e.g. O_TMPFILE support). Closes: #968 Approved by: cgwalters --- .papr.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.papr.yml b/.papr.yml index b2eeb972..be4c4f04 100644 --- a/.papr.yml +++ b/.papr.yml @@ -31,18 +31,22 @@ artifacts: - test-suite.log --- -context: c7-build +context: c7-primary inherit: true required: true -container: - image: registry.centos.org/centos/centos:7 +packages: + +host: + distro: centos/7/atomic env: CFLAGS: '' tests: - - ci/build-check.sh + - docker run --privileged -v $PWD:$PWD --workdir $PWD + registry.centos.org/centos/centos:7 sh -c + 'yum install -y git && ci/build-check.sh' --- From 79f285d188eef288cecb35ed7977bf6e219cc396 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 27 Jun 2017 14:25:56 -0700 Subject: [PATCH 54/82] test-switchroot.sh: skip if no busybox Closes: #968 Approved by: cgwalters --- tests/test-switchroot.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test-switchroot.sh b/tests/test-switchroot.sh index fd5a30d0..bc3ec38b 100755 --- a/tests/test-switchroot.sh +++ b/tests/test-switchroot.sh @@ -127,6 +127,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then . $(dirname $0)/libtest.sh unshare -m true || \ skip "this test needs to set up mount namespaces, rerun as root" + [ -f /bin/busybox ] || \ + skip "this test needs busybox" echo "1..3" test_that_prepare_root_sets_sysroot_up_correctly_with_initrd From 5776d5dcc09e5aabae1b5c1b8854c0b66522accd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 24 Jun 2017 14:06:53 +0000 Subject: [PATCH 55/82] Port to GLnxTmpfile There's lots of mechanically replacing `OtTmpFile` with `GLnxTmpfile`; the biggest changes are in the commit path. Symlink commits are now very clearly separated from regular files. Symlinks are `OtCleanupUnlinkat`, and regular files are `GLnxTmpfile`. The commit codepath separates those as `_ostree_repo_commit_path_final()` and `_ostree_repo_commit_tmpf_final()`. A nice aspect of all of this is that they both *consume* the temporary on success. This avoids an extra spurious `unlink()` call. One of the biggest bits of code motion is in `commit_loose_regfile_object()`, which no longer needs to care about symlinks. For the most parth though it's just removing conditionals. Update submodule: libglnx Closes: #958 Approved by: jlebon --- libglnx | 2 +- src/libostree/ostree-fetcher-curl.c | 16 +- src/libostree/ostree-impl-system-generator.c | 14 +- src/libostree/ostree-repo-checkout.c | 24 +- src/libostree/ostree-repo-commit.c | 382 +++++++++--------- src/libostree/ostree-repo-private.h | 26 +- src/libostree/ostree-repo-pull.c | 9 +- .../ostree-repo-static-delta-compilation.c | 54 ++- .../ostree-repo-static-delta-processing.c | 4 +- src/libostree/ostree-repo.c | 9 +- src/libotutil/ot-fs-utils.c | 58 +-- src/libotutil/ot-fs-utils.h | 26 -- src/ostree/ot-remote-cookie-util.c | 12 +- 13 files changed, 278 insertions(+), 358 deletions(-) diff --git a/libglnx b/libglnx index 32231fdb..caa51ac2 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 32231fdb5273dd2a812c61549d6c0e681c0f5d59 +Subproject commit caa51ac24ffcdffcb610bc6ccc9da964d4be74ee diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index f483a6bb..5d35e7b7 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -99,7 +99,7 @@ struct FetcherRequest { OstreeFetcherRequestFlags flags; gboolean is_membuf; GError *caught_write_error; - OtTmpfile tmpf; + GLnxTmpfile tmpf; GString *output_buf; CURL *easy; @@ -270,9 +270,9 @@ ensure_tmpfile (FetcherRequest *req, GError **error) { if (!req->tmpf.initialized) { - if (!ot_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".", - O_WRONLY | O_CLOEXEC, &req->tmpf, - error)) + if (!glnx_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".", + O_WRONLY | O_CLOEXEC, &req->tmpf, + error)) return FALSE; } return TRUE; @@ -390,9 +390,9 @@ check_multi_info (OstreeFetcher *fetcher) glnx_set_error_from_errno (error); g_task_return_error (task, g_steal_pointer (&local_error)); } - else if (!ot_link_tmpfile_at (&req->tmpf, GLNX_LINK_TMPFILE_REPLACE, - fetcher->tmpdir_dfd, tmpfile_path, - error)) + else if (!glnx_link_tmpfile_at (&req->tmpf, GLNX_LINK_TMPFILE_REPLACE, + fetcher->tmpdir_dfd, tmpfile_path, + error)) g_task_return_error (task, g_steal_pointer (&local_error)); else { @@ -616,7 +616,7 @@ request_unref (FetcherRequest *req) g_ptr_array_unref (req->mirrorlist); g_free (req->filename); g_clear_error (&req->caught_write_error); - ot_tmpfile_clear (&req->tmpf); + glnx_tmpfile_clear (&req->tmpf); if (req->output_buf) g_string_free (req->output_buf, TRUE); curl_easy_cleanup (req->easy); diff --git a/src/libostree/ostree-impl-system-generator.c b/src/libostree/ostree-impl-system-generator.c index 72f52bc5..5630348c 100644 --- a/src/libostree/ostree-impl-system-generator.c +++ b/src/libostree/ostree-impl-system-generator.c @@ -172,12 +172,11 @@ _ostree_impl_system_generator (const char *ostree_cmdline, /* Generate our bind mount unit */ const char *stateroot_var_path = glnx_strjoina ("/sysroot/ostree/deploy/", stateroot, "/var"); - glnx_fd_close int tmpfd = -1; - g_autofree char *tmppath = NULL; + g_auto(GLnxTmpfile) tmpf = { 0, }; if (!glnx_open_tmpfile_linkable_at (normal_dir_dfd, ".", O_WRONLY | O_CLOEXEC, - &tmpfd, &tmppath, error)) + &tmpf, error)) return FALSE; - g_autoptr(GOutputStream) outstream = g_unix_output_stream_new (tmpfd, FALSE); + g_autoptr(GOutputStream) outstream = g_unix_output_stream_new (tmpf.fd, FALSE); gsize bytes_written; /* This code is inspired by systemd's fstab-generator.c. * @@ -204,12 +203,11 @@ _ostree_impl_system_generator (const char *ostree_cmdline, return FALSE; g_clear_object (&outstream); /* It should be readable */ - if (fchmod (tmpfd, 0644) < 0) + if (fchmod (tmpf.fd, 0644) < 0) return glnx_throw_errno_prefix (error, "fchmod"); /* Error out if somehow it already exists, that'll help us debug conflicts */ - if (!glnx_link_tmpfile_at (normal_dir_dfd, GLNX_LINK_TMPFILE_NOREPLACE, - tmpfd, tmppath, normal_dir_dfd, - "var.mount", error)) + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE, + normal_dir_dfd, "var.mount", error)) return FALSE; /* And ensure it's required; newer systemd will auto-inject fs dependencies diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index af5c021f..ac8d5b46 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -60,9 +60,9 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, guint32 file_mode = g_file_info_get_attribute_uint32 (src_info, "unix::mode"); file_mode &= ~(S_ISUID|S_ISGID); - g_auto(OtTmpfile) tmpf = { 0, }; - if (!ot_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY | O_CLOEXEC, - &tmpf, error)) + g_auto(GLnxTmpfile) tmpf = { 0, }; + if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY | O_CLOEXEC, + &tmpf, error)) return FALSE; g_autoptr(GOutputStream) temp_out = g_unix_output_stream_new (tmpf.fd, FALSE); @@ -89,9 +89,9 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, cancellable, error)) return FALSE; - if (!ot_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, - self->uncompressed_objects_dir_fd, loose_path, - error)) + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, + self->uncompressed_objects_dir_fd, loose_path, + error)) return FALSE; return TRUE; @@ -254,11 +254,11 @@ create_file_copy_from_input_at (OstreeRepo *repo, } else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { - g_auto(OtTmpfile) tmpf = { 0, }; + g_auto(GLnxTmpfile) tmpf = { 0, }; GLnxLinkTmpfileReplaceMode replace_mode; - if (!ot_open_tmpfile_linkable_at (destination_dfd, ".", O_WRONLY | O_CLOEXEC, - &tmpf, error)) + if (!glnx_open_tmpfile_linkable_at (destination_dfd, ".", O_WRONLY | O_CLOEXEC, + &tmpf, error)) return FALSE; if (sepolicy_enabled && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) @@ -285,9 +285,9 @@ create_file_copy_from_input_at (OstreeRepo *repo, else replace_mode = GLNX_LINK_TMPFILE_NOREPLACE; - if (!ot_link_tmpfile_at (&tmpf, replace_mode, - destination_dfd, destination_name, - error)) + if (!glnx_link_tmpfile_at (&tmpf, replace_mode, + destination_dfd, destination_name, + error)) return FALSE; } else diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 916b99d2..d64f6481 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -134,21 +134,19 @@ ot_security_smack_reset_fd (int fd) #endif } +/* Given an O_TMPFILE regular file, link it into place. */ gboolean -_ostree_repo_commit_loose_final (OstreeRepo *self, - const char *checksum, - OstreeObjectType objtype, - int temp_dfd, - int fd, - const char *temp_filename, - GCancellable *cancellable, - GError **error) +_ostree_repo_commit_tmpf_final (OstreeRepo *self, + const char *checksum, + OstreeObjectType objtype, + GLnxTmpfile *tmpf, + GCancellable *cancellable, + GError **error) { - int dest_dfd; char tmpbuf[_OSTREE_LOOSE_PATH_MAX]; - _ostree_loose_path (tmpbuf, checksum, objtype, self->mode); + int dest_dfd; if (self->in_transaction) dest_dfd = self->commit_stagedir_fd; else @@ -158,41 +156,64 @@ _ostree_repo_commit_loose_final (OstreeRepo *self, cancellable, error)) return FALSE; - if (fd != -1) + return glnx_link_tmpfile_at (tmpf, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, + dest_dfd, tmpbuf, error); +} + +/* Given a dfd+path combination (may be regular file or symlink), + * rename it into place. + */ +gboolean +_ostree_repo_commit_path_final (OstreeRepo *self, + const char *checksum, + OstreeObjectType objtype, + OtCleanupUnlinkat *tmp_path, + GCancellable *cancellable, + GError **error) +{ + /* The final renameat() */ + char tmpbuf[_OSTREE_LOOSE_PATH_MAX]; + _ostree_loose_path (tmpbuf, checksum, objtype, self->mode); + + int dest_dfd; + if (self->in_transaction) + dest_dfd = self->commit_stagedir_fd; + else + dest_dfd = self->objects_dir_fd; + + if (!_ostree_repo_ensure_loose_objdir_at (dest_dfd, tmpbuf, + cancellable, error)) + return FALSE; + + if (renameat (tmp_path->dfd, tmp_path->path, + dest_dfd, tmpbuf) == -1) { - if (!glnx_link_tmpfile_at (temp_dfd, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, - fd, temp_filename, dest_dfd, tmpbuf, error)) - return FALSE; + if (errno != EEXIST) + return glnx_throw_errno_prefix (error, "Storing file '%s'", tmp_path->path); + /* Otherwise, the caller's cleanup will unlink+free */ } else { - if (G_UNLIKELY (renameat (temp_dfd, temp_filename, - dest_dfd, tmpbuf) == -1)) - { - if (errno != EEXIST) - return glnx_throw_errno_prefix (error, "Storing file '%s'", temp_filename); - else - (void) unlinkat (temp_dfd, temp_filename, 0); - } + /* The tmp path was consumed */ + ot_cleanup_unlinkat_clear (tmp_path); } return TRUE; } + /* Given either a file or symlink, apply the final metadata to it depending on * the repository mode. Note that @checksum is assumed to have been validated by * the caller. */ static gboolean -commit_loose_content_object (OstreeRepo *self, +commit_loose_regfile_object (OstreeRepo *self, const char *checksum, - const char *temp_filename, - gboolean object_is_symlink, + GLnxTmpfile *tmpf, guint32 uid, guint32 gid, guint32 mode, GVariant *xattrs, - int fd, GCancellable *cancellable, GError **error) { @@ -200,115 +221,80 @@ commit_loose_content_object (OstreeRepo *self, * automatically inherit the non-root ownership. */ if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 - && self->target_owner_uid != -1) + && self->target_owner_uid != -1) { - if (fd != -1) - { - if (fchown (fd, self->target_owner_uid, self->target_owner_gid) < 0) - return glnx_throw_errno_prefix (error, "fchown"); - } - else if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename, - self->target_owner_uid, - self->target_owner_gid, - AT_SYMLINK_NOFOLLOW) == -1)) - return glnx_throw_errno_prefix (error, "fchownat"); + if (fchown (tmpf->fd, self->target_owner_uid, self->target_owner_gid) < 0) + return glnx_throw_errno_prefix (error, "fchown"); } + else if (self->mode == OSTREE_REPO_MODE_BARE) + { + if (TEMP_FAILURE_RETRY (fchown (tmpf->fd, uid, gid)) < 0) + return glnx_throw_errno_prefix (error, "fchown"); - /* Special handling for symlinks in bare repositories */ - if (object_is_symlink && self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) - { - /* We don't store the metadata in bare-user-only, so we're done. */ + if (TEMP_FAILURE_RETRY (fchmod (tmpf->fd, mode)) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + + if (xattrs) + { + ot_security_smack_reset_fd (tmpf->fd); + if (!glnx_fd_set_all_xattrs (tmpf->fd, xattrs, cancellable, error)) + return FALSE; + } } - else if (object_is_symlink && self->mode == OSTREE_REPO_MODE_BARE) + else if (self->mode == OSTREE_REPO_MODE_BARE_USER) { - /* Now that we know the checksum is valid, apply uid/gid, mode bits, - * and extended attributes. + if (!write_file_metadata_to_xattr (tmpf->fd, uid, gid, mode, xattrs, error)) + return FALSE; + + /* Note that previously this path added `| 0755` which made every + * file executable, see + * https://github.com/ostreedev/ostree/issues/907 * - * Note, this does not apply for bare-user repos, as they store symlinks - * as regular files. + * Again here, symlinks in bare-user are a hairy special case; only do a + * chmod for a *real* regular file, otherwise we'll take the default 0644. */ - if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename, - uid, gid, - AT_SYMLINK_NOFOLLOW) == -1)) - return glnx_throw_errno_prefix (error, "fchownat"); - - if (xattrs != NULL) + if (S_ISREG (mode)) { - ot_security_smack_reset_dfd_name (self->tmp_dir_fd, temp_filename); - if (!glnx_dfd_name_set_all_xattrs (self->tmp_dir_fd, temp_filename, - xattrs, cancellable, error)) - return FALSE; + const mode_t content_mode = (mode & (S_IFREG | 0775)); + if (fchmod (tmpf->fd, content_mode) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); } + else + g_assert (S_ISLNK (mode)); } - else + else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) { - if (self->mode == OSTREE_REPO_MODE_BARE) - { - if (TEMP_FAILURE_RETRY (fchown (fd, uid, gid)) < 0) - return glnx_throw_errno_prefix (error, "fchown"); + guint32 invalid_modebits = (mode & ~S_IFMT) & ~0775; + if (invalid_modebits > 0) + return glnx_throw (error, "Invalid mode 0%04o with bits 0%04o in bare-user-only repository", + mode, invalid_modebits); - if (TEMP_FAILURE_RETRY (fchmod (fd, mode)) < 0) - return glnx_throw_errno_prefix (error, "fchmod"); - - if (xattrs) - { - ot_security_smack_reset_fd (fd); - if (!glnx_fd_set_all_xattrs (fd, xattrs, cancellable, error)) - return FALSE; - } - } - else if (self->mode == OSTREE_REPO_MODE_BARE_USER) - { - if (!write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error)) - return FALSE; - - if (!object_is_symlink) - { - /* Note that previously this path added `| 0755` which made every - * file executable, see - * https://github.com/ostreedev/ostree/issues/907 - */ - const mode_t content_mode = (mode & (S_IFREG | 0775)); - if (fchmod (fd, content_mode) < 0) - return glnx_throw_errno_prefix (error, "fchmod"); - } - } - else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY - && !object_is_symlink) - { - guint32 invalid_modebits = (mode & ~S_IFMT) & ~0775; - if (invalid_modebits > 0) - return glnx_throw (error, "Invalid mode 0%04o with bits 0%04o in bare-user-only repository", - mode, invalid_modebits); - - if (fchmod (fd, mode) < 0) - return glnx_throw_errno_prefix (error, "fchmod"); - } - - if (_ostree_repo_mode_is_bare (self->mode)) - { - /* To satisfy tools such as guile which compare mtimes - * to determine whether or not source files need to be compiled, - * set the modification time to OSTREE_TIMESTAMP. - */ - const struct timespec times[2] = { { OSTREE_TIMESTAMP, UTIME_OMIT }, { OSTREE_TIMESTAMP, 0} }; - if (TEMP_FAILURE_RETRY (futimens (fd, times)) < 0) - return glnx_throw_errno_prefix (error, "futimens"); - } - - /* Ensure that in case of a power cut, these files have the data we - * want. See http://lwn.net/Articles/322823/ - */ - if (!self->in_transaction && !self->disable_fsync) - { - if (fsync (fd) == -1) - return glnx_throw_errno_prefix (error, "fsync"); - } + if (fchmod (tmpf->fd, mode) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); } - if (!_ostree_repo_commit_loose_final (self, checksum, OSTREE_OBJECT_TYPE_FILE, - self->tmp_dir_fd, fd, temp_filename, - cancellable, error)) + if (_ostree_repo_mode_is_bare (self->mode)) + { + /* To satisfy tools such as guile which compare mtimes + * to determine whether or not source files need to be compiled, + * set the modification time to OSTREE_TIMESTAMP. + */ + const struct timespec times[2] = { { OSTREE_TIMESTAMP, UTIME_OMIT }, { OSTREE_TIMESTAMP, 0} }; + if (TEMP_FAILURE_RETRY (futimens (tmpf->fd, times)) < 0) + return glnx_throw_errno_prefix (error, "futimens"); + } + + /* Ensure that in case of a power cut, these files have the data we + * want. See http://lwn.net/Articles/322823/ + */ + if (!self->in_transaction && !self->disable_fsync) + { + if (fsync (tmpf->fd) == -1) + return glnx_throw_errno_prefix (error, "fsync"); + } + + if (!_ostree_repo_commit_tmpf_final (self, checksum, OSTREE_OBJECT_TYPE_FILE, + tmpf, cancellable, error)) return FALSE; return TRUE; @@ -453,7 +439,7 @@ gboolean _ostree_repo_open_content_bare (OstreeRepo *self, const char *checksum, guint64 content_len, - OtTmpfile *out_tmpf, + GLnxTmpfile *out_tmpf, gboolean *out_have_object, GCancellable *cancellable, GError **error) @@ -471,14 +457,14 @@ _ostree_repo_open_content_bare (OstreeRepo *self, return TRUE; } - return ot_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, - out_tmpf, error); + return glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, + out_tmpf, error); } gboolean _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, const char *checksum, - OtTmpfile *tmpf, + GLnxTmpfile *tmpf, guint32 uid, guint32 gid, guint32 mode, @@ -492,57 +478,45 @@ _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, if (!tmpf->initialized || tmpf->fd == -1) return TRUE; - if (!commit_loose_content_object (self, checksum, - tmpf->path, - FALSE, uid, gid, mode, - xattrs, tmpf->fd, - cancellable, error)) - return glnx_prefix_error (error, "Writing object %s.%s", checksum, ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE)); - /* The path was consumed */ - g_clear_pointer (&tmpf->path, g_free); - tmpf->initialized = FALSE; - return TRUE; + return commit_loose_regfile_object (self, checksum, + tmpf, uid, gid, mode, xattrs, + cancellable, error); } static gboolean create_regular_tmpfile_linkable_with_content (OstreeRepo *self, guint64 length, GInputStream *input, - int *out_fd, - char **out_path, + GLnxTmpfile *out_tmpf, GCancellable *cancellable, GError **error) { - glnx_fd_close int temp_fd = -1; - g_autofree char *temp_filename = NULL; - + g_auto(GLnxTmpfile) tmpf = { 0, }; if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, - &temp_fd, &temp_filename, - error)) + &tmpf, error)) return FALSE; - if (!ot_fallocate (temp_fd, length, error)) + if (!ot_fallocate (tmpf.fd, length, error)) return FALSE; if (G_IS_FILE_DESCRIPTOR_BASED (input)) { int infd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*) input); - if (glnx_regfile_copy_bytes (infd, temp_fd, (off_t)length, TRUE) < 0) + if (glnx_regfile_copy_bytes (infd, tmpf.fd, (off_t)length, TRUE) < 0) return glnx_throw_errno_prefix (error, "regfile copy"); } else { - g_autoptr(GOutputStream) temp_out = g_unix_output_stream_new (temp_fd, FALSE); + g_autoptr(GOutputStream) temp_out = g_unix_output_stream_new (tmpf.fd, FALSE); if (g_output_stream_splice (temp_out, input, 0, cancellable, error) < 0) return FALSE; } - if (fchmod (temp_fd, 0644) < 0) + if (fchmod (tmpf.fd, 0644) < 0) return glnx_throw_errno_prefix (error, "fchmod"); - *out_fd = temp_fd; temp_fd = -1; - *out_path = g_steal_pointer (&temp_filename); + *out_tmpf = tmpf; tmpf.initialized = FALSE; return TRUE; } @@ -615,47 +589,46 @@ write_content_object (OstreeRepo *self, * could potentially cause the system to make a temporary setuid * binary with trailing garbage, creating a window on the local * system where a malicious setuid binary exists. - */ - /* These variables are almost equivalent to OtTmpfile, except - * temp_filename might also be a symlink. Hence the CleanupUnlinkat - * which handles that case. + * + * We use GLnxTmpfile for regular files, and OtCleanupUnlinkat for symlinks. */ g_auto(OtCleanupUnlinkat) tmp_unlinker = { self->tmp_dir_fd, NULL }; - glnx_fd_close int temp_fd = -1; + g_auto(GLnxTmpfile) tmpf = { 0, }; gssize unpacked_size = 0; gboolean indexable = FALSE; - if ((_ostree_repo_mode_is_bare (repo_mode)) && !phys_object_is_symlink) + /* Is it a symlink physically? */ + if (phys_object_is_symlink) { - if (!create_regular_tmpfile_linkable_with_content (self, size, file_input, - &temp_fd, &tmp_unlinker.path, - cancellable, error)) - return FALSE; - } - else if (_ostree_repo_mode_is_bare (repo_mode) && phys_object_is_symlink) - { - /* Note: This will not be hit for bare-user mode because its converted to a - regular file and take the branch above */ + /* This will not be hit for bare-user or archive */ + g_assert (self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY); if (!_ostree_make_temporary_symlink_at (self->tmp_dir_fd, g_file_info_get_symlink_target (file_info), &tmp_unlinker.path, cancellable, error)) return FALSE; } - else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2) + else if (repo_mode != OSTREE_REPO_MODE_ARCHIVE_Z2) + { + if (!create_regular_tmpfile_linkable_with_content (self, size, file_input, + &tmpf, cancellable, error)) + return FALSE; + } + else { g_autoptr(GVariant) file_meta = NULL; g_autoptr(GConverter) zlib_compressor = NULL; g_autoptr(GOutputStream) compressed_out_stream = NULL; g_autoptr(GOutputStream) temp_out = NULL; + g_assert (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2); + if (self->generate_sizes) indexable = TRUE; if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, - &temp_fd, &tmp_unlinker.path, - error)) + &tmpf, error)) return FALSE; - temp_out = g_unix_output_stream_new (temp_fd, FALSE); + temp_out = g_unix_output_stream_new (tmpf.fd, FALSE); file_meta = _ostree_zlib_file_header_new (file_info, xattrs); @@ -679,7 +652,7 @@ write_content_object (OstreeRepo *self, if (!g_output_stream_flush (temp_out, cancellable, error)) return FALSE; - if (fchmod (temp_fd, 0644) < 0) + if (fchmod (tmpf.fd, 0644) < 0) return glnx_throw_errno_prefix (error, "fchmod"); } @@ -720,23 +693,61 @@ write_content_object (OstreeRepo *self, const guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid"); const guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid"); const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); - if (!commit_loose_content_object (self, actual_checksum, - tmp_unlinker.path, - object_file_type == G_FILE_TYPE_SYMBOLIC_LINK, - uid, gid, mode, - xattrs, temp_fd, - cancellable, error)) - return glnx_prefix_error (error, "Writing object %s.%s", actual_checksum, - ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE)); - /* Clear the unlinker, it was consumed */ - ot_cleanup_unlinkat_clear (&tmp_unlinker); + /* Is it "physically" a symlink? */ + if (phys_object_is_symlink) + { + if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + { + /* We don't store the metadata in bare-user-only, so we're done. */ + } + else if (self->mode == OSTREE_REPO_MODE_BARE) + { + /* Now that we know the checksum is valid, apply uid/gid, mode bits, + * and extended attributes. + * + * Note, this does not apply for bare-user repos, as they store symlinks + * as regular files. + */ + if (G_UNLIKELY (fchownat (self->tmp_dir_fd, tmp_unlinker.path, + uid, gid, AT_SYMLINK_NOFOLLOW) == -1)) + return glnx_throw_errno_prefix (error, "fchownat"); + + if (xattrs != NULL) + { + ot_security_smack_reset_dfd_name (self->tmp_dir_fd, tmp_unlinker.path); + if (!glnx_dfd_name_set_all_xattrs (self->tmp_dir_fd, tmp_unlinker.path, + xattrs, cancellable, error)) + return FALSE; + } + } + else + { + /* We don't do symlinks in archive or bare-user */ + g_assert_not_reached (); + } + + if (!_ostree_repo_commit_path_final (self, actual_checksum, OSTREE_OBJECT_TYPE_FILE, + &tmp_unlinker, + cancellable, error)) + return FALSE; + } + else + { + /* This path is for regular files */ + if (!commit_loose_regfile_object (self, actual_checksum, &tmpf, + uid, gid, mode, + xattrs, + cancellable, error)) + return glnx_prefix_error (error, "Writing object %s.%s", actual_checksum, + ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE)); + } /* Update size metadata if configured */ if (indexable && object_file_type == G_FILE_TYPE_REGULAR) { struct stat stbuf; - if (!glnx_fstat (temp_fd, &stbuf, error)) + if (!glnx_fstat (tmpf.fd, &stbuf, error)) return FALSE; repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); @@ -832,9 +843,9 @@ write_metadata_object (OstreeRepo *self, } /* Write the metadata to a temporary file */ - g_auto(OtTmpfile) tmpf = { 0, }; - if (!ot_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, - &tmpf, error)) + g_auto(GLnxTmpfile) tmpf = { 0, }; + if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, + &tmpf, error)) return FALSE; if (!ot_fallocate (tmpf.fd, len, error)) return FALSE; @@ -844,12 +855,9 @@ write_metadata_object (OstreeRepo *self, return glnx_throw_errno_prefix (error, "fchmod"); /* And commit it into place */ - if (!_ostree_repo_commit_loose_final (self, actual_checksum, objtype, - self->tmp_dir_fd, tmpf.fd, tmpf.path, - cancellable, error)) + if (!_ostree_repo_commit_tmpf_final (self, actual_checksum, objtype, + &tmpf, cancellable, error)) return FALSE; - /* The temp path was consumed */ - g_clear_pointer (&tmpf.path, g_free); if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 1bd797fa..d518e52b 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -296,20 +296,26 @@ _ostree_repo_verify_commit_internal (OstreeRepo *self, GError **error); gboolean -_ostree_repo_commit_loose_final (OstreeRepo *self, - const char *checksum, - OstreeObjectType objtype, - int temp_dfd, - int fd, - const char *temp_filename, - GCancellable *cancellable, - GError **error); +_ostree_repo_commit_tmpf_final (OstreeRepo *self, + const char *checksum, + OstreeObjectType objtype, + GLnxTmpfile *tmpf, + GCancellable *cancellable, + GError **error); + +gboolean +_ostree_repo_commit_path_final (OstreeRepo *self, + const char *checksum, + OstreeObjectType objtype, + OtCleanupUnlinkat *tmp_path, + GCancellable *cancellable, + GError **error); gboolean _ostree_repo_open_content_bare (OstreeRepo *self, const char *checksum, guint64 content_len, - OtTmpfile *out_tmpf, + GLnxTmpfile *out_tmpf, gboolean *out_have_object, GCancellable *cancellable, GError **error); @@ -317,7 +323,7 @@ _ostree_repo_open_content_bare (OstreeRepo *self, gboolean _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, const char *checksum, - OtTmpfile *tmpf, + GLnxTmpfile *tmpf, guint32 uid, guint32 gid, guint32 mode, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index fc29a2ec..5479d74b 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -941,12 +941,10 @@ content_fetch_on_complete (GObject *object, if (!have_object) { - if (!_ostree_repo_commit_loose_final (pull_data->repo, checksum, OSTREE_OBJECT_TYPE_FILE, - tmp_unlinker.dfd, -1, tmp_unlinker.path, - cancellable, error)) + if (!_ostree_repo_commit_path_final (pull_data->repo, checksum, objtype, + &tmp_unlinker, + cancellable, error)) goto out; - /* The path was consumed */ - ot_cleanup_unlinkat_clear (&tmp_unlinker); } pull_data->n_fetched_content++; } @@ -954,6 +952,7 @@ content_fetch_on_complete (GObject *object, { /* Non-mirroring path */ + /* If it appears corrupted, we'll delete it below */ if (!ostree_content_file_parse_at (TRUE, _ostree_fetcher_get_dfd (fetcher), tmp_unlinker.path, FALSE, &file_in, &file_info, &xattrs, diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 7bcf5a92..6d26a1e5 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -437,29 +437,25 @@ get_unpacked_unlinked_content (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - g_autofree char *tmpname = NULL; - glnx_fd_close int fd = -1; + g_auto(GLnxTmpfile) tmpf = { 0, }; g_autoptr(GBytes) ret_content = NULL; g_autoptr(GInputStream) istream = NULL; g_autoptr(GOutputStream) out = NULL; if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, "/tmp", O_RDWR | O_CLOEXEC, - &fd, &tmpname, error)) + &tmpf, error)) return FALSE; - /* We don't need the file name */ - if (tmpname) - (void) unlinkat (AT_FDCWD, tmpname, 0); if (!ostree_repo_load_file (repo, checksum, &istream, NULL, NULL, cancellable, error)) return FALSE; - out = g_unix_output_stream_new (fd, FALSE); + out = g_unix_output_stream_new (tmpf.fd, FALSE); if (g_output_stream_splice (out, istream, G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, cancellable, error) < 0) return FALSE; - { g_autoptr(GMappedFile) mfile = g_mapped_file_new_from_fd (fd, FALSE, error); + { g_autoptr(GMappedFile) mfile = g_mapped_file_new_from_fd (tmpf.fd, FALSE, error); if (!mfile) return FALSE; ret_content = g_mapped_file_get_bytes (mfile); @@ -1163,6 +1159,15 @@ get_fallback_headers (OstreeRepo *self, return TRUE; } +static inline void +glnx_tmpfile_clear_p (void *ptr) +{ + GLnxTmpfile *tmpf = ptr; + if (!tmpf) + return; + glnx_tmpfile_clear (tmpf); +} + /** * ostree_repo_static_delta_generate: * @self: Repo @@ -1213,7 +1218,6 @@ ostree_repo_static_delta_generate (OstreeRepo *self, guint64 total_compressed_size = 0; guint64 total_uncompressed_size = 0; g_autoptr(GVariantBuilder) part_headers = NULL; - g_autoptr(GArray) part_temp_fds = NULL; g_autoptr(GPtrArray) part_temp_paths = NULL; g_autoptr(GVariant) delta_descriptor = NULL; g_autoptr(GVariant) to_commit = NULL; @@ -1325,8 +1329,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } part_headers = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT)); - part_temp_paths = g_ptr_array_new_with_free_func (g_free); - part_temp_fds = g_array_new (FALSE, TRUE, sizeof(int)); + part_temp_paths = g_ptr_array_new_with_free_func (glnx_tmpfile_clear_p); for (i = 0; i < builder.parts->len; i++) { OstreeStaticDeltaPartBuilder *part_builder = builder.parts->pdata[i]; @@ -1399,17 +1402,15 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } else { - char *part_tempfile; - int part_temp_fd; + GLnxTmpfile *part_tmpf = g_new0 (GLnxTmpfile, 1); if (!glnx_open_tmpfile_linkable_at (tmp_dfd, ".", O_WRONLY | O_CLOEXEC, - &part_temp_fd, &part_tempfile, error)) + part_tmpf, error)) goto out; - /* Transfer tempfile ownership to arrays */ - g_array_append_val (part_temp_fds, part_temp_fd); - g_ptr_array_add (part_temp_paths, g_steal_pointer (&part_tempfile)); - part_temp_outstream = g_unix_output_stream_new (part_temp_fd, FALSE); + /* Transfer tempfile ownership */ + part_temp_outstream = g_unix_output_stream_new (part_tmpf->fd, FALSE); + g_ptr_array_add (part_temp_paths, g_steal_pointer (&part_tmpf)); } part_in = ot_variant_read (delta_part); @@ -1468,17 +1469,16 @@ ostree_repo_static_delta_generate (OstreeRepo *self, { g_autofree char *partstr = g_strdup_printf ("%u", i); /* Take ownership of the path/fd here */ - g_autofree char *path = g_steal_pointer (&part_temp_paths->pdata[i]); - glnx_fd_close int fd = g_array_index (part_temp_fds, int, i); - g_array_index (part_temp_fds, int, i) = -1; + g_auto(GLnxTmpfile) tmpf = *((GLnxTmpfile*)part_temp_paths->pdata[i]); + g_clear_pointer (&(part_temp_paths->pdata[i]), g_free); - if (fchmod (fd, 0644) < 0) + if (fchmod (tmpf.fd, 0644) < 0) { glnx_set_error_from_errno (error); goto out; } - if (!glnx_link_tmpfile_at (tmp_dfd, GLNX_LINK_TMPFILE_REPLACE, fd, path, + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, descriptor_dfd, partstr, error)) goto out; } @@ -1538,14 +1538,6 @@ ostree_repo_static_delta_generate (OstreeRepo *self, ret = TRUE; out: - if (part_temp_fds) - for (i = 0; i < part_temp_fds->len; i++) - { - int fd = g_array_index (part_temp_fds, int, i); - if (fd == -1) - continue; - (void) close (fd); - } g_clear_pointer (&builder.parts, g_ptr_array_unref); g_clear_pointer (&builder.fallback_objects, g_ptr_array_unref); return ret; diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index b8bd6587..9141f659 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -56,7 +56,7 @@ typedef struct { GError **async_error; OstreeObjectType output_objtype; - OtTmpfile tmpf; + GLnxTmpfile tmpf; guint64 content_size; GOutputStream *content_out; GChecksum *content_checksum; @@ -281,7 +281,7 @@ _ostree_static_delta_part_execute (OstreeRepo *repo, ret = TRUE; out: - ot_tmpfile_clear (&state->tmpf); + glnx_tmpfile_clear (&state->tmpf); g_clear_object (&state->content_out); g_clear_pointer (&state->content_checksum, g_checksum_free); return ret; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 6dcb70de..20f866b9 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3862,8 +3862,7 @@ sign_data (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - glnx_fd_close int tmp_fd = -1; - g_autofree char *tmp_path = NULL; + g_auto(GLnxTmpfile) tmpf = { 0, }; g_autoptr(GOutputStream) tmp_signature_output = NULL; gpgme_ctx_t context = NULL; g_autoptr(GBytes) ret_signature = NULL; @@ -3874,9 +3873,9 @@ sign_data (OstreeRepo *self, g_autoptr(GMappedFile) signature_file = NULL; if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_RDWR | O_CLOEXEC, - &tmp_fd, &tmp_path, error)) + &tmpf, error)) goto out; - tmp_signature_output = g_unix_output_stream_new (tmp_fd, FALSE); + tmp_signature_output = g_unix_output_stream_new (tmpf.fd, FALSE); context = ot_gpgme_new_ctx (homedir, error); if (!context) @@ -3930,7 +3929,7 @@ sign_data (OstreeRepo *self, if (!g_output_stream_close (tmp_signature_output, cancellable, error)) goto out; - signature_file = g_mapped_file_new_from_fd (tmp_fd, FALSE, error); + signature_file = g_mapped_file_new_from_fd (tmpf.fd, FALSE, error); if (!signature_file) goto out; ret_signature = g_mapped_file_get_bytes (signature_file); diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c index 8ed88984..d2bde837 100644 --- a/src/libotutil/ot-fs-utils.c +++ b/src/libotutil/ot-fs-utils.c @@ -25,61 +25,6 @@ #include #include -/* Before https://github.com/GNOME/libglnx/commit/9929adc, the libglnx - * tmpfile API made it hard to clean up tmpfiles in failure cases. - * it's API breaking. Carry the fix here until we're ready to fully port. - */ -void -ot_tmpfile_clear (OtTmpfile *tmpf) -{ - if (!tmpf->initialized) - return; - if (tmpf->fd == -1) - return; - (void) close (tmpf->fd); - /* If ->path is set, we're likely aborting due to an error. Clean it up */ - if (tmpf->path) - { - (void) unlinkat (tmpf->src_dfd, tmpf->path, 0); - g_free (tmpf->path); - } -} - -gboolean -ot_open_tmpfile_linkable_at (int dfd, - const char *subpath, - int flags, - OtTmpfile *out_tmpf, - GError **error) -{ - if (!glnx_open_tmpfile_linkable_at (dfd, subpath, flags, &out_tmpf->fd, &out_tmpf->path, error)) - return FALSE; - out_tmpf->initialized = TRUE; - out_tmpf->src_dfd = dfd; - return TRUE; -} - -gboolean -ot_link_tmpfile_at (OtTmpfile *tmpf, - GLnxLinkTmpfileReplaceMode mode, - int target_dfd, - const char *target, - GError **error) -{ - g_return_val_if_fail (tmpf->initialized, FALSE); - glnx_fd_close int fd = glnx_steal_fd (&tmpf->fd); - if (!glnx_link_tmpfile_at (tmpf->src_dfd, mode, fd, tmpf->path, - target_dfd, target, error)) - { - if (tmpf->path) - (void) unlinkat (tmpf->src_dfd, tmpf->path, 0); - tmpf->initialized = FALSE; - return FALSE; - } - tmpf->initialized = FALSE; - return TRUE; -} - /* Convert a fd-relative path to a GFile* - use * for legacy code. */ @@ -213,9 +158,8 @@ ot_dfd_iter_init_allow_noent (int dfd, *out_exists = FALSE; return TRUE; } - if (!glnx_dirfd_iterator_init_take_fd (fd, dfd_iter, error)) + if (!glnx_dirfd_iterator_init_take_fd (&fd, dfd_iter, error)) return FALSE; - fd = -1; *out_exists = TRUE; return TRUE; } diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h index 0f62cb7f..3b40d337 100644 --- a/src/libotutil/ot-fs-utils.h +++ b/src/libotutil/ot-fs-utils.h @@ -25,31 +25,6 @@ G_BEGIN_DECLS -/* This is a copy of https://github.com/GNOME/libglnx/pull/46 until we - * can do a full port; see https://github.com/ostreedev/ostree/pull/861 */ -typedef struct { - gboolean initialized; - int src_dfd; - int fd; - char *path; -} OtTmpfile; -void ot_tmpfile_clear (OtTmpfile *tmpf); -G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OtTmpfile, ot_tmpfile_clear); - -gboolean -ot_open_tmpfile_linkable_at (int dfd, - const char *subpath, - int flags, - OtTmpfile *out_tmpf, - GError **error); -gboolean -ot_link_tmpfile_at (OtTmpfile *tmpf, - GLnxLinkTmpfileReplaceMode flags, - int target_dfd, - const char *target, - GError **error); - - /* A little helper to call unlinkat() as a cleanup * function. Mostly only necessary to handle * deletion of temporary symlinks. @@ -76,7 +51,6 @@ ot_cleanup_unlinkat (OtCleanupUnlinkat *cleanup) } G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OtCleanupUnlinkat, ot_cleanup_unlinkat); - GFile * ot_fdrel_to_gfile (int dfd, const char *path); gboolean ot_readlinkat_gfile_info (int dfd, diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c index 9e152e18..a33b38bf 100644 --- a/src/ostree/ot-remote-cookie-util.c +++ b/src/ostree/ot-remote-cookie-util.c @@ -202,7 +202,7 @@ ot_delete_cookie_at (int dfd, const char *jar_path, { gboolean found = FALSE; #ifdef HAVE_LIBCURL - g_auto(OtTmpfile) tmpf = { 0, }; + g_auto(GLnxTmpfile) tmpf = { 0, }; g_autofree char *dnbuf = NULL; const char *dn = NULL; g_autoptr(OtCookieParser) parser = NULL; @@ -212,8 +212,8 @@ ot_delete_cookie_at (int dfd, const char *jar_path, dnbuf = g_strdup (jar_path); dn = dirname (dnbuf); - if (!ot_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, - &tmpf, error)) + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tmpf, error)) return FALSE; while (ot_parse_cookies_next (parser)) @@ -232,9 +232,9 @@ ot_delete_cookie_at (int dfd, const char *jar_path, return glnx_throw_errno_prefix (error, "write"); } - if (!ot_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, - AT_FDCWD, jar_path, - error)) + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, + AT_FDCWD, jar_path, + error)) return FALSE; #else GSList *cookies; From ba918e49c5dca5ee927f0ed5a9c6156e85170c9b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 27 Jun 2017 21:26:17 -0400 Subject: [PATCH 56/82] tree-wide: Misc porting to newer libglnx APIs - Use the new tmpfile bits - `glnx_try_fallocate` - `glnx_renameat()` Depends: https://github.com/GNOME/libglnx/pull/57 Update submodule: libglnx Closes: #970 Approved by: jlebon --- libglnx | 2 +- src/libostree/ostree-repo-commit.c | 26 ++++--------------- .../ostree-repo-static-delta-compilation.c | 14 ++-------- src/libostree/ostree-sysroot-deploy.c | 24 +++++------------ 4 files changed, 15 insertions(+), 51 deletions(-) diff --git a/libglnx b/libglnx index caa51ac2..01e934c1 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit caa51ac24ffcdffcb610bc6ccc9da964d4be74ee +Subproject commit 01e934c18efdbac071ebc19a8a95916d324970c9 diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index d64f6481..ad847e89 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -416,22 +416,6 @@ add_size_index_to_metadata (OstreeRepo *self, return ret; } -static gboolean -ot_fallocate (int fd, goffset size, GError **error) -{ - if (size == 0) - return TRUE; - - int r = posix_fallocate (fd, 0, size); - if (r != 0) - { - /* posix_fallocate is a weird deviation from errno standards */ - errno = r; - return glnx_throw_errno_prefix (error, "fallocate"); - } - return TRUE; -} - /* Combines a check for whether or not we already have the object with * allocating a tempfile if we don't. Used by the static delta code. */ @@ -496,7 +480,7 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self, &tmpf, error)) return FALSE; - if (!ot_fallocate (tmpf.fd, length, error)) + if (!glnx_try_fallocate (tmpf.fd, 0, length, error)) return FALSE; if (G_IS_FILE_DESCRIPTOR_BASED (input)) @@ -847,7 +831,7 @@ write_metadata_object (OstreeRepo *self, if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, &tmpf, error)) return FALSE; - if (!ot_fallocate (tmpf.fd, len, error)) + if (!glnx_try_fallocate (tmpf.fd, 0, len, error)) return FALSE; if (glnx_loop_write (tmpf.fd, bufp, len) < 0) return glnx_throw_errno_prefix (error, "write()"); @@ -1179,9 +1163,9 @@ rename_pending_loose_objects (OstreeRepo *self, cancellable, error)) return FALSE; - if (G_UNLIKELY (renameat (child_dfd_iter.fd, loose_objpath + 3, - self->objects_dir_fd, loose_objpath) < 0)) - return glnx_throw_errno (error); + if (!glnx_renameat (child_dfd_iter.fd, loose_objpath + 3, + self->objects_dir_fd, loose_objpath, error)) + return FALSE; renamed_some_object = TRUE; } diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 6d26a1e5..8a1bb1d2 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -442,8 +442,7 @@ get_unpacked_unlinked_content (OstreeRepo *repo, g_autoptr(GInputStream) istream = NULL; g_autoptr(GOutputStream) out = NULL; - if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, "/tmp", O_RDWR | O_CLOEXEC, - &tmpf, error)) + if (!glnx_open_anonymous_tmpfile (O_RDWR | O_CLOEXEC, &tmpf, error)) return FALSE; if (!ostree_repo_load_file (repo, checksum, &istream, NULL, NULL, @@ -1159,15 +1158,6 @@ get_fallback_headers (OstreeRepo *self, return TRUE; } -static inline void -glnx_tmpfile_clear_p (void *ptr) -{ - GLnxTmpfile *tmpf = ptr; - if (!tmpf) - return; - glnx_tmpfile_clear (tmpf); -} - /** * ostree_repo_static_delta_generate: * @self: Repo @@ -1329,7 +1319,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } part_headers = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT)); - part_temp_paths = g_ptr_array_new_with_free_func (glnx_tmpfile_clear_p); + part_temp_paths = g_ptr_array_new_with_free_func ((GDestroyNotify)glnx_tmpfile_clear); for (i = 0; i < builder.parts->len; i++) { OstreeStaticDeltaPartBuilder *part_builder = builder.parts->pdata[i]; diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index ed4831c1..a1584f8c 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -73,15 +73,9 @@ symlink_at_replace (const char *oldpath, goto out; } - /* Rename it into place */ - do - res = renameat (parent_dfd, temppath, parent_dfd, newpath); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + /* Rename it into place */ + if (!glnx_renameat (parent_dfd, temppath, parent_dfd, newpath, error)) + goto out; ret = TRUE; out: @@ -819,8 +813,8 @@ merge_configuration (OstreeSysroot *sysroot, else if (etc_exists) { /* Compatibility hack */ - if (renameat (deployment_dfd, "etc", deployment_dfd, "usr/etc") < 0) - return glnx_throw_errno_prefix (error, "renameat"); + if (!glnx_renameat (deployment_dfd, "etc", deployment_dfd, "usr/etc", error)) + return FALSE; usretc_exists = TRUE; etc_exists = FALSE; } @@ -1424,7 +1418,6 @@ swap_bootloader (OstreeSysroot *sysroot, GError **error) { glnx_fd_close int boot_dfd = -1; - int res; g_assert ((current_bootversion == 0 && new_bootversion == 1) || (current_bootversion == 1 && new_bootversion == 0)); @@ -1436,11 +1429,8 @@ swap_bootloader (OstreeSysroot *sysroot, * its data is in place. Renaming now should give us atomic semantics; * see https://bugzilla.gnome.org/show_bug.cgi?id=755595 */ - do - res = renameat (boot_dfd, "loader.tmp", boot_dfd, "loader"); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1) - return glnx_throw_errno (error); + if (!glnx_renameat (boot_dfd, "loader.tmp", boot_dfd, "loader", error)) + return FALSE; /* Now we explicitly fsync this directory, even though it * isn't required for atomicity, for two reasons: From 373dc4b66c7966bd917039addf134a1dc556db3f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 28 Jun 2017 08:06:50 -0700 Subject: [PATCH 57/82] codebase: start using GLNX_HASH_TABLE_FOREACH macros Use the new macros introduced recently in libglnx to make iterating over hash tables cleaner. This is just a start, it does not migrate the whole tree. Update submodule: libglnx Closes: #971 Approved by: cgwalters --- src/libostree/ostree-bootconfig-parser.c | 17 ++---- src/libostree/ostree-fetcher-curl.c | 11 +--- src/libostree/ostree-fetcher-soup.c | 14 ++--- src/libostree/ostree-mutable-tree.c | 7 +-- src/libostree/ostree-repo-checkout.c | 11 ++-- src/libostree/ostree-repo-commit.c | 51 +++++------------ src/libostree/ostree-repo-libarchive.c | 13 ++--- src/libostree/ostree-repo-prune.c | 28 ++-------- src/libostree/ostree-repo-pull.c | 70 ++++++------------------ 9 files changed, 59 insertions(+), 163 deletions(-) diff --git a/src/libostree/ostree-bootconfig-parser.c b/src/libostree/ostree-bootconfig-parser.c index a7c473dd..9de0e56b 100644 --- a/src/libostree/ostree-bootconfig-parser.c +++ b/src/libostree/ostree-bootconfig-parser.c @@ -48,15 +48,11 @@ OstreeBootconfigParser * ostree_bootconfig_parser_clone (OstreeBootconfigParser *self) { OstreeBootconfigParser *parser = ostree_bootconfig_parser_new (); - guint i; - GHashTableIter hashiter; - gpointer k, v; - for (i = 0; i < self->lines->len; i++) + for (guint i = 0; i < self->lines->len; i++) g_ptr_array_add (parser->lines, g_variant_ref (self->lines->pdata[i])); - g_hash_table_iter_init (&hashiter, self->options); - while (g_hash_table_iter_next (&hashiter, &k, &v)) + GLNX_HASH_TABLE_FOREACH_KV (self->options, const char*, k, const char*, v) g_hash_table_replace (parser->options, g_strdup (k), g_strdup (v)); return parser; @@ -183,14 +179,11 @@ ostree_bootconfig_parser_write_at (OstreeBootconfigParser *self, } } - GHashTableIter hashiter; - gpointer hashkey, hashvalue; - g_hash_table_iter_init (&hashiter, self->options); - while (g_hash_table_iter_next (&hashiter, &hashkey, &hashvalue)) + GLNX_HASH_TABLE_FOREACH_KV (self->options, const char*, k, const char*, v) { - if (g_hash_table_lookup (written_overrides, hashkey)) + if (g_hash_table_lookup (written_overrides, k)) continue; - write_key (self, buf, hashkey, hashvalue); + write_key (self, buf, k, v); } if (!glnx_file_replace_contents_at (dfd, path, (guint8*)buf->str, buf->len, diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index 5d35e7b7..77844ec7 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -690,9 +690,6 @@ static void adopt_steal_mainctx (OstreeFetcher *self, GMainContext *mainctx) { - GHashTableIter hiter; - gpointer key, value; - g_assert (self->mainctx == NULL); self->mainctx = mainctx; /* Transfer */ @@ -706,12 +703,8 @@ adopt_steal_mainctx (OstreeFetcher *self, update_timeout_cb (self->multi, timeout_micros / 1000, self); } - g_hash_table_iter_init (&hiter, self->sockets); - while (g_hash_table_iter_next (&hiter, &key, &value)) - { - SockInfo *fdp = key; - setsock (fdp, fdp->sockfd, fdp->action, self); - } + GLNX_HASH_TABLE_FOREACH (self->sockets, SockInfo*, fdp) + setsock (fdp, fdp->sockfd, fdp->action, self); } static void diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c index 1ca2e771..b877d27c 100644 --- a/src/libostree/ostree-fetcher-soup.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -1323,24 +1323,18 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher *self) { - GHashTableIter hiter; - gpointer key, value; - guint64 ret; - g_return_val_if_fail (OSTREE_IS_FETCHER (self), 0); g_mutex_lock (&self->thread_closure->output_stream_set_lock); - ret = self->thread_closure->total_downloaded; + guint64 ret = self->thread_closure->total_downloaded; - g_hash_table_iter_init (&hiter, self->thread_closure->output_stream_set); - while (g_hash_table_iter_next (&hiter, &key, &value)) + GLNX_HASH_TABLE_FOREACH (self->thread_closure->output_stream_set, + GFileOutputStream*, stream) { - GFileOutputStream *stream = key; - struct stat stbuf; - if (G_IS_FILE_DESCRIPTOR_BASED (stream)) { + struct stat stbuf; if (glnx_stream_fstat ((GFileDescriptorBased*)stream, &stbuf, NULL)) ret += stbuf.st_size; } diff --git a/src/libostree/ostree-mutable-tree.c b/src/libostree/ostree-mutable-tree.c index 5540cc7c..c263fbe4 100644 --- a/src/libostree/ostree-mutable-tree.c +++ b/src/libostree/ostree-mutable-tree.c @@ -114,9 +114,6 @@ ostree_mutable_tree_set_contents_checksum (OstreeMutableTree *self, const char * ostree_mutable_tree_get_contents_checksum (OstreeMutableTree *self) { - GHashTableIter iter; - gpointer key, value; - if (!self->contents_checksum) return NULL; @@ -127,10 +124,8 @@ ostree_mutable_tree_get_contents_checksum (OstreeMutableTree *self) * * However, we only call this function once right now. */ - g_hash_table_iter_init (&iter, self->subdirs); - while (g_hash_table_iter_next (&iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_V (self->subdirs, OstreeMutableTree*, subdir) { - OstreeMutableTree *subdir = value; if (!ostree_mutable_tree_get_contents_checksum (subdir)) { g_free (self->contents_checksum); diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index ac8d5b46..14b25540 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -1088,13 +1088,12 @@ ostree_repo_checkout_gc (OstreeRepo *self, self->updated_uncompressed_dirs = g_hash_table_new (NULL, NULL); g_mutex_unlock (&self->cache_lock); - GHashTableIter iter; - gpointer key, value; - if (to_clean_dirs) - g_hash_table_iter_init (&iter, to_clean_dirs); - while (to_clean_dirs && g_hash_table_iter_next (&iter, &key, &value)) + if (!to_clean_dirs) + return TRUE; /* Note early return */ + + GLNX_HASH_TABLE_FOREACH (to_clean_dirs, guint, prefix) { - g_autofree char *objdir_name = g_strdup_printf ("%02x", GPOINTER_TO_UINT (key)); + g_autofree char *objdir_name = g_strdup_printf ("%02x", prefix); g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (self->uncompressed_objects_dir_fd, objdir_name, FALSE, diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index ad847e89..5753edca 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -362,33 +362,24 @@ add_size_index_to_metadata (OstreeRepo *self, { gboolean ret = FALSE; g_autoptr(GVariantBuilder) builder = NULL; - + /* original_metadata may be NULL */ builder = ot_util_variant_builder_from_variant (original_metadata, G_VARIANT_TYPE ("a{sv}")); if (self->object_sizes && g_hash_table_size (self->object_sizes) > 0) { - GHashTableIter entries = { 0 }; - gchar *e_checksum = NULL; - OstreeContentSizeCacheEntry *e_size = NULL; GVariantBuilder index_builder; - guint i; - g_autoptr(GPtrArray) sorted_keys = NULL; - - g_hash_table_iter_init (&entries, self->object_sizes); g_variant_builder_init (&index_builder, G_VARIANT_TYPE ("a" _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE)); /* Sort the checksums so we can bsearch if desired */ - sorted_keys = g_ptr_array_new (); - while (g_hash_table_iter_next (&entries, - (gpointer *) &e_checksum, - (gpointer *) &e_size)) - g_ptr_array_add (sorted_keys, e_checksum); + g_autoptr(GPtrArray) sorted_keys = g_ptr_array_new (); + GLNX_HASH_TABLE_FOREACH (self->object_sizes, const char*, e_checksum) + g_ptr_array_add (sorted_keys, (gpointer)e_checksum); g_ptr_array_sort (sorted_keys, compare_ascii_checksums_for_sorting); - for (i = 0; i < sorted_keys->len; i++) + for (guint i = 0; i < sorted_keys->len; i++) { guint8 csum[OSTREE_SHA256_DIGEST_LEN]; const char *e_checksum = sorted_keys->pdata[i]; @@ -397,18 +388,19 @@ add_size_index_to_metadata (OstreeRepo *self, ostree_checksum_inplace_to_bytes (e_checksum, csum); g_string_append_len (buffer, (char*)csum, sizeof (csum)); - e_size = g_hash_table_lookup (self->object_sizes, e_checksum); + OstreeContentSizeCacheEntry *e_size = + g_hash_table_lookup (self->object_sizes, e_checksum); _ostree_write_varuint64 (buffer, e_size->archived); _ostree_write_varuint64 (buffer, e_size->unpacked); g_variant_builder_add (&index_builder, "@ay", ot_gvariant_new_bytearray ((guint8*)buffer->str, buffer->len)); } - + g_variant_builder_add (builder, "{sv}", "ostree.sizes", g_variant_builder_end (&index_builder)); } - + ret = TRUE; *out_metadata = g_variant_builder_end (builder); g_variant_ref_sink (*out_metadata); @@ -2234,8 +2226,6 @@ create_tree_variant_from_hashes (GHashTable *file_checksums, GHashTable *dir_contents_checksums, GHashTable *dir_metadata_checksums) { - GHashTableIter hash_iter; - gpointer key, value; GVariantBuilder files_builder; GVariantBuilder dirs_builder; GSList *sorted_filenames = NULL; @@ -2245,11 +2235,8 @@ create_tree_variant_from_hashes (GHashTable *file_checksums, g_variant_builder_init (&files_builder, G_VARIANT_TYPE ("a(say)")); g_variant_builder_init (&dirs_builder, G_VARIANT_TYPE ("a(sayay)")); - g_hash_table_iter_init (&hash_iter, file_checksums); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH (file_checksums, const char*, name) { - const char *name = key; - /* Should have been validated earlier, but be paranoid */ g_assert (ot_util_filename_validate (name, NULL)); @@ -2270,13 +2257,8 @@ create_tree_variant_from_hashes (GHashTable *file_checksums, g_slist_free (sorted_filenames); sorted_filenames = NULL; - - g_hash_table_iter_init (&hash_iter, dir_metadata_checksums); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *name = key; - sorted_filenames = g_slist_prepend (sorted_filenames, (char*)name); - } + GLNX_HASH_TABLE_FOREACH (dir_metadata_checksums, const char*, name) + sorted_filenames = g_slist_prepend (sorted_filenames, (char*)name); sorted_filenames = g_slist_sort (sorted_filenames, (GCompareFunc)strcmp); @@ -2993,15 +2975,10 @@ ostree_repo_write_mtree (OstreeRepo *self, dir_metadata_checksums = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, (GDestroyNotify)g_free); - GHashTableIter hash_iter; - gpointer key, value; - g_hash_table_iter_init (&hash_iter, ostree_mutable_tree_get_subdirs (mtree)); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_KV (ostree_mutable_tree_get_subdirs (mtree), + const char*, name, OstreeMutableTree*, child_dir) { - const char *name = key; g_autoptr(GFile) child_file = NULL; - OstreeMutableTree *child_dir = value; - if (!ostree_repo_write_mtree (self, child_dir, &child_file, cancellable, error)) return FALSE; diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 02f1364e..1109d5d7 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -773,14 +773,11 @@ aic_import_deferred_hardlinks (OstreeRepoArchiveImportContext *ctx, GCancellable *cancellable, GError **error) { - GHashTableIter iter; - gpointer key, value; - - g_hash_table_iter_init (&iter, ctx->deferred_hardlinks); - while (g_hash_table_iter_next (&iter, &key, &value)) - if (!aic_import_deferred_hardlinks_for (ctx, key, value, error)) - return FALSE; - + GLNX_HASH_TABLE_FOREACH_KV (ctx->deferred_hardlinks, const char*, target, GSList*, links) + { + if (!aic_import_deferred_hardlinks_for (ctx, target, links, error)) + return FALSE; + } return TRUE; } diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index c0da7121..7cd9eb2c 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -232,8 +232,6 @@ repo_prune_internal (OstreeRepo *self, GCancellable *cancellable, GError **error) { - GHashTableIter hash_iter; - gpointer key, value; OtPruneData data = { 0, }; data.repo = self; @@ -241,11 +239,8 @@ repo_prune_internal (OstreeRepo *self, g_autoptr(GHashTable) reachable_owned = g_hash_table_ref (options->reachable); data.reachable = reachable_owned; - g_hash_table_iter_init (&hash_iter, objects); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_KV (objects, GVariant*, serialized_key, GVariant*, objdata) { - GVariant *serialized_key = key; - GVariant *objdata = value; const char *checksum; OstreeObjectType objtype; gboolean is_loose; @@ -309,13 +304,10 @@ ostree_repo_prune (OstreeRepo *self, GCancellable *cancellable, GError **error) { - GHashTableIter hash_iter; - gpointer key, value; g_autoptr(GHashTable) objects = NULL; - g_autoptr(GHashTable) reachable = NULL; gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY; - reachable = ostree_repo_traverse_new_reachable (); + g_autoptr(GHashTable) reachable = ostree_repo_traverse_new_reachable (); /* This original prune API has fixed logic for traversing refs or all commits * combined with actually deleting content. The newer backend API just does @@ -331,12 +323,8 @@ ostree_repo_prune (OstreeRepo *self, cancellable, error)) return FALSE; - g_hash_table_iter_init (&hash_iter, all_refs); - - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_V (all_refs, const char*, checksum) { - const char *checksum = value; - g_debug ("Finding objects to keep for commit %s", checksum); if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, cancellable, error)) @@ -350,12 +338,8 @@ ostree_repo_prune (OstreeRepo *self, cancellable, error)) return FALSE; - g_hash_table_iter_init (&hash_iter, all_collection_refs); - - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_V (all_collection_refs, const char*, checksum) { - const char *checksum = value; - g_debug ("Finding objects to keep for commit %s", checksum); if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, cancellable, error)) @@ -369,10 +353,8 @@ ostree_repo_prune (OstreeRepo *self, if (!refs_only) { - g_hash_table_iter_init (&hash_iter, objects); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH (objects, GVariant*, serialized_key) { - GVariant *serialized_key = key; const char *checksum; OstreeObjectType objtype; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 5479d74b..040eb421 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2023,23 +2023,19 @@ get_best_static_delta_start_for (OtPullData *pull_data, GCancellable *cancellable, GError **error) { - GHashTableIter hiter; - gpointer hkey, hvalue; /* Array of possible from checksums */ g_autoptr(GPtrArray) candidates = g_ptr_array_new_with_free_func (g_free); const char *newest_candidate = NULL; guint64 newest_candidate_timestamp = 0; g_assert (pull_data->summary_deltas_checksums != NULL); - g_hash_table_iter_init (&hiter, pull_data->summary_deltas_checksums); *out_have_scratch_delta = FALSE; /* Loop over all deltas known from the summary file, * finding ones which go to to_revision */ - while (g_hash_table_iter_next (&hiter, &hkey, &hvalue)) + GLNX_HASH_TABLE_FOREACH (pull_data->summary_deltas_checksums, const char*, delta_name) { - const char *delta_name = hkey; g_autofree char *cur_from_rev = NULL; g_autofree char *cur_to_rev = NULL; @@ -2893,8 +2889,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - GHashTableIter hash_iter; - gpointer key, value; g_autoptr(GBytes) bytes_summary = NULL; g_autofree char *metalink_url_str = NULL; g_autoptr(GHashTable) requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ @@ -3539,15 +3533,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, /* Resolve the checksum for each ref. This has to be done into a new hash table, * since we can’t modify the keys of @requested_refs_to_fetch while iterating * over it, and we need to ensure the collection IDs are resolved too. */ - g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); updated_requested_refs_to_fetch = g_hash_table_new_full (ostree_collection_ref_hash, ostree_collection_ref_equal, (GDestroyNotify) ostree_collection_ref_free, g_free); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_KV (requested_refs_to_fetch, const OstreeCollectionRef*, ref, + const char*, override_commitid) { - const OstreeCollectionRef *ref = key; - const char *override_commitid = value; g_autofree char *contents = NULL; /* Support specifying "" for an override commitid */ @@ -3621,20 +3613,16 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_debug ("resuming legacy transaction"); /* Initiate requests for explicit commit revisions */ - g_hash_table_iter_init (&hash_iter, commits_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_V (commits_to_fetch, const char*, commit) { - const char *commit = value; if (!initiate_request (pull_data, NULL, commit, error)) goto out; } /* Initiate requests for refs */ - g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_KV (requested_refs_to_fetch, const OstreeCollectionRef*, ref, + const char*, to_revision) { - const OstreeCollectionRef *ref = key; - const char *to_revision = value; if (!initiate_request (pull_data, ref, to_revision, error)) goto out; } @@ -3671,11 +3659,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_assert_cmpint (pull_data->n_outstanding_content_fetches, ==, 0); g_assert_cmpint (pull_data->n_outstanding_content_write_requests, ==, 0); - g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_KV (requested_refs_to_fetch, const OstreeCollectionRef*, ref, + const char*, checksum) { - const OstreeCollectionRef *ref = key; - const char *checksum = value; g_autofree char *remote_ref = NULL; g_autofree char *original_rev = NULL; @@ -3762,22 +3748,16 @@ ostree_repo_pull_with_options (OstreeRepo *self, /* iterate over commits fetched and delete any commitpartial files */ if (pull_data->dirs == NULL && !pull_data->is_commit_only) { - g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_V (requested_refs_to_fetch, const char*, checksum) { - const char *checksum = value; g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum); - if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0)) goto out; } - g_hash_table_iter_init (&hash_iter, commits_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH_V (commits_to_fetch, const char*, commit) { - const char *commit = value; g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (commit); - if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0)) goto out; } @@ -4326,18 +4306,16 @@ find_remotes_cb (GObject *obj, GCancellable *cancellable; const FindRemotesData *data; const OstreeCollectionRef * const *refs; - OstreeAsyncProgress *progress; + /* FIXME: We currently do nothing with @progress. Comment out to assuage -Wunused-variable */ + /* OstreeAsyncProgress *progress; */ g_autoptr(GError) error = NULL; g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ gsize i; - GHashTableIter iter; - CommitMetadata *commit_metadata; g_autoptr(PointerTable) refs_and_remotes_table = NULL; /* (element-type commit-checksum) */ g_autoptr(GHashTable) commit_metadatas = NULL; /* (element-type commit-checksum CommitMetadata) */ g_autoptr(OstreeFetcher) fetcher = NULL; g_autofree const gchar **ref_to_latest_commit = NULL; /* indexed as @refs; (element-type commit-checksum) */ gsize n_refs; - const gchar *checksum; g_autoptr(GPtrArray) remotes_to_remove = NULL; /* (element-type OstreeRemote) */ g_autoptr(GPtrArray) final_results = NULL; /* (element-type OstreeRepoFinderResult) */ @@ -4347,7 +4325,7 @@ find_remotes_cb (GObject *obj, data = g_task_get_task_data (task); refs = (const OstreeCollectionRef * const *) data->refs; - progress = data->progress; + /* progress = data->progress; */ /* Finish finding the remotes. */ results = ostree_repo_finder_resolve_all_finish (result, &error); @@ -4383,8 +4361,6 @@ find_remotes_cb (GObject *obj, * estimation for the actual pull operation. This should check the * disable-static-deltas option first. */ - /* FIXME: We currently do nothing with @progress. */ - /* Each key must be a pointer to the #CommitMetadata.checksum field of its value. */ commit_metadatas = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify) commit_metadata_free); @@ -4496,9 +4472,7 @@ find_remotes_cb (GObject *obj, * the commit metadata from the remotes. The ‘most recent commits’ are the * set of head commits pointed to by the refs we just resolved from the * summary files. */ - g_hash_table_iter_init (&iter, commit_metadatas); - - while (g_hash_table_iter_next (&iter, (gpointer *) &checksum, (gpointer *) &commit_metadata)) + GLNX_HASH_TABLE_FOREACH_V (commit_metadatas, CommitMetadata*, commit_metadata) { char buf[_OSTREE_LOOSE_PATH_MAX]; g_autofree gchar *commit_filename = NULL; @@ -4853,9 +4827,6 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, g_autoptr(GHashTable) refs_pulled = NULL; /* (element-type OstreeCollectionRef gboolean) */ gsize i, j; g_autoptr(GString) refs_unpulled_string = NULL; - GHashTableIter iter; - const OstreeCollectionRef *ref; - gpointer is_pulled_pointer; g_autoptr(GError) local_error = NULL; g_auto(GVariantDict) options_dict = OT_VARIANT_BUILDER_INITIALIZER; OstreeRepoPullFlags flags; @@ -4900,16 +4871,14 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, g_auto(GVariantBuilder) refs_to_pull_builder = OT_VARIANT_BUILDER_INITIALIZER; g_auto(GVariantDict) local_options_dict = OT_VARIANT_BUILDER_INITIALIZER; g_autoptr(GVariant) local_options = NULL; - const gchar *checksum; gboolean remove_remote; refs_to_pull = g_ptr_array_new_with_free_func (NULL); refs_to_pull_str = g_string_new (""); g_variant_builder_init (&refs_to_pull_builder, G_VARIANT_TYPE ("a(sss)")); - g_hash_table_iter_init (&iter, result->ref_to_checksum); - - while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum)) + GLNX_HASH_TABLE_FOREACH_KV (result->ref_to_checksum, const OstreeCollectionRef*, ref, + const char*, checksum) { if (checksum != NULL && !GPOINTER_TO_INT (g_hash_table_lookup (refs_pulled, ref))) @@ -4996,12 +4965,9 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, } /* Any refs left un-downloaded? If so, we’ve failed. */ - g_hash_table_iter_init (&iter, refs_pulled); - - while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &is_pulled_pointer)) + GLNX_HASH_TABLE_FOREACH_KV (refs_pulled, const OstreeCollectionRef*, ref, + gboolean, is_pulled) { - gboolean is_pulled = GPOINTER_TO_INT (is_pulled_pointer); - if (is_pulled) continue; From 9d10bdfd0d90d6ea6a31bbd89243f3b23ff51e8d Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 28 Jun 2017 08:59:36 -0700 Subject: [PATCH 58/82] ci: unconditionally turn on -Werror Closes: #971 Approved by: cgwalters --- .papr.yml | 2 +- ci/build-check.sh | 6 +++--- ci/build.sh | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.papr.yml b/.papr.yml index be4c4f04..ac7c3278 100644 --- a/.papr.yml +++ b/.papr.yml @@ -14,7 +14,7 @@ packages: env: # Enable all the sanitizers for this primary build. # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain - CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2 -Werror=maybe-uninitialized' + CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' # Only for CI with a known g-ir-scanner GI_SCANNERFLAGS: '--warn-error' ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc diff --git a/ci/build-check.sh b/ci/build-check.sh index da03f3cc..677515b3 100755 --- a/ci/build-check.sh +++ b/ci/build-check.sh @@ -16,9 +16,9 @@ fi if test -x /usr/bin/clang; then git clean -dfx && git submodule foreach git clean -dfx - # And now a clang build to find unused variables; perhaps - # in the future these could parallelize + # And now a clang build to find unused variables because it does a better + # job than gcc for vars with cleanups; perhaps in the future these could + # parallelize export CC=clang - export CFLAGS='-Werror=unused-variable' build fi diff --git a/ci/build.sh b/ci/build.sh index eefb7c3c..a9a8ac3f 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -12,6 +12,9 @@ pkg_install sudo which attr fuse \ elfutils pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang +# always fail on warnings; https://github.com/ostreedev/ostree/pull/971 +export CFLAGS="-Werror ${CFLAGS:-}" + DETECTED_CONFIGOPTS= if test -x /usr/bin/gnome-desktop-testing-runner; then DETECTED_CONFIGOPTS="${DETECTED_CONFIGOPTS} --enable-installed-tests=exclusive" From 6f2ea23e8a96cf7dddadf0eae1b4f831db88d67f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 13:41:52 -0400 Subject: [PATCH 59/82] libutil: Add a helper for O_TMPFILE + mmap() I added `glnx_open_anonymous_tmpfile()`, but then later noticed that the usage of this was really to be combined with `mmap()`, and we had two versions of that in the delta code. Add a helper. (Bigger picture...how is this different from glibc's "mmap() of /dev/zero" approach for large chunks? One advantage is the storage can be "swapped" to `/var/tmp`, but still deleted automatically, rather than requiring swap space) Closes: #973 Approved by: jlebon --- .../ostree-repo-static-delta-compilation.c | 20 ++--------- src/libostree/ostree-repo-static-delta-core.c | 35 +++---------------- src/libotutil/ot-fs-utils.c | 27 ++++++++++++++ src/libotutil/ot-fs-utils.h | 5 +++ 4 files changed, 38 insertions(+), 49 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 8a1bb1d2..851f3fcb 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -437,31 +437,15 @@ get_unpacked_unlinked_content (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - g_auto(GLnxTmpfile) tmpf = { 0, }; - g_autoptr(GBytes) ret_content = NULL; g_autoptr(GInputStream) istream = NULL; - g_autoptr(GOutputStream) out = NULL; - - if (!glnx_open_anonymous_tmpfile (O_RDWR | O_CLOEXEC, &tmpf, error)) - return FALSE; if (!ostree_repo_load_file (repo, checksum, &istream, NULL, NULL, cancellable, error)) return FALSE; - out = g_unix_output_stream_new (tmpf.fd, FALSE); - if (g_output_stream_splice (out, istream, G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, - cancellable, error) < 0) + *out_content = ot_map_anonymous_tmpfile_from_content (istream, cancellable, error); + if (!*out_content) return FALSE; - - { g_autoptr(GMappedFile) mfile = g_mapped_file_new_from_fd (tmpf.fd, FALSE, error); - if (!mfile) - return FALSE; - ret_content = g_mapped_file_get_bytes (mfile); - } - - if (out_content) - *out_content = g_steal_pointer (&ret_content); return TRUE; } diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 878d607d..a409f9b6 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -473,41 +473,14 @@ _ostree_static_delta_part_open (GInputStream *part_in, break; case 'x': { - g_autofree char *tmppath = g_strdup ("/var/tmp/ostree-delta-XXXXXX"); g_autoptr(GConverter) decomp = (GConverter*) _ostree_lzma_decompressor_new (); g_autoptr(GInputStream) convin = g_converter_input_stream_new (source_in, decomp); - g_autoptr(GOutputStream) unpacked_out = NULL; - glnx_fd_close int unpacked_fd = -1; - gssize n_bytes_written; - - unpacked_fd = g_mkstemp_full (tmppath, O_RDWR | O_CLOEXEC, 0640); - if (unpacked_fd < 0) - { - glnx_set_error_from_errno (error); - goto out; - } - - /* Now make it autocleanup on process exit - in the future, we - * should consider caching unpacked deltas as well. - */ - if (unlink (tmppath) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } - - unpacked_out = g_unix_output_stream_new (unpacked_fd, FALSE); - - n_bytes_written = g_output_stream_splice (unpacked_out, convin, - G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | - G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, - cancellable, error); - if (n_bytes_written < 0) + g_autoptr(GBytes) buf = ot_map_anonymous_tmpfile_from_content (convin, cancellable, error); + if (!buf) goto out; - if (!ot_util_variant_map_fd (unpacked_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), - trusted, &ret_part, error)) - goto out; + ret_part = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), + buf, FALSE); } break; default: diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c index d2bde837..2f0ae19c 100644 --- a/src/libotutil/ot-fs-utils.c +++ b/src/libotutil/ot-fs-utils.c @@ -24,6 +24,7 @@ #include "libglnx.h" #include #include +#include /* Convert a fd-relative path to a GFile* - use * for legacy code. @@ -181,3 +182,29 @@ ot_file_mapat_bytes (int dfd, return g_mapped_file_get_bytes (mfile); } + +/* Given an input stream, splice it to an anonymous file (O_TMPFILE). + * Useful for potentially large but transient files. + */ +GBytes * +ot_map_anonymous_tmpfile_from_content (GInputStream *instream, + GCancellable *cancellable, + GError **error) +{ + g_auto(GLnxTmpfile) tmpf = { 0, }; + if (!glnx_open_anonymous_tmpfile (O_RDWR | O_CLOEXEC, &tmpf, error)) + return NULL; + + g_autoptr(GOutputStream) out = g_unix_output_stream_new (tmpf.fd, FALSE); + gssize n_bytes_written = g_output_stream_splice (out, instream, + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | + G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, + cancellable, error); + if (n_bytes_written < 0) + return NULL; + + g_autoptr(GMappedFile) mfile = g_mapped_file_new_from_fd (tmpf.fd, FALSE, error); + if (!mfile) + return NULL; + return g_mapped_file_get_bytes (mfile); +} diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h index 3b40d337..43bc6942 100644 --- a/src/libotutil/ot-fs-utils.h +++ b/src/libotutil/ot-fs-utils.h @@ -86,6 +86,11 @@ gboolean ot_dfd_iter_init_allow_noent (int dfd, gboolean *out_exists, GError **error); +GBytes * +ot_map_anonymous_tmpfile_from_content (GInputStream *instream, + GCancellable *cancellable, + GError **error); + GBytes *ot_file_mapat_bytes (int dfd, const char *path, GError **error); From ab9fef52792a6deddce49be31606b124335d54f5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 13:54:00 -0400 Subject: [PATCH 60/82] lib/commit: Refactor non-failable size indexing function It can't throw, so remove the `GError` machinery. Closes: #973 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 5753edca..45d10163 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -353,14 +353,10 @@ compare_ascii_checksums_for_sorting (gconstpointer a_pp, /* * Create sizes metadata GVariant and add it to the metadata variant given. */ -static gboolean +static GVariant * add_size_index_to_metadata (OstreeRepo *self, - GVariant *original_metadata, - GVariant **out_metadata, - GCancellable *cancellable, - GError **error) + GVariant *original_metadata) { - gboolean ret = FALSE; g_autoptr(GVariantBuilder) builder = NULL; /* original_metadata may be NULL */ @@ -401,11 +397,7 @@ add_size_index_to_metadata (OstreeRepo *self, g_variant_builder_end (&index_builder)); } - ret = TRUE; - *out_metadata = g_variant_builder_end (builder); - g_variant_ref_sink (*out_metadata); - - return ret; + return g_variant_ref_sink (g_variant_builder_end (builder)); } /* Combines a check for whether or not we already have the object with @@ -2093,10 +2085,7 @@ ostree_repo_write_commit_with_time (OstreeRepo *self, OstreeRepoFile *repo_root = OSTREE_REPO_FILE (root); /* Add sizes information to our metadata object */ - g_autoptr(GVariant) new_metadata = NULL; - if (!add_size_index_to_metadata (self, metadata, &new_metadata, - cancellable, error)) - return FALSE; + g_autoptr(GVariant) new_metadata = add_size_index_to_metadata (self, metadata); g_autoptr(GVariant) commit = g_variant_new ("(@a{sv}@ay@a(say)sst@ay@ay)", From 064d7bffef1a1821cb972b31caa137ecd15cc541 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 13:55:19 -0400 Subject: [PATCH 61/82] lib/deltas: More porting to new code style Just noticed some of this while working on the previous tmpfile bits. Closes: #973 Approved by: jlebon --- src/libostree/ostree-repo-static-delta-core.c | 73 ++++++------------- 1 file changed, 24 insertions(+), 49 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index a409f9b6..c8cefdef 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -65,7 +65,7 @@ _ostree_static_delta_parse_checksum_array (GVariant *array, * * This function synchronously enumerates all static deltas in the * repository, returning its result in @out_deltas. - */ + */ gboolean ostree_repo_list_static_delta_names (OstreeRepo *self, GPtrArray **out_deltas, @@ -409,14 +409,12 @@ _ostree_static_delta_part_open (GInputStream *part_in, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; const gboolean trusted = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED) > 0; const gboolean skip_checksum = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM) > 0; gsize bytes_read; guint8 comptype; g_autoptr(GChecksum) checksum = NULL; g_autoptr(GInputStream) checksum_in = NULL; - g_autoptr(GVariant) ret_part = NULL; GInputStream *source_in; /* We either take a fd or a GBytes reference */ @@ -438,13 +436,11 @@ _ostree_static_delta_part_open (GInputStream *part_in, /* First byte is compression type */ if (!g_input_stream_read_all (source_in, buf, sizeof(buf), &bytes_read, cancellable, error)) - { - g_prefix_error (error, "Reading initial compression flag byte: "); - goto out; - } + return glnx_prefix_error (error, "Reading initial compression flag byte"); comptype = buf[0]; } + g_autoptr(GVariant) ret_part = NULL; switch (comptype) { case 0: @@ -455,7 +451,7 @@ _ostree_static_delta_part_open (GInputStream *part_in, /* No compression, no checksums - a fast path */ if (!ot_util_variant_map_fd (part_fd, 1, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), trusted, &ret_part, error)) - goto out; + return FALSE; } else { @@ -469,7 +465,7 @@ _ostree_static_delta_part_open (GInputStream *part_in, if (!skip_checksum) g_checksum_update (checksum, g_variant_get_data (ret_part), g_variant_get_size (ret_part)); - + break; case 'x': { @@ -477,16 +473,14 @@ _ostree_static_delta_part_open (GInputStream *part_in, g_autoptr(GInputStream) convin = g_converter_input_stream_new (source_in, decomp); g_autoptr(GBytes) buf = ot_map_anonymous_tmpfile_from_content (convin, cancellable, error); if (!buf) - goto out; + return FALSE; ret_part = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), buf, FALSE); } break; default: - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid compression type '%u'", comptype); - goto out; + return glnx_throw (error, "Invalid compression type '%u'", comptype); } if (checksum) @@ -494,18 +488,12 @@ _ostree_static_delta_part_open (GInputStream *part_in, const char *actual_checksum = g_checksum_get_string (checksum); g_assert (expected_checksum != NULL); if (strcmp (actual_checksum, expected_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Checksum mismatch in static delta part; expected=%s actual=%s", - expected_checksum, actual_checksum); - goto out; - } + return glnx_throw (error, "Checksum mismatch in static delta part; expected=%s actual=%s", + expected_checksum, actual_checksum); } - - ret = TRUE; + *out_part = g_steal_pointer (&ret_part); - out: - return ret; + return TRUE; } /* @@ -524,15 +512,12 @@ show_one_part (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; + g_autoptr(GVariant) part = NULL; + g_autofree char *part_path = _ostree_get_relative_static_delta_part_path (from, to, i); + guint32 version; guint64 size, usize; g_autoptr(GVariant) objects = NULL; - g_autoptr(GInputStream) part_in = NULL; - g_autoptr(GVariant) part = NULL; - g_autofree char *part_path = _ostree_get_relative_static_delta_part_path (from, to, i); - gint part_fd = -1; - g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects); size = maybe_swap_endian_u64 (swap_endian, size); usize = maybe_swap_endian_u64 (swap_endian, usize); @@ -541,21 +526,17 @@ show_one_part (OstreeRepo *self, g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n", i, (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN), size, usize); - part_fd = openat (self->repo_dir_fd, part_path, O_RDONLY | O_CLOEXEC); + glnx_fd_close gint part_fd = openat (self->repo_dir_fd, part_path, O_RDONLY | O_CLOEXEC); if (part_fd < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "openat(%s)", part_path); + g_autoptr(GInputStream) part_in = g_unix_input_stream_new (part_fd, FALSE); - part_in = g_unix_input_stream_new (part_fd, FALSE); - - if (!_ostree_static_delta_part_open (part_in, NULL, + if (!_ostree_static_delta_part_open (part_in, NULL, OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM, NULL, &part, cancellable, error)) - goto out; + return FALSE; { g_autoptr(GVariant) modes = NULL; g_autoptr(GVariant) xattrs = NULL; @@ -580,7 +561,7 @@ show_one_part (OstreeRepo *self, if (!_ostree_static_delta_part_execute (self, objects, part, TRUE, &stats, cancellable, error)) - goto out; + return FALSE; { const guint *n_ops = stats.n_ops_executed; g_print ("PartPayloadOps%u: openspliceclose=%u open=%u write=%u setread=%u " @@ -588,10 +569,8 @@ show_one_part (OstreeRepo *self, i, n_ops[0], n_ops[1], n_ops[2], n_ops[3], n_ops[4], n_ops[5], n_ops[6]); } } - - ret = TRUE; - out: - return ret; + + return TRUE; } OstreeDeltaEndianness @@ -748,13 +727,12 @@ _ostree_repo_static_delta_query_exists (OstreeRepo *self, { g_autofree char *from = NULL; g_autofree char *to = NULL; - g_autofree char *superblock_path = NULL; struct stat stbuf; if (!_ostree_parse_delta_name (delta_id, &from, &to, error)) return FALSE; - superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); + g_autofree char *superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); if (fstatat (self->repo_dir_fd, superblock_path, &stbuf, 0) < 0) { @@ -764,10 +742,7 @@ _ostree_repo_static_delta_query_exists (OstreeRepo *self, return TRUE; } else - { - glnx_set_error_from_errno (error); - return FALSE; - } + return glnx_throw_errno_prefix (error, "fstatat(%s)", superblock_path); } *out_exists = TRUE; return TRUE; From 250e305f738dc8ce6d80b54bbb8b649b1a09a118 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 21:50:10 -0400 Subject: [PATCH 62/82] lib/repo: Port bareuser-conversion stat to bare load I noticed this is a simple call that's useful to port to the new internal-only non-allocating API. Closes: #977 Approved by: jlebon --- src/libostree/ostree-repo.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 20f866b9..a53a056e 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3074,25 +3074,24 @@ import_one_object_link (OstreeRepo *self, */ if (import_is_bareuser_only_conversion (source, self, objtype)) { - g_autoptr(GFileInfo) finfo = NULL; + struct stat stbuf; - if (!ostree_repo_load_file (source, checksum, NULL, &finfo, NULL, - cancellable, error)) + if (!_ostree_repo_load_file_bare (source, checksum, NULL, &stbuf, + NULL, NULL, cancellable, error)) return FALSE; - switch (g_file_info_get_file_type (finfo)) + if (S_ISREG (stbuf.st_mode)) { - case G_FILE_TYPE_REGULAR: /* This is OK, we'll drop through and try a hardlink */ - break; - case G_FILE_TYPE_SYMBOLIC_LINK: + } + else if (S_ISLNK (stbuf.st_mode)) + { /* NOTE early return */ *out_was_supported = FALSE; return TRUE; - default: - g_assert_not_reached (); - break; } + else + g_assert_not_reached (); } if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_path_buf, cancellable, error)) From ea15025c192978420b42f654d028ae74a4f7a3ce Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 22:28:29 -0400 Subject: [PATCH 63/82] lib/pull: Some small style porting I'd mostly been avoiding this file since there's always patches outstanding, but these few functions shouldn't conflict much. Closes: #979 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 75 +++++++++++--------------------- 1 file changed, 26 insertions(+), 49 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 040eb421..a82448e1 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1489,22 +1489,17 @@ scan_one_metadata_object_c (OtPullData *pull_data, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GVariant) object = NULL; - g_autofree char *tmp_checksum = NULL; - gboolean is_requested; - gboolean is_stored; - - tmp_checksum = ostree_checksum_from_bytes (csum); - object = ostree_object_name_serialize (tmp_checksum, objtype); + g_autofree char *tmp_checksum = ostree_checksum_from_bytes (csum); + g_autoptr(GVariant) object = ostree_object_name_serialize (tmp_checksum, objtype); if (g_hash_table_lookup (pull_data->scanned_metadata, object)) return TRUE; - is_requested = g_hash_table_lookup (pull_data->requested_metadata, object) != NULL; + gboolean is_requested = g_hash_table_lookup (pull_data->requested_metadata, object) != NULL; + gboolean is_stored; if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored, cancellable, error)) - goto out; + return FALSE; if (pull_data->remote_repo_local) { @@ -1513,12 +1508,12 @@ scan_one_metadata_object_c (OtPullData *pull_data, if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!write_commitpartial_for (pull_data, tmp_checksum, error)) - goto out; + return FALSE; } if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, objtype, tmp_checksum, !pull_data->is_untrusted, cancellable, error)) - goto out; + return FALSE; } is_stored = TRUE; is_requested = TRUE; @@ -1544,7 +1539,7 @@ scan_one_metadata_object_c (OtPullData *pull_data, { if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth, pull_data->cancellable, error)) - goto out; + return FALSE; g_hash_table_add (pull_data->scanned_metadata, g_variant_ref (object)); pull_data->n_scanned_metadata++; @@ -1554,15 +1549,13 @@ scan_one_metadata_object_c (OtPullData *pull_data, { if (!scan_dirtree_object (pull_data, tmp_checksum, path, recursion_depth, pull_data->cancellable, error)) - goto out; + return FALSE; g_hash_table_add (pull_data->scanned_metadata, g_variant_ref (object)); pull_data->n_scanned_metadata++; } - ret = TRUE; - out: - return ret; + return TRUE; } static void @@ -1677,26 +1670,21 @@ load_remote_repo_config (OtPullData *pull_data, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autofree char *contents = NULL; - GKeyFile *ret_keyfile = NULL; if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher, pull_data->meta_mirrorlist, "config", &contents, cancellable, error)) - goto out; + return FALSE; - ret_keyfile = g_key_file_new (); + g_autoptr(GKeyFile) ret_keyfile = g_key_file_new (); if (!g_key_file_load_from_data (ret_keyfile, contents, strlen (contents), 0, error)) - goto out; + return FALSE; - ret = TRUE; ot_transfer_out_value (out_keyfile, &ret_keyfile); - out: - g_clear_pointer (&ret_keyfile, (GDestroyNotify) g_key_file_unref); - return ret; + return TRUE; } static gboolean @@ -1706,20 +1694,17 @@ process_one_static_delta_fallback (OtPullData *pull_data, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GVariant) csum_v = NULL; - g_autofree char *checksum = NULL; guint8 objtype_y; - OstreeObjectType objtype; - gboolean is_stored; + g_autoptr(GVariant) csum_v = NULL; guint64 compressed_size, uncompressed_size; g_variant_get (fallback_object, "(y@aytt)", &objtype_y, &csum_v, &compressed_size, &uncompressed_size); + if (!ostree_validate_structureof_objtype (objtype_y, error)) - goto out; + return FALSE; if (!ostree_validate_structureof_csum_v (csum_v, error)) - goto out; + return FALSE; compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size); uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size); @@ -1728,33 +1713,27 @@ process_one_static_delta_fallback (OtPullData *pull_data, pull_data->total_deltapart_size += compressed_size; pull_data->total_deltapart_usize += uncompressed_size; - objtype = (OstreeObjectType)objtype_y; - checksum = ostree_checksum_from_bytes_v (csum_v); + OstreeObjectType objtype = (OstreeObjectType)objtype_y; + g_autofree char *checksum = ostree_checksum_from_bytes_v (csum_v); + gboolean is_stored; if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored, cancellable, error)) - goto out; + return FALSE; if (is_stored) pull_data->fetched_deltapart_size += compressed_size; if (pull_data->dry_run) - { - ret = TRUE; - goto out; - } + return TRUE; /* Note early return */ if (!is_stored) { /* The delta compiler never did this, there's no reason to support it */ if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Found metadata object as fallback: %s.%s", checksum, - ostree_object_type_to_string (objtype)); - goto out; - } + return glnx_throw (error, "Found metadata object as fallback: %s.%s", checksum, + ostree_object_type_to_string (objtype)); else { if (!g_hash_table_lookup (pull_data->requested_content, checksum)) @@ -1771,9 +1750,7 @@ process_one_static_delta_fallback (OtPullData *pull_data, } } - ret = TRUE; - out: - return ret; + return TRUE; } static void From 2013db0527dc2f85ab21cdc37047154a26fd161c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 21:39:16 -0400 Subject: [PATCH 64/82] tests: Fix assert_files_hardlinked It was always succeeding because we were trying to stat the inode number, and failing, and thus getting the empty string for both, which compared as true. Regression from: Noticed this while working on and looking at the test results. Closes: #976 Approved by: jlebon --- tests/libtest.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index ee3f4af8..6f33c5b5 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -117,15 +117,14 @@ else fi files_are_hardlinked() { - f1=$(stat -c %i $1) - f2=$(stat -c %i $2) - [ "$f1" == "$f2" ] + inode1=$(stat -c %i $1) + inode2=$(stat -c %i $2) + test -n "${inode1}" && test -n "${inode2}" + [ "${inode1}" == "${inode2}" ] } assert_files_hardlinked() { - f1=$(stat -c %i $1) - f2=$(stat -c %i $2) - if ! files_are_hardlinked "$f1" "$f2"; then + if ! files_are_hardlinked "$1" "$2"; then fatal "Files '$1' and '$2' are not hardlinked" fi } From 8d586a9da0b3736c745b3441b213af20ee183109 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 22:23:17 -0400 Subject: [PATCH 65/82] lib/pull: Don't fetch detached metadata twice for local pulls Obviously very minor, but I noticed this while working on `pull --reference`. If we have a local repo, we'll have already done a hardlink and copied the detached metadata too, so there's no reason to request it again via the fetcher path. Closes: #978 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index a82448e1..31c825b8 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1514,6 +1514,11 @@ scan_one_metadata_object_c (OtPullData *pull_data, objtype, tmp_checksum, !pull_data->is_untrusted, cancellable, error)) return FALSE; + /* The import API will fetch both the commit and detached metadata, so + * add it to the hash to avoid re-fetching it below. + */ + if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum)); } is_stored = TRUE; is_requested = TRUE; From 90e0d5633244ad818eb40473d1d6f2d50cbd8c14 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 22:52:40 -0400 Subject: [PATCH 66/82] =?UTF-8?q?tree-wide:=20Replace=20various=20uses=20o?= =?UTF-8?q?f=20`archive-z2`=20=E2=86=92=20`archive`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `-z2` is annoying now since it's really a legacy; we've long since supported typing `archive`. Convert the docs fully and explain that. Also do some (but not all) of the tests just to encourage newer tests to use `archive` too. Closes: #980 Approved by: jlebon --- Makefile-tests.am | 2 +- docs/manual/buildsystem-and-repos.md | 8 ++++---- docs/manual/formats.md | 17 +++++++++-------- docs/manual/related-projects.md | 2 +- docs/manual/repo.md | 6 +++--- docs/manual/repository-management.md | 14 +++++++------- src/libostree/ostree-repo-checkout.c | 4 ++-- tests/basic-test.sh | 4 ++-- tests/pull-test.sh | 10 +++++----- ...t-pull-archive-z.sh => test-pull-archive.sh} | 2 +- tests/test-pull-c.c | 2 +- 11 files changed, 36 insertions(+), 35 deletions(-) rename tests/{test-pull-archive-z.sh => test-pull-archive.sh} (95%) diff --git a/Makefile-tests.am b/Makefile-tests.am index 2fe0a235..348a8e4a 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -67,7 +67,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-help.sh \ tests/test-libarchive.sh \ tests/test-parent.sh \ - tests/test-pull-archive-z.sh \ + tests/test-pull-archive.sh \ tests/test-pull-commit-only.sh \ tests/test-pull-depth.sh \ tests/test-pull-mirror-summary.sh \ diff --git a/docs/manual/buildsystem-and-repos.md b/docs/manual/buildsystem-and-repos.md index 1f14b181..5f9c0cd7 100644 --- a/docs/manual/buildsystem-and-repos.md +++ b/docs/manual/buildsystem-and-repos.md @@ -21,11 +21,11 @@ side, but aiming to do client side too). ## Initializing For this initial discussion, we're assuming you have a single -`archive-z2` repository: +`archive` repository: ``` mkdir repo -ostree --repo=repo init --mode=archive-z2 +ostree --repo=repo init --mode=archive ``` You can export this via a static webserver, and configure clients to @@ -107,7 +107,7 @@ equivalent to RPMs/debs. Further, in order to make things fast, we will need a separate `bare-user` repository in order to perform checkouts quickly via -hardlinks. We'll then export content into the `archive-z2` repository +hardlinks. We'll then export content into the `archive` repository for use by client systems. ``` @@ -161,7 +161,7 @@ checksum). Now that we have content in our `build-repo` repository (in `bare-user` mode), we need to move the `exampleos/x86_64/standard` -branch content into the repository just named `repo` (in `archive-z2` +branch content into the repository just named `repo` (in `archive` mode) for export, which will involve zlib compression of new objects. We likely want to generate static deltas after that as well. diff --git a/docs/manual/formats.md b/docs/manual/formats.md index e3fe00e1..bdd4bb56 100644 --- a/docs/manual/formats.md +++ b/docs/manual/formats.md @@ -14,22 +14,23 @@ primary advantages are security and compute efficiency. Services like Amazon S3 and CDNs are a canonical target, as well as a stock static nginx server. -## The archive-z2 format +## The archive format In the [repo](repo) section, the concept of objects was introduced, where file/content objects are checksummed and managed individually. (Unlike a package system, which operates on compressed aggregates). -The archive-z2 format simply gzip-compresses each content object. +The `archive` format simply gzip-compresses each content object. Metadata objects are stored uncompressed. This means that it's easy -to serve via static HTTP. +to serve via static HTTP. Note: this format used to be called `archive-z2` +for historical reasons. When you commit new content, you will see new `.filez` files appearing in `objects/`. -## archive-z2 efficiency +## archive efficiency -The advantages of `archive-z2`: +The advantages of `archive`: - It's easy to understand and implement - Can be served directly over plain HTTP by a static webserver @@ -47,7 +48,7 @@ such that large data that changes infrequently (e.g. graphic images) are stored separately from small frequently changing data (application code). -Other disadvantages of `archive-z2`: +Other disadvantages of `archive`: - It's quite bad when clients are performing an initial pull (without HTTP/2), - One doesn't know the total size (compressed or uncompressed) of content @@ -55,7 +56,7 @@ Other disadvantages of `archive-z2`: ## Aside: the bare and bare-user formats -The most common operation is to pull from an `archive-z2` repository +The most common operation is to pull from an `archive` repository into a `bare` or `bare-user` formatted repository. These latter two are not compressed on disk. In other words, pulling to them is similar to unpacking (but not installing) an RPM/deb package. @@ -72,7 +73,7 @@ client systems are expected to update regularly. However, many OS vendors would like to supply content that's updated e.g. once a month or less often. For this model, we can do a lot better to support batched updates than -a basic `archive-z2` repo. However, we still want to preserve the +a basic `archive` repo. However, we still want to preserve the model of "static webserver only". Given this, OSTree has gained the concept of a "static delta". diff --git a/docs/manual/related-projects.md b/docs/manual/related-projects.md index 38608b94..50076ee6 100644 --- a/docs/manual/related-projects.md +++ b/docs/manual/related-projects.md @@ -300,7 +300,7 @@ Docker (natively) only shares storage via layering. The biggest feature OSTree has over Docker though is support for (static) deltas, and even without pre-configured static deltas, the -archive-z2 format has "natural" deltas. Particularly for a "base +`archive` format has "natural" deltas. Particularly for a "base operating system", one really wants on-wire deltas. It'd likely be possible to extend Docker with this concept. diff --git a/docs/manual/repo.md b/docs/manual/repo.md index 6b547be6..abfc4911 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -66,8 +66,8 @@ is reverted to use zero again. # Repository types and locations -Also unlike git, an OSTree repository can be in one of three separate -modes: `bare`, `bare-user`, and `archive-z2`. A bare repository is +Also unlike git, an OSTree repository can be in one of four separate +modes: `bare`, `bare-user`, `bare-user-only`, and `archive`. A bare repository is one where content files are just stored as regular files; it's designed to be the source of a "hardlink farm", where each operating system checkout is merely links into it. If you want to store files @@ -88,7 +88,7 @@ information is not applied anyway. The main advantage of `bare-user-only` is that repos can be stored on filesystems which do not support extended attributes, such as tmpfs. -In contrast, the `archive-z2` mode is designed for serving via plain +In contrast, the `archive` mode is designed for serving via plain HTTP. Like tar files, it can be read/written by non-root users. On an OSTree-deployed system, the "system repository" is diff --git a/docs/manual/repository-management.md b/docs/manual/repository-management.md index a9db65ff..713ca4cf 100644 --- a/docs/manual/repository-management.md +++ b/docs/manual/repository-management.md @@ -29,14 +29,14 @@ It's very common to want to perform a full or partial mirror, in particular across organizational boundaries (e.g. an upstream OS provider, and a user that wants offline and faster access to the content). OSTree supports both full and partial mirroring of the base -`archive-z2` content, although not yet of static deltas. +`archive` content, although not yet of static deltas. -To create a mirror, first create an `archive-z2` repository (you don't +To create a mirror, first create an `archive` repository (you don't need to run this as root), then add the upstream as a remote, then use `pull --mirror`. ``` -ostree --repo=repo init --mode=archive-z2 +ostree --repo=repo init --mode=archive ostree --repo=repo remote add exampleos https://exampleos.com/ostree/repo ostree --repo=repo pull --mirror exampleos:exampleos/x86_64/standard ``` @@ -66,7 +66,7 @@ unlikely most of your content consumers (i.e. not developers/testers) will be interested in all of it. The original vision of OSTree was to fulfill this "dev" role, and in -particular the "archive-z2" format was designed for it. +particular the "archive" format was designed for it. Then, what you'll want to do is promote content from "dev" to "prod". We'll discuss this later, but first, let's talk about promotion @@ -138,7 +138,7 @@ We'll have other business requirements such as writing release notes In [Build Systems](buildsystem-and-repos.md) we saw how the `pull-local` command can be used to migrate content from the "build" -repository (in `bare-user` mode) into an `archive-z2` repository for +repository (in `bare-user` mode) into an `archive` repository for serving to client systems. Following this section, we now have three repositories, let's call @@ -146,7 +146,7 @@ them `repo-build`, `repo-dev`, and `repo-prod`. We've been pulling content from `repo-build` into `repo-dev` (which involves gzip compression among other things since it is a format change). -When using `pull-local` to migrate content between two `archive-z2` +When using `pull-local` to migrate content between two `archive` repositories, the binary content is taken unmodified. Let's go ahead and generate a new commit in our prod repository: @@ -178,7 +178,7 @@ default. ## Derived data - static deltas and the summary file -As discussed in [Formats](formats.md), the `archive-z2` repository we +As discussed in [Formats](formats.md), the `archive` repository we use for "prod" requires one HTTP fetch per client request by default. If we're only performing a release e.g. once a week, it's appropriate to use "static deltas" to speed up client updates. diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 14b25540..8729b81c 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -446,7 +446,7 @@ checkout_one_file_at (OstreeRepo *repo, /* But only under these conditions */ if (is_bare || is_archive_z2_with_cache) { - /* Override repo mode; for archive-z2 we're looking in + /* Override repo mode; for archive we're looking in the cache, which is in "bare" form */ _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, OSTREE_REPO_MODE_BARE); if (!checkout_file_hardlink (current_repo, @@ -488,7 +488,7 @@ checkout_one_file_at (OstreeRepo *repo, g_autoptr(GInputStream) input = NULL; g_autoptr(GVariant) xattrs = NULL; - /* Ok, if we're archive-z2 and we didn't find an object, uncompress + /* Ok, if we're archive and we didn't find an object, uncompress * it now, stick it in the cache, and then hardlink to that. */ if (can_cache diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 6f237696..012a98e0 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -512,7 +512,7 @@ find repo3/objects -name '*.filez' > file-objects if test -s file-objects; then assert_not_reached "prune didn't delete all objects" fi -echo "ok prune in archive-z2 deleted everything" +echo "ok prune in archive deleted everything" cd ${test_tmpdir} rm -rf test2-checkout @@ -730,7 +730,7 @@ $OSTREE commit ${COMMIT_ARGS} -b test2 -s "Unfsynced commit" --fsync=false if test "$(id -u)" != "0"; then cd ${test_tmpdir} rm -f expected-fail error-message - $OSTREE init --mode=archive-z2 --repo=repo-noperm + $OSTREE init --mode=archive --repo=repo-noperm chmod -w repo-noperm/objects $OSTREE --repo=repo-noperm pull-local repo 2> error-message || touch expected-fail chmod +w repo-noperm/objects diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 24eb16a0..7b4fbe94 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -50,7 +50,7 @@ echo "ok pull contents" cd ${test_tmpdir} mkdir mirrorrepo -ostree_repo_init mirrorrepo --mode=archive-z2 +ostree_repo_init mirrorrepo --mode=archive ${CMD_PREFIX} ostree --repo=mirrorrepo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo ${CMD_PREFIX} ostree --repo=mirrorrepo pull --mirror origin main ${CMD_PREFIX} ostree --repo=mirrorrepo fsck @@ -63,7 +63,7 @@ ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b otherbranch --tree=di ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u rm mirrorrepo -rf # All refs -ostree_repo_init mirrorrepo --mode=archive-z2 +ostree_repo_init mirrorrepo --mode=archive ${CMD_PREFIX} ostree --repo=mirrorrepo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo ${CMD_PREFIX} ostree --repo=mirrorrepo pull --mirror origin ${CMD_PREFIX} ostree --repo=mirrorrepo fsck @@ -73,7 +73,7 @@ done echo "ok pull mirror (all refs)" rm mirrorrepo -rf -ostree_repo_init mirrorrepo --mode=archive-z2 +ostree_repo_init mirrorrepo --mode=archive ${CMD_PREFIX} ostree --repo=mirrorrepo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo # Generate a summary in the mirror ${CMD_PREFIX} ostree --repo=mirrorrepo summary -u @@ -152,7 +152,7 @@ echo "ok pull commit metadata only (should not apply deltas)" cd ${test_tmpdir} mkdir mirrorrepo-local -ostree_repo_init mirrorrepo-local --mode=archive-z2 +ostree_repo_init mirrorrepo-local --mode=archive ${CMD_PREFIX} ostree --repo=mirrorrepo-local remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo ${CMD_PREFIX} ostree --repo=mirrorrepo-local pull --mirror origin main ${CMD_PREFIX} ostree --repo=mirrorrepo-local fsck @@ -170,7 +170,7 @@ echo "ok pull detached metadata" cd ${test_tmpdir} mkdir parentpullrepo -ostree_repo_init parentpullrepo --mode=archive-z2 +ostree_repo_init parentpullrepo --mode=archive ${CMD_PREFIX} ostree --repo=parentpullrepo remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo parent_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^) rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) diff --git a/tests/test-pull-archive-z.sh b/tests/test-pull-archive.sh similarity index 95% rename from tests/test-pull-archive-z.sh rename to tests/test-pull-archive.sh index 66f8873e..d8b0f099 100755 --- a/tests/test-pull-archive-z.sh +++ b/tests/test-pull-archive.sh @@ -21,6 +21,6 @@ set -euo pipefail . $(dirname $0)/libtest.sh -setup_fake_remote_repo1 "archive-z2" +setup_fake_remote_repo1 "archive" . ${test_srcdir}/pull-test.sh diff --git a/tests/test-pull-c.c b/tests/test-pull-c.c index c2d64dcc..2b0cef6d 100644 --- a/tests/test-pull-c.c +++ b/tests/test-pull-c.c @@ -43,7 +43,7 @@ test_data_init (TestData *td) if (!td->repo) goto out; - if (!ot_test_run_libtest ("setup_fake_remote_repo1 archive-z2", error)) + if (!ot_test_run_libtest ("setup_fake_remote_repo1 archive", error)) goto out; if (!g_file_get_contents ("httpd-address", &http_address, NULL, error)) From 1a9a473580e9066f5d4a969932d703b5b104b2b7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 29 Jun 2017 10:23:32 -0400 Subject: [PATCH 67/82] cmdline/pull: Print final status even if noninteractive Previously, `ostree pull` was silent if not on a tty. I don't see a reason not to print the final status line at least. This is prep for more work in the test suite, so I can write assertions on the output. But it should also be nicer for people who e.g. do an `ostree pull` in a Jenkins job or whatever. Closes: #981 Approved by: jlebon --- src/ostree/ot-builtin-pull.c | 20 ++++++++++++++++++-- tests/pull-test.sh | 6 ++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index bd3ac115..edb09550 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -125,6 +125,13 @@ dry_run_console_progress_changed (OstreeAsyncProgress *progress, g_print ("%s\n", buf->str); } +static void +noninteractive_console_progress_changed (OstreeAsyncProgress *progress, + gpointer user_data) +{ + /* We do nothing here - we just want the final status */ +} + gboolean ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **error) { @@ -302,6 +309,8 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** { if (console.is_tty) progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); + else + progress = ostree_async_progress_new_and_connect (noninteractive_console_progress_changed, &console); } else { @@ -321,8 +330,15 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** progress, cancellable, error)) goto out; - if (progress) - ostree_async_progress_finish (progress); + if (!console.is_tty && !opt_dry_run) + { + g_assert (progress); + const char *status = ostree_async_progress_get_status (progress); + if (status) + g_print ("%s\n", status); + } + + ostree_async_progress_finish (progress); if (opt_dry_run) g_assert (printed_console_progress); diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 7b4fbe94..1da0fa4c 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -39,8 +39,10 @@ echo "1..25" # Try both syntaxes repo_init --no-gpg-verify -${CMD_PREFIX} ostree --repo=repo pull origin main -${CMD_PREFIX} ostree --repo=repo pull origin:main +${CMD_PREFIX} ostree --repo=repo pull origin main >out.txt +assert_file_has_content out.txt "[1-9][0-9]* metadata, [1-9][0-9]* content objects fetched" +${CMD_PREFIX} ostree --repo=repo pull origin:main > out.txt +assert_not_file_has_content out.txt "content objects fetched" ${CMD_PREFIX} ostree --repo=repo fsck echo "ok pull" From d57410a7e62dcb89321807dcb2d91c85f9d26df7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 16:09:37 -0400 Subject: [PATCH 68/82] =?UTF-8?q?lib:=20Add=20a=20helper=20to=20convert=20?= =?UTF-8?q?struct=20stat=20=E2=86=92=20GFileInfo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's more natural for a few calling places. Prep for patches to go the other way, which in turn are prep for adding a commit filter v2 that takes `struct stat`. `ot_gfile_type_for_mode()` was only used in this function, so inline it here. Closes: #974 Approved by: jlebon --- src/libostree/ostree-core-private.h | 4 +- src/libostree/ostree-core.c | 40 +++++++++++++++---- src/libostree/ostree-repo-commit.c | 8 ++-- src/libostree/ostree-repo-libarchive.c | 29 +++++--------- .../ostree-repo-static-delta-processing.c | 16 +++----- src/libostree/ostree-repo.c | 15 ++----- src/libotutil/ot-gio-utils.c | 16 -------- src/libotutil/ot-gio-utils.h | 2 - 8 files changed, 58 insertions(+), 72 deletions(-) diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 76c76cc7..799bd228 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -21,6 +21,7 @@ #pragma once #include "ostree-core.h" +#include G_BEGIN_DECLS @@ -88,7 +89,8 @@ _ostree_make_temporary_symlink_at (int tmp_dirfd, GCancellable *cancellable, GError **error); -GFileInfo * _ostree_header_gfile_info_new (mode_t mode, uid_t uid, gid_t gid); +GFileInfo * _ostree_stbuf_to_gfileinfo (const struct stat *stbuf); +GFileInfo * _ostree_mode_uidgid_to_gfileinfo (mode_t mode, uid_t uid, gid_t gid); static inline void _ostree_checksum_inplace_from_bytes_v (GVariant *csum_v, char *buf) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index c0b7cbe3..6442bd52 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1493,7 +1493,7 @@ _ostree_loose_path (char *buf, } /** - * _ostree_header_gfile_info_new: + * _ostree_stbuf_to_gfileinfo: * @mode: File mode * @uid: File uid * @gid: File gid @@ -1506,17 +1506,42 @@ _ostree_loose_path (char *buf, * Returns: (transfer full): A new #GFileInfo mapping a subset of @stbuf. */ GFileInfo * -_ostree_header_gfile_info_new (mode_t mode, uid_t uid, gid_t gid) +_ostree_stbuf_to_gfileinfo (const struct stat *stbuf) { GFileInfo *ret = g_file_info_new (); - g_file_info_set_attribute_uint32 (ret, "standard::type", ot_gfile_type_for_mode (mode)); + GFileType ftype; + const mode_t mode = stbuf->st_mode; + if (S_ISDIR (mode)) + ftype = G_FILE_TYPE_DIRECTORY; + else if (S_ISREG (mode)) + ftype = G_FILE_TYPE_REGULAR; + else if (S_ISLNK (mode)) + ftype = G_FILE_TYPE_SYMBOLIC_LINK; + else if (S_ISBLK (mode) || S_ISCHR(mode) || S_ISFIFO(mode)) + ftype = G_FILE_TYPE_SPECIAL; + else + ftype = G_FILE_TYPE_UNKNOWN; + g_file_info_set_attribute_uint32 (ret, "standard::type", ftype); g_file_info_set_attribute_boolean (ret, "standard::is-symlink", S_ISLNK (mode)); - g_file_info_set_attribute_uint32 (ret, "unix::uid", uid); - g_file_info_set_attribute_uint32 (ret, "unix::gid", gid); + g_file_info_set_attribute_uint32 (ret, "unix::uid", stbuf->st_uid); + g_file_info_set_attribute_uint32 (ret, "unix::gid", stbuf->st_gid); g_file_info_set_attribute_uint32 (ret, "unix::mode", mode); + if (S_ISREG (mode)) + g_file_info_set_attribute_uint64 (ret, "standard::size", stbuf->st_size); + return ret; } +GFileInfo * +_ostree_mode_uidgid_to_gfileinfo (mode_t mode, uid_t uid, gid_t gid) +{ + struct stat stbuf; + stbuf.st_mode = mode; + stbuf.st_uid = uid; + stbuf.st_gid = gid; + return _ostree_stbuf_to_gfileinfo (&stbuf); +} + /* * _ostree_get_relative_object_path: * @checksum: ASCII checksum string @@ -1680,8 +1705,7 @@ file_header_parse (GVariant *metadata, uid = GUINT32_FROM_BE (uid); gid = GUINT32_FROM_BE (gid); mode = GUINT32_FROM_BE (mode); - - g_autoptr(GFileInfo) ret_file_info = _ostree_header_gfile_info_new (mode, uid, gid); + g_autoptr(GFileInfo) ret_file_info = _ostree_mode_uidgid_to_gfileinfo (mode, uid, gid); if (S_ISREG (mode)) { @@ -1731,7 +1755,7 @@ zlib_file_header_parse (GVariant *metadata, uid = GUINT32_FROM_BE (uid); gid = GUINT32_FROM_BE (gid); mode = GUINT32_FROM_BE (mode); - g_autoptr(GFileInfo) ret_file_info = _ostree_header_gfile_info_new (mode, uid, gid); + g_autoptr(GFileInfo) ret_file_info = _ostree_mode_uidgid_to_gfileinfo (mode, uid, gid); g_file_info_set_size (ret_file_info, GUINT64_FROM_BE (size)); if (S_ISREG (mode)) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 45d10163..171fa562 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2748,7 +2748,7 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self, if (fstat (src_dfd_iter->fd, &dir_stbuf) != 0) return glnx_throw_errno (error); - child_info = _ostree_header_gfile_info_new (dir_stbuf.st_mode, dir_stbuf.st_uid, dir_stbuf.st_gid); + child_info = _ostree_stbuf_to_gfileinfo (&dir_stbuf); if (modifier != NULL) { @@ -2809,13 +2809,11 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self, continue; } - child_info = _ostree_header_gfile_info_new (stbuf.st_mode, stbuf.st_uid, stbuf.st_gid); + child_info = _ostree_stbuf_to_gfileinfo (&stbuf); g_file_info_set_name (child_info, dent->d_name); if (S_ISREG (stbuf.st_mode)) - { - g_file_info_set_size (child_info, stbuf.st_size); - } + ; else if (S_ISLNK (stbuf.st_mode)) { if (!ot_readlinkat_gfile_info (src_dfd_iter->fd, dent->d_name, diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 1109d5d7..2e1696b5 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -126,33 +126,24 @@ squash_trailing_slashes (char *path) static GFileInfo * file_info_from_archive_entry (struct archive_entry *entry) { - g_autoptr(GFileInfo) info = NULL; - const struct stat *st = NULL; - guint32 file_type; - mode_t mode; - - st = archive_entry_stat (entry); - mode = st->st_mode; + const struct stat *st = archive_entry_stat (entry); + struct stat st_copy; /* Some archives only store the permission mode bits in hardlink entries, so * let's just make it into a regular file. Yes, this hack will work even if * it's a hardlink to a symlink. */ if (archive_entry_hardlink (entry)) - mode |= S_IFREG; - - info = _ostree_header_gfile_info_new (mode, st->st_uid, st->st_gid); - - file_type = ot_gfile_type_for_mode (mode); - if (file_type == G_FILE_TYPE_REGULAR) { - g_file_info_set_attribute_uint64 (info, "standard::size", st->st_size); - } - else if (file_type == G_FILE_TYPE_SYMBOLIC_LINK) - { - g_file_info_set_attribute_byte_string (info, "standard::symlink-target", - archive_entry_symlink (entry)); + st_copy = *st; + st_copy.st_mode |= S_IFREG; + st = &st_copy; } + g_autoptr(GFileInfo) info = _ostree_stbuf_to_gfileinfo (st); + if (S_ISLNK (st->st_mode)) + g_file_info_set_attribute_byte_string (info, "standard::symlink-target", + archive_entry_symlink (entry)); + return g_steal_pointer (&info); } diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index 9141f659..028c700d 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -515,18 +515,15 @@ dispatch_bspatch (OstreeRepo *repo, static gboolean handle_untrusted_content_checksum (OstreeRepo *repo, StaticDeltaExecutionState *state, - GCancellable *cancellable, + GCancellable *cancellable, GError **error) { - g_autoptr(GVariant) header = NULL; - g_autoptr(GFileInfo) finfo = NULL; - gsize bytes_written; - - finfo = _ostree_header_gfile_info_new (state->mode, state->uid, state->gid); - header = _ostree_file_header_new (finfo, state->xattrs); + g_autoptr(GFileInfo) finfo = _ostree_mode_uidgid_to_gfileinfo (state->mode, state->uid, state->gid); + g_autoptr(GVariant) header = _ostree_file_header_new (finfo, state->xattrs); state->content_checksum = g_checksum_new (G_CHECKSUM_SHA256); + gsize bytes_written; if (!_ostree_write_variant_with_size (NULL, header, 0, &bytes_written, state->content_checksum, cancellable, error)) return FALSE; @@ -629,9 +626,8 @@ dispatch_open_splice_and_close (OstreeRepo *repo, else { /* Slower path, for symlinks and unpacking deltas into archive-z2 */ - g_autoptr(GFileInfo) finfo = NULL; - - finfo = _ostree_header_gfile_info_new (state->mode, state->uid, state->gid); + g_autoptr(GFileInfo) finfo = + _ostree_mode_uidgid_to_gfileinfo (state->mode, state->uid, state->gid); if (S_ISLNK (state->mode)) { diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index a53a056e..8f3a8bd0 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2791,18 +2791,11 @@ ostree_repo_load_file (OstreeRepo *self, } if (out_file_info) { - *out_file_info = _ostree_header_gfile_info_new (stbuf.st_mode, stbuf.st_uid, stbuf.st_gid); - if (S_ISREG (stbuf.st_mode)) - { - g_file_info_set_size (*out_file_info, stbuf.st_size); - } - else if (S_ISLNK (stbuf.st_mode)) - { - g_file_info_set_size (*out_file_info, 0); - g_file_info_set_symlink_target (*out_file_info, symlink_target); - } + *out_file_info = _ostree_stbuf_to_gfileinfo (&stbuf); + if (S_ISLNK (stbuf.st_mode)) + g_file_info_set_symlink_target (*out_file_info, symlink_target); else - g_assert_not_reached (); + g_assert (S_ISREG (stbuf.st_mode)); } ot_transfer_out_value (out_xattrs, &ret_xattrs); diff --git a/src/libotutil/ot-gio-utils.c b/src/libotutil/ot-gio-utils.c index a4b61842..af2abb63 100644 --- a/src/libotutil/ot-gio-utils.c +++ b/src/libotutil/ot-gio-utils.c @@ -35,22 +35,6 @@ #define O_BINARY 0 #endif -GFileType -ot_gfile_type_for_mode (guint32 mode) -{ - if (S_ISDIR (mode)) - return G_FILE_TYPE_DIRECTORY; - else if (S_ISREG (mode)) - return G_FILE_TYPE_REGULAR; - else if (S_ISLNK (mode)) - return G_FILE_TYPE_SYMBOLIC_LINK; - else if (S_ISBLK (mode) || S_ISCHR(mode) || S_ISFIFO(mode)) - return G_FILE_TYPE_SPECIAL; - else - return G_FILE_TYPE_UNKNOWN; -} - - GFile * ot_gfile_resolve_path_printf (GFile *path, const char *format, diff --git a/src/libotutil/ot-gio-utils.h b/src/libotutil/ot-gio-utils.h index 0fd3ddf0..a69b744d 100644 --- a/src/libotutil/ot-gio-utils.h +++ b/src/libotutil/ot-gio-utils.h @@ -33,8 +33,6 @@ G_BEGIN_DECLS #define OSTREE_GIO_FAST_QUERYINFO ("standard::name,standard::type,standard::size,standard::is-symlink,standard::symlink-target," \ "unix::device,unix::inode,unix::mode,unix::uid,unix::gid,unix::rdev") -GFileType ot_gfile_type_for_mode (guint32 mode); - GFile * ot_gfile_resolve_path_printf (GFile *path, const char *format, ...) G_GNUC_PRINTF(2, 3); From aa26db825f492c7ada9bf4d820f6337681ef089e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 29 Jun 2017 16:54:38 -0400 Subject: [PATCH 69/82] lib/commit: Port a few minor functions to new style Not sure why these weren't converted before. Closes: #984 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 60 ++++++++++-------------------- 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 171fa562..ae3760c0 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -866,22 +866,18 @@ scan_one_loose_devino (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - int res; g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - if (!glnx_dirfd_iterator_init_at (object_dir_fd, ".", FALSE, &dfd_iter, error)) - goto out; - + return FALSE; + while (TRUE) { struct dirent *dent; g_auto(GLnxDirFdIterator) child_dfd_iter = { 0, }; if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error)) - goto out; - + return FALSE; if (dent == NULL) break; @@ -891,25 +887,20 @@ scan_one_loose_devino (OstreeRepo *self, if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE, &child_dfd_iter, error)) - goto out; + return FALSE; while (TRUE) { - struct stat stbuf; - OstreeDevIno *key; struct dirent *child_dent; - const char *dot; - gboolean skip; - const char *name; if (!glnx_dirfd_iterator_next_dent (&child_dfd_iter, &child_dent, cancellable, error)) - goto out; - + return FALSE; if (child_dent == NULL) break; - name = child_dent->d_name; + const char *name = child_dent->d_name; + gboolean skip; switch (self->mode) { case OSTREE_REPO_MODE_ARCHIVE_Z2: @@ -924,36 +915,29 @@ scan_one_loose_devino (OstreeRepo *self, if (skip) continue; - dot = strrchr (name, '.'); + const char *dot = strrchr (name, '.'); g_assert (dot); - + /* Skip anything that doesn't look like a 64 character checksum */ if ((dot - name) != 62) continue; - do - res = fstatat (child_dfd_iter.fd, child_dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + struct stat stbuf; + if (!glnx_fstatat (child_dfd_iter.fd, child_dent->d_name, + &stbuf, AT_SYMLINK_NOFOLLOW, error)) + return FALSE; - key = g_new (OstreeDevIno, 1); + OstreeDevIno *key = g_new (OstreeDevIno, 1); key->dev = stbuf.st_dev; key->ino = stbuf.st_ino; memcpy (key->checksum, dent->d_name, 2); memcpy (key->checksum + 2, name, 62); key->checksum[sizeof(key->checksum)-1] = '\0'; - g_hash_table_add (devino_cache, key); } } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -962,28 +946,24 @@ scan_loose_devino (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - if (self->parent_repo) { if (!scan_loose_devino (self->parent_repo, devino_cache, cancellable, error)) - goto out; + return FALSE; } if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2) { if (!scan_one_loose_devino (self, self->uncompressed_objects_dir_fd, devino_cache, cancellable, error)) - goto out; + return FALSE; } if (!scan_one_loose_devino (self, self->objects_dir_fd, devino_cache, cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; + return FALSE; + + return TRUE; } static const char * From acace571efb0fe6410e6464e715d1f50f832a9a7 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 30 Jun 2017 00:43:31 +0100 Subject: [PATCH 70/82] lib/repo: Fix repo-finder deleting remote configs when run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An inverted condition in _ostree_repo_add_remote() was causing the OstreeRepoFinder to delete precisely the wrong remote configurations from memory once it was finished. It’s supposed to delete the ones which it transiently added; but was instead deleting all the existing remote configurations. Signed-off-by: Philip Withnall Closes: #985 Approved by: cgwalters --- src/libostree/ostree-repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 8f3a8bd0..a02214e1 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -181,7 +181,7 @@ _ostree_repo_add_remote (OstreeRepo *self, g_mutex_lock (&self->remotes_lock); - already_existed = g_hash_table_replace (self->remotes, remote->name, ostree_remote_ref (remote)); + already_existed = !g_hash_table_replace (self->remotes, remote->name, ostree_remote_ref (remote)); g_mutex_unlock (&self->remotes_lock); From d5dd576d2095d085ed69f4c1efe7ecdfd9716fb7 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 30 Jun 2017 09:01:56 -0700 Subject: [PATCH 71/82] pull: fix GLNX_HASH_TABLE_FOREACH_KV regressions These are regression from #971. We were stuffing a pointer size inside a variable of integer size. So the assignment was spilling over into other variables' storage space. Actually use a gpointer and GPOINTER_TO_[U]INT as was done originally. Also bump libglnx which has static checks for this error in the future. Update submodule: libglnx Closes: #990 Approved by: cgwalters --- libglnx | 2 +- src/libostree/ostree-repo-checkout.c | 4 ++-- src/libostree/ostree-repo-pull.c | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libglnx b/libglnx index 01e934c1..a37e6727 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 01e934c18efdbac071ebc19a8a95916d324970c9 +Subproject commit a37e672739197b8a7f3bdfe3f17099fe402f9a98 diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 8729b81c..9ab9a623 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -1091,9 +1091,9 @@ ostree_repo_checkout_gc (OstreeRepo *self, if (!to_clean_dirs) return TRUE; /* Note early return */ - GLNX_HASH_TABLE_FOREACH (to_clean_dirs, guint, prefix) + GLNX_HASH_TABLE_FOREACH (to_clean_dirs, gpointer, prefix) { - g_autofree char *objdir_name = g_strdup_printf ("%02x", prefix); + g_autofree char *objdir_name = g_strdup_printf ("%02x", GPOINTER_TO_UINT (prefix)); g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (self->uncompressed_objects_dir_fd, objdir_name, FALSE, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 31c825b8..fd9e9aff 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -4948,8 +4948,10 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, /* Any refs left un-downloaded? If so, we’ve failed. */ GLNX_HASH_TABLE_FOREACH_KV (refs_pulled, const OstreeCollectionRef*, ref, - gboolean, is_pulled) + gpointer, is_pulled_pointer) { + gboolean is_pulled = GPOINTER_TO_INT (is_pulled_pointer); + if (is_pulled) continue; From 1782a1c2797da38f69b147908d48711ecef8946f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 29 Jun 2017 15:01:16 -0400 Subject: [PATCH 72/82] lib/pull: Move check for requested content earlier This is prep for a later patch; currently the logic is unchanged, but we'll need this if we make local imports async. Closes: #982 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index fd9e9aff..fb0c9e82 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -702,13 +702,17 @@ scan_dirtree_object (OtPullData *pull_data, if (file_is_stored) continue; + /* Already have a request pending? If so, move on to the next */ + if (g_hash_table_lookup (pull_data->requested_content, file_checksum)) + continue; + /* Is this a local repo? */ if (pull_data->remote_repo_local) { if (!import_one_local_content_object (pull_data, file_checksum, cancellable, error)) return FALSE; } - else if (!g_hash_table_lookup (pull_data->requested_content, file_checksum)) + else { /* In this case we're doing HTTP pulls */ g_hash_table_add (pull_data->requested_content, file_checksum); From 4273e670eaec12b86a23608deeaf593799306b51 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 22:19:15 -0400 Subject: [PATCH 73/82] Add "pull --localcache-repo" This is a lot like `git clone --reference`, but we chose "localcache" as the term "reference" is already used. The main use case I'm targeting this for is the Fedora Atomic Host installer case where we embed the repo content in the installer, but we may want to kickstart and download newer content. There, while we want to get a newer ref, we can still use the local repo as an object cache, since we have it sitting there in memory anyways. Another case is where one has a host ostree (say e.g. Fedora Atomic Workstation), and one wants to create a local archive mirror of FAH. Then one can use `pull --reference /ostree/repo` and pull the common objects (e.g. contents of `bash.rpm` etc.) Closes: https://github.com/ostreedev/ostree/issues/975 Closes: #982 Approved by: jlebon --- Makefile-tests.am | 1 + man/ostree-pull.xml | 10 +++ src/libostree/ostree-repo-pull.c | 102 ++++++++++++++++++++++++++++--- src/ostree/ot-builtin-pull.c | 5 ++ tests/test-pull-localcache.sh | 45 ++++++++++++++ 5 files changed, 153 insertions(+), 10 deletions(-) create mode 100755 tests/test-pull-localcache.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 348a8e4a..444b6636 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -78,6 +78,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-repeated.sh \ tests/test-pull-untrusted.sh \ tests/test-pull-override-url.sh \ + tests/test-pull-localcache.sh \ tests/test-local-pull.sh \ tests/test-local-pull-depth.sh \ tests/test-gpg-signed-commit.sh \ diff --git a/man/ostree-pull.xml b/man/ostree-pull.xml index 24ab0b72..394a29c2 100644 --- a/man/ostree-pull.xml +++ b/man/ostree-pull.xml @@ -73,6 +73,16 @@ Boston, MA 02111-1307, USA. + + + + + Like git's clone --reference. Reuse the provided + OSTree repo as a local object cache of objects when doing HTTP fetches. + May be specified multiple times. + + + diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index fb0c9e82..7346b299 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -64,6 +64,7 @@ typedef struct { GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */ GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */ OstreeRepo *remote_repo_local; + GPtrArray *localcache_repos; /* Array */ GMainContext *main_context; GCancellable *cancellable; @@ -118,6 +119,9 @@ typedef struct { guint n_fetched_deltapart_fallbacks; guint n_fetched_metadata; guint n_fetched_content; + /* Objects from pull --localcache-repo */ + guint n_fetched_localcache_metadata; + guint n_fetched_localcache_content; int maxdepth; guint64 start_time; @@ -236,6 +240,8 @@ update_progress (gpointer user_data) "scanned-metadata", "u", n_scanned_metadata, "bytes-transferred", "t", bytes_transferred, "start-time", "t", start_time, + "metadata-fetched-localcache", "u", pull_data->n_fetched_localcache_metadata, + "content-fetched-localcache", "u", pull_data->n_fetched_localcache_content, /* Deltas */ "fetched-delta-parts", "u", pull_data->n_fetched_deltaparts, @@ -606,16 +612,15 @@ validate_bareuseronly_mode (OtPullData *pull_data, */ static gboolean import_one_local_content_object (OtPullData *pull_data, + OstreeRepo *src_repo, const char *checksum, GCancellable *cancellable, GError **error) { - g_assert (pull_data->remote_repo_local); - const gboolean trusted = !pull_data->is_untrusted; if (trusted && !pull_data->is_bareuseronly_files) { - if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, + if (!ostree_repo_import_object_from_with_trust (pull_data->repo, src_repo, OSTREE_OBJECT_TYPE_FILE, checksum, trusted, cancellable, error)) @@ -630,7 +635,7 @@ import_one_local_content_object (OtPullData *pull_data, g_autoptr(GFileInfo) content_finfo = NULL; g_autoptr(GVariant) content_xattrs = NULL; - if (!ostree_repo_load_file (pull_data->remote_repo_local, checksum, + if (!ostree_repo_load_file (src_repo, checksum, &content_input, &content_finfo, &content_xattrs, cancellable, error)) return FALSE; @@ -709,16 +714,43 @@ scan_dirtree_object (OtPullData *pull_data, /* Is this a local repo? */ if (pull_data->remote_repo_local) { - if (!import_one_local_content_object (pull_data, file_checksum, cancellable, error)) + if (!import_one_local_content_object (pull_data, pull_data->remote_repo_local, + file_checksum, cancellable, error)) return FALSE; + + /* Note early loop continue */ + continue; } - else + + /* We're doing HTTP, but see if we have the object in a local cache first */ + gboolean did_import_from_cache_repo = FALSE; + if (pull_data->localcache_repos) { - /* In this case we're doing HTTP pulls */ - g_hash_table_add (pull_data->requested_content, file_checksum); - enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE); - file_checksum = NULL; /* Transfer ownership */ + for (guint j = 0; j < pull_data->localcache_repos->len; j++) + { + OstreeRepo *localcache_repo = pull_data->localcache_repos->pdata[j]; + gboolean localcache_repo_has_obj; + + if (!ostree_repo_has_object (localcache_repo, OSTREE_OBJECT_TYPE_FILE, file_checksum, + &localcache_repo_has_obj, cancellable, error)) + return FALSE; + if (!localcache_repo_has_obj) + continue; + if (!import_one_local_content_object (pull_data, localcache_repo, file_checksum, + cancellable, error)) + return FALSE; + did_import_from_cache_repo = TRUE; + pull_data->n_fetched_localcache_content++; + break; + } } + if (did_import_from_cache_repo) + continue; /* Note early continue */ + + /* Not available locally, queue a HTTP request */ + g_hash_table_add (pull_data->requested_content, file_checksum); + enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE); + file_checksum = NULL; /* Transfer ownership */ } g_autoptr(GVariant) dirs_variant = g_variant_get_child_value (tree, 1); @@ -1527,6 +1559,33 @@ scan_one_metadata_object_c (OtPullData *pull_data, is_stored = TRUE; is_requested = TRUE; } + /* Do we have any localcache repos? */ + else if (!is_stored && pull_data->localcache_repos) + { + for (guint i = 0; i < pull_data->localcache_repos->len; i++) + { + OstreeRepo *refd_repo = pull_data->localcache_repos->pdata[i]; + gboolean localcache_repo_has_obj; + + if (!ostree_repo_has_object (refd_repo, objtype, tmp_checksum, + &localcache_repo_has_obj, cancellable, error)) + return FALSE; + if (!localcache_repo_has_obj) + continue; + if (!ostree_repo_import_object_from_with_trust (pull_data->repo, refd_repo, + objtype, tmp_checksum, + !pull_data->is_untrusted, + cancellable, error)) + return FALSE; + /* See comment above */ + if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum)); + is_stored = TRUE; + is_requested = TRUE; + pull_data->n_fetched_localcache_metadata++; + break; + } + } if (!is_stored && !is_requested) { @@ -2865,6 +2924,7 @@ initiate_request (OtPullData *pull_data, * * inherit-transaction (b): Don't initiate, finish or abort a transaction, useful to do multiple pulls in one transaction. * * http-headers (a(ss)): Additional headers to add to all HTTP requests * * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid + * * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -2903,6 +2963,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean inherit_transaction = FALSE; g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ int i; + g_autofree char **opt_localcache_repos = NULL; if (options) { @@ -2929,6 +2990,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction); (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers); (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency); + (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); @@ -2993,6 +3055,20 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)fetch_object_data_free); pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL); + if (opt_localcache_repos && *opt_localcache_repos) + { + pull_data->localcache_repos = g_ptr_array_new_with_free_func (g_object_unref); + for (char **it = opt_localcache_repos; it && *it; it++) + { + const char *localcache_path = *it; + g_autoptr(GFile) localcache_file = g_file_new_for_path (localcache_path); + g_autoptr(OstreeRepo) cacherepo = ostree_repo_new (localcache_file); + if (!ostree_repo_open (cacherepo, cancellable, error)) + goto out; + g_ptr_array_add (pull_data->localcache_repos, g_steal_pointer (&cacherepo)); + } + } + if (dir_to_pull != NULL || dirs_to_pull != NULL) { pull_data->dirs = g_ptr_array_new_with_free_func (g_free); @@ -3722,6 +3798,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, else g_string_append_printf (buf, "%u metadata, %u content objects fetched", pull_data->n_fetched_metadata, pull_data->n_fetched_content); + if (pull_data->n_fetched_localcache_metadata || + pull_data->n_fetched_localcache_content) + g_string_append_printf (buf, " (%u meta, %u content local)", + pull_data->n_fetched_localcache_metadata, + pull_data->n_fetched_localcache_content); g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds", (guint64)(bytes_transferred / shift), @@ -3769,6 +3850,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_object (&pull_data->fetcher); g_clear_pointer (&pull_data->extra_headers, (GDestroyNotify)g_variant_unref); g_clear_object (&pull_data->cancellable); + g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref); g_clear_object (&pull_data->remote_repo_local); g_free (pull_data->remote_name); g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index edb09550..7898e107 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -41,6 +41,7 @@ static char* opt_cache_dir; static int opt_depth = 0; static int opt_frequency = 0; static char* opt_url; +static char** opt_localcache_repos; static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, @@ -57,6 +58,7 @@ static GOptionEntry options[] = { { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL }, { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" }, { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" }, + { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" }, { NULL } }; @@ -281,6 +283,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (override_commit_ids) g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); + if (opt_localcache_repos) + g_variant_builder_add (&builder, "{s@v}", "localcache-repos", + g_variant_new_variant (g_variant_new_strv ((const char*const*)opt_localcache_repos, -1))); if (opt_http_headers) { diff --git a/tests/test-pull-localcache.sh b/tests/test-pull-localcache.sh new file mode 100755 index 00000000..5f810acb --- /dev/null +++ b/tests/test-pull-localcache.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Copyright (C) 2017 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive" + +echo '1..1' + +cd ${test_tmpdir} +gnomerepo_url="$(cat httpd-address)/ostree/gnomerepo" +ostree_repo_init repo --mode "archive" +ostree_repo_init repo-local --mode "archive" +for repo in repo{,-local}; do + ${CMD_PREFIX} ostree --repo=${repo} remote add --set=gpg-verify=false origin ${gnomerepo_url} +done + +# Pull the contents to our local cache +${CMD_PREFIX} ostree --repo=repo-local pull origin main +rm files -rf +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main files +echo anewfile > files/anewfile +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main --tree=dir=files + +${CMD_PREFIX} ostree --repo=repo pull -L repo-local origin main >out.txt +assert_file_has_content out.txt '3 metadata, 1 content objects fetched (4 meta, 5 content local)' +echo "ok pull --reference" From 192e7b888f8cfcf4f02b0b5b6b502e679e91a4a1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 29 Jun 2017 21:45:26 -0400 Subject: [PATCH 74/82] lib/commit: Fix a tmpfile fd leak in static delta processing I had thought `glnx_link_tmpfile_at()` actually consumed the tmpfile; it does consume the *path* but not the fd. In the non-delta path things were fine since we used the autocleanup. But the delta code had a tmpfile allocated in its struct that got reused, and hence leaked the fd. Fix this by making the commit API actually consume the tmpfile fully, just like the path path. Closes: #986 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index ae3760c0..bc55fd6f 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -156,8 +156,12 @@ _ostree_repo_commit_tmpf_final (OstreeRepo *self, cancellable, error)) return FALSE; - return glnx_link_tmpfile_at (tmpf, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, - dest_dfd, tmpbuf, error); + if (!glnx_link_tmpfile_at (tmpf, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, + dest_dfd, tmpbuf, error)) + return FALSE; + /* We're done with the fd */ + glnx_tmpfile_clear (tmpf); + return TRUE; } /* Given a dfd+path combination (may be regular file or symlink), From e3a540a606fe75c588e0b5fc7fc234e86d49f244 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 30 Jun 2017 09:19:43 -0400 Subject: [PATCH 75/82] bin/commit: Port helper functions to new style Prep for more work here. Can't yet port the main function without a cleanup for transactions. Closes: #988 Approved by: jlebon --- src/ostree/ot-builtin-commit.c | 105 ++++++++++----------------------- 1 file changed, 30 insertions(+), 75 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index c14cbec5..77dafdfb 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -63,12 +63,10 @@ parse_fsync_cb (const char *option_name, GError **error) { gboolean val; - if (!ot_parse_boolean (value, &val, error)) return FALSE; - - opt_disable_fsync = !val; + opt_disable_fsync = !val; return TRUE; } @@ -109,16 +107,12 @@ parse_file_by_line (const char *path, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autofree char *contents = NULL; - g_autoptr(GFile) file = NULL; - char **lines = NULL; + g_autofree char *contents = + glnx_file_get_contents_utf8_at (AT_FDCWD, path, NULL, cancellable, error); + if (!contents) + return FALSE; - file = g_file_new_for_path (path); - if (!g_file_load_contents (file, cancellable, &contents, NULL, NULL, error)) - goto out; - - lines = g_strsplit (contents, "\n", -1); + g_auto(GStrv) lines = g_strsplit (contents, "\n", -1); for (char **iter = lines; iter && *iter; iter++) { /* skip empty lines at least */ @@ -126,13 +120,10 @@ parse_file_by_line (const char *path, continue; if (!cb (*iter, cbdata, error)) - goto out; + return FALSE; } - ret = TRUE; -out: - g_strfreev (lines); - return ret; + return TRUE; } static gboolean @@ -141,16 +132,11 @@ handle_statoverride_line (const char *line, GError **error) { GHashTable *files = data; - const char *spc; - guint mode_add; - - spc = strchr (line, ' '); + const char *spc = strchr (line, ' '); if (spc == NULL) - { - return glnx_throw (error, "Malformed statoverride file (no space found)"); - } + return glnx_throw (error, "Malformed statoverride file (no space found)"); - mode_add = (guint32)(gint32)g_ascii_strtod (line, NULL); + guint mode_add = (guint32)(gint32)g_ascii_strtod (line, NULL); g_hash_table_insert (files, g_strdup (spc + 1), GUINT_TO_POINTER((gint32)mode_add)); return TRUE; @@ -213,14 +199,7 @@ commit_editor (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - g_autofree char *input = NULL; - g_autofree char *output = NULL; - gboolean ret = FALSE; - g_autoptr(GString) bodybuf = NULL; - char **lines = NULL; - int i; - - input = g_strdup_printf ("\n" + g_autofree char *input = g_strdup_printf ("\n" "# Please enter the commit message for your changes. The first line will\n" "# become the subject, and the remainder the body. Lines starting\n" "# with '#' will be ignored, and an empty message aborts the commit." @@ -233,12 +212,13 @@ commit_editor (OstreeRepo *repo, *subject = NULL; *body = NULL; - output = ot_editor_prompt (repo, input, cancellable, error); + g_autofree char *output = ot_editor_prompt (repo, input, cancellable, error); if (output == NULL) - goto out; + return FALSE; - lines = g_strsplit (output, "\n", -1); - for (i = 0; lines[i] != NULL; i++) + g_auto(GStrv) lines = g_strsplit (output, "\n", -1); + g_autoptr(GString) bodybuf = NULL; + for (guint i = 0; lines[i] != NULL; i++) { g_strchomp (lines[i]); @@ -269,24 +249,15 @@ commit_editor (OstreeRepo *repo, } if (!*subject) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Aborting commit due to empty commit subject."); - goto out; - } + return glnx_throw (error, "Aborting commit due to empty commit subject."); if (bodybuf) { - *body = g_string_free (bodybuf, FALSE); + *body = g_string_free (g_steal_pointer (&bodybuf), FALSE); g_strchomp (*body); - bodybuf = NULL; } - ret = TRUE; - -out: - g_strfreev (lines); - return ret; + return TRUE; } static gboolean @@ -294,38 +265,22 @@ parse_keyvalue_strings (char **strings, GVariant **out_metadata, GError **error) { - gboolean ret = FALSE; - char **iter; - g_autoptr(GVariantBuilder) builder = NULL; + g_autoptr(GVariantBuilder) builder = + g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - - for (iter = strings; *iter; iter++) + for (char ** iter = strings; *iter; iter++) { - const char *s; - const char *eq; - g_autofree char *key = NULL; - - s = *iter; - - eq = strchr (s, '='); + const char *s = *iter; + const char *eq = strchr (s, '='); if (!eq) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Missing '=' in KEY=VALUE metadata '%s'", s); - goto out; - } - - key = g_strndup (s, eq - s); + return glnx_throw (error, "Missing '=' in KEY=VALUE metadata '%s'", s); + g_autofree char *key = g_strndup (s, eq - s); g_variant_builder_add (builder, "{sv}", key, g_variant_new_string (eq + 1)); } - ret = TRUE; - *out_metadata = g_variant_builder_end (builder); - g_variant_ref_sink (*out_metadata); - out: - return ret; + *out_metadata = g_variant_ref_sink (g_variant_builder_end (builder)); + return TRUE; } gboolean @@ -387,7 +342,7 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError &detached_metadata, error)) goto out; } - + if (!(opt_branch || opt_orphan)) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, From cd7d35945af7c11891919b2f2ffae65532b0af31 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 30 Jun 2017 09:40:37 -0400 Subject: [PATCH 76/82] bin/commit: Add '=' to --statoverride Previously, we only supported additions in the statoverride file; it was mainly for adding the setuid bit without having that physically on disk. However, for testing a change to `bare-user` handling around *unreadable* files (which happens for `/etc/shadow` in host content), I need a way to write that into a repo in the test suite. I'm not actually aware of a non-test-suite use case for this; a more sophisticated user is going to be using the API directly, which can already do this. But we need it for tests at least. Closes: #989 Approved by: jlebon --- src/ostree/ot-builtin-commit.c | 44 +++++++++++++++++++++++++--------- tests/basic-test.sh | 4 ++++ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 77dafdfb..8e8b9233 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -126,19 +126,35 @@ parse_file_by_line (const char *path, return TRUE; } +struct CommitFilterData { + GHashTable *mode_adds; + GHashTable *mode_overrides; + GHashTable *skip_list; +}; + static gboolean handle_statoverride_line (const char *line, void *data, GError **error) { - GHashTable *files = data; + struct CommitFilterData *cf = data; const char *spc = strchr (line, ' '); if (spc == NULL) return glnx_throw (error, "Malformed statoverride file (no space found)"); + const char *fn = spc + 1; - guint mode_add = (guint32)(gint32)g_ascii_strtod (line, NULL); - g_hash_table_insert (files, g_strdup (spc + 1), - GUINT_TO_POINTER((gint32)mode_add)); + if (g_str_has_prefix (line, "=")) + { + guint mode_override = (guint32)(gint32)g_ascii_strtod (line+1, NULL); + g_hash_table_insert (cf->mode_overrides, g_strdup (fn), + GUINT_TO_POINTER((gint32)mode_override)); + } + else + { + guint mode_add = (guint32)(gint32)g_ascii_strtod (line, NULL); + g_hash_table_insert (cf->mode_adds, g_strdup (fn), + GUINT_TO_POINTER((gint32)mode_add)); + } return TRUE; } @@ -152,11 +168,6 @@ handle_skiplist_line (const char *line, return TRUE; } -struct CommitFilterData { - GHashTable *mode_adds; - GHashTable *skip_list; -}; - static OstreeRepoCommitFilterResult commit_filter (OstreeRepo *self, const char *path, @@ -165,6 +176,7 @@ commit_filter (OstreeRepo *self, { struct CommitFilterData *data = user_data; GHashTable *mode_adds = data->mode_adds; + GHashTable *mode_overrides = data->mode_overrides; GHashTable *skip_list = data->skip_list; gpointer value; @@ -181,6 +193,14 @@ commit_filter (OstreeRepo *self, current_mode | mode_add); g_hash_table_remove (mode_adds, path); } + else if (mode_overrides && g_hash_table_lookup_extended (mode_overrides, path, NULL, &value)) + { + guint current_fmt = g_file_info_get_attribute_uint32 (file_info, "unix::mode") & S_IFMT; + guint mode_override = GPOINTER_TO_UINT (value); + g_file_info_set_attribute_uint32 (file_info, "unix::mode", + current_fmt | mode_override); + g_hash_table_remove (mode_adds, path); + } if (skip_list && g_hash_table_contains (skip_list, path)) { @@ -299,6 +319,7 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError glnx_unref_object OstreeMutableTree *mtree = NULL; g_autofree char *tree_type = NULL; g_autoptr(GHashTable) mode_adds = NULL; + g_autoptr(GHashTable) mode_overrides = NULL; g_autoptr(GHashTable) skip_list = NULL; OstreeRepoCommitModifierFlags flags = 0; OstreeRepoCommitModifier *modifier = NULL; @@ -316,9 +337,10 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError if (opt_statoverride_file) { - mode_adds = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + filter_data.mode_adds = mode_adds = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + filter_data.mode_overrides = mode_overrides = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); if (!parse_file_by_line (opt_statoverride_file, handle_statoverride_line, - mode_adds, cancellable, error)) + &filter_data, cancellable, error)) goto out; } diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 012a98e0..07bbeddf 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -342,8 +342,11 @@ cd ${test_tmpdir} cat > test-statoverride.txt < a/readable-only +chmod 664 a/readable-only $OSTREE commit ${COMMIT_ARGS} -b test2-override -s "with statoverride" --statoverride=../test-statoverride.txt cd ${test_tmpdir} $OSTREE checkout test2-override checkout-test2-override @@ -354,6 +357,7 @@ else test '!' -g checkout-test2-override/a/nested/2 test '!' -u checkout-test2-override/a/nested/3 fi +assert_file_has_mode checkout-test2-override/a/readable-only 600 echo "ok commit statoverride" cd ${test_tmpdir} From 3348baf6eb69be0ec8ac3138e696e2828a0cc9e3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 30 Jun 2017 09:40:47 -0400 Subject: [PATCH 77/82] lib/commit: Ensure bare-user objects are always user-readable Some of the Jenkins jobs for Fedora Atomic Host broke after updating to 2017.7, and it turns out that we regressed handling unreadable files in `bare-user` mode. An example of this is `/etc/shadow`, which ends up in the ostree-as-host content as `/usr/etc/shadow`. Now there are better fixes here; we should probably delete it and create it during the config merge if it doesn't exist. In general, having secret files in ostree really isn't supported, so it doesn't make sense to include them. But let's fix this regression - when operating as an unprivileged user we don't have `CAP_DAC_OVERRIDE` and hence will fail to open un-user-readable objects. (We still preserve the actual `0` mode of course in the xattr and will apply it in `bare`) Closes: #989 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 4 +++- tests/test-basic-user.sh | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index bc55fd6f..1ecbc178 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -253,13 +253,15 @@ commit_loose_regfile_object (OstreeRepo *self, /* Note that previously this path added `| 0755` which made every * file executable, see * https://github.com/ostreedev/ostree/issues/907 + * We then changed it to mask by 0775, but we always need at least read + * permission when running as non-root, so explicitly mask that in. * * Again here, symlinks in bare-user are a hairy special case; only do a * chmod for a *real* regular file, otherwise we'll take the default 0644. */ if (S_ISREG (mode)) { - const mode_t content_mode = (mode & (S_IFREG | 0775)); + const mode_t content_mode = (mode & (S_IFREG | 0775)) | S_IRUSR; if (fchmod (tmpf->fd, content_mode) < 0) return glnx_throw_errno_prefix (error, "fchmod"); } diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh index fa802df6..94866550 100755 --- a/tests/test-basic-user.sh +++ b/tests/test-basic-user.sh @@ -25,7 +25,7 @@ skip_without_user_xattrs setup_test_repository "bare-user" -extra_basic_tests=3 +extra_basic_tests=4 . $(dirname $0)/basic-test.sh # Reset things so we don't inherit a lot of state from earlier tests @@ -64,3 +64,16 @@ $OSTREE checkout -U -H test2-unwritable test2-checkout cd test2-checkout assert_file_has_mode unwritable 400 echo "ok bare-user unwritable" + +rm test2-checkout -rf +$OSTREE checkout -U -H test2 test2-checkout +cat > statoverride.txt < Date: Thu, 29 Jun 2017 15:58:05 -0400 Subject: [PATCH 78/82] lib/commit: Use provided length when doing writes This is prep for storage space checks, where we look at free space after parsing the metadata, before we write anything. We did length-limited writes in the fd-based input path, but not for the `GInputStream` path which in practice is used for HTTP pulls. Closes: #987 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 1ecbc178..9cc028e0 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -481,10 +481,27 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self, } else { - g_autoptr(GOutputStream) temp_out = g_unix_output_stream_new (tmpf.fd, FALSE); - if (g_output_stream_splice (temp_out, input, 0, - cancellable, error) < 0) - return FALSE; + /* We used to do a g_output_stream_splice(), but there are two issues with that: + * - We want to honor the size provided, to avoid malicious content that says it's + * e.g. 10 bytes but is actually gigabytes. + * - Due to GLib bugs that pointlessly calls `poll()` on the output fd for every write + */ + char buf[8192]; + guint64 remaining = length; + while (remaining > 0) + { + const gssize bytes_read = + g_input_stream_read (input, buf, MIN (remaining, sizeof (buf)), cancellable, error); + if (bytes_read < 0) + return FALSE; + else if (G_UNLIKELY (bytes_read == 0 && remaining > 0)) + return glnx_throw (error, "Unexpected EOF with %" G_GUINT64_FORMAT "/%" G_GUINT64_FORMAT " bytes remaining", remaining, length); + else if (bytes_read == 0) + break; + if (glnx_loop_write (tmpf.fd, buf, bytes_read) < 0) + return glnx_throw_errno_prefix (error, "write"); + remaining -= bytes_read; + } } if (fchmod (tmpf.fd, 0644) < 0) From 1f5ce1a9f789d9c0de5d6fbdf79540bf71c5bc9b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 29 Jun 2017 16:51:56 -0400 Subject: [PATCH 79/82] lib/repo: Add min-free-space-percent option, default 3% For ostree-as-host, we're the superuser, so we'll blow past any reserved free space by default. While deltas have size metadata, if one happens to do a loose fetch, we can fill up the disk. Another case is flatpak: the system helper has similar concerns here as ostree-as-host, and for `flatpak --user`, we also want to be nice and avoid filling up the user's quota. Closes: https://github.com/ostreedev/ostree/issues/962 Closes: #987 Approved by: jlebon --- man/ostree.repo-config.xml | 8 ++++++ src/libostree/ostree-repo-commit.c | 42 +++++++++++++++++++++++++++++ src/libostree/ostree-repo-private.h | 5 ++++ src/libostree/ostree-repo.c | 15 +++++++++++ tests/installed/itest-pull-space.sh | 24 +++++++++++++++++ 5 files changed, 94 insertions(+) create mode 100755 tests/installed/itest-pull-space.sh diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index d187f89f..60458dfa 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -190,6 +190,14 @@ Boston, MA 02111-1307, USA. unconfigured-state If set, pulls from this remote will fail with the configured text. This is intended for OS vendors which have a subscription process to access content. + + + min-free-space-percent + Integer percentage value (0-99) that specifies a minimum + percentage of total space (in blocks) in the underlying filesystem to + keep free. The default value is 3. + + diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 9cc028e0..185ab82f 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -23,6 +23,7 @@ #include "config.h" #include +#include #include #include #include @@ -574,6 +575,24 @@ write_content_object (OstreeRepo *self, else size = 0; + /* Free space check; only applies during transactions */ + if (self->min_free_space_percent > 0 && self->in_transaction) + { + g_mutex_lock (&self->txn_stats_lock); + g_assert_cmpint (self->txn_blocksize, >, 0); + const fsblkcnt_t object_blocks = (size / self->txn_blocksize) + 1; + if (object_blocks > self->max_txn_blocks) + { + g_mutex_unlock (&self->txn_stats_lock); + g_autofree char *formatted_required = g_format_size ((guint64)object_blocks * self->txn_blocksize); + return glnx_throw (error, "min-free-space-percent '%u%%' would be exceeded, %s more required", + self->min_free_space_percent, formatted_required); + } + /* This is the main bit that needs mutex protection */ + self->max_txn_blocks -= object_blocks; + g_mutex_unlock (&self->txn_stats_lock); + } + /* For regular files, we create them with default mode, and only * later apply any xattrs and setuid bits. The rationale here * is that an attacker on the network with the ability to MITM @@ -1080,6 +1099,29 @@ ostree_repo_prepare_transaction (OstreeRepo *self, memset (&self->txn_stats, 0, sizeof (OstreeRepoTransactionStats)); self->in_transaction = TRUE; + if (self->min_free_space_percent > 0) + { + struct statvfs stvfsbuf; + if (TEMP_FAILURE_RETRY (fstatvfs (self->repo_dir_fd, &stvfsbuf)) < 0) + return glnx_throw_errno_prefix (error, "fstatvfs"); + g_mutex_lock (&self->txn_stats_lock); + self->txn_blocksize = stvfsbuf.f_bsize; + /* Convert fragment to blocks to compute the total */ + guint64 total_blocks = (stvfsbuf.f_frsize * stvfsbuf.f_blocks) / stvfsbuf.f_bsize; + /* Use the appropriate free block count if we're unprivileged */ + guint64 bfree = (getuid () != 0 ? stvfsbuf.f_bavail : stvfsbuf.f_bfree); + guint64 reserved_blocks = ((double)total_blocks) * (self->min_free_space_percent/100.0); + if (bfree > reserved_blocks) + self->max_txn_blocks = bfree - reserved_blocks; + else + { + g_mutex_unlock (&self->txn_stats_lock); + g_autofree char *formatted_free = g_format_size (bfree * self->txn_blocksize); + return glnx_throw (error, "min-free-space-percent '%u%%' would be exceeded, %s available", + self->min_free_space_percent, formatted_free); + } + g_mutex_unlock (&self->txn_stats_lock); + } gboolean ret_transaction_resume = FALSE; if (!_ostree_repo_allocate_tmpdir (self->tmp_dir_fd, diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index d518e52b..9e00cf40 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -20,6 +20,7 @@ #pragma once +#include #include "ostree-ref.h" #include "ostree-repo.h" #include "ostree-remote-private.h" @@ -102,6 +103,9 @@ struct OstreeRepo { GHashTable *txn_collection_refs; /* (element-type OstreeCollectionRef utf8) */ GMutex txn_stats_lock; OstreeRepoTransactionStats txn_stats; + /* Implementation of min-free-space-percent */ + gulong txn_blocksize; + fsblkcnt_t max_txn_blocks; GMutex cache_lock; guint dirmeta_cache_refcount; @@ -123,6 +127,7 @@ struct OstreeRepo { uid_t owner_uid; uid_t target_owner_uid; gid_t target_owner_gid; + guint min_free_space_percent; guint test_error_flags; /* OstreeRepoTestErrorFlags */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index a02214e1..e29b8fca 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -44,6 +44,7 @@ #include #include #include +#include /** * SECTION:ostree-repo @@ -1964,6 +1965,20 @@ reload_core_config (OstreeRepo *self, self->zlib_compression_level = OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL; } + { g_autofree char *min_free_space_percent_str = NULL; + /* If changing this, be sure to change the man page too */ + const char *default_min_free_space = "3"; + + if (!ot_keyfile_get_value_with_default (self->config, "core", "min-free-space-percent", + default_min_free_space, + &min_free_space_percent_str, error)) + return FALSE; + + self->min_free_space_percent = g_ascii_strtoull (min_free_space_percent_str, NULL, 10); + if (self->min_free_space_percent > 99) + return glnx_throw (error, "Invalid min-free-space-percent '%s'", min_free_space_percent_str); + } + { g_clear_pointer (&self->collection_id, g_free); if (!ot_keyfile_get_value_with_default (self->config, "core", "collection-id", diff --git a/tests/installed/itest-pull-space.sh b/tests/installed/itest-pull-space.sh new file mode 100755 index 00000000..36703a40 --- /dev/null +++ b/tests/installed/itest-pull-space.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Test min-free-space-percent using loopback devices + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libinsttest.sh + +test_tmpdir=$(prepare_tmpdir) +trap _tmpdir_cleanup EXIT + +cd ${test_tmpdir} +truncate -s 100MB testblk.img +blkdev=$(losetup --find --show $(pwd)/testblk.img) +mkfs.xfs ${blkdev} +mkdir mnt +mount ${blkdev} mnt +ostree --repo=mnt/repo init --mode=bare-user +if ostree --repo=mnt/repo pull-local /ostree/repo ${host_commit} 2>err.txt; then + fatal "succeeded in doing a pull with no free space" +fi +assert_file_has_content err.txt "min-free-space-percent" +umount mnt +losetup -d ${blkdev} From 3234295324fe63f488cc68632066a784d4d11f53 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 4 Jul 2017 06:55:51 -0400 Subject: [PATCH 80/82] pull: Cleanly error when doing local pulls of remote-prefixed refs In the storage PR I was trying to do a `pull-local` of the whole `/ostree/repo` on the system, which ended up triggering a `g_critical()` in the collections code, since we tried to parse a remote-prefixed ref `fedora:fedora/26/x86_64/atomic-host` as a ref. I'm not sure offhand what our behavior in this case *should* be. I think git only clones local refs, but I need to check. This corner case arises only with `pull-local`. But in any case, while we were previously saying this is programmer error, since it's so easy to pass various unchecked input into the pull machinery, make invalid refs an explicit error. Closes: #992 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 2 ++ tests/pull-test.sh | 22 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 7346b299..b51220bb 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3561,6 +3561,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, } else { + if (!ostree_validate_rev (branch, error)) + goto out; char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; g_hash_table_insert (requested_refs_to_fetch, ostree_collection_ref_new (NULL, branch), commitid); diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 1da0fa4c..aaba2a7b 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -35,7 +35,7 @@ function verify_initial_contents() { assert_file_has_content baz/cow '^moo$' } -echo "1..25" +echo "1..26" # Try both syntaxes repo_init --no-gpg-verify @@ -161,6 +161,26 @@ ${CMD_PREFIX} ostree --repo=mirrorrepo-local fsck $OSTREE show main >/dev/null echo "ok pull local mirror" +cd ${test_tmpdir} +# This is more of a known issue; test that we give a clean error right now +rm otherrepo -rf +ostree_repo_init otherrepo --mode=archive +rm checkout-origin-main -rf +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main checkout-origin-main +${CMD_PREFIX} ostree --repo=otherrepo commit -b localbranch --tree=dir=checkout-origin-main +${CMD_PREFIX} ostree --repo=otherrepo remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo +${CMD_PREFIX} ostree --repo=otherrepo pull origin main +rm mirrorrepo-local -rf +ostree_repo_init mirrorrepo-local --mode=archive +if ${CMD_PREFIX} ostree --repo=mirrorrepo-local pull-local otherrepo 2>err.txt; then + fatal "pull with mixed refs succeeded?" +fi +assert_file_has_content err.txt "error: Invalid ref name origin:main" +${CMD_PREFIX} ostree --repo=mirrorrepo-local pull-local otherrepo localbranch +${CMD_PREFIX} ostree --repo=mirrorrepo-local rev-parse localbranch +${CMD_PREFIX} ostree --repo=mirrorrepo-local fsck +echo "ok pull-local mirror errors with mixed refs" + cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main -s "Metadata string" --add-detached-metadata-string=SIGNATURE=HANCOCK --tree=ref=main ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u From 23b93a3eb6d4483a3be22757b887bcbf6147cd4c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 5 Jul 2017 10:50:34 -0400 Subject: [PATCH 81/82] lib/repo: Immediately error creating bare-user repo on tmpfs And in general, if for some reason we can't write `user.` xattrs, provide an error immediately rather than doing it during a later pull. This way the failure cause is a lot more obvious. Related: https://github.com/ostreedev/ostree/issues/991 Closes: #993 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 38 +++++++------------ src/libostree/ostree-repo-private.h | 8 ++++ src/libostree/ostree-repo.c | 15 ++++++++ .../installed/itest-bareuser-nouserxattrs.sh | 24 ++++++++++++ 4 files changed, 61 insertions(+), 24 deletions(-) create mode 100755 tests/installed/itest-bareuser-nouserxattrs.sh diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 185ab82f..baeef3f9 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -82,31 +82,21 @@ create_file_metadata (guint32 uid, return ret_metadata; } -static gboolean -write_file_metadata_to_xattr (int fd, - guint32 uid, - guint32 gid, - guint32 mode, - GVariant *xattrs, - GError **error) +gboolean +_ostree_write_bareuser_metadata (int fd, + guint32 uid, + guint32 gid, + guint32 mode, + GVariant *xattrs, + GError **error) { - g_autoptr(GVariant) filemeta = NULL; - int res; + g_autoptr(GVariant) filemeta = create_file_metadata (uid, gid, mode, xattrs); - filemeta = create_file_metadata (uid, gid, mode, xattrs); - - do - res = fsetxattr (fd, "user.ostreemeta", - (char*)g_variant_get_data (filemeta), - g_variant_get_size (filemeta), - 0); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - g_prefix_error (error, "Unable to set xattr: "); - return FALSE; - } + if (TEMP_FAILURE_RETRY (fsetxattr (fd, "user.ostreemeta", + (char*)g_variant_get_data (filemeta), + g_variant_get_size (filemeta), + 0)) != 0) + return glnx_throw_errno_prefix (error, "fsetxattr(user.ostreemeta)"); return TRUE; } @@ -248,7 +238,7 @@ commit_loose_regfile_object (OstreeRepo *self, } else if (self->mode == OSTREE_REPO_MODE_BARE_USER) { - if (!write_file_metadata_to_xattr (tmpf->fd, uid, gid, mode, xattrs, error)) + if (!_ostree_write_bareuser_metadata (tmpf->fd, uid, gid, mode, xattrs, error)) return FALSE; /* Note that previously this path added `| 0755` which made every diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 9e00cf40..407e2cb3 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -216,6 +216,14 @@ _ostree_repo_has_loose_object (OstreeRepo *self, GCancellable *cancellable, GError **error); +gboolean +_ostree_write_bareuser_metadata (int fd, + guint32 uid, + guint32 gid, + guint32 mode, + GVariant *xattrs, + GError **error); + gboolean _ostree_repo_write_directory_meta (OstreeRepo *self, GFileInfo *file_info, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index e29b8fca..815d2d65 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1752,6 +1752,21 @@ ostree_repo_create (OstreeRepo *self, } } + /* Test that the fs supports user xattrs now, so we get an error early rather + * than during an object write later. + */ + if (mode == OSTREE_REPO_MODE_BARE_USER) + { + g_auto(GLnxTmpfile) tmpf = { 0, }; + + if (!glnx_open_tmpfile_linkable_at (dfd, ".", O_RDWR|O_CLOEXEC, &tmpf, error)) + return FALSE; + if (fchmod (tmpf.fd, 0600) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + if (!_ostree_write_bareuser_metadata (tmpf.fd, 0, 0, 644, NULL, error)) + return FALSE; + } + if (!ostree_repo_open (self, cancellable, error)) return FALSE; diff --git a/tests/installed/itest-bareuser-nouserxattrs.sh b/tests/installed/itest-bareuser-nouserxattrs.sh new file mode 100755 index 00000000..06d3e109 --- /dev/null +++ b/tests/installed/itest-bareuser-nouserxattrs.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Test that initializing a bare-user repo on tmpfs fails +# Maybe at some point this will be fixed in the kernel +# but I doubt it'll be soon +# https://www.spinics.net/lists/linux-mm/msg109775.html + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libinsttest.sh + +test_tmpdir=$(prepare_tmpdir) +trap _tmpdir_cleanup EXIT +cd ${test_tmpdir} + +mkdir mnt +mount -t tmpfs tmpfs mnt +if ostree --repo=mnt/repo init --mode=bare-user 2>err.txt; then + umount mnt + assert_not_reached "bare-user on tmpfs worked?" +fi +umount mnt +assert_file_has_content err.txt "Operation not supported" From 5a5e465492aca13937dab7a2df39f25da94e6e36 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 5 Jul 2017 16:42:34 -0400 Subject: [PATCH 82/82] Release 2017.8 Closes: #994 Approved by: jlebon --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 4 ---- src/libostree/libostree-released.sym | 5 +++++ tests/test-symbols.sh | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index 67e70b3e..30ae793b 100644 --- a/configure.ac +++ b/configure.ac @@ -7,7 +7,7 @@ m4_define([year_version], [2017]) m4_define([release_version], [8]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index d2bc8399..01f182f6 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,10 +18,6 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2017.8 { -global: - ostree_validate_remote_name; -} LIBOSTREE_2017.7; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index 5fc8a9b1..b68d6a5e 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -408,6 +408,11 @@ global: ostree_sysroot_query_deployments_for; } LIBOSTREE_2017.6; +LIBOSTREE_2017.8 { +global: + ostree_validate_remote_name; +} LIBOSTREE_2017.7; + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 4a11183e..fcc345f0 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -52,7 +52,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt <