From 79c2d76e9f56a4eb0338641702d5804ad5056923 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 11 May 2018 10:40:59 -0400 Subject: [PATCH 01/45] Post-release version bump Closes: #1586 Approved by: jlebon --- configure.ac | 4 ++-- src/libostree/libostree-devel.sym | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index afedc030..5fd95da4 100644 --- a/configure.ac +++ b/configure.ac @@ -4,10 +4,10 @@ dnl update libostree-released.sym from libostree-devel.sym, and update the check dnl in test-symbols.sh, and also set is_release_build=yes below. Then make dnl another post-release commit to bump the version, and set is_release_build=no. m4_define([year_version], [2018]) -m4_define([release_version], [5]) +m4_define([release_version], [6]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 46ad280c..b217e3e5 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,13 +18,15 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ +LIBOSTREE_2018.6 { +} LIBOSTREE_2018.5; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION * with whatever the next version with new symbols will be. -LIBOSTREE_2017.$NEWVERSION { +LIBOSTREE_2018.$NEWVERSION { global: someostree_symbol_deleteme; -} LIBOSTREE_2017.$LASTSTABLE; +} LIBOSTREE_2018.$LASTSTABLE; */ From 7a1ac8ef155f36f9c395cd29c5741af0e48e4f69 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 11 May 2018 15:03:58 -0400 Subject: [PATCH 02/45] ci: Add .papr-ex.yaml I'd like to start working towards migrating the OSTree CI towards the new PAPR model running in PACI. In order to take it for a test drive (pun intended) without yet replacing our current CI, the new PAPR now knows to look for a `.papr-ex.yaml` file. So as to not conflict with the current PAPR, no GitHub statuses will be output other than a `required-ex` label, which plays the same role as `required`. For more information, see: https://github.com/projectatomic/papr/commit/85b1d0a91eba6597d01b52b674de38a876f030fe The `.papr-ex.yaml` version is essentially the same except for: 1. the `FAH27-insttests` suite runs directly in the scheduled container (notice the new `kvm: true` key) 2. the `host:` based contexts (rpm-ostree and flatpak) are missing since we don't support provisioning a host just yet Keeping them in sync could get annoying, though building confidence in the new approach will allow us to migrate faster. Closes: #1587 Approved by: cgwalters --- .papr-ex.yaml | 131 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 .papr-ex.yaml diff --git a/.papr-ex.yaml b/.papr-ex.yaml new file mode 100644 index 00000000..900aae7b --- /dev/null +++ b/.papr-ex.yaml @@ -0,0 +1,131 @@ +# https://fedoraproject.org/wiki/CI/Tests +branches: + - master + - auto + - try + +context: FAH27-insttests +required: true + +container: + image: registry.fedoraproject.org/fedora:27 + kvm: true + +tests: + - ci/fah27-insttests.sh + +artifacts: + - tests/installed/artifacts/ + +--- + +# This suite skips the RPMs and does the build+unit tests in a container +inherit: false +branches: + - master + - auto + - try +required: true +container: + image: registry.fedoraproject.org/fedora:27 +context: f27-primary +env: + # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain + CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' + # Only for CI with a known g-ir-scanner + GI_SCANNERFLAGS: '--warn-error' + ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc + # TODO when we're doing leak checks: G_SLICE: "always-malloc" + CONFIGOPTS: '--with-curl --with-openssl' + +tests: + - ci/ci-commitmessage-submodules.sh + - ci/build-check.sh + - ci/ci-release-build.sh + +artifacts: + - test-suite.log + - config.log + - gdtr-results + +--- +# And now the contexts below here are variant container builds + +context: f27-rust +inherit: true +container: + image: registry.fedoraproject.org/fedora:27 +env: + CONFIGOPTS: '--enable-rust' + CI_PKGS: cargo + +tests: + - ci/build.sh + - make check TESTS=tests/test-rollsum + +--- + +context: f27-gnutls +inherit: true +container: + image: registry.fedoraproject.org/fedora:27 +env: + CONFIGOPTS: '--with-crypto=gnutls' + CI_PKGS: pkgconfig(gnutls) + +tests: + - ci/build.sh + - make check TESTS=tests/test-basic.sh + +--- + +inherit: true + +context: f27-experimental-api +env: + CONFIGOPTS: '--enable-experimental-api' + +tests: + - ci/build-check.sh + +--- + +inherit: true + +context: f27-minimal +env: + CONFIGOPTS: '--without-curl --without-soup --disable-gtk-doc --disable-man + --disable-rust --without-libarchive --without-selinux --without-smack + --without-openssl --without-avahi --without-libmount --disable-rofiles-fuse + --disable-experimental-api' + +tests: + - ci/build.sh + +--- + +inherit: true +required: true + +context: f27-libsoup + +env: + CONFIGOPTS: "--without-curl --without-openssl --with-libsoup" + +tests: + - ci/build-check.sh + +--- + +inherit: true +required: true + +context: f27-introspection-tests + +env: + # ASAN conflicts with introspection testing; + # See https://github.com/ostreedev/ostree/issues/1014 + INSTALLED_TESTS_PATTERN: "libostree/test-sizes.js libostree/test-sysroot.js libostree/test-core.js" + +tests: + - ci/build-check.sh From 4b330feed66dacf62c7193a1e84e20d9d1e34ba3 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Mon, 14 May 2018 11:36:14 +0700 Subject: [PATCH 03/45] docs: var/local -> var/usrlocal Signed-off-by: Arnaud Rebillout Closes: #1588 Approved by: jlebon --- docs/manual/adapting-existing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manual/adapting-existing.md b/docs/manual/adapting-existing.md index 79bf7be2..f73c2264 100644 --- a/docs/manual/adapting-existing.md +++ b/docs/manual/adapting-existing.md @@ -32,7 +32,7 @@ recommended set: - `/opt` → `/var/opt` - `/srv` → `/var/srv` - `/root` → `/var/roothome` - - `/usr/local` → `/var/local` + - `/usr/local` → `/var/usrlocal` - `/mnt` → `/var/mnt` - `/tmp` → `/sysroot/tmp` From ce2995e1dc1557c4d97ef5af807eacf3ef4a22d8 Mon Sep 17 00:00:00 2001 From: Andrea Galbusera Date: Tue, 15 May 2018 08:29:28 +0200 Subject: [PATCH 04/45] u-boot: add support for devicetree Likewise the corresponding support for syslinux introduced by commit c5112c25e4519835c4cd53f4350c1b2f2a477746, this one enables writing devicetree filename into the uEnv.txt environment file for u-boot. Since u-boot does not strictly defines variable names, here 'fdt_file' was chosen as it appear to be one the most frequently adopted names in u-boot default environments. Outer boot logic should of course comply with this choice and use $fdt_file as the device tree file name to pass to boot commands. This was tested on a custom board booting with u-boot. Closes: #1590 Approved by: cgwalters --- src/libostree/ostree-bootloader-uboot.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libostree/ostree-bootloader-uboot.c b/src/libostree/ostree-bootloader-uboot.c index 262681b1..531a3756 100644 --- a/src/libostree/ostree-bootloader-uboot.c +++ b/src/libostree/ostree-bootloader-uboot.c @@ -140,6 +140,10 @@ create_config_from_boot_loader_entries (OstreeBootloaderUboot *self, if (val) g_ptr_array_add (new_lines, g_strdup_printf ("ramdisk_image%s=%s", index_suffix, val)); + val = ostree_bootconfig_parser_get (config, "devicetree"); + if (val) + g_ptr_array_add (new_lines, g_strdup_printf ("fdt_file%s=%s", index_suffix, val)); + val = ostree_bootconfig_parser_get (config, "options"); if (val) { From 371081d123a9263d8dbdd4dad69c0468e2db427d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Apr 2018 14:31:33 +0000 Subject: [PATCH 05/45] lib: Add a public helper method for pruning to find all ref'd commits Prep for reworking how we do sysroot cleanup. We're going to start doing more lowlevel pruning work there, and I wanted to avoid duplicating the ref enumeration. Closes: #1566 Approved by: jlebon --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-repo-prune.c | 87 +++++++++++++++++++++---------- src/libostree/ostree-repo.h | 8 +++ 4 files changed, 69 insertions(+), 28 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 5162b2f7..6d4a3423 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -413,6 +413,7 @@ ostree_repo_commit_traverse_iter_next OstreeRepoPruneFlags ostree_repo_prune ostree_repo_prune_static_deltas +ostree_repo_traverse_reachable_refs ostree_repo_prune_from_reachable OstreeRepoPullFlags ostree_repo_pull diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index b217e3e5..eb3b3211 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,6 +19,7 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2018.6 { + ostree_repo_traverse_reachable_refs; } LIBOSTREE_2018.5; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index 4c883542..b93d35ac 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -301,6 +301,64 @@ repo_prune_internal (OstreeRepo *self, return TRUE; } +/** + * ostree_repo_traverse_reachable_refs: + * @self: Repo + * @depth: Depth of traversal + * @reachable: (element-type GVariant GVariant): Set of reachable objects (will be modified) + * @cancellable: Cancellable + * @error: Error + * + * Add all commit objects directly reachable via a ref to @reachable. + * + * Locking: shared + * Since: 2018.6 + */ +gboolean +ostree_repo_traverse_reachable_refs (OstreeRepo *self, + guint depth, + GHashTable *reachable, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(OstreeRepoAutoLock) lock = + _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_SHARED, cancellable, error); + if (!lock) + return FALSE; + + /* Ignoring collections. */ + g_autoptr(GHashTable) all_refs = NULL; /* (element-type utf8 utf8) */ + + if (!ostree_repo_list_refs (self, NULL, &all_refs, + cancellable, error)) + return FALSE; + + GLNX_HASH_TABLE_FOREACH_V (all_refs, const char*, checksum) + { + g_debug ("Finding objects to keep for commit %s", checksum); + if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, + cancellable, error)) + return FALSE; + } + + /* Using collections. */ + g_autoptr(GHashTable) all_collection_refs = NULL; /* (element-type OstreeChecksumRef utf8) */ + + if (!ostree_repo_list_collection_refs (self, NULL, &all_collection_refs, + OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES, cancellable, error)) + return FALSE; + + GLNX_HASH_TABLE_FOREACH_V (all_collection_refs, const char*, checksum) + { + g_debug ("Finding objects to keep for commit %s", checksum); + if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, + cancellable, error)) + return FALSE; + } + + return TRUE; +} + /** * ostree_repo_prune: * @self: Repo @@ -355,35 +413,8 @@ ostree_repo_prune (OstreeRepo *self, if (refs_only) { - /* Ignoring collections. */ - g_autoptr(GHashTable) all_refs = NULL; /* (element-type utf8 utf8) */ - - if (!ostree_repo_list_refs (self, NULL, &all_refs, - cancellable, error)) + if (!ostree_repo_traverse_reachable_refs (self, depth, reachable, cancellable, error)) return FALSE; - - GLNX_HASH_TABLE_FOREACH_V (all_refs, const char*, checksum) - { - g_debug ("Finding objects to keep for commit %s", checksum); - if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, - cancellable, error)) - return FALSE; - } - - /* Using collections. */ - g_autoptr(GHashTable) all_collection_refs = NULL; /* (element-type OstreeChecksumRef utf8) */ - - if (!ostree_repo_list_collection_refs (self, NULL, &all_collection_refs, - OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES, cancellable, error)) - return FALSE; - - GLNX_HASH_TABLE_FOREACH_V (all_collection_refs, const char*, checksum) - { - g_debug ("Finding objects to keep for commit %s", checksum); - if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, - cancellable, error)) - return FALSE; - } } if (!ostree_repo_list_objects (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 8d3a7a6f..d86d241e 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1202,6 +1202,14 @@ struct _OstreeRepoPruneOptions { typedef struct _OstreeRepoPruneOptions OstreeRepoPruneOptions; +_OSTREE_PUBLIC +gboolean +ostree_repo_traverse_reachable_refs (OstreeRepo *self, + guint depth, + GHashTable *reachable, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_prune_from_reachable (OstreeRepo *self, OstreeRepoPruneOptions *options, From 9131d8a4cc28ea498c2ebc058aac73f70d41841c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Apr 2018 11:19:17 -0400 Subject: [PATCH 06/45] lib/sysroot: Add wrapper API to prune system repository The initial motivation for this is that the "staging" code currently didn't rewrite the deployment refs, meaning that the staged commit could be pruned. Hence first, this new API ensures that deployments also hold a strong ref to their commit, without relying on the magical "deployment refs" that we inject. That has always been a weird artifact of the strict layering separation between OstreeSysroot and OstreeRepo. I also plan to change rpm-ostree to start using this API to hold references to base layers for client-side layering; it also today generates various refs. That said, if we still want to support multiple processes writing to a single repo (as happens on EndlessOS today) we still need to write refs; perhaps later we could add a concept of "generators" or something that create refs based on whatever logic? Another minor thing this fixes is that we had a printf inside the library; this propagates the pruned data to the higher level which can log however it likes. Closes: #1566 Approved by: jlebon --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-sysroot-cleanup.c | 95 ++++++++++++++++++++++---- src/libostree/ostree-sysroot.h | 10 +++ 4 files changed, 92 insertions(+), 15 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 6d4a3423..ad1db32c 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -509,6 +509,7 @@ ostree_sysroot_get_deployment_dirpath ostree_sysroot_get_deployment_origin_path ostree_sysroot_cleanup ostree_sysroot_prepare_cleanup +ostree_sysroot_cleanup_prune_repo ostree_sysroot_repo ostree_sysroot_get_repo ostree_sysroot_get_staged_deployment diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index eb3b3211..06544bb6 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -20,6 +20,7 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2018.6 { ostree_repo_traverse_reachable_refs; + ostree_sysroot_cleanup_prune_repo; } LIBOSTREE_2018.5; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index 1d46222b..7a352e6b 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -420,25 +420,76 @@ generate_deployment_refs (OstreeSysroot *self, return TRUE; } -static gboolean -prune_repo (OstreeRepo *repo, - GCancellable *cancellable, - GError **error) +/** + * ostree_sysroot_cleanup_prune_repo: + * @sysroot: Sysroot + * @options: Flags controlling pruning + * @out_objects_total: (out): Number of objects found + * @out_objects_pruned: (out): Number of objects deleted + * @out_pruned_object_size_total: (out): Storage size in bytes of objects deleted + * @cancellable: Cancellable + * @error: Error + * + * Prune the system repository. This is a thin wrapper + * around ostree_repo_prune_from_reachable(); the primary + * addition is that this function automatically gathers + * all deployed commits into the reachable set. + * + * You generally want to at least set the `OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY` + * flag in @options. A commit traversal depth of `0` is assumed. + * + * Locking: exclusive + * Since: 2018.6 + */ +gboolean +ostree_sysroot_cleanup_prune_repo (OstreeSysroot *sysroot, + OstreeRepoPruneOptions *options, + gint *out_objects_total, + gint *out_objects_pruned, + guint64 *out_pruned_object_size_total, + GCancellable *cancellable, + GError **error) { - gint n_objects_total; - gint n_objects_pruned; - guint64 freed_space; - if (!ostree_repo_prune (repo, OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, 0, - &n_objects_total, &n_objects_pruned, &freed_space, - cancellable, error)) + GLNX_AUTO_PREFIX_ERROR ("Pruning system repository", error); + OstreeRepo *repo = ostree_sysroot_repo (sysroot); + const guint depth = 0; /* Historical default */ + + /* Hold an exclusive lock by default across gathering refs and doing + * the prune. + */ + g_autoptr(OstreeRepoAutoLock) lock = + _ostree_repo_auto_lock_push (repo, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, error); + if (!lock) return FALSE; - if (freed_space > 0) + /* Ensure reachable has refs, but default to depth 0. This is + * what we've always done for the system repo, but perhaps down + * the line we could add a depth flag to the repo config or something? + */ + if (!ostree_repo_traverse_reachable_refs (repo, depth, options->reachable, cancellable, error)) + return FALSE; + + /* Since ostree was created we've been generating "deployment refs" in + * generate_deployment_refs() that look like ostree/0/1 etc. to ensure that + * anything doing a direct prune won't delete commits backing deployments. + * This bit might allow us to eventually drop that behavior, although we'd + * have to be very careful to ensure that all software is updated to use + * `ostree_sysroot_cleanup_prune_repo()`. + */ + for (guint i = 0; i < sysroot->deployments->len; i++) { - g_autofree char *freed_space_str = g_format_size_full (freed_space, 0); - g_print ("Freed objects: %s\n", freed_space_str); + const char *checksum = ostree_deployment_get_csum (sysroot->deployments->pdata[i]); + if (!ostree_repo_traverse_commit_union (repo, checksum, depth, options->reachable, + cancellable, error)) + return FALSE; } + if (!ostree_repo_prune_from_reachable (repo, options, + out_objects_total, out_objects_pruned, + out_pruned_object_size_total, + cancellable, error)) + return FALSE; + return TRUE; } @@ -501,8 +552,22 @@ _ostree_sysroot_cleanup_internal (OstreeSysroot *self, if (do_prune_repo) { - if (!prune_repo (repo, cancellable, error)) - return glnx_prefix_error (error, "Pruning repo"); + gint n_objects_total; + gint n_objects_pruned; + guint64 freed_space; + g_autoptr(GHashTable) reachable = ostree_repo_traverse_new_reachable (); + OstreeRepoPruneOptions opts = { OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, reachable }; + if (!ostree_sysroot_cleanup_prune_repo (self, &opts, &n_objects_total, + &n_objects_pruned, &freed_space, + cancellable, error)) + return FALSE; + + /* TODO remove printf in library */ + if (freed_space > 0) + { + g_autofree char *freed_space_str = g_format_size_full (freed_space, 0); + g_print ("Freed objects: %s\n", freed_space_str); + } } return TRUE; diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index 47cbb022..502cd750 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -122,6 +122,16 @@ gboolean ostree_sysroot_prepare_cleanup (OstreeSysroot *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean +ostree_sysroot_cleanup_prune_repo (OstreeSysroot *sysroot, + OstreeRepoPruneOptions *options, + gint *out_objects_total, + gint *out_objects_pruned, + guint64 *out_pruned_object_size_total, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_sysroot_write_origin_file (OstreeSysroot *sysroot, OstreeDeployment *deployment, From 0d53e5d30940c567ddfa8a7655cc69deaf22df18 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Thu, 24 May 2018 00:08:57 -0700 Subject: [PATCH 07/45] bash-completion: Don't add a space after files and directories Currently if ostree is completing a file or directory for you it adds a space to the end, but this is an inconvenience when it's an intermediate directory in a tree. It's better to let the user add the space after the final directory, so this commit changes the bash completion to avoid adding a space when completing files or directories. Closes: #1598 Approved by: jlebon --- bash/ostree | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bash/ostree b/bash/ostree index 218e4254..46baa51b 100644 --- a/bash/ostree +++ b/bash/ostree @@ -87,10 +87,12 @@ __ostree_compreply_all_options() { } __ostree_compreply_all_files() { + compopt -o nospace COMPREPLY+=( $( compgen -f "$cur" ) ) } __ostree_compreply_dirs_only() { + compopt -o nospace COMPREPLY+=( $( compgen -d "$cur" ) ) } From ef86abe2282325f28a3507076f9124862840da10 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 17 May 2018 10:05:49 +0200 Subject: [PATCH 08/45] fsck: Include the commits that contain the corruption in errors This makes it easier to figure out what the corruption affects. Closes: #1591 Approved by: cgwalters --- src/ostree/ot-builtin-fsck.c | 22 ++++++++++++++++++---- tests/test-corruption.sh | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 81124d3f..b3f06e4d 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -63,16 +63,31 @@ fsck_one_object (OstreeRepo *repo, if (!ostree_repo_fsck_object (repo, objtype, checksum, cancellable, &temp_error)) { gboolean object_missing = FALSE; + g_auto(GStrv) parent_commits = NULL; + g_autofree char *parent_commits_str = NULL; + + if (object_parents) + { + parent_commits = ostree_repo_traverse_parents_get_commits (object_parents, key); + parent_commits_str = g_strjoinv (", ", parent_commits); + } if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) { g_clear_error (&temp_error); - g_printerr ("Object missing: %s.%s\n", checksum, - ostree_object_type_to_string (objtype)); + if (parent_commits_str) + g_printerr ("Object missing in commits %s: %s.%s\n", parent_commits_str, checksum, + ostree_object_type_to_string (objtype)); + else + g_printerr ("Object missing: %s.%s\n", checksum, + ostree_object_type_to_string (objtype)); object_missing = TRUE; } else { + if (parent_commits_str) + g_prefix_error (&temp_error, "In commits %s: ", parent_commits_str); + if (opt_delete) { g_printerr ("%s\n", temp_error->message); @@ -90,9 +105,8 @@ fsck_one_object (OstreeRepo *repo, { *out_found_corruption = TRUE; - if (object_parents != NULL && objtype != OSTREE_OBJECT_TYPE_COMMIT) + if (parent_commits != NULL && objtype != OSTREE_OBJECT_TYPE_COMMIT) { - g_auto(GStrv) parent_commits = ostree_repo_traverse_parents_get_commits (object_parents, key); int i; /* The commit was missing or deleted, mark the commit partial */ diff --git a/tests/test-corruption.sh b/tests/test-corruption.sh index f26ed2d2..4994e277 100755 --- a/tests/test-corruption.sh +++ b/tests/test-corruption.sh @@ -104,7 +104,7 @@ assert_not_has_file repo/state/${rev}.commitpartial if $OSTREE fsck -q 2>err.txt; then assert_not_reached "fsck unexpectedly succeeded" fi -assert_file_has_content_literal err.txt "Object missing:" +assert_file_has_content_literal err.txt "Object missing" assert_file_has_content_literal err.txt "Marking commit as partial: $rev" assert_has_file repo/state/${rev}.commitpartial From 5f82503a01b715cdc29c033189045712d6cff181 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 17 May 2018 10:06:24 +0200 Subject: [PATCH 09/45] fsck: Add --all to print all corrupted object Stopping on the first error is nice if you just want to know if everything is ok, but if you want to figure out all that is wrong its nice to be able to continue and print all corruptions. Closes: #1591 Approved by: cgwalters --- src/ostree/ot-builtin-fsck.c | 7 +++++++ tests/test-corruption.sh | 25 ++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index b3f06e4d..83000c6e 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -31,6 +31,7 @@ static gboolean opt_quiet; static gboolean opt_delete; +static gboolean opt_all; static gboolean opt_add_tombstones; static gboolean opt_verify_bindings; static gboolean opt_verify_back_refs; @@ -43,6 +44,7 @@ static gboolean opt_verify_back_refs; static GOptionEntry options[] = { { "add-tombstones", 0, 0, G_OPTION_ARG_NONE, &opt_add_tombstones, "Add tombstones for missing commits", NULL }, { "quiet", 'q', 0, G_OPTION_ARG_NONE, &opt_quiet, "Only print error messages", NULL }, + { "all", 'a', 0, G_OPTION_ARG_NONE, &opt_all, "Don't stop on first error", NULL }, { "delete", 0, 0, G_OPTION_ARG_NONE, &opt_delete, "Remove corrupted objects", NULL }, { "verify-bindings", 0, 0, G_OPTION_ARG_NONE, &opt_verify_bindings, "Verify ref bindings", NULL }, { "verify-back-refs", 0, 0, G_OPTION_ARG_NONE, &opt_verify_back_refs, "Verify back-references (implies --verify-bindings)", NULL }, @@ -94,6 +96,11 @@ fsck_one_object (OstreeRepo *repo, (void) ostree_repo_delete_object (repo, objtype, checksum, cancellable, NULL); object_missing = TRUE; } + else if (opt_all) + { + *out_found_corruption = TRUE; + g_printerr ("%s\n", temp_error->message); + } else { g_propagate_error (error, g_steal_pointer (&temp_error)); diff --git a/tests/test-corruption.sh b/tests/test-corruption.sh index 4994e277..2b5e61bb 100755 --- a/tests/test-corruption.sh +++ b/tests/test-corruption.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..8" +echo "1..9" . $(dirname $0)/libtest.sh @@ -118,6 +118,7 @@ rev=$($OSTREE rev-parse test2) filechecksum=$(ostree_file_path_to_checksum repo test2 /firstfile) echo corrupted >> repo/$(ostree_checksum_to_relative_object_path repo $filechecksum) +unset filechecksum assert_not_has_file repo/state/${rev}.commitpartial @@ -129,3 +130,25 @@ assert_file_has_content_literal err.txt "Marking commit as partial: $rev" assert_has_file repo/state/${rev}.commitpartial echo "ok corrupt file" + +cd ${test_tmpdir} +rm repo files -rf +setup_test_repository "bare" + +rev=$($OSTREE rev-parse test2) +firstfilechecksum=$(ostree_file_path_to_checksum repo test2 /firstfile) +echo corrupted >> repo/$(ostree_checksum_to_relative_object_path repo $firstfilechecksum) +secondfilechecksum=$(ostree_file_path_to_checksum repo test2 /baz/cow) +echo corrupted >> repo/$(ostree_checksum_to_relative_object_path repo $secondfilechecksum) + +assert_not_has_file repo/state/${rev}.commitpartial + +if $OSTREE fsck -a --delete 2>err.txt; then + assert_not_reached "fsck unexpectedly succeeded" +fi +assert_file_has_content err.txt "Corrupted file object.*${firstfilechecksum}" +assert_file_has_content err.txt "Corrupted file object.*${secondfilechecksum}" +assert_file_has_content_literal err.txt "Marking commit as partial: $rev" +assert_has_file repo/state/${rev}.commitpartial + +echo "ok fsck --all" From 8593357902ef3695b4eb9996e0567a07e3dcd006 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 22 May 2018 12:13:08 +0100 Subject: [PATCH 10/45] lib/repo-pull: Use values from struct in enqueue_one_object_request() This introduces no functional changes, but will make some upcoming refactoring a little easier. Signed-off-by: Philip Withnall Closes: #1599 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f5745f86..4410f998 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1939,12 +1939,11 @@ enqueue_one_object_request (OtPullData *pull_data, { g_debug ("queuing fetch of %s.%s%s", checksum, ostree_object_type_to_string (objtype), - is_detached_meta ? " (detached)" : ""); + fetch_data->is_detached_meta ? " (detached)" : ""); if (is_meta) { - GVariant *objname = ostree_object_name_serialize (checksum, objtype); - g_hash_table_insert (pull_data->pending_fetch_metadata, objname, fetch_data); + g_hash_table_insert (pull_data->pending_fetch_metadata, g_variant_ref (fetch_data->object), fetch_data); } else { From 66777b1b7593fc0d15cfb0b5ffb433a9b622f96d Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 25 May 2018 01:01:52 +0100 Subject: [PATCH 11/45] lib/repo-pull: Use GCC `?:` ternary shortcut where appropriate This introduces no functional changes; just makes the code a bit shorter in a few places. https://gcc.gnu.org/onlinedocs/gcc/Conditionals.html Signed-off-by: Philip Withnall Closes: #1599 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 4410f998..3ce5cea7 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -897,7 +897,7 @@ fetch_ref_contents (OtPullData *pull_data, if (!ostree_validate_checksum_string (ret_contents, error)) return glnx_prefix_error (error, "Fetching checksum for ref (%s, %s)", - ref->collection_id ? ref->collection_id : "(empty)", + ref->collection_id ?: "(empty)", ref->ref_name); ot_transfer_out_value (out_contents, &ret_contents); @@ -2239,7 +2239,7 @@ process_one_static_delta (OtPullData *pull_data, if (have_all) { g_debug ("Have all objects from static delta %s-%s part %u", - from_revision ? from_revision : "empty", to_revision, + from_revision ?: "empty", to_revision, i); pull_data->fetched_deltapart_size += size; pull_data->n_fetched_deltaparts++; @@ -2505,7 +2505,7 @@ on_superblock_fetched (GObject *src, else { g_autoptr(GVariant) delta_superblock = NULL; - g_autofree gchar *delta = g_strconcat (from_revision ? from_revision : "", from_revision ? "-" : "", to_revision, NULL); + g_autofree gchar *delta = g_strconcat (from_revision ?: "", from_revision ? "-" : "", to_revision, NULL); const guchar *expected_summary_digest = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta); guint8 actual_summary_digest[OSTREE_SHA256_DIGEST_LEN]; @@ -3150,7 +3150,7 @@ initiate_request (OtPullData *pull_data, if (pull_data->remote_name != NULL) refspec = g_strdup_printf ("%s:%s", pull_data->remote_name, ref->ref_name); if (!ostree_repo_resolve_rev (pull_data->repo, - (refspec != NULL) ? refspec : ref->ref_name, TRUE, + refspec ?: ref->ref_name, TRUE, &delta_from_revision, error)) return FALSE; @@ -3913,7 +3913,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, else if (refs_to_fetch != NULL) { char **strviter = refs_to_fetch; - char **commitid_strviter = override_commit_ids ? override_commit_ids : NULL; + char **commitid_strviter = override_commit_ids ?: NULL; while (*strviter) { @@ -4139,7 +4139,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, ref, checksum); else ostree_repo_transaction_set_ref (pull_data->repo, - (pull_data->remote_refspec_name != NULL) ? pull_data->remote_refspec_name : pull_data->remote_name, + pull_data->remote_refspec_name ?: pull_data->remote_name, ref->ref_name, checksum); } } From fc10354404123d76910bb7eca4d6bae8d9f58da3 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 22 May 2018 12:14:22 +0100 Subject: [PATCH 12/45] lib/repo-pull: Factor out free function for FetchDeltaSuperData This introduces no functional changes, but does make the code a little cleaner. Signed-off-by: Philip Withnall Closes: #1599 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 3ce5cea7..00b161f6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2461,6 +2461,16 @@ typedef struct { OstreeCollectionRef *requested_ref; /* (nullable) */ } FetchDeltaSuperData; +static void +fetch_delta_super_data_free (FetchDeltaSuperData *fetch_data) +{ + g_free (fetch_data->from_revision); + g_free (fetch_data->to_revision); + if (fetch_data->requested_ref) + ostree_collection_ref_free (fetch_data->requested_ref); + g_free (fetch_data); +} + static void set_required_deltas_error (GError **error, const char *from_revision, @@ -2541,15 +2551,12 @@ on_superblock_fetched (GObject *src, } out: - g_free (fdata->from_revision); - g_free (fdata->to_revision); - if (fdata->requested_ref) - ostree_collection_ref_free (fdata->requested_ref); - g_free (fdata); g_assert (pull_data->n_outstanding_metadata_fetches > 0); pull_data->n_outstanding_metadata_fetches--; pull_data->n_fetched_metadata++; check_outstanding_requests_handle_error (pull_data, &local_error); + + g_clear_pointer (&fetch_data, fetch_delta_super_data_free); } static gboolean From 6d21f83813c69468e4d48018ecbbad46d4de2439 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 22 May 2018 12:16:06 +0100 Subject: [PATCH 13/45] lib/repo-pull: Rename a variable Rename from `fdata` to `fetch_data` to clarify things and make it consistent with other similar functionality in the file. This introduces no functional changes. Signed-off-by: Philip Withnall Closes: #1599 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 00b161f6..64035ec5 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2487,13 +2487,13 @@ on_superblock_fetched (GObject *src, gpointer data) { - FetchDeltaSuperData *fdata = data; - OtPullData *pull_data = fdata->pull_data; + FetchDeltaSuperData *fetch_data = data; + OtPullData *pull_data = fetch_data->pull_data; g_autoptr(GError) local_error = NULL; GError **error = &local_error; g_autoptr(GBytes) delta_superblock_data = NULL; - const char *from_revision = fdata->from_revision; - const char *to_revision = fdata->to_revision; + const char *from_revision = fetch_data->from_revision; + const char *to_revision = fetch_data->to_revision; if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src, res, @@ -2510,7 +2510,7 @@ on_superblock_fetched (GObject *src, goto out; } - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, fdata->requested_ref); + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, fetch_data->requested_ref); } else { @@ -2545,7 +2545,7 @@ on_superblock_fetched (GObject *src, delta_superblock_data, FALSE)); g_ptr_array_add (pull_data->static_delta_superblocks, g_variant_ref (delta_superblock)); - if (!process_one_static_delta (pull_data, from_revision, to_revision, delta_superblock, fdata->requested_ref, + if (!process_one_static_delta (pull_data, from_revision, to_revision, delta_superblock, fetch_data->requested_ref, pull_data->cancellable, error)) goto out; } From 8b8952ebee1824162f3f282fbab0af54b1cf1eb4 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 22 May 2018 12:17:38 +0100 Subject: [PATCH 14/45] lib/repo-pull: Factor out enqueue function for ScanObjectQueueData This introduces no functional changes, but will make upcoming support for retrying downloads easier to add. Signed-off-by: Philip Withnall Closes: #1599 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 64035ec5..87dd322d 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -215,6 +215,8 @@ static void queue_scan_one_metadata_object (OtPullData *pull_data guint recursion_depth, const OstreeCollectionRef *ref); +static void queue_scan_one_metadata_object_s (OtPullData *pull_data, + ScanObjectQueueData *scan_data); static void queue_scan_one_metadata_object_c (OtPullData *pull_data, const guchar *csum, OstreeObjectType objtype, @@ -1764,6 +1766,14 @@ queue_scan_one_metadata_object (OtPullData *pull_data, queue_scan_one_metadata_object_c (pull_data, buf, objtype, path, recursion_depth, ref); } +static void +queue_scan_one_metadata_object_s (OtPullData *pull_data, + ScanObjectQueueData *scan_data) +{ + g_queue_push_tail (&pull_data->scan_object_queue, scan_data); + ensure_idle_queued (pull_data); +} + static void queue_scan_one_metadata_object_c (OtPullData *pull_data, const guchar *csum, @@ -1780,8 +1790,7 @@ queue_scan_one_metadata_object_c (OtPullData *pull_data, scan_data->recursion_depth = recursion_depth; scan_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; - g_queue_push_tail (&pull_data->scan_object_queue, scan_data); - ensure_idle_queued (pull_data); + queue_scan_one_metadata_object_s (pull_data, g_steal_pointer (&scan_data)); } /* Called out of the main loop to look at metadata objects which can have From dc1fde704da768e7dc5ab73a91e3892a6e576cf3 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 22 May 2018 12:19:15 +0100 Subject: [PATCH 15/45] lib/repo-pull: Factor out enqueue function for FetchObjectData This introduces no functional changes, but will make upcoming support for retrying downloads easier to add. Signed-off-by: Philip Withnall Closes: #1599 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 61 ++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 87dd322d..b9cafa9d 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -224,6 +224,9 @@ static void queue_scan_one_metadata_object_c (OtPullData *pull_da guint recursion_depth, const OstreeCollectionRef *ref); +static void enqueue_one_object_request_s (OtPullData *pull_data, + FetchObjectData *fetch_data); + static gboolean scan_one_metadata_object (OtPullData *pull_data, const char *checksum, OstreeObjectType objtype, @@ -1917,31 +1920,14 @@ scan_one_metadata_object (OtPullData *pull_data, } static void -enqueue_one_object_request (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - const char *path, - gboolean is_detached_meta, - gboolean object_is_stored, - const OstreeCollectionRef *ref) +enqueue_one_object_request_s (OtPullData *pull_data, + FetchObjectData *fetch_data) { - gboolean is_meta; - FetchObjectData *fetch_data; + const char *checksum; + OstreeObjectType objtype; - is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); - - fetch_data = g_new0 (FetchObjectData, 1); - fetch_data->pull_data = pull_data; - fetch_data->object = ostree_object_name_serialize (checksum, objtype); - fetch_data->path = g_strdup (path); - fetch_data->is_detached_meta = is_detached_meta; - fetch_data->object_is_stored = object_is_stored; - fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; - - if (is_meta) - pull_data->n_requested_metadata++; - else - pull_data->n_requested_content++; + ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype); + gboolean is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); /* Are too many requests are in flight? */ if (fetcher_queue_is_full (pull_data)) @@ -1965,6 +1951,35 @@ enqueue_one_object_request (OtPullData *pull_data, } } +static void +enqueue_one_object_request (OtPullData *pull_data, + const char *checksum, + OstreeObjectType objtype, + const char *path, + gboolean is_detached_meta, + gboolean object_is_stored, + const OstreeCollectionRef *ref) +{ + FetchObjectData *fetch_data; + + fetch_data = g_new0 (FetchObjectData, 1); + fetch_data->pull_data = pull_data; + fetch_data->object = ostree_object_name_serialize (checksum, objtype); + fetch_data->path = g_strdup (path); + fetch_data->is_detached_meta = is_detached_meta; + fetch_data->object_is_stored = object_is_stored; + fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; + + gboolean is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); + + if (is_meta) + pull_data->n_requested_metadata++; + else + pull_data->n_requested_content++; + + enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data)); +} + static void start_fetch (OtPullData *pull_data, FetchObjectData *fetch) From 23d4beead1fc13ef200ec71d5e6bd5682d6d0368 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 22 May 2018 12:20:15 +0100 Subject: [PATCH 16/45] lib/repo-pull: Factor out enqueue function for FetchStaticDeltaData This introduces no functional changes, but will make upcoming support for retrying downloads easier to add. Signed-off-by: Philip Withnall Closes: #1599 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index b9cafa9d..f05f8835 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -226,6 +226,8 @@ static void queue_scan_one_metadata_object_c (OtPullData *pull_da static void enqueue_one_object_request_s (OtPullData *pull_data, FetchObjectData *fetch_data); +static void enqueue_one_static_delta_part_request_s (OtPullData *pull_data, + FetchStaticDeltaData *fetch_data); static gboolean scan_one_metadata_object (OtPullData *pull_data, const char *checksum, @@ -2131,6 +2133,24 @@ process_one_static_delta_fallback (OtPullData *pull_data, return TRUE; } +static void +enqueue_one_static_delta_part_request_s (OtPullData *pull_data, + FetchStaticDeltaData *fetch_data) +{ + if (fetcher_queue_is_full (pull_data)) + { + g_debug ("queuing fetch of static delta %s-%s part %u", + fetch_data->from_revision ?: "empty", + fetch_data->to_revision, fetch_data->i); + + g_hash_table_add (pull_data->pending_fetch_deltaparts, fetch_data); + } + else + { + start_fetch_deltapart (pull_data, fetch_data); + } +} + static void start_fetch_deltapart (OtPullData *pull_data, FetchStaticDeltaData *fetch) @@ -2313,12 +2333,7 @@ process_one_static_delta (OtPullData *pull_data, } else { - if (!fetcher_queue_is_full (pull_data)) - start_fetch_deltapart (pull_data, fetch_data); - else - { - g_hash_table_add (pull_data->pending_fetch_deltaparts, fetch_data); - } + enqueue_one_static_delta_part_request_s (pull_data, g_steal_pointer (&fetch_data)); } } From f6e4543a246ede94032416795cfe473b4b8f9557 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 28 May 2018 15:20:06 -0400 Subject: [PATCH 17/45] ci: Stop layering oci-kvm-hook Since we're explicitly using `--device /dev/kvm`, there's no need to install `oci-kvm-hook`. Closes: #1601 Approved by: cgwalters --- .papr.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.papr.yml b/.papr.yml index 7d76d247..2d3dfe5e 100644 --- a/.papr.yml +++ b/.papr.yml @@ -17,8 +17,6 @@ host: # image: registry.fedoraproject.org/fedora:27 tests: - - cd /etc/yum.repos.d/ && curl -L -O https://copr.fedorainfracloud.org/coprs/walters/oci-kvm-hook/repo/fedora-27/walters-oci-kvm-hook-fedora-27.repo - - rpm-ostree install oci-kvm-hook && rpm-ostree ex livefs - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.fedoraproject.org/fedora:27 /bin/sh -c "cd /srv/code && ./ci/fah27-insttests.sh" artifacts: From 0f124e2584cc6bb86d870adea56654a30cfb1f13 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 28 May 2018 15:21:19 -0400 Subject: [PATCH 18/45] ci: Bump to f28, except flatpak We successfully switched to F28 in rpm-ostree. Time to follow suit here. We exclude flatpak for now since we're still hitting test issues there on F28. Closes: #1601 Approved by: cgwalters --- .papr-ex.yaml | 26 +++++++------- .papr.yml | 34 +++++++++---------- ci/build-check.sh | 4 +++ ci/{fah27-insttests.sh => fah28-insttests.sh} | 0 ci/rpmostree.sh | 5 +-- 5 files changed, 37 insertions(+), 32 deletions(-) rename ci/{fah27-insttests.sh => fah28-insttests.sh} (100%) diff --git a/.papr-ex.yaml b/.papr-ex.yaml index 900aae7b..e4370fcf 100644 --- a/.papr-ex.yaml +++ b/.papr-ex.yaml @@ -4,15 +4,15 @@ branches: - auto - try -context: FAH27-insttests +context: FAH28-insttests required: true container: - image: registry.fedoraproject.org/fedora:27 + image: registry.fedoraproject.org/fedora:28 kvm: true tests: - - ci/fah27-insttests.sh + - ci/fah28-insttests.sh artifacts: - tests/installed/artifacts/ @@ -27,8 +27,8 @@ branches: - try required: true container: - image: registry.fedoraproject.org/fedora:27 -context: f27-primary + image: registry.fedoraproject.org/fedora:28 +context: f28-primary env: # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' @@ -51,10 +51,10 @@ artifacts: --- # And now the contexts below here are variant container builds -context: f27-rust +context: f28-rust inherit: true container: - image: registry.fedoraproject.org/fedora:27 + image: registry.fedoraproject.org/fedora:28 env: CONFIGOPTS: '--enable-rust' CI_PKGS: cargo @@ -65,10 +65,10 @@ tests: --- -context: f27-gnutls +context: f28-gnutls inherit: true container: - image: registry.fedoraproject.org/fedora:27 + image: registry.fedoraproject.org/fedora:28 env: CONFIGOPTS: '--with-crypto=gnutls' CI_PKGS: pkgconfig(gnutls) @@ -81,7 +81,7 @@ tests: inherit: true -context: f27-experimental-api +context: f28-experimental-api env: CONFIGOPTS: '--enable-experimental-api' @@ -92,7 +92,7 @@ tests: inherit: true -context: f27-minimal +context: f28-minimal env: CONFIGOPTS: '--without-curl --without-soup --disable-gtk-doc --disable-man --disable-rust --without-libarchive --without-selinux --without-smack @@ -107,7 +107,7 @@ tests: inherit: true required: true -context: f27-libsoup +context: f28-libsoup env: CONFIGOPTS: "--without-curl --without-openssl --with-libsoup" @@ -120,7 +120,7 @@ tests: inherit: true required: true -context: f27-introspection-tests +context: f28-introspection-tests env: # ASAN conflicts with introspection testing; diff --git a/.papr.yml b/.papr.yml index 2d3dfe5e..7e6d6ce8 100644 --- a/.papr.yml +++ b/.papr.yml @@ -4,20 +4,20 @@ branches: - auto - try -context: FAH27-insttests +context: FAH28-insttests required: true # FIXME; temporary workaround # https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162 host: - distro: fedora/27/atomic + distro: fedora/28/atomic specs: ram: 4096 #container: -# image: registry.fedoraproject.org/fedora:27 +# image: registry.fedoraproject.org/fedora:28 tests: - - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.fedoraproject.org/fedora:27 /bin/sh -c "cd /srv/code && ./ci/fah27-insttests.sh" + - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.fedoraproject.org/fedora:28 /bin/sh -c "cd /srv/code && ./ci/fah28-insttests.sh" artifacts: - tests/installed/artifacts/ @@ -32,8 +32,8 @@ branches: - try required: true container: - image: registry.fedoraproject.org/fedora:27 -context: f27-primary + image: registry.fedoraproject.org/fedora:28 +context: f28-primary env: # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' @@ -56,10 +56,10 @@ artifacts: --- # And now the contexts below here are variant container builds -context: f27-rust +context: f28-rust inherit: true container: - image: registry.fedoraproject.org/fedora:27 + image: registry.fedoraproject.org/fedora:28 env: CONFIGOPTS: '--enable-rust' CI_PKGS: cargo @@ -70,10 +70,10 @@ tests: --- -context: f27-gnutls +context: f28-gnutls inherit: true container: - image: registry.fedoraproject.org/fedora:27 + image: registry.fedoraproject.org/fedora:28 env: CONFIGOPTS: '--with-crypto=gnutls' CI_PKGS: pkgconfig(gnutls) @@ -86,7 +86,7 @@ tests: inherit: true -context: f27-experimental-api +context: f28-experimental-api env: CONFIGOPTS: '--enable-experimental-api' @@ -97,7 +97,7 @@ tests: inherit: true -context: f27-minimal +context: f28-minimal env: CONFIGOPTS: '--without-curl --without-soup --disable-gtk-doc --disable-man --disable-rust --without-libarchive --without-selinux --without-smack @@ -112,7 +112,7 @@ tests: inherit: true required: true -context: f27-libsoup +context: f28-libsoup env: CONFIGOPTS: "--without-curl --without-openssl --with-libsoup" @@ -125,7 +125,7 @@ tests: inherit: true required: true -context: f27-introspection-tests +context: f28-introspection-tests env: # ASAN conflicts with introspection testing; @@ -172,15 +172,15 @@ branches: - auto - try -context: f27-rpmostree +context: f28-rpmostree required: true cluster: hosts: - name: vmcheck - distro: fedora/27/atomic + distro: fedora/28/atomic container: - image: registry.fedoraproject.org/fedora:27 + image: registry.fedoraproject.org/fedora:28 env: HOSTS: vmcheck diff --git a/ci/build-check.sh b/ci/build-check.sh index c09e9a52..ed129c74 100755 --- a/ci/build-check.sh +++ b/ci/build-check.sh @@ -22,11 +22,15 @@ make install # job than gcc for vars with cleanups; perhaps in the future these could # parallelize if test -x /usr/bin/clang; then + if grep -q -e 'static inline.*_GLIB_AUTOPTR_LIST_FUNC_NAME' /usr/include/glib-2.0/glib/gmacros.h; then + echo 'Skipping clang check, see https://bugzilla.gnome.org/show_bug.cgi?id=796346' + else # Except for clang-4.0: error: argument unused during compilation: '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' [-Werror,-Wunused-command-line-argument] export CFLAGS="-Wall -Werror -Wno-error=unused-command-line-argument ${CFLAGS:-}" git clean -dfx && git submodule foreach git clean -dfx export CC=clang build + fi fi copy_out_gdtr_artifacts() { diff --git a/ci/fah27-insttests.sh b/ci/fah28-insttests.sh similarity index 100% rename from ci/fah27-insttests.sh rename to ci/fah28-insttests.sh diff --git a/ci/rpmostree.sh b/ci/rpmostree.sh index e8189c40..f766a368 100755 --- a/ci/rpmostree.sh +++ b/ci/rpmostree.sh @@ -6,7 +6,7 @@ set -xeuo pipefail # Frozen to a tag for now to help predictability; it's # also useful to test building *older* versions since # that must work. -RPMOSTREE_TAG=v2017.11 +RPMOSTREE_TAG=v2018.5 dn=$(dirname $0) . ${dn}/libpaprci/libbuild.sh @@ -21,7 +21,7 @@ pkg_install rpm-ostree && rpm -e rpm-ostree # Duplicate of deps from build.sh in rpm-ostree for tests pkg_install ostree{,-devel,-grub2} createrepo_c /usr/bin/jq PyYAML \ libubsan libasan libtsan elfutils fuse sudo python-gobject-base \ - selinux-policy-devel selinux-policy-targeted + selinux-policy-devel selinux-policy-targeted openssh-clients ansible # This one is in the papr.yml pkg_install rsync @@ -38,6 +38,7 @@ build # We want to capture automake results cleanup() { mv test-suite.log ${codedir} || true + mv vmcheck ${codedir} || true } trap cleanup EXIT make -j 8 check From 517140f1101d1d72544ed7a4a9c965e828122f6a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 28 May 2018 15:22:14 -0400 Subject: [PATCH 19/45] ci: Drop gpgme and libgcrypt hacks These shouldn't be needed anymore in F28. Closes: #1601 Approved by: cgwalters --- ci/libpaprci/libbuild.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ci/libpaprci/libbuild.sh b/ci/libpaprci/libbuild.sh index 8b8af3f5..a7c774e7 100644 --- a/ci/libpaprci/libbuild.sh +++ b/ci/libpaprci/libbuild.sh @@ -49,16 +49,6 @@ pkg_builddep() { # This is sadly the only case where it's a different command if test -x /usr/bin/dnf; then dnf builddep -y "$@" - - # XXX: tmp hack -- see - # https://github.com/ostreedev/ostree/pull/1539 - if rpm -q gpgme | grep -q gpgme-1.9.0-6.fc27; then - dnf install -y https://kojipkgs.fedoraproject.org//packages/gpgme/1.10.0/4.fc27/x86_64/{gpgme{,-devel},python{2,3}-gpg}-1.10.0-4.fc27.x86_64.rpm - fi - # https://bugzilla.redhat.com/show_bug.cgi?id=1542453 - if rpm -q libgcrypt | grep -q libgcrypt-1.8.2-1.fc27; then - dnf install -y https://fedorapeople.org/~walters/libgcrypt-1.8.2-2.fc27.x86_64.rpm - fi else yum-builddep -y "$@" fi From ee0d9bbd5bd4ec9731534563b347cd8965b7775b Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Tue, 29 May 2018 06:38:49 -0700 Subject: [PATCH 20/45] Update libglnx to get g_autoptr backports Update submodule: libglnx Closes: #1603 Approved by: jlebon --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index 03e16afa..e1a78cf2 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 03e16afa7f2cf7ce517b604035683fa0409f730c +Subproject commit e1a78cf2f5351d5394ccfb79f3f5a7b4917f73f3 From 4f8944b7191947ab9dd67f30426918aae6a043f8 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 29 May 2018 10:23:31 -0400 Subject: [PATCH 21/45] .gitmodules: Update URL for libglnx Same thing as https://github.com/projectatomic/rpm-ostree/pull/1374. Otherwise we get an annoying "warning: redirecting to" message. Update submodule: libglnx Closes: #1604 Approved by: cgwalters --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 32c403ce..1da1f9b2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "libglnx"] path = libglnx - url = https://git.gnome.org/browse/libglnx + url = https://gitlab.gnome.org/GNOME/libglnx.git [submodule "bsdiff"] path = bsdiff url = https://github.com/mendsley/bsdiff From 2b32de38cf7ad8298b334ec2657864d6de9844c6 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Fri, 25 May 2018 15:46:34 -0700 Subject: [PATCH 22/45] Allow compiling with experimental API but without curl/soup This commit rearranges a few things in ostree-repo-pull.c so that OSTree will successfully compile with experimental API enabled and without libsoup, libcurl, or avahi: ./autogen.sh --enable-experimental-api --without-soup --without-curl --without-avahi This is accomplished with two sets of changes: 1. Move ostree_repo_resolve_keyring_for_collection() so it can be used even without libsoup or libcurl. 2. Add stub functions for ostree_repo_find_remotes_async() and ostree_repo_pull_from_remotes_async(), and their _finish() counterparts, so they return an error when libsoup or libcurl isn't available. Closes: #1605 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 383 +++++++++++++++++++------------ 1 file changed, 230 insertions(+), 153 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f05f8835..16e9efda 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -30,11 +30,11 @@ #include "ostree.h" #include "otutil.h" #include "ostree-repo-pull-private.h" +#include "ostree-repo-private.h" #ifdef HAVE_LIBCURL_OR_LIBSOUP #include "ostree-core-private.h" -#include "ostree-repo-private.h" #include "ostree-repo-static-delta-private.h" #include "ostree-metalink.h" #include "ostree-fetcher-util.h" @@ -1435,51 +1435,6 @@ commitstate_is_partial (OtPullData *pull_data, || (commitstate & OSTREE_REPO_COMMIT_STATE_PARTIAL) > 0; } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API -/* Reads the collection-id of a given remote from the repo - * configuration. - */ -static char * -get_real_remote_repo_collection_id (OstreeRepo *repo, - const gchar *remote_name) -{ - /* remote_name == NULL can happen for pull-local */ - if (!remote_name) - return NULL; - - g_autofree gchar *remote_collection_id = NULL; - if (!ostree_repo_get_remote_option (repo, remote_name, "collection-id", NULL, - &remote_collection_id, NULL) || - (remote_collection_id == NULL) || - (remote_collection_id[0] == '\0')) - return NULL; - - return g_steal_pointer (&remote_collection_id); -} - -/* Reads the collection-id of the remote repo. Where it will be read - * from depends on whether we pull from the "local" remote repo (the - * "file://" URL) or "remote" remote repo (likely the "http(s)://" - * URL). - */ -static char * -get_remote_repo_collection_id (OtPullData *pull_data) -{ - if (pull_data->remote_repo_local != NULL) - { - const char *remote_collection_id = - ostree_repo_get_collection_id (pull_data->remote_repo_local); - if ((remote_collection_id == NULL) || - (remote_collection_id[0] == '\0')) - return NULL; - return g_strdup (remote_collection_id); - } - - return get_real_remote_repo_collection_id (pull_data->repo, - pull_data->remote_name); -} -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - #endif /* HAVE_LIBCURL_OR_LIBSOUP */ /** @@ -1587,6 +1542,164 @@ _ostree_repo_verify_bindings (const char *collection_id, return TRUE; } +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + +/* Reads the collection-id of a given remote from the repo + * configuration. + */ +static char * +get_real_remote_repo_collection_id (OstreeRepo *repo, + const gchar *remote_name) +{ + /* remote_name == NULL can happen for pull-local */ + if (!remote_name) + return NULL; + + g_autofree gchar *remote_collection_id = NULL; + if (!ostree_repo_get_remote_option (repo, remote_name, "collection-id", NULL, + &remote_collection_id, NULL) || + (remote_collection_id == NULL) || + (remote_collection_id[0] == '\0')) + return NULL; + + return g_steal_pointer (&remote_collection_id); +} + +#ifdef HAVE_LIBCURL_OR_LIBSOUP + +/* Reads the collection-id of the remote repo. Where it will be read + * from depends on whether we pull from the "local" remote repo (the + * "file://" URL) or "remote" remote repo (likely the "http(s)://" + * URL). + */ +static char * +get_remote_repo_collection_id (OtPullData *pull_data) +{ + if (pull_data->remote_repo_local != NULL) + { + const char *remote_collection_id = + ostree_repo_get_collection_id (pull_data->remote_repo_local); + if ((remote_collection_id == NULL) || + (remote_collection_id[0] == '\0')) + return NULL; + return g_strdup (remote_collection_id); + } + + return get_real_remote_repo_collection_id (pull_data->repo, + pull_data->remote_name); +} + +#endif /* HAVE_LIBCURL_OR_LIBSOUP */ + +/* Check whether the given remote exists, has a `collection-id` key set, and it + * equals @collection_id. If so, return %TRUE. Otherwise, %FALSE. */ +static gboolean +check_remote_matches_collection_id (OstreeRepo *repo, + const gchar *remote_name, + const gchar *collection_id) +{ + g_autofree gchar *remote_collection_id = NULL; + + remote_collection_id = get_real_remote_repo_collection_id (repo, remote_name); + if (remote_collection_id == NULL) + return FALSE; + + return g_str_equal (remote_collection_id, collection_id); +} + +/** + * ostree_repo_resolve_keyring_for_collection: + * @self: an #OstreeRepo + * @collection_id: the collection ID to look up a keyring for + * @cancellable: (nullable): a #GCancellable, or %NULL + * @error: return location for a #GError, or %NULL + * + * Find the GPG keyring for the given @collection_id, using the local + * configuration from the given #OstreeRepo. This will search the configured + * remotes for ones whose `collection-id` key matches @collection_id, and will + * return the first matching remote. + * + * If multiple remotes match and have different keyrings, a debug message will + * be emitted, and the first result will be returned. It is expected that the + * keyrings should match. + * + * If no match can be found, a %G_IO_ERROR_NOT_FOUND error will be returned. + * + * Returns: (transfer full): #OstreeRemote containing the GPG keyring for + * @collection_id + * Since: 2017.8 + */ +OstreeRemote * +ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, + const gchar *collection_id, + GCancellable *cancellable, + GError **error) +{ + gsize i; + g_auto(GStrv) remotes = NULL; + g_autoptr(OstreeRemote) keyring_remote = NULL; + + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + g_return_val_if_fail (ostree_validate_collection_id (collection_id, NULL), NULL); + g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + /* Look through all the currently configured remotes for the given collection. */ + remotes = ostree_repo_remote_list (self, NULL); + + for (i = 0; remotes != NULL && remotes[i] != NULL; i++) + { + g_autoptr(GError) local_error = NULL; + + if (!check_remote_matches_collection_id (self, remotes[i], collection_id)) + continue; + + if (keyring_remote == NULL) + { + g_debug ("%s: Found match for collection ‘%s’ in remote ‘%s’.", + G_STRFUNC, collection_id, remotes[i]); + keyring_remote = _ostree_repo_get_remote_inherited (self, remotes[i], &local_error); + + if (keyring_remote == NULL) + { + g_debug ("%s: Error loading remote ‘%s’: %s", + G_STRFUNC, remotes[i], local_error->message); + continue; + } + + if (g_strcmp0 (keyring_remote->keyring, "") == 0 || + g_strcmp0 (keyring_remote->keyring, "/dev/null") == 0) + { + g_debug ("%s: Ignoring remote ‘%s’ as it has no keyring configured.", + G_STRFUNC, remotes[i]); + g_clear_object (&keyring_remote); + continue; + } + + /* continue so we can catch duplicates */ + } + else + { + g_debug ("%s: Duplicate keyring for collection ‘%s’ in remote ‘%s’." + "Keyring will be loaded from remote ‘%s’.", + G_STRFUNC, collection_id, remotes[i], + keyring_remote->name); + } + } + + if (keyring_remote != NULL) + return g_steal_pointer (&keyring_remote); + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "No keyring found configured locally for collection ‘%s’", + collection_id); + return NULL; + } +} + +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + #ifdef HAVE_LIBCURL_OR_LIBSOUP /* Look at a commit object, and determine whether there are @@ -5694,113 +5807,6 @@ ostree_repo_pull_from_remotes_finish (OstreeRepo *self, return g_task_propagate_boolean (G_TASK (result), error); } -/* Check whether the given remote exists, has a `collection-id` key set, and it - * equals @collection_id. If so, return %TRUE. Otherwise, %FALSE. */ -static gboolean -check_remote_matches_collection_id (OstreeRepo *repo, - const gchar *remote_name, - const gchar *collection_id) -{ - g_autofree gchar *remote_collection_id = NULL; - - remote_collection_id = get_real_remote_repo_collection_id (repo, remote_name); - if (remote_collection_id == NULL) - return FALSE; - - return g_str_equal (remote_collection_id, collection_id); -} - -/** - * ostree_repo_resolve_keyring_for_collection: - * @self: an #OstreeRepo - * @collection_id: the collection ID to look up a keyring for - * @cancellable: (nullable): a #GCancellable, or %NULL - * @error: return location for a #GError, or %NULL - * - * Find the GPG keyring for the given @collection_id, using the local - * configuration from the given #OstreeRepo. This will search the configured - * remotes for ones whose `collection-id` key matches @collection_id, and will - * return the first matching remote. - * - * If multiple remotes match and have different keyrings, a debug message will - * be emitted, and the first result will be returned. It is expected that the - * keyrings should match. - * - * If no match can be found, a %G_IO_ERROR_NOT_FOUND error will be returned. - * - * Returns: (transfer full): #OstreeRemote containing the GPG keyring for - * @collection_id - * Since: 2017.8 - */ -OstreeRemote * -ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, - const gchar *collection_id, - GCancellable *cancellable, - GError **error) -{ - gsize i; - g_auto(GStrv) remotes = NULL; - g_autoptr(OstreeRemote) keyring_remote = NULL; - - g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); - g_return_val_if_fail (ostree_validate_collection_id (collection_id, NULL), NULL); - g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), NULL); - g_return_val_if_fail (error == NULL || *error == NULL, NULL); - - /* Look through all the currently configured remotes for the given collection. */ - remotes = ostree_repo_remote_list (self, NULL); - - for (i = 0; remotes != NULL && remotes[i] != NULL; i++) - { - g_autoptr(GError) local_error = NULL; - - if (!check_remote_matches_collection_id (self, remotes[i], collection_id)) - continue; - - if (keyring_remote == NULL) - { - g_debug ("%s: Found match for collection ‘%s’ in remote ‘%s’.", - G_STRFUNC, collection_id, remotes[i]); - keyring_remote = _ostree_repo_get_remote_inherited (self, remotes[i], &local_error); - - if (keyring_remote == NULL) - { - g_debug ("%s: Error loading remote ‘%s’: %s", - G_STRFUNC, remotes[i], local_error->message); - continue; - } - - if (g_strcmp0 (keyring_remote->keyring, "") == 0 || - g_strcmp0 (keyring_remote->keyring, "/dev/null") == 0) - { - g_debug ("%s: Ignoring remote ‘%s’ as it has no keyring configured.", - G_STRFUNC, remotes[i]); - g_clear_object (&keyring_remote); - continue; - } - - /* continue so we can catch duplicates */ - } - else - { - g_debug ("%s: Duplicate keyring for collection ‘%s’ in remote ‘%s’." - "Keyring will be loaded from remote ‘%s’.", - G_STRFUNC, collection_id, remotes[i], - keyring_remote->name); - } - } - - if (keyring_remote != NULL) - return g_steal_pointer (&keyring_remote); - else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "No keyring found configured locally for collection ‘%s’", - collection_id); - return NULL; - } -} - #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ /** @@ -5952,4 +5958,75 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, return FALSE; } +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + +void +ostree_repo_find_remotes_async (OstreeRepo *self, + const OstreeCollectionRef * const *refs, + GVariant *options, + OstreeRepoFinder **finders, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_return_if_fail (OSTREE_IS_REPO (self)); + + g_task_report_new_error (self, callback, user_data, ostree_repo_find_remotes_async, + G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree was built without libsoup or libcurl, and cannot fetch over HTTP"); +} + +OstreeRepoFinderResult ** +ostree_repo_find_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error) +{ + g_autoptr(GPtrArray) results = NULL; + + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + g_return_val_if_fail (g_task_is_valid (result, self), NULL); + g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_find_remotes_async), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + results = g_task_propagate_pointer (G_TASK (result), error); + g_assert (results == NULL); + return NULL; +} + +void +ostree_repo_pull_from_remotes_async (OstreeRepo *self, + const OstreeRepoFinderResult * const *results, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_return_if_fail (OSTREE_IS_REPO (self)); + + g_task_report_new_error (self, callback, user_data, ostree_repo_find_remotes_async, + G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree was built without libsoup or libcurl, and cannot fetch over HTTP"); +} + +gboolean +ostree_repo_pull_from_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error) +{ + gboolean success; + + g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_pull_from_remotes_async), FALSE); + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + + success = g_task_propagate_boolean (G_TASK (result), error); + g_assert (!success); + return FALSE; +} + +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + #endif /* HAVE_LIBCURL_OR_LIBSOUP */ From 7412e65f7da5c9d53ad057073404d251bbd644c8 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Fri, 25 May 2018 17:48:39 -0700 Subject: [PATCH 23/45] Allow compiling OstreeRepoFinderAvahi without libsoup Closes: #1605 Approved by: cgwalters --- src/libostree/ostree-repo-finder-avahi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-finder-avahi.c b/src/libostree/ostree-repo-finder-avahi.c index 1e77a6e0..fcf62048 100644 --- a/src/libostree/ostree-repo-finder-avahi.c +++ b/src/libostree/ostree-repo-finder-avahi.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #endif /* HAVE_AVAHI */ @@ -55,6 +54,7 @@ #include "ostree-repo-private.h" #include "ostree-repo.h" #include "ostree-repo-finder-avahi-private.h" +#include "ostree-soup-uri.h" #include "otutil.h" #endif /* HAVE_AVAHI */ From 6ec19a6953aae3808f29fc9d1de479b9d3b79ddd Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Fri, 25 May 2018 18:01:35 -0700 Subject: [PATCH 24/45] find-remotes: Fix unused variable warnings Closes: #1605 Approved by: cgwalters --- src/ostree/ot-builtin-find-remotes.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-find-remotes.c b/src/ostree/ot-builtin-find-remotes.c index 77b7a8da..f255501a 100644 --- a/src/ostree/ot-builtin-find-remotes.c +++ b/src/ostree/ot-builtin-find-remotes.c @@ -179,14 +179,15 @@ ostree_builtin_find_remotes (int argc, g_autoptr(GPtrArray) finders = NULL; /* (element-type OstreeRepoFinder) */ g_autoptr(OstreeRepoFinder) finder_config = NULL; g_autoptr(OstreeRepoFinder) finder_mount = NULL; +#ifdef HAVE_AVAHI g_autoptr(OstreeRepoFinder) finder_avahi = NULL; +#endif /* HAVE_AVAHI */ g_autoptr(OstreeAsyncProgress) progress = NULL; gsize i; g_autoptr(GAsyncResult) find_result = NULL, pull_result = NULL; g_auto(OstreeRepoFinderResultv) results = NULL; g_auto(GLnxConsoleRef) console = { 0, }; g_autoptr(GHashTable) refs_found = NULL; /* set (element-type OstreeCollectionRef) */ - g_auto(GStrv) finders_strings = NULL; context = g_option_context_new ("COLLECTION-ID REF [COLLECTION-ID REF...]"); From f31087137ed2ce23b8cd886f2ab37b0b27cbfc61 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Mon, 28 May 2018 18:03:42 -0700 Subject: [PATCH 25/45] Fix building P2P code against old glib versions When building the OstreeBloom code against old versions of glib, we have to have the libglnx headers included so that it defines G_DEFINE_AUTOPTR_CLEANUP_FUNC and friends for us. This is similarly true for test-repo-finder-mount.c which indirectly includes ostree-autocleanups.h. Closes: #1605 Approved by: cgwalters --- src/libostree/ostree-bloom-private.h | 2 ++ tests/test-repo-finder-mount.c | 1 + 2 files changed, 3 insertions(+) diff --git a/src/libostree/ostree-bloom-private.h b/src/libostree/ostree-bloom-private.h index ed7ab359..1c5acb8f 100644 --- a/src/libostree/ostree-bloom-private.h +++ b/src/libostree/ostree-bloom-private.h @@ -28,6 +28,8 @@ #include #include +#include "libglnx.h" + G_BEGIN_DECLS /** diff --git a/tests/test-repo-finder-mount.c b/tests/test-repo-finder-mount.c index 948a3245..55eb47fb 100644 --- a/tests/test-repo-finder-mount.c +++ b/tests/test-repo-finder-mount.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "libostreetest.h" From 938055392fd455027a69398c441b992ae521aa87 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 22 May 2018 12:21:45 +0100 Subject: [PATCH 26/45] lib/repo-pull: Support retrying requests on transient network errors Allow network requests to be re-queued if they failed with a transient error, such as a socket timeout. Retry each request up to a limit (default: 5), and only then fail the entire pull and propagate the error to the caller. Add a new ostree_repo_pull_with_options() option, n-network-retries, to control the number of retries (including setting it back to the old default of 0, if the caller wants). Currently, retries are not supported for FetchDeltaSuperData requests, as they are not queued. Once they are queued, adding support for retries should be trivial. A FIXME comment has been left for this. Signed-off-by: Philip Withnall Closes: #1594 Approved by: jlebon --- src/libostree/ostree-fetcher-util.c | 94 +++++++++++++++++--- src/libostree/ostree-fetcher-util.h | 4 + src/libostree/ostree-metalink.c | 10 ++- src/libostree/ostree-metalink.h | 3 +- src/libostree/ostree-repo-pull.c | 129 ++++++++++++++++++++++------ 5 files changed, 202 insertions(+), 38 deletions(-) diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c index aca31358..9cdb82c6 100644 --- a/src/libostree/ostree-fetcher-util.c +++ b/src/libostree/ostree-fetcher-util.c @@ -52,15 +52,15 @@ fetch_uri_sync_on_complete (GObject *object, data->done = TRUE; } -gboolean -_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, - GPtrArray *mirrorlist, - const char *filename, - OstreeFetcherRequestFlags flags, - GBytes **out_contents, - guint64 max_size, - GCancellable *cancellable, - GError **error) +static gboolean +_ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fetcher, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + GBytes **out_contents, + guint64 max_size, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; g_autoptr(GMainContext) mainctx = NULL; @@ -108,11 +108,42 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, return ret; } +gboolean +_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + guint n_network_retries, + GBytes **out_contents, + guint64 max_size, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GError) local_error = NULL; + guint n_retries_remaining = n_network_retries; + + do + { + g_clear_error (&local_error); + if (_ostree_fetcher_mirrored_request_to_membuf_once (fetcher, mirrorlist, + filename, flags, + out_contents, max_size, + cancellable, &local_error)) + return TRUE; + } + while (_ostree_fetcher_should_retry_request (local_error, n_retries_remaining--)); + + g_assert (local_error != NULL); + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; +} + /* Helper for callers who just want to fetch single one-off URIs */ gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, OstreeFetcherURI *uri, OstreeFetcherRequestFlags flags, + guint n_network_retries, GBytes **out_contents, guint64 max_size, GCancellable *cancellable, @@ -121,7 +152,7 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new (); g_ptr_array_add (mirrorlist, uri); /* no transfer */ return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, flags, - out_contents, max_size, + n_network_retries, out_contents, max_size, cancellable, error); } @@ -144,3 +175,46 @@ _ostree_fetcher_journal_failure (const char *remote_name, NULL); #endif } + +/* Check whether a particular operation should be retried. This is entirely + * based on how it failed (if at all) last time, and whether the operation has + * some retries left. The retry count is set when the operation is first + * created, and must be decremented by the caller. (@n_retries_remaining == 0) + * will always return %FALSE from this function. + * + * FIXME: In future, we may decide to use transient failures like this as a hint + * to prioritise other mirrors for a particular pull operation (for example). */ +gboolean +_ostree_fetcher_should_retry_request (const GError *error, + guint n_retries_remaining) +{ + if (error == NULL) + g_debug ("%s: error: unset, n_retries_remaining: %u", + G_STRFUNC, n_retries_remaining); + else + g_debug ("%s: error: %u:%u %s, n_retries_remaining: %u", + G_STRFUNC, error->domain, error->code, error->message, + n_retries_remaining); + + if (error == NULL || n_retries_remaining == 0) + return FALSE; + + /* Return TRUE for transient errors. */ + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_TIMED_OUT) || + g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_NOT_FOUND) || + g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_UNREACHABLE) || +#if !GLIB_CHECK_VERSION(2, 44, 0) + g_error_matches (error, G_IO_ERROR, G_IO_ERROR_BROKEN_PIPE) || +#else + g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CONNECTION_CLOSED) || +#endif + g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_NOT_FOUND) || + g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_TEMPORARY_FAILURE)) + { + g_debug ("Should retry request (remaining: %u retries), due to transient error: %s", + n_retries_remaining, error->message); + return TRUE; + } + + return FALSE; +} diff --git a/src/libostree/ostree-fetcher-util.h b/src/libostree/ostree-fetcher-util.h index 1e2dabe5..5f62ad45 100644 --- a/src/libostree/ostree-fetcher-util.h +++ b/src/libostree/ostree-fetcher-util.h @@ -56,6 +56,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + guint n_network_retries, GBytes **out_contents, guint64 max_size, GCancellable *cancellable, @@ -64,6 +65,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, OstreeFetcherURI *uri, OstreeFetcherRequestFlags flags, + guint n_network_retries, GBytes **out_contents, guint64 max_size, GCancellable *cancellable, @@ -73,6 +75,8 @@ void _ostree_fetcher_journal_failure (const char *remote_name, const char *url, const char *msg); +gboolean _ostree_fetcher_should_retry_request (const GError *error, + guint n_retries_remaining); G_END_DECLS diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c index 7cb879f7..cb8a50e3 100644 --- a/src/libostree/ostree-metalink.c +++ b/src/libostree/ostree-metalink.c @@ -50,6 +50,7 @@ struct OstreeMetalink OstreeFetcher *fetcher; char *requested_file; guint64 max_size; + guint n_network_retries; }; G_DEFINE_TYPE (OstreeMetalink, _ostree_metalink, G_TYPE_OBJECT) @@ -401,7 +402,8 @@ OstreeMetalink * _ostree_metalink_new (OstreeFetcher *fetcher, const char *requested_file, guint64 max_size, - OstreeFetcherURI *uri) + OstreeFetcherURI *uri, + guint n_network_retries) { OstreeMetalink *self = (OstreeMetalink*)g_object_new (OSTREE_TYPE_METALINK, NULL); @@ -409,6 +411,7 @@ _ostree_metalink_new (OstreeFetcher *fetcher, self->requested_file = g_strdup (requested_file); self->max_size = max_size; self->uri = _ostree_fetcher_uri_clone (uri); + self->n_network_retries = n_network_retries; return self; } @@ -432,7 +435,9 @@ try_one_url (OstreeMetalinkRequest *self, gssize n_bytes; if (!_ostree_fetcher_request_uri_to_membuf (self->metalink->fetcher, - uri, 0, &bytes, + uri, 0, + self->metalink->n_network_retries, + &bytes, self->metalink->max_size, self->cancellable, error)) @@ -613,6 +618,7 @@ _ostree_metalink_request_sync (OstreeMetalink *self, request.parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, &request, NULL); if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher, self->uri, 0, + self->n_network_retries, &contents, self->max_size, cancellable, error)) goto out; diff --git a/src/libostree/ostree-metalink.h b/src/libostree/ostree-metalink.h index a9a090b8..1a48945f 100644 --- a/src/libostree/ostree-metalink.h +++ b/src/libostree/ostree-metalink.h @@ -48,7 +48,8 @@ GType _ostree_metalink_get_type (void) G_GNUC_CONST; OstreeMetalink *_ostree_metalink_new (OstreeFetcher *fetcher, const char *requested_file, guint64 max_size, - OstreeFetcherURI *uri); + OstreeFetcherURI *uri, + guint n_network_retries); gboolean _ostree_metalink_request_sync (OstreeMetalink *self, OstreeFetcherURI **out_target_uri, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 16e9efda..668968e6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -61,6 +61,12 @@ #define OSTREE_REPO_PULL_CONTENT_PRIORITY (OSTREE_FETCHER_DEFAULT_PRIORITY) #define OSTREE_REPO_PULL_METADATA_PRIORITY (OSTREE_REPO_PULL_CONTENT_PRIORITY - 100) +/* Arbitrarily chosen number of retries for all download operations when they + * receive a transient network error (such as a socket timeout) — see + * _ostree_fetcher_should_retry_request(). This is the default value for the + * `n-network-retries` pull option. */ +#define DEFAULT_N_NETWORK_RETRIES 5 + typedef enum { OSTREE_FETCHER_SECURITY_STATE_CA_PINNED, OSTREE_FETCHER_SECURITY_STATE_TLS, @@ -92,6 +98,7 @@ typedef struct { gboolean dry_run; gboolean dry_run_emitted_progress; gboolean legacy_transaction_resuming; + guint n_network_retries; enum { OSTREE_PULL_PHASE_FETCHING_REFS, OSTREE_PULL_PHASE_FETCHING_OBJECTS @@ -177,6 +184,7 @@ typedef struct { gboolean object_is_stored; OstreeCollectionRef *requested_ref; /* (nullable) */ + guint n_retries_remaining; } FetchObjectData; typedef struct { @@ -187,6 +195,7 @@ typedef struct { char *to_revision; guint i; guint64 size; + guint n_retries_remaining; } FetchStaticDeltaData; typedef struct { @@ -502,6 +511,8 @@ idle_worker (gpointer user_data) scan_one_metadata_object (pull_data, checksum, scan_data->objtype, scan_data->path, scan_data->recursion_depth, scan_data->requested_ref, pull_data->cancellable, &error); + + /* No need to retry scan tasks, since they’re local. */ check_outstanding_requests_handle_error (pull_data, &error); scan_object_queue_data_free (scan_data); @@ -532,6 +543,7 @@ static gboolean fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher, GPtrArray *mirrorlist, const char *filename, + guint n_network_retries, char **out_contents, GCancellable *cancellable, GError **error) @@ -539,6 +551,7 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher, g_autoptr(GBytes) bytes = NULL; if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION, + n_network_retries, &bytes, OSTREE_MAX_METADATA_SIZE, cancellable, error)) @@ -557,6 +570,7 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher, static gboolean fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher, OstreeFetcherURI *uri, + guint n_network_retries, char **out_contents, GCancellable *cancellable, GError **error) @@ -564,7 +578,8 @@ fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher, g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new (); g_ptr_array_add (mirrorlist, uri); /* no transfer */ return fetch_mirrored_uri_contents_utf8_sync (fetcher, mirrorlist, - NULL, out_contents, + NULL, n_network_retries, + out_contents, cancellable, error); } @@ -718,6 +733,7 @@ on_local_object_imported (GObject *object, pull_data->n_imported_content++; g_assert_cmpint (pull_data->n_outstanding_content_write_requests, >, 0); pull_data->n_outstanding_content_write_requests--; + /* No retries for local reads. */ check_outstanding_requests_handle_error (pull_data, &local_error); } @@ -893,7 +909,8 @@ fetch_ref_contents (OtPullData *pull_data, if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher, pull_data->meta_mirrorlist, - filename, &ret_contents, + filename, pull_data->n_network_retries, + &ret_contents, cancellable, error)) return FALSE; @@ -1017,6 +1034,7 @@ content_fetch_on_write_complete (GObject *object, pull_data->n_fetched_deltapart_fallbacks++; out: pull_data->n_outstanding_content_write_requests--; + /* No retries for local writes. */ check_outstanding_requests_handle_error (pull_data, &local_error); fetch_object_data_free (fetch_data); } @@ -1102,9 +1120,14 @@ content_fetch_on_complete (GObject *object, out: pull_data->n_outstanding_content_fetches--; - check_outstanding_requests_handle_error (pull_data, &local_error); + + if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--)) + enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data)); + else + check_outstanding_requests_handle_error (pull_data, &local_error); + if (free_fetch_data) - fetch_object_data_free (fetch_data); + g_clear_pointer (&fetch_data, fetch_object_data_free); } static void @@ -1148,6 +1171,7 @@ on_metadata_written (GObject *object, pull_data->n_outstanding_metadata_write_requests--; fetch_object_data_free (fetch_data); + /* No need to retry local write operations. */ check_outstanding_requests_handle_error (pull_data, &local_error); } @@ -1285,10 +1309,17 @@ meta_fetch_on_complete (GObject *object, out: g_assert (pull_data->n_outstanding_metadata_fetches > 0); pull_data->n_outstanding_metadata_fetches--; - pull_data->n_fetched_metadata++; - check_outstanding_requests_handle_error (pull_data, &local_error); + + if (local_error == NULL) + pull_data->n_fetched_metadata++; + + if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--)) + enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data)); + else + check_outstanding_requests_handle_error (pull_data, &local_error); + if (free_fetch_data) - fetch_object_data_free (fetch_data); + g_clear_pointer (&fetch_data, fetch_object_data_free); } static void @@ -1320,6 +1351,7 @@ on_static_delta_written (GObject *object, out: g_assert (pull_data->n_outstanding_deltapart_write_requests > 0); pull_data->n_outstanding_deltapart_write_requests--; + /* No need to retry on failure to write locally. */ check_outstanding_requests_handle_error (pull_data, &local_error); /* Always free state */ fetch_static_delta_data_free (fetch_data); @@ -1365,10 +1397,17 @@ static_deltapart_fetch_on_complete (GObject *object, out: g_assert (pull_data->n_outstanding_deltapart_fetches > 0); pull_data->n_outstanding_deltapart_fetches--; - pull_data->n_fetched_deltaparts++; - check_outstanding_requests_handle_error (pull_data, &local_error); + + if (local_error == NULL) + pull_data->n_fetched_deltaparts++; + + if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--)) + enqueue_one_static_delta_part_request_s (pull_data, g_steal_pointer (&fetch_data)); + else + check_outstanding_requests_handle_error (pull_data, &local_error); + if (free_fetch_data) - fetch_static_delta_data_free (fetch_data); + g_clear_pointer (&fetch_data, fetch_static_delta_data_free); } static gboolean @@ -2084,10 +2123,9 @@ enqueue_one_object_request (OtPullData *pull_data, fetch_data->is_detached_meta = is_detached_meta; fetch_data->object_is_stored = object_is_stored; fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; + fetch_data->n_retries_remaining = pull_data->n_network_retries; - gboolean is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); - - if (is_meta) + if (OSTREE_OBJECT_TYPE_IS_META (objtype)) pull_data->n_requested_metadata++; else pull_data->n_requested_content++; @@ -2167,8 +2205,8 @@ load_remote_repo_config (OtPullData *pull_data, if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher, pull_data->meta_mirrorlist, - "config", &contents, - cancellable, error)) + "config", pull_data->n_network_retries, + &contents, cancellable, error)) return FALSE; g_autoptr(GKeyFile) ret_keyfile = g_key_file_new (); @@ -2350,6 +2388,7 @@ process_one_static_delta (OtPullData *pull_data, fetch_data->is_detached_meta = FALSE; fetch_data->object_is_stored = FALSE; fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; + fetch_data->n_retries_remaining = pull_data->n_network_retries; ostree_repo_write_metadata_async (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, to_checksum, to_commit, @@ -2423,6 +2462,7 @@ process_one_static_delta (OtPullData *pull_data, fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v); fetch_data->size = size; fetch_data->i = i; + fetch_data->n_retries_remaining = pull_data->n_network_retries; if (inline_part_bytes != NULL) { @@ -2705,7 +2745,11 @@ on_superblock_fetched (GObject *src, out: g_assert (pull_data->n_outstanding_metadata_fetches > 0); pull_data->n_outstanding_metadata_fetches--; - pull_data->n_fetched_metadata++; + + if (local_error == NULL) + pull_data->n_fetched_metadata++; + + /* FIXME: This should check _ostree_fetcher_should_retry_request(). */ check_outstanding_requests_handle_error (pull_data, &local_error); g_clear_pointer (&fetch_data, fetch_delta_super_data_free); @@ -2938,6 +2982,7 @@ _ostree_preload_metadata_file (OstreeRepo *self, GPtrArray *mirrorlist, const char *filename, gboolean is_metalink, + guint n_network_retries, GBytes **out_bytes, GCancellable *cancellable, GError **error) @@ -2951,7 +2996,7 @@ _ostree_preload_metadata_file (OstreeRepo *self, g_autoptr(OstreeMetalink) metalink = _ostree_metalink_new (fetcher, filename, OSTREE_MAX_METADATA_SIZE, - mirrorlist->pdata[0]); + mirrorlist->pdata[0], n_network_retries); _ostree_metalink_request_sync (metalink, NULL, out_bytes, cancellable, &local_error); @@ -2973,6 +3018,7 @@ _ostree_preload_metadata_file (OstreeRepo *self, { return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + n_network_retries, out_bytes, OSTREE_MAX_METADATA_SIZE, cancellable, error); } @@ -2981,6 +3027,7 @@ _ostree_preload_metadata_file (OstreeRepo *self, static gboolean fetch_mirrorlist (OstreeFetcher *fetcher, const char *mirrorlist_url, + guint n_network_retries, GPtrArray **out_mirrorlist, GCancellable *cancellable, GError **error) @@ -2993,8 +3040,8 @@ fetch_mirrorlist (OstreeFetcher *fetcher, return FALSE; g_autofree char *contents = NULL; - if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, &contents, - cancellable, error)) + if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, n_network_retries, + &contents, cancellable, error)) return glnx_prefix_error (error, "While fetching mirrorlist '%s'", mirrorlist_url); @@ -3040,8 +3087,8 @@ fetch_mirrorlist (OstreeFetcher *fetcher, GError *local_error = NULL; g_autoptr(OstreeFetcherURI) config_uri = _ostree_fetcher_uri_new_subpath (mirror_uri, "config"); - if (fetch_uri_contents_utf8_sync (fetcher, config_uri, NULL, - cancellable, &local_error)) + if (fetch_uri_contents_utf8_sync (fetcher, config_uri, n_network_retries, + NULL, cancellable, &local_error)) g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri)); else { @@ -3083,12 +3130,14 @@ repo_remote_fetch_summary (OstreeRepo *self, g_autoptr(GVariant) extra_headers = NULL; g_autoptr(GPtrArray) mirrorlist = NULL; const char *append_user_agent = NULL; + guint n_network_retries = DEFAULT_N_NETWORK_RETRIES; if (options) { (void) g_variant_lookup (options, "override-url", "&s", &url_override); (void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers); (void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent); + (void) g_variant_lookup (options, "n-network-retries", "&u", &n_network_retries); } mainctx = g_main_context_new (); @@ -3117,7 +3166,7 @@ repo_remote_fetch_summary (OstreeRepo *self, g_str_has_prefix (url_string, "mirrorlist=")) { if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="), - &mirrorlist, cancellable, error)) + n_network_retries, &mirrorlist, cancellable, error)) goto out; } else @@ -3143,6 +3192,7 @@ repo_remote_fetch_summary (OstreeRepo *self, mirrorlist, "summary.sig", metalink_url_string ? TRUE : FALSE, + n_network_retries, out_signatures, cancellable, error)) @@ -3168,6 +3218,7 @@ repo_remote_fetch_summary (OstreeRepo *self, mirrorlist, "summary", metalink_url_string ? TRUE : FALSE, + n_network_retries, out_summary, cancellable, error)) @@ -3390,6 +3441,9 @@ initiate_request (OtPullData *pull_data, * * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid * * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches * * append-user-agent (s): Additional string to append to the user agent + * * n-network-retries (u): Number of times to retry each download on receiving + * a transient network error, such as a socket timeout; default is 5, 0 + * means return errors without retrying */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -3423,6 +3477,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; gboolean opt_collection_refs_set = FALSE; + gboolean opt_n_network_retries_set = FALSE; const char *main_collection_id = NULL; const char *url_override = NULL; gboolean inherit_transaction = FALSE; @@ -3462,6 +3517,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos); (void) g_variant_lookup (options, "timestamp-check", "b", &pull_data->timestamp_check); (void) g_variant_lookup (options, "append-user-agent", "s", &pull_data->append_user_agent); + opt_n_network_retries_set = + g_variant_lookup (options, "n-network-retries", "u", &pull_data->n_network_retries); if (pull_data->remote_refspec_name != NULL) pull_data->remote_name = g_strdup (pull_data->remote_refspec_name); @@ -3502,6 +3559,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->main_context = g_main_context_ref_thread_default (); pull_data->flags = flags; + if (!opt_n_network_retries_set) + pull_data->n_network_retries = DEFAULT_N_NETWORK_RETRIES; + pull_data->repo = self; pull_data->progress = progress; @@ -3647,6 +3707,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, { if (!fetch_mirrorlist (pull_data->fetcher, baseurl + strlen ("mirrorlist="), + pull_data->n_network_retries, &pull_data->meta_mirrorlist, cancellable, error)) goto out; @@ -3673,7 +3734,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; metalink = _ostree_metalink_new (pull_data->fetcher, "summary", - OSTREE_MAX_METADATA_SIZE, metalink_uri); + OSTREE_MAX_METADATA_SIZE, metalink_uri, + pull_data->n_network_retries); if (! _ostree_metalink_request_sync (metalink, &target_uri, @@ -3719,6 +3781,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, { if (!fetch_mirrorlist (pull_data->fetcher, contenturl + strlen ("mirrorlist="), + pull_data->n_network_retries, &pull_data->content_mirrorlist, cancellable, error)) goto out; @@ -3865,6 +3928,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + pull_data->n_network_retries, &bytes_sig, OSTREE_MAX_METADATA_SIZE, cancellable, error)) @@ -3889,6 +3953,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + pull_data->n_network_retries, &bytes_summary, OSTREE_MAX_METADATA_SIZE, cancellable, error)) @@ -4663,6 +4728,7 @@ typedef struct GVariant *options; OstreeAsyncProgress *progress; OstreeRepoFinder *default_finder_avahi; + guint n_network_retries; } FindRemotesData; static void @@ -4682,7 +4748,8 @@ static FindRemotesData * find_remotes_data_new (const OstreeCollectionRef * const *refs, GVariant *options, OstreeAsyncProgress *progress, - OstreeRepoFinder *default_finder_avahi) + OstreeRepoFinder *default_finder_avahi, + guint n_network_retries) { g_autoptr(FindRemotesData) data = NULL; @@ -4691,6 +4758,7 @@ find_remotes_data_new (const OstreeCollectionRef * const *refs, data->options = (options != NULL) ? g_variant_ref (options) : NULL; data->progress = (progress != NULL) ? g_object_ref (progress) : NULL; data->default_finder_avahi = (default_finder_avahi != NULL) ? g_object_ref (default_finder_avahi) : NULL; + data->n_network_retries = n_network_retries; return g_steal_pointer (&data); } @@ -4770,6 +4838,9 @@ static void find_remotes_cb (GObject *obj, * * `override-commit-ids` (`as`): Array of specific commit IDs to fetch. The nth * commit ID applies to the nth ref, so this must be the same length as @refs, if * provided. + * * `n-network-retries` (`u`): Number of times to retry each download on + * receiving a transient network error, such as a socket timeout; default is + * 5, 0 means return errors without retrying. * * @finders must be a non-empty %NULL-terminated array of the #OstreeRepoFinder * instances to use, or %NULL to use the system default set of finders, which @@ -4799,6 +4870,7 @@ ostree_repo_find_remotes_async (OstreeRepo *self, g_autoptr(OstreeRepoFinder) finder_mount = NULL; g_autoptr(OstreeRepoFinder) finder_avahi = NULL; g_autofree char **override_commit_ids = NULL; + guint n_network_retries = DEFAULT_N_NETWORK_RETRIES; g_return_if_fail (OSTREE_IS_REPO (self)); g_return_if_fail (is_valid_collection_ref_array (refs)); @@ -4812,6 +4884,8 @@ ostree_repo_find_remotes_async (OstreeRepo *self, { (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); g_return_if_fail (override_commit_ids == NULL || g_strv_length ((gchar **) refs) == g_strv_length (override_commit_ids)); + + (void) g_variant_lookup (options, "n-network-retries", "u", &n_network_retries); } /* Set up a task for the whole operation. */ @@ -4867,7 +4941,7 @@ ostree_repo_find_remotes_async (OstreeRepo *self, /* We need to keep a pointer to the default Avahi finder so we can stop it * again after the operation, which happens implicitly by dropping the final * ref. */ - data = find_remotes_data_new (refs, options, progress, finder_avahi); + data = find_remotes_data_new (refs, options, progress, finder_avahi, n_network_retries); g_task_set_task_data (task, g_steal_pointer (&data), (GDestroyNotify) find_remotes_data_free); /* Asynchronously resolve all possible remotes for the given refs. */ @@ -5260,6 +5334,7 @@ find_remotes_cb (GObject *obj, mirrorlist, commit_filename, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + data->n_network_retries, &commit_bytes, 0, /* no maximum size */ cancellable, @@ -5710,6 +5785,7 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, copy_option (&options_dict, &local_options_dict, "subdirs", G_VARIANT_TYPE ("as")); copy_option (&options_dict, &local_options_dict, "update-frequency", G_VARIANT_TYPE ("u")); copy_option (&options_dict, &local_options_dict, "append-user-agent", G_VARIANT_TYPE ("s")); + copy_option (&options_dict, &local_options_dict, "n-network-retries", G_VARIANT_TYPE ("u")); local_options = g_variant_dict_end (&local_options_dict); @@ -5827,6 +5903,9 @@ ostree_repo_pull_from_remotes_finish (OstreeRepo *self, * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options * - http-headers (a(ss)): Additional headers to add to all HTTP requests * - append-user-agent (s): Additional string to append to the user agent + * - n-network-retries (u): Number of times to retry each download on receiving + * a transient network error, such as a socket timeout; default is 5, 0 + * means return errors without retrying * * Returns: %TRUE on success, %FALSE on failure */ From 224f3cdd24e81de6f490fdcd93b114bbb7d4ab8a Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 25 May 2018 15:49:18 +0100 Subject: [PATCH 27/45] lib/fetcher-soup: Map more SoupStatus codes to known GIOErrors This allows the retry code in ostree-repo-pull.c to recover from (for example) timeouts at the libsoup layer in the stack, as well as from the GSocket layer in the stack. Signed-off-by: Philip Withnall Closes: #1594 Approved by: jlebon --- src/libostree/ostree-fetcher-soup.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c index 08a0a700..43794275 100644 --- a/src/libostree/ostree-fetcher-soup.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -1072,11 +1072,28 @@ on_request_sent (GObject *object, GIOErrorEnum code; switch (msg->status_code) { - case 404: - case 403: - case 410: + case SOUP_STATUS_NOT_FOUND: + case SOUP_STATUS_FORBIDDEN: + case SOUP_STATUS_GONE: code = G_IO_ERROR_NOT_FOUND; break; + case SOUP_STATUS_CANCELLED: + code = G_IO_ERROR_CANCELLED; + break; + case SOUP_STATUS_REQUEST_TIMEOUT: + code = G_IO_ERROR_TIMED_OUT; + break; + case SOUP_STATUS_CANT_RESOLVE: + case SOUP_STATUS_CANT_CONNECT: + code = G_IO_ERROR_HOST_NOT_FOUND; + break; + case SOUP_STATUS_IO_ERROR: +#if !GLIB_CHECK_VERSION(2, 44, 0) + code = G_IO_ERROR_BROKEN_PIPE; +#else + code = G_IO_ERROR_CONNECTION_CLOSED; +#endif + break; default: code = G_IO_ERROR_FAILED; } From 97c348d65b989993cdf995e37a28936faa4ec1b2 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 28 May 2018 16:24:33 +0100 Subject: [PATCH 28/45] ostree/trivial-httpd: Add --random-408s command line option This is exactly like the --random-500s option, except that it will cause error 408 (request timeout) to be returned, rather than error 500 (internal server error). This will be used in a following commit to test pull behaviour when timeouts occur. Signed-off-by: Philip Withnall Closes: #1594 Approved by: jlebon --- src/ostree/ostree-trivial-httpd.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index 5da5a6ab..1b7aa2a2 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -45,11 +45,14 @@ static int opt_random_500s_percentage; /* We have a strong upper bound for any unlikely * cases involving repeated random 500s. */ static int opt_random_500s_max = 100; +static int opt_random_408s_percentage; +static int opt_random_408s_max = 100; static gint opt_port = 0; static gchar **opt_expected_cookies; static gchar **opt_expected_headers; static guint emitted_random_500s_count = 0; +static guint emitted_random_408s_count = 0; typedef struct { int root_dfd; @@ -70,6 +73,8 @@ static GOptionEntry options[] = { { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL }, { "random-500s", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_percentage, "Generate random HTTP 500 errors approximately for PERCENTAGE requests", "PERCENTAGE" }, { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" }, + { "random-408s", 0, 0, G_OPTION_ARG_INT, &opt_random_408s_percentage, "Generate random HTTP 408 errors approximately for PERCENTAGE requests", "PERCENTAGE" }, + { "random-408s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_408s_max, "Limit HTTP 408 errors to MAX (default 100)", "MAX" }, { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here (use - for stdout)", "PATH" }, { "expected-cookies", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_cookies, "Expect given cookies in the http request", "KEY=VALUE" }, { "expected-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_headers, "Expect given headers in the http request", "KEY=VALUE" }, @@ -291,6 +296,14 @@ do_get (OtTrivialHttpd *self, soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR); goto out; } + else if (opt_random_408s_percentage > 0 && + emitted_random_408s_count < opt_random_408s_max && + g_random_int_range (0, 100) < opt_random_408s_percentage) + { + emitted_random_408s_count++; + soup_message_set_status (msg, SOUP_STATUS_REQUEST_TIMEOUT); + goto out; + } while (path[0] == '/') path++; @@ -511,6 +524,13 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) goto out; } + if (!(opt_random_408s_percentage >= 0 && opt_random_408s_percentage <= 99)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid --random-408s=%u", opt_random_408s_percentage); + goto out; + } + if (opt_log) { GOutputStream *stream = NULL; From c9619a490490a8ed29ea002a2f358bbebfb80ef4 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 28 May 2018 16:25:53 +0100 Subject: [PATCH 29/45] tests: Test pull behaviour when network timeouts occur MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend test-pull-repeated.sh to test error 408 as well as error 500, to ensure that the new retry-on-network-timeout code in ostree-repo-pull.c correctly retries. Rather than the 200 iterations needed for the error 500 tests, only do 5 iterations. The pull code internally does 5 retries (by default), which means a full iteration count of 25. That seems to be sufficient to make the tests reliably pass, in my testing — we can always bump it up to 200 / 5 = 40 in future if needed (to put it in parity with the error 500 tests). Signed-off-by: Philip Withnall Closes: #1594 Approved by: jlebon --- tests/test-pull-repeated.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test-pull-repeated.sh b/tests/test-pull-repeated.sh index f43f720f..bfc7148a 100755 --- a/tests/test-pull-repeated.sh +++ b/tests/test-pull-repeated.sh @@ -23,12 +23,14 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..1" +echo "1..2" COMMIT_SIGN="--gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1}" + +# Test pulling from a repo which gives error 500 (internal server error) a lot of the time. setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-500s=50 -cd ${test_tmpdir} +pushd ${test_tmpdir} ostree_repo_init repo --mode=archive ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo for x in $(seq 200); do @@ -42,4 +44,26 @@ done ${CMD_PREFIX} ostree --repo=repo fsck ${CMD_PREFIX} ostree --repo=repo rev-parse main +popd echo "ok repeated pull after 500s" + +# Now test from a repo which gives error 408 (request timeout) a lot of the time. +rm ostree-srv httpd repo -rf +setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-408s=50 + +pushd ${test_tmpdir} +ostree_repo_init repo --mode=archive +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo +for x in $(seq 40); do + if ${CMD_PREFIX} ostree --repo=repo pull --mirror origin main 2>err.txt; then + echo "Success on iteration ${x}" + break; + fi + assert_file_has_content err.txt "\(408.*Request Timeout\)\|\(HTTP 408\)" +done + +${CMD_PREFIX} ostree --repo=repo fsck +${CMD_PREFIX} ostree --repo=repo rev-parse main + +popd +echo "ok repeated pull after 408s" From 78f40136dbf8411a66ce3f6d8263d8e72d951163 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 30 May 2018 12:00:22 +0100 Subject: [PATCH 30/45] lib/repo-pull: Add some missing assertions for progress statistics Various of the counters already have assertions like this; add some more for total paranoia. Signed-off-by: Philip Withnall Closes: #1594 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 668968e6..83ccff5c 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1119,6 +1119,7 @@ content_fetch_on_complete (GObject *object, } out: + g_assert (pull_data->n_outstanding_content_fetches > 0); pull_data->n_outstanding_content_fetches--; if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--)) @@ -1168,6 +1169,7 @@ on_metadata_written (GObject *object, queue_scan_one_metadata_object_c (pull_data, csum, objtype, fetch_data->path, 0, fetch_data->requested_ref); out: + g_assert (pull_data->n_outstanding_metadata_write_requests > 0); pull_data->n_outstanding_metadata_write_requests--; fetch_object_data_free (fetch_data); From 197644c406deec76a6567e6639ebd6e06dbc1013 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 30 May 2018 12:20:49 +0100 Subject: [PATCH 31/45] lib/fetcher: Factor out HTTP status code handling from soup and curl Use the same G_IO_ERROR_* values for HTTP status codes in both fetchers. The libsoup fetcher still handles a few more internal error codes than the libcurl one; this could be built on in future. Signed-off-by: Philip Withnall Closes: #1594 Approved by: jlebon --- src/libostree/ostree-fetcher-curl.c | 14 +------------- src/libostree/ostree-fetcher-soup.c | 13 ++++--------- src/libostree/ostree-fetcher-util.c | 19 +++++++++++++++++++ src/libostree/ostree-fetcher-util.h | 2 ++ 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index c0f38131..2e090cfa 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -337,19 +337,7 @@ check_multi_info (OstreeFetcher *fetcher) curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response); if (!is_file && !(response >= 200 && response < 300)) { - GIOErrorEnum giocode; - - /* TODO - share with soup */ - switch (response) - { - case 404: - case 403: - case 410: - giocode = G_IO_ERROR_NOT_FOUND; - break; - default: - giocode = G_IO_ERROR_FAILED; - } + GIOErrorEnum giocode = _ostree_fetcher_http_status_code_to_io_error (response); if (req->idx + 1 == req->mirrorlist->len) { diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c index 43794275..3951a927 100644 --- a/src/libostree/ostree-fetcher-soup.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -1070,19 +1070,13 @@ on_request_sent (GObject *object, soup_uri_to_string (soup_request_get_uri (pending->request), FALSE); GIOErrorEnum code; + switch (msg->status_code) { - case SOUP_STATUS_NOT_FOUND: - case SOUP_STATUS_FORBIDDEN: - case SOUP_STATUS_GONE: - code = G_IO_ERROR_NOT_FOUND; - break; + /* These statuses are internal to libsoup, and not standard HTTP ones: */ case SOUP_STATUS_CANCELLED: code = G_IO_ERROR_CANCELLED; break; - case SOUP_STATUS_REQUEST_TIMEOUT: - code = G_IO_ERROR_TIMED_OUT; - break; case SOUP_STATUS_CANT_RESOLVE: case SOUP_STATUS_CANT_CONNECT: code = G_IO_ERROR_HOST_NOT_FOUND; @@ -1095,7 +1089,8 @@ on_request_sent (GObject *object, #endif break; default: - code = G_IO_ERROR_FAILED; + code = _ostree_fetcher_http_status_code_to_io_error (msg->status_code); + break; } { diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c index 9cdb82c6..6f759c86 100644 --- a/src/libostree/ostree-fetcher-util.c +++ b/src/libostree/ostree-fetcher-util.c @@ -218,3 +218,22 @@ _ostree_fetcher_should_retry_request (const GError *error, return FALSE; } + +/* Convert a HTTP status code representing an error from libsoup or libcurl to + * a #GIOError. This will return %G_IO_ERROR_FAILED if the status code is + * unknown or otherwise unhandled. */ +GIOError +_ostree_fetcher_http_status_code_to_io_error (guint status_code) +{ + switch (status_code) + { + case 403: /* SOUP_STATUS_FORBIDDEN */ + case 404: /* SOUP_STATUS_NOT_FOUND */ + case 410: /* SOUP_STATUS_GONE */ + return G_IO_ERROR_NOT_FOUND; + case 408: /* SOUP_STATUS_REQUEST_TIMEOUT */ + return G_IO_ERROR_TIMED_OUT; + default: + return G_IO_ERROR_FAILED; + } +} diff --git a/src/libostree/ostree-fetcher-util.h b/src/libostree/ostree-fetcher-util.h index 5f62ad45..1cade068 100644 --- a/src/libostree/ostree-fetcher-util.h +++ b/src/libostree/ostree-fetcher-util.h @@ -78,6 +78,8 @@ void _ostree_fetcher_journal_failure (const char *remote_name, gboolean _ostree_fetcher_should_retry_request (const GError *error, guint n_retries_remaining); +GIOError _ostree_fetcher_http_status_code_to_io_error (guint status_code); + G_END_DECLS #endif From f342e66c116b8731e161b03fb1abcd4504f7098e Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 28 May 2018 16:55:05 +0100 Subject: [PATCH 32/45] lib/repo-pull: Support queuing delta superblock requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just like all the other requests made for delta parts and objects by the pull code, use a queue for delta superblocks. Currently this doesn’t do any prioritisation or retries after transient failures, but it could do in future. This means that delta superblocks are now subject to the parallel request limit in the fetcher, which was a problem highlighted here: https://github.com/ostreedev/ostree/pull/1453#discussion_r168321706. Signed-off-by: Philip Withnall Closes: #1600 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 118 +++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 36 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 83ccff5c..7d05c10f 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -127,6 +127,7 @@ typedef struct { GHashTable *requested_fallback_content; /* Maps checksum to itself */ GHashTable *pending_fetch_metadata; /* Map */ GHashTable *pending_fetch_content; /* Map */ + GHashTable *pending_fetch_delta_superblocks; /* Set */ GHashTable *pending_fetch_deltaparts; /* Set */ guint n_outstanding_metadata_fetches; guint n_outstanding_metadata_write_requests; @@ -206,6 +207,13 @@ typedef struct { OstreeCollectionRef *requested_ref; /* (nullable) */ } ScanObjectQueueData; +typedef struct { + OtPullData *pull_data; + char *from_revision; + char *to_revision; + OstreeCollectionRef *requested_ref; /* (nullable) */ +} FetchDeltaSuperData; + static void variant_or_null_unref (gpointer data) { @@ -216,6 +224,8 @@ variant_or_null_unref (gpointer data) static void start_fetch (OtPullData *pull_data, FetchObjectData *fetch); static void start_fetch_deltapart (OtPullData *pull_data, FetchStaticDeltaData *fetch); +static void start_fetch_delta_superblock (OtPullData *pull_data, + FetchDeltaSuperData *fetch_data); static gboolean fetcher_queue_is_full (OtPullData *pull_data); static void queue_scan_one_metadata_object (OtPullData *pull_data, const char *csum, @@ -235,6 +245,8 @@ static void queue_scan_one_metadata_object_c (OtPullData *pull_da static void enqueue_one_object_request_s (OtPullData *pull_data, FetchObjectData *fetch_data); +static void enqueue_one_static_delta_superblock_request_s (OtPullData *pull_data, + FetchDeltaSuperData *fetch_data); static void enqueue_one_static_delta_part_request_s (OtPullData *pull_data, FetchStaticDeltaData *fetch_data); @@ -391,6 +403,7 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL); g_queue_clear (&pull_data->scan_object_queue); g_hash_table_remove_all (pull_data->pending_fetch_metadata); + g_hash_table_remove_all (pull_data->pending_fetch_delta_superblocks); g_hash_table_remove_all (pull_data->pending_fetch_deltaparts); g_hash_table_remove_all (pull_data->pending_fetch_content); } @@ -423,6 +436,16 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_variant_unref (objname); } + /* Next, process delta superblock requests */ + g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_superblocks); + while (!fetcher_queue_is_full (pull_data) && + g_hash_table_iter_next (&hiter, &key, &value)) + { + FetchDeltaSuperData *fetch = key; + g_hash_table_iter_steal (&hiter); + start_fetch_delta_superblock (pull_data, g_steal_pointer (&fetch)); + } + /* Now, process deltapart requests */ g_hash_table_iter_init (&hiter, pull_data->pending_fetch_deltaparts); while (!fetcher_queue_is_full (pull_data) && @@ -2648,13 +2671,6 @@ get_best_static_delta_start_for (OtPullData *pull_data, return TRUE; } -typedef struct { - OtPullData *pull_data; - char *from_revision; - char *to_revision; - OstreeCollectionRef *requested_ref; /* (nullable) */ -} FetchDeltaSuperData; - static void fetch_delta_super_data_free (FetchDeltaSuperData *fetch_data) { @@ -2757,6 +2773,59 @@ on_superblock_fetched (GObject *src, g_clear_pointer (&fetch_data, fetch_delta_super_data_free); } +static void +start_fetch_delta_superblock (OtPullData *pull_data, + FetchDeltaSuperData *fetch_data) +{ + g_autofree char *delta_name = + _ostree_get_relative_static_delta_superblock_path (fetch_data->from_revision, + fetch_data->to_revision); + _ostree_fetcher_request_to_membuf (pull_data->fetcher, + pull_data->content_mirrorlist, + delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + OSTREE_MAX_METADATA_SIZE, + 0, pull_data->cancellable, + on_superblock_fetched, + g_steal_pointer (&fetch_data)); + pull_data->n_outstanding_metadata_fetches++; + pull_data->n_requested_metadata++; +} + +static void +enqueue_one_static_delta_superblock_request_s (OtPullData *pull_data, + FetchDeltaSuperData *fetch_data) +{ + if (fetcher_queue_is_full (pull_data)) + { + g_debug ("queuing fetch of static delta superblock %s-%s", + fetch_data->from_revision ?: "empty", + fetch_data->to_revision); + + g_hash_table_add (pull_data->pending_fetch_delta_superblocks, + g_steal_pointer (&fetch_data)); + } + else + { + start_fetch_delta_superblock (pull_data, g_steal_pointer (&fetch_data)); + } +} + +/* Start a request for a static delta */ +static void +enqueue_one_static_delta_superblock_request (OtPullData *pull_data, + const char *from_revision, + const char *to_revision, + const OstreeCollectionRef *ref) +{ + FetchDeltaSuperData *fdata = g_new0(FetchDeltaSuperData, 1); + fdata->pull_data = pull_data; + fdata->from_revision = g_strdup (from_revision); + fdata->to_revision = g_strdup (to_revision); + fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; + + enqueue_one_static_delta_superblock_request_s (pull_data, g_steal_pointer (&fdata)); +} + static gboolean validate_variant_is_csum (GVariant *csum, GError **error) @@ -3260,31 +3329,6 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, return TRUE; } -/* Start a request for a static delta */ -static void -initiate_delta_request (OtPullData *pull_data, - const char *from_revision, - const char *to_revision, - const OstreeCollectionRef *ref) -{ - g_autofree char *delta_name = - _ostree_get_relative_static_delta_superblock_path (from_revision, to_revision); - FetchDeltaSuperData *fdata = g_new0(FetchDeltaSuperData, 1); - fdata->pull_data = pull_data; - fdata->from_revision = g_strdup (from_revision); - fdata->to_revision = g_strdup (to_revision); - fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; - - _ostree_fetcher_request_to_membuf (pull_data->fetcher, - pull_data->content_mirrorlist, - delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - OSTREE_MAX_METADATA_SIZE, - 0, pull_data->cancellable, - on_superblock_fetched, fdata); - pull_data->n_outstanding_metadata_fetches++; - pull_data->n_requested_metadata++; -} - /* * initiate_request: * @ref: Optional ref name and collection ID @@ -3335,10 +3379,10 @@ initiate_request (OtPullData *pull_data, } break; case DELTA_SEARCH_RESULT_FROM: - initiate_delta_request (pull_data, deltares.from_revision, to_revision, ref); + enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref); break; case DELTA_SEARCH_RESULT_SCRATCH: - initiate_delta_request (pull_data, NULL, to_revision, ref); + enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref); break; case DELTA_SEARCH_RESULT_UNCHANGED: { @@ -3390,14 +3434,14 @@ initiate_request (OtPullData *pull_data, if (delta_from_revision && g_str_equal (delta_from_revision, to_revision)) queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); else - initiate_delta_request (pull_data, delta_from_revision ?: NULL, to_revision, ref); + enqueue_one_static_delta_superblock_request (pull_data, delta_from_revision ?: NULL, to_revision, ref); } else { /* Legacy path without a summary file - let's try a scratch delta, if that * doesn't work, it'll drop down to object requests. */ - initiate_delta_request (pull_data, NULL, to_revision, NULL); + enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, NULL); } return TRUE; @@ -3597,6 +3641,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, (GDestroyNotify)fetch_object_data_free); + pull_data->pending_fetch_delta_superblocks = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_super_data_free, NULL); pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL); if (opt_localcache_repos && *opt_localcache_repos) @@ -4560,6 +4605,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->pending_fetch_delta_superblocks, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref); g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL); g_queue_clear (&pull_data->scan_object_queue); From bf1af263d67c0f7e49355088528f02d78fa68e90 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 28 May 2018 17:06:46 +0100 Subject: [PATCH 33/45] lib/repo-pull: Support retries for delta superblocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the recently introduced architecture for retrying network requests on transient failure to do the same for delta superblock requests, now that they’re queued. Signed-off-by: Philip Withnall Closes: #1600 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 7d05c10f..550a33de 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -212,6 +212,7 @@ typedef struct { char *from_revision; char *to_revision; OstreeCollectionRef *requested_ref; /* (nullable) */ + guint n_retries_remaining; } FetchDeltaSuperData; static void @@ -2767,8 +2768,10 @@ on_superblock_fetched (GObject *src, if (local_error == NULL) pull_data->n_fetched_metadata++; - /* FIXME: This should check _ostree_fetcher_should_retry_request(). */ - check_outstanding_requests_handle_error (pull_data, &local_error); + if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--)) + enqueue_one_static_delta_superblock_request_s (pull_data, g_steal_pointer (&fetch_data)); + else + check_outstanding_requests_handle_error (pull_data, &local_error); g_clear_pointer (&fetch_data, fetch_delta_super_data_free); } @@ -2822,6 +2825,7 @@ enqueue_one_static_delta_superblock_request (OtPullData *pull_dat fdata->from_revision = g_strdup (from_revision); fdata->to_revision = g_strdup (to_revision); fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; + fdata->n_retries_remaining = pull_data->n_network_retries; enqueue_one_static_delta_superblock_request_s (pull_data, g_steal_pointer (&fdata)); } From 6a7620b58d901ecbee021e829b34a9645d2e8a6d Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Tue, 29 May 2018 18:24:54 -0700 Subject: [PATCH 34/45] ci: Bump flatpak build to F28 Closes: #1607 Approved by: jlebon --- .papr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.papr.yml b/.papr.yml index 7e6d6ce8..ab639021 100644 --- a/.papr.yml +++ b/.papr.yml @@ -144,19 +144,19 @@ branches: - auto - try -context: f27-flatpak +context: f28-flatpak required: true # This test case wants an "unprivileged container with bubblewrap", # which we don't have right now; so just provision a VM and do a # docker --privileged run. host: - distro: fedora/27/atomic + distro: fedora/28/atomic specs: ram: 4096 # build-bundle is a static delta, which needs RAM right now tests: - - docker run --rm --privileged -v $(pwd):/srv/code registry.fedoraproject.org/fedora:27 /bin/sh -c "cd /srv/code && ./ci/flatpak.sh" + - docker run --rm --privileged -v $(pwd):/srv/code registry.fedoraproject.org/fedora:28 /bin/sh -c "cd /srv/code && ./ci/flatpak.sh" artifacts: - test-suite.log From bf3525adcbdcafa67597c216ed4465d4bceadd54 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Tue, 29 May 2018 19:09:07 -0700 Subject: [PATCH 35/45] ci: Use master branch as flatpak tag Closes: #1607 Approved by: jlebon --- ci/flatpak.sh | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/ci/flatpak.sh b/ci/flatpak.sh index 015cd390..c47171f2 100755 --- a/ci/flatpak.sh +++ b/ci/flatpak.sh @@ -3,38 +3,24 @@ set -xeuo pipefail -# Frozen to a tag for now to help predictability; it's -# also useful to test building *older* versions since -# that must work. -FLATPAK_TAG=0.10.2.1 +FLATPAK_TAG=master dn=$(dirname $0) . ${dn}/libpaprci/libbuild.sh codedir=$(pwd) -pkg_upgrade -pkg_install_buildroot -pkg_builddep ostree flatpak -pkg_install gettext-devel # A new dependency -# Copy of builddeps from build.sh in flatpak -pkg_install sudo which attr fuse \ - libubsan libasan libtsan \ - elfutils git gettext-devel libappstream-glib-devel \ - /usr/bin/{update-mime-database,update-desktop-database,gtk-update-icon-cache} \ - hicolor-icon-theme -pkg_install flatpak && rpm -e flatpak - # Build and install ostree cd ${codedir} -build +ci/build.sh make install + +# Build flatpak tmpd=$(mktemp -d) cd ${tmpd} -# Frozen to a tag for now on general principle git clone --recursive --depth=1 -b ${FLATPAK_TAG} https://github.com/flatpak/flatpak cd ${tmpd}/flatpak -build +ci/build.sh # We want to capture automake results from flatpak cleanup() { mv test-suite.log ${codedir} || true From f200efdb8ae7189e8e50e2e934bfaad3f14eaa9b Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 3 Jun 2018 00:25:31 +0200 Subject: [PATCH 36/45] tests: Fix locale detection When a locale with C.utf8 in its name (e.g. es_EC.utf8) was installed on a system, the C.utf8 locale was chosen, even when it was not available. This patch fixes the grep pattern to match whole lines returned by locale -a. See: #1592 Closes: #1611 Approved by: cgwalters --- tests/libtest-core.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh index 6991a86d..f6f94288 100644 --- a/tests/libtest-core.sh +++ b/tests/libtest-core.sh @@ -37,9 +37,9 @@ assert_not_reached () { # Some tests look for specific English strings. Use a UTF-8 version # of the C (POSIX) locale if we have one, or fall back to POSIX # (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8) -if locale -a | grep C.UTF-8 >/dev/null; then +if locale -a | grep '^C.UTF-8$' >/dev/null; then export LC_ALL=C.UTF-8 -elif locale -a | grep C.utf8 >/dev/null; then +elif locale -a | grep '^C.utf8$' >/dev/null; then export LC_ALL=C.utf8 else export LC_ALL=C From f1d9196076d4aea1f64e0d2cbd17bfa2891b8c4c Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 30 May 2018 17:20:08 -0700 Subject: [PATCH 37/45] lib: Fix building against old glib versions This commit includes libglnx.h in ostree-autocleanups.h, so we get the g_autoptr backports wherever they're needed. Also, remove the "#include libglnx.h" lines elsewhere that are no longer needed. Closes: #1596 Approved by: cgwalters --- src/libostree/ostree-autocleanups.h | 1 + src/libostree/ostree-ref.c | 1 - src/libostree/ostree-repo-finder-config.c | 1 - src/libostree/ostree-repo-finder-mount.c | 1 - src/libostree/ostree-repo-finder-override.c | 1 - src/libostree/ostree-repo.c | 1 - tests/test-repo-finder-mount.c | 1 - 7 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index 504954e0..d1c94410 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -24,6 +24,7 @@ #pragma once #include +#include G_BEGIN_DECLS diff --git a/src/libostree/ostree-ref.c b/src/libostree/ostree-ref.c index 0954cb0e..9a410499 100644 --- a/src/libostree/ostree-ref.c +++ b/src/libostree/ostree-ref.c @@ -27,7 +27,6 @@ #include #include #include -#include #include "ostree-autocleanups.h" #include "ostree-core.h" diff --git a/src/libostree/ostree-repo-finder-config.c b/src/libostree/ostree-repo-finder-config.c index 5d1e1595..58cf321c 100644 --- a/src/libostree/ostree-repo-finder-config.c +++ b/src/libostree/ostree-repo-finder-config.c @@ -28,7 +28,6 @@ #include #include #include -#include #include "ostree-autocleanups.h" #include "ostree-remote-private.h" diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index 09e85035..117033df 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include "ostree-autocleanups.h" diff --git a/src/libostree/ostree-repo-finder-override.c b/src/libostree/ostree-repo-finder-override.c index 5bad9ace..0ee0270d 100644 --- a/src/libostree/ostree-repo-finder-override.c +++ b/src/libostree/ostree-repo-finder-override.c @@ -28,7 +28,6 @@ #include #include #include -#include #include "ostree-autocleanups.h" #include "ostree-remote-private.h" diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 3251880f..43c18767 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -28,7 +28,6 @@ #include #include #include -#include "libglnx.h" #include "otutil.h" #include diff --git a/tests/test-repo-finder-mount.c b/tests/test-repo-finder-mount.c index 55eb47fb..948a3245 100644 --- a/tests/test-repo-finder-mount.c +++ b/tests/test-repo-finder-mount.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include "libostreetest.h" From 8fbf19c9f54ba34ae7ebbc540b08566b8dd18848 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Tue, 22 May 2018 15:55:14 -0700 Subject: [PATCH 38/45] Make P2P API public (no longer experimental) Currently the API that allows P2P operations (e.g. pulling an ostree ref from a LAN or USB source) is hidden behind the configure flag --enable-experimental-api. This commit makes the API public and makes that flag essentially a no-op (leaving it in place in case we want to use it again in the future). The P2P API has been tested over the last several months and proven to work. This means that since we're no longer using the "experimental" feature flag, P2P builds of Flatpak will fail when using versions of OSTree from this commit onwards, until Flatpak is patched in the near future. If you want to build Flatpak < 0.11.8 with P2P enabled and link against OSTree 2018.6, you'll have to patch Flatpak. However, since Flatpak won't yet have a hard dependency on OSTree 2018.6, it needs a new way to determine if the P2P API in OSTree is available, so this commit adds a "p2p" feature flag. This way the feature set is more semantically correct than if we had continued to use the "experimental" feature flag. In addition to making the P2P API public, this commit makes the P2P unit tests run by default, removes the f27-experimental CI instance that's no longer needed, changes a few man pages to reflect the changes, and updates the bash completion script to accept the new commands and options. Closes: #1596 Approved by: cgwalters --- .papr-ex.yaml | 11 -- .papr.yml | 11 -- Makefile-libostree-defines.am | 5 - Makefile-libostree.am | 29 ++--- Makefile-man.am | 15 ++- Makefile-ostree.am | 4 +- Makefile-tests.am | 22 ++-- apidoc/ostree-experimental-sections.txt | 101 ------------------ apidoc/ostree-sections.txt | 102 ++++++++++++++++++ bash/ostree | 111 ++++++++++++++++++++ configure.ac | 3 + man/ostree-find-remotes.xml | 6 +- man/ostree-summary.xml | 2 - man/ostree.xml | 6 +- src/libostree/libostree-devel.sym | 49 +++++++++ src/libostree/libostree-experimental.sym | 97 ----------------- src/libostree/ostree-autocleanups.h | 2 - src/libostree/ostree-core-private.h | 28 ----- src/libostree/ostree-core.c | 1 + src/libostree/ostree-core.h | 6 +- src/libostree/ostree-ref.c | 14 +-- src/libostree/ostree-ref.h | 4 +- src/libostree/ostree-remote-private.h | 4 - src/libostree/ostree-remote.c | 12 +-- src/libostree/ostree-remote.h | 6 +- src/libostree/ostree-repo-commit.c | 4 +- src/libostree/ostree-repo-finder-avahi.c | 8 +- src/libostree/ostree-repo-finder-config.c | 4 +- src/libostree/ostree-repo-finder-mount.c | 6 +- src/libostree/ostree-repo-finder-override.c | 6 +- src/libostree/ostree-repo-finder.c | 18 ++-- src/libostree/ostree-repo-finder.h | 4 +- src/libostree/ostree-repo-private.h | 29 ----- src/libostree/ostree-repo-pull.c | 32 +----- src/libostree/ostree-repo-refs.c | 10 +- src/libostree/ostree-repo.c | 4 +- src/libostree/ostree-repo.h | 28 +---- src/libostree/ostree-types.h | 3 - src/libostree/ostree.h | 6 -- src/ostree/main.c | 2 - src/ostree/ot-builtin-commit.c | 4 - src/ostree/ot-builtin-fsck.c | 6 -- src/ostree/ot-builtin-init.c | 6 -- src/ostree/ot-builtin-prune.c | 4 - src/ostree/ot-builtin-refs.c | 8 -- src/ostree/ot-builtin-summary.c | 9 -- src/ostree/ot-builtins.h | 2 - src/ostree/ot-remote-builtin-add.c | 6 -- 48 files changed, 347 insertions(+), 513 deletions(-) delete mode 100644 src/libostree/libostree-experimental.sym diff --git a/.papr-ex.yaml b/.papr-ex.yaml index e4370fcf..3ebbdb40 100644 --- a/.papr-ex.yaml +++ b/.papr-ex.yaml @@ -81,17 +81,6 @@ tests: inherit: true -context: f28-experimental-api -env: - CONFIGOPTS: '--enable-experimental-api' - -tests: - - ci/build-check.sh - ---- - -inherit: true - context: f28-minimal env: CONFIGOPTS: '--without-curl --without-soup --disable-gtk-doc --disable-man diff --git a/.papr.yml b/.papr.yml index ab639021..b99a1f70 100644 --- a/.papr.yml +++ b/.papr.yml @@ -86,17 +86,6 @@ tests: inherit: true -context: f28-experimental-api -env: - CONFIGOPTS: '--enable-experimental-api' - -tests: - - ci/build-check.sh - ---- - -inherit: true - context: f28-minimal env: CONFIGOPTS: '--without-curl --without-soup --disable-gtk-doc --disable-man diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index 58f77a2c..77748a41 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -38,10 +38,6 @@ libostree_public_headers = \ src/libostree/ostree-deployment.h \ src/libostree/ostree-bootconfig-parser.h \ src/libostree/ostree-repo-deprecated.h \ - $(NULL) - -if ENABLE_EXPERIMENTAL_API -libostree_public_headers += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ src/libostree/ostree-repo-finder.h \ @@ -50,7 +46,6 @@ libostree_public_headers += \ src/libostree/ostree-repo-finder-mount.h \ src/libostree/ostree-repo-finder-override.h \ $(NULL) -endif # This one is generated via configure.ac, and the gtk-doc # code hence needs to look in the builddir. diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 9dc81b2f..01a209d1 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -140,6 +140,13 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-gpg-verify-result.c \ src/libostree/ostree-gpg-verify-result-private.h \ src/libostree/ostree-autocleanups.h \ + src/libostree/ostree-bloom.c \ + src/libostree/ostree-bloom-private.h \ + src/libostree/ostree-repo-finder.c \ + src/libostree/ostree-repo-finder-avahi.c \ + src/libostree/ostree-repo-finder-config.c \ + src/libostree/ostree-repo-finder-mount.c \ + src/libostree/ostree-repo-finder-override.c \ $(NULL) if USE_LIBARCHIVE libostree_1_la_SOURCES += src/libostree/ostree-libarchive-input-stream.h \ @@ -154,26 +161,13 @@ libostree_1_la_SOURCES += \ $(NULL) endif libostree_experimental_headers = \ - src/libostree/ostree-ref.h \ - src/libostree/ostree-remote.h \ - src/libostree/ostree-repo-finder.h \ - src/libostree/ostree-repo-finder-avahi.h \ - src/libostree/ostree-repo-finder-config.h \ - src/libostree/ostree-repo-finder-mount.h \ - src/libostree/ostree-repo-finder-override.h \ $(NULL) if !ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += $(libostree_experimental_headers) else # if ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += \ - src/libostree/ostree-bloom.c \ - src/libostree/ostree-bloom-private.h \ - src/libostree/ostree-repo-finder.c \ - src/libostree/ostree-repo-finder-avahi.c \ - src/libostree/ostree-repo-finder-config.c \ - src/libostree/ostree-repo-finder-mount.c \ - src/libostree/ostree-repo-finder-override.c \ $(NULL) +endif if USE_AVAHI libostree_1_la_SOURCES += \ @@ -181,20 +175,15 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-repo-finder-avahi-private.h \ $(NULL) endif # USE_AVAHI -endif symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym if BUILDOPT_IS_DEVEL_BUILD symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym endif -if ENABLE_EXPERIMENTAL_API -symbol_files += $(top_srcdir)/src/libostree/libostree-experimental.sym -endif # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html wl_versionscript_arg = -Wl,--version-script= EXTRA_DIST += \ $(top_srcdir)/src/libostree/libostree-devel.sym \ - $(top_srcdir)/src/libostree/libostree-experimental.sym \ $(top_srcdir)/src/libostree/libostree-released.sym \ $(NULL) @@ -216,12 +205,10 @@ libostree_1_la_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_LIBARCHIVE_LIBS) endif -if ENABLE_EXPERIMENTAL_API if USE_AVAHI libostree_1_la_CFLAGS += $(OT_DEP_AVAHI_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_AVAHI_LIBS) endif -endif if BUILDOPT_LIBSYSTEMD libostree_1_la_CFLAGS += $(LIBSYSTEMD_CFLAGS) diff --git a/Makefile-man.am b/Makefile-man.am index 342af520..8ccbba8c 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -28,11 +28,12 @@ ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 \ ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin-unlock.1 \ ostree-admin-pin.1 \ ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 \ -ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 \ -ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 \ -ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 \ -ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 \ -ostree-summary.1 ostree-static-delta.1 +ostree-commit.1 ostree-create-usb.1 ostree-export.1 ostree-gpg-sign.1 \ +ostree-config.1 ostree-diff.1 ostree-find-remotes.1 ostree-fsck.1 \ +ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 \ +ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 \ +ostree-rev-parse.1 ostree-show.1 ostree-summary.1 \ +ostree-static-delta.1 if BUILDOPT_TRIVIAL_HTTPD man1_files += ostree-trivial-httpd.1 else @@ -44,10 +45,6 @@ if BUILDOPT_FUSE man1_files += rofiles-fuse.1 endif -if ENABLE_EXPERIMENTAL_API -man1_files += ostree-find-remotes.1 ostree-create-usb.1 -endif - man5_files = ostree.repo.5 ostree.repo-config.5 man1_MANS = $(addprefix man/,$(man1_files)) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 91d8383d..8d352e38 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -29,8 +29,10 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-checkout.c \ src/ostree/ot-builtin-checksum.c \ src/ostree/ot-builtin-commit.c \ + src/ostree/ot-builtin-create-usb.c \ src/ostree/ot-builtin-diff.c \ src/ostree/ot-builtin-export.c \ + src/ostree/ot-builtin-find-remotes.c \ src/ostree/ot-builtin-fsck.c \ src/ostree/ot-builtin-gpg-sign.c \ src/ostree/ot-builtin-init.c \ @@ -60,8 +62,6 @@ nodist_ostree_SOURCES = \ if ENABLE_EXPERIMENTAL_API ostree_SOURCES += \ - src/ostree/ot-builtin-create-usb.c \ - src/ostree/ot-builtin-find-remotes.c \ $(NULL) endif diff --git a/Makefile-tests.am b/Makefile-tests.am index 48363b9a..1bdad4c1 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -54,7 +54,11 @@ dist_uninstalled_test_scripts = tests/test-symbols.sh tests/coccinelle.sh # This overrides the glib-tap.mk emphasis on doing both, if we'd # used e.g. `dist_test_scripts`. dist_test_scripts = $(NULL) -test_programs = $(NULL) +test_programs = \ + tests/test-bloom \ + tests/test-repo-finder-config \ + tests/test-repo-finder-mount \ + $(NULL) _installed_or_uninstalled_test_scripts = \ tests/test-basic.sh \ tests/test-basic-user.sh \ @@ -119,9 +123,6 @@ _installed_or_uninstalled_test_scripts = \ tests/test-summary-update.sh \ tests/test-summary-view.sh \ tests/test-no-initramfs.sh \ - $(NULL) - -experimental_test_scripts = \ tests/test-create-usb.sh \ tests/test-find-remotes.sh \ tests/test-fsck-collections.sh \ @@ -133,8 +134,12 @@ experimental_test_scripts = \ tests/test-summary-collections.sh \ tests/test-pull-collections.sh \ $(NULL) + +experimental_test_scripts = \ + $(NULL) test_extra_programs = \ tests/get-byte-order \ + tests/repo-finder-mount \ $(NULL) tests_get_byte_order_SOURCES = tests/get-byte-order.c @@ -147,7 +152,6 @@ tests_repo_finder_mount_LDADD = $(common_tests_ldadd) libostreetest.la if ENABLE_EXPERIMENTAL_API _installed_or_uninstalled_test_scripts += $(experimental_test_scripts) -test_extra_programs += tests/repo-finder-mount else EXTRA_DIST += $(experimental_test_scripts) endif @@ -240,17 +244,9 @@ _installed_or_uninstalled_test_programs = tests/test-varint tests/test-ot-unix-u tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum \ tests/test-basic-c tests/test-sysroot-c tests/test-pull-c tests/test-repo -if ENABLE_EXPERIMENTAL_API -test_programs += \ - tests/test-bloom \ - tests/test-repo-finder-config \ - tests/test-repo-finder-mount \ - $(NULL) - if USE_AVAHI test_programs += tests/test-repo-finder-avahi endif -endif # An interactive tool noinst_PROGRAMS += tests/test-rollsum-cli diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 60daaca5..e69de29b 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -1,101 +0,0 @@ -
-ostree-ref -OstreeCollectionRef -ostree_collection_ref_new -ostree_collection_ref_dup -ostree_collection_ref_free -ostree_collection_ref_hash -ostree_collection_ref_equal -OstreeCollectionRefv -ostree_collection_ref_dupv -ostree_collection_ref_freev - -ostree_collection_ref_get_type -
- -
-ostree-remote -OstreeRemote -ostree_remote_ref -ostree_remote_unref -ostree_remote_get_name -ostree_remote_get_url - -ostree_remote_get_type -
- -
-ostree-repo-experimental -ostree_repo_find_remotes_async -ostree_repo_find_remotes_finish -ostree_repo_pull_from_remotes_async -ostree_repo_pull_from_remotes_finish -ostree_repo_resolve_keyring_for_collection -OSTREE_REPO_METADATA_REF -
- -
-ostree-repo-finder -OstreeRepoFinder -ostree_repo_finder_resolve_async -ostree_repo_finder_resolve_finish -ostree_repo_finder_resolve_all_async -ostree_repo_finder_resolve_all_finish -OstreeRepoFinderResult -ostree_repo_finder_result_new -ostree_repo_finder_result_dup -ostree_repo_finder_result_free -ostree_repo_finder_result_compare -OstreeRepoFinderResultv -ostree_repo_finder_result_freev - -ostree_repo_finder_get_type -ostree_repo_finder_result_get_type -
- -
-ostree-repo-finder-avahi -OstreeRepoFinderAvahi -ostree_repo_finder_avahi_new -ostree_repo_finder_avahi_start -ostree_repo_finder_avahi_stop - -ostree_repo_finder_avahi_get_type -
- -
-ostree-repo-finder-config -OstreeRepoFinderConfig -ostree_repo_finder_config_new - -ostree_repo_finder_config_get_type -
- -
-ostree-repo-finder-mount -OstreeRepoFinderMount -ostree_repo_finder_mount_new - -ostree_repo_finder_mount_get_type -
- -
-ostree-repo-finder-override -OstreeRepoFinderOverride -ostree_repo_finder_override_new -ostree_repo_finder_override_add_uri - -ostree_repo_finder_override_get_type -
- -
-ostree-misc-experimental -ostree_repo_get_collection_id -ostree_repo_set_collection_id -ostree_validate_collection_id -ostree_repo_list_collection_refs -ostree_repo_remote_list_collection_refs -ostree_repo_set_collection_ref_immediate -ostree_repo_transaction_set_collection_ref -ostree_repo_resolve_collection_ref -
diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index ad1db32c..74c1fba0 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -558,3 +558,105 @@ OSTREE_TYPE_SYSROOT_UPGRADER ostree_sysroot_upgrader_get_type ostree_sysroot_upgrader_flags_get_type + +
+ostree-ref +OstreeCollectionRef +ostree_collection_ref_new +ostree_collection_ref_dup +ostree_collection_ref_free +ostree_collection_ref_hash +ostree_collection_ref_equal +OstreeCollectionRefv +ostree_collection_ref_dupv +ostree_collection_ref_freev + +ostree_collection_ref_get_type +
+ +
+ostree-remote +OstreeRemote +ostree_remote_ref +ostree_remote_unref +ostree_remote_get_name +ostree_remote_get_url + +ostree_remote_get_type +
+ +
+ostree-repo-experimental +ostree_repo_find_remotes_async +ostree_repo_find_remotes_finish +ostree_repo_pull_from_remotes_async +ostree_repo_pull_from_remotes_finish +ostree_repo_resolve_keyring_for_collection +OSTREE_REPO_METADATA_REF +
+ +
+ostree-repo-finder +OstreeRepoFinder +ostree_repo_finder_resolve_async +ostree_repo_finder_resolve_finish +ostree_repo_finder_resolve_all_async +ostree_repo_finder_resolve_all_finish +OstreeRepoFinderResult +ostree_repo_finder_result_new +ostree_repo_finder_result_dup +ostree_repo_finder_result_free +ostree_repo_finder_result_compare +OstreeRepoFinderResultv +ostree_repo_finder_result_freev + +ostree_repo_finder_get_type +ostree_repo_finder_result_get_type +
+ +
+ostree-repo-finder-avahi +OstreeRepoFinderAvahi +ostree_repo_finder_avahi_new +ostree_repo_finder_avahi_start +ostree_repo_finder_avahi_stop + +ostree_repo_finder_avahi_get_type +
+ +
+ostree-repo-finder-config +OstreeRepoFinderConfig +ostree_repo_finder_config_new + +ostree_repo_finder_config_get_type +
+ +
+ostree-repo-finder-mount +OstreeRepoFinderMount +ostree_repo_finder_mount_new + +ostree_repo_finder_mount_get_type +
+ +
+ostree-repo-finder-override +OstreeRepoFinderOverride +ostree_repo_finder_override_new +ostree_repo_finder_override_add_uri + +ostree_repo_finder_override_get_type +
+ +
+ostree-misc-experimental +ostree_repo_get_collection_id +ostree_repo_set_collection_id +ostree_validate_collection_id +ostree_repo_list_collection_refs +ostree_repo_remote_list_collection_refs +ostree_repo_set_collection_ref_immediate +ostree_repo_transaction_set_collection_ref +ostree_repo_resolve_collection_ref +
diff --git a/bash/ostree b/bash/ostree index 46baa51b..f3aef686 100644 --- a/bash/ostree +++ b/bash/ostree @@ -120,12 +120,25 @@ __ostree_compreply_oses() { fi } +# Find refs associated with the given collection ID under $repo_path. +__ostree_compreply_collection_refs() { + local collection_id=( $1 ) + refs=$( ostree refs --repo $repo_path --collections $collection_id 2>/dev/null | cut -d ' ' -f 2 | sed -e 's/)$//' ) + COMPREPLY+=( $( compgen -W "$refs" -- "$cur" ) ) +} + # Find refs under $repo_path. __ostree_compreply_refs() { refs=$( ostree refs --repo $repo_path 2>/dev/null ) COMPREPLY+=( $( compgen -W "$refs" -- "$cur" ) ) } +# Find collection IDs under $repo_path. +__ostree_compreply_collection_ids() { + collection_ids=$( ostree refs --repo $repo_path --collections 2>/dev/null | cut -d ',' -f 1 | sed -e 's/^(//' | sort | uniq ) + COMPREPLY+=( $( compgen -W "$collection_ids" -- "$cur" ) ) +} + # Find remotes under $repo_path. __ostree_compreply_remotes() { remotes=$( ostree remote list --repo $repo_path 2> /dev/null ) @@ -410,6 +423,49 @@ _ostree_config() { return 0 } +_ostree_create_usb() { + local boolean_options=" + $main_boolean_options + --disable-fsync + " + + local options_with_args=" + --destination-repo + --repo + " + + local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) + + case "$prev" in + --destination-repo|--repo) + __ostree_compreply_dirs_only + return 0 + ;; + $options_with_args_glob ) + return 0 + ;; + esac + + case "$cur" in + -*) + local all_options="$boolean_options $options_with_args" + __ostree_compreply_all_options + ;; + *) + local argpos=$( __ostree_pos_first_nonflag $( __ostree_to_alternatives "$options_with_args" ) ) + + if [ $cword -eq $argpos ]; then + __ostree_compreply_dirs_only + elif [ $cword -gt $argpos ] && [ $(((cword - argpos) % 2)) -eq 1 ]; then + __ostree_compreply_collection_ids + elif [ $cword -gt $argpos ] && [ $(((cword - argpos) % 2)) -eq 0 ]; then + __ostree_compreply_collection_refs "${words[$cword - 1]}" + fi + esac + + return 0 +} + _ostree_diff() { local boolean_options=" $main_boolean_options @@ -495,6 +551,56 @@ _ostree_export() { return 0 } +_ostree_find_remotes() { + local boolean_options=" + $main_boolean_options + --disable-fsync + --pull + " + + local options_with_args=" + --finders + --repo + " + + local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) + + case "$prev" in + --finders) + local choices="config lan mount" + local config_first="config,lan config,mount config,lan,mount config,mount,lan" + local lan_first="lan,config lan,mount lan,config,mount lan,mount,config" + local mount_first="mount,config mount,lan mount,lan,config mount,config,lan" + COMPREPLY+=( $( compgen -W "$choices $config_first $lan_first $mount_first" -- "$cur" ) ) + return 0 + ;; + --repo) + __ostree_compreply_dirs_only + return 0 + ;; + $options_with_args_glob ) + return 0 + ;; + esac + + case "$cur" in + -*) + local all_options="$boolean_options $options_with_args" + __ostree_compreply_all_options + ;; + *) + local argpos=$( __ostree_pos_first_nonflag $( __ostree_to_alternatives "$options_with_args" ) ) + + if [ $cword -ge $argpos ] && [ $(((cword - argpos) % 2)) -eq 0 ]; then + __ostree_compreply_collection_ids + elif [ $cword -ge $argpos ] && [ $(((cword - argpos) % 2)) -eq 1 ]; then + __ostree_compreply_collection_refs "${words[$cword - 1]}" + fi + esac + + return 0 +} + _ostree_fsck() { local boolean_options=" $main_boolean_options @@ -576,6 +682,7 @@ _ostree_init() { " local options_with_args=" + --collection-id --mode --repo " @@ -839,6 +946,7 @@ _ostree_refs() { local boolean_options=" $main_boolean_options --alias -A + --collections -c --delete --list " @@ -878,6 +986,7 @@ _ostree_remote_add() { " local options_with_args=" + --collection-id --contenturl --gpg-import --repo @@ -1615,8 +1724,10 @@ _ostree() { checksum commit config + create-usb diff export + find-remotes fsck gpg-sign init diff --git a/configure.ac b/configure.ac index 5fd95da4..1f6ea34c 100644 --- a/configure.ac +++ b/configure.ac @@ -572,6 +572,9 @@ AM_COND_IF([BUILDOPT_IS_DEVEL_BUILD], release_build_type=release) OSTREE_FEATURES="$OSTREE_FEATURES $release_build_type" +# P2P API is public in OSTree >= 2018.6 +OSTREE_FEATURES="$OSTREE_FEATURES p2p" + AC_CONFIG_FILES([ Makefile apidoc/Makefile diff --git a/man/ostree-find-remotes.xml b/man/ostree-find-remotes.xml index 15f5a516..b3796c31 100644 --- a/man/ostree-find-remotes.xml +++ b/man/ostree-find-remotes.xml @@ -58,13 +58,11 @@ Boston, MA 02111-1307, USA. Description - OSTree has the ability do pulls not just from configured remote servers but also from peer computers on the LAN and from mounted - filesystems such as USB drives. Currently this requires compiling - OSTree with experimental API enabled, and it requires the use of - collection IDs and GPG verification. + filesystems such as USB drives. This functionality requires the use + of collection IDs and GPG verification. The find-remotes command searches for remotes diff --git a/man/ostree-summary.xml b/man/ostree-summary.xml index d0f92882..387dacd7 100644 --- a/man/ostree-summary.xml +++ b/man/ostree-summary.xml @@ -84,12 +84,10 @@ Boston, MA 02111-1307, USA. fields to the summary. - diff --git a/man/ostree.xml b/man/ostree.xml index f8d3e2fc..d1c15665 100644 --- a/man/ostree.xml +++ b/man/ostree.xml @@ -283,14 +283,13 @@ Boston, MA 02111-1307, USA. - + ostree-diff1 @@ -301,14 +300,13 @@ Boston, MA 02111-1307, USA. - + ostree-fsck1 diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 06544bb6..832ab84c 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,8 +19,57 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2018.6 { +global: + ostree_collection_ref_dup; + ostree_collection_ref_dupv; + ostree_collection_ref_equal; + ostree_collection_ref_free; + ostree_collection_ref_freev; + ostree_collection_ref_get_type; + ostree_collection_ref_hash; + ostree_collection_ref_new; + ostree_remote_get_name; + ostree_remote_get_type; + ostree_remote_get_url; + ostree_remote_ref; + ostree_remote_unref; + ostree_repo_find_remotes_async; + ostree_repo_find_remotes_finish; + ostree_repo_finder_avahi_get_type; + ostree_repo_finder_avahi_new; + ostree_repo_finder_avahi_start; + ostree_repo_finder_avahi_stop; + ostree_repo_finder_config_get_type; + ostree_repo_finder_config_new; + ostree_repo_finder_get_type; + ostree_repo_finder_mount_get_type; + ostree_repo_finder_mount_new; + ostree_repo_finder_override_add_uri; + ostree_repo_finder_override_get_type; + ostree_repo_finder_override_new; + ostree_repo_finder_resolve_all_async; + ostree_repo_finder_resolve_all_finish; + ostree_repo_finder_resolve_async; + ostree_repo_finder_resolve_finish; + ostree_repo_finder_result_compare; + ostree_repo_finder_result_dup; + ostree_repo_finder_result_free; + ostree_repo_finder_result_freev; + ostree_repo_finder_result_get_type; + ostree_repo_finder_result_new; + ostree_repo_get_collection_id; + ostree_repo_list_collection_refs; + ostree_repo_pull_from_remotes_async; + ostree_repo_pull_from_remotes_finish; + ostree_repo_remote_list_collection_refs; + ostree_repo_resolve_collection_ref; + ostree_repo_resolve_keyring_for_collection; + ostree_repo_set_collection_id; + ostree_repo_set_collection_ref_immediate; + ostree_repo_transaction_set_collection_ref; ostree_repo_traverse_reachable_refs; ostree_sysroot_cleanup_prune_repo; + ostree_validate_collection_id; } LIBOSTREE_2018.5; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym deleted file mode 100644 index b83ad1b0..00000000 --- a/src/libostree/libostree-experimental.sym +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright © 2017 Endless Mobile, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 02111-1307, USA. - * - * Authors: - * - Philip Withnall - */ - -/* Symbols in this file are added to the build if OSTree is configured with - * --enable-experimental-api. They are not stable or officially supported, and - * might disappear or change in future releases. */ - -LIBOSTREE_2017.6_EXPERIMENTAL { -global: - ostree_remote_ref; - ostree_remote_unref; -} LIBOSTREE_2017.6; - -LIBOSTREE_2017.7_EXPERIMENTAL { -global: - ostree_remote_get_name; -} LIBOSTREE_2017.6_EXPERIMENTAL; - -LIBOSTREE_2017.8_EXPERIMENTAL { -global: - ostree_collection_ref_dup; - ostree_collection_ref_dupv; - ostree_collection_ref_equal; - ostree_collection_ref_free; - ostree_collection_ref_freev; - ostree_collection_ref_get_type; - ostree_collection_ref_hash; - ostree_collection_ref_new; - ostree_repo_find_remotes_async; - ostree_repo_find_remotes_finish; - ostree_repo_finder_avahi_get_type; - ostree_repo_finder_avahi_new; - ostree_repo_finder_avahi_start; - ostree_repo_finder_avahi_stop; - ostree_repo_finder_config_get_type; - ostree_repo_finder_config_new; - ostree_repo_finder_get_type; - ostree_repo_finder_mount_get_type; - ostree_repo_finder_mount_new; - ostree_repo_finder_resolve_async; - ostree_repo_finder_resolve_all_async; - ostree_repo_finder_resolve_all_finish; - ostree_repo_finder_resolve_finish; - ostree_repo_finder_result_compare; - ostree_repo_finder_result_dup; - ostree_repo_finder_result_free; - ostree_repo_finder_result_freev; - ostree_repo_finder_result_get_type; - ostree_repo_finder_result_new; - ostree_repo_get_collection_id; - ostree_repo_list_collection_refs; - ostree_repo_pull_from_remotes_async; - ostree_repo_pull_from_remotes_finish; - ostree_repo_remote_list_collection_refs; - ostree_repo_resolve_keyring_for_collection; - ostree_repo_set_collection_id; - ostree_repo_set_collection_ref_immediate; - ostree_repo_transaction_set_collection_ref; - ostree_validate_collection_id; -} LIBOSTREE_2017.7_EXPERIMENTAL; - -LIBOSTREE_2017.12_EXPERIMENTAL { -global: - ostree_repo_resolve_collection_ref; -} LIBOSTREE_2017.8_EXPERIMENTAL; - -LIBOSTREE_2017.13_EXPERIMENTAL { -global: - ostree_repo_finder_override_add_uri; - ostree_repo_finder_override_get_type; - ostree_repo_finder_override_new; -} LIBOSTREE_2017.12_EXPERIMENTAL; - -LIBOSTREE_2017.14_EXPERIMENTAL { -global: - ostree_remote_get_type; - ostree_remote_get_url; -} LIBOSTREE_2017.13_EXPERIMENTAL; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index d1c94410..ae678721 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -61,7 +61,6 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSysrootUpgrader, g_object_unref) G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_commit_traverse_iter_clear) -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) @@ -72,7 +71,6 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderMount, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderOverride, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #endif diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index b5f65d08..dd2cbc7e 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -207,10 +207,6 @@ _ostree_raw_file_to_archive_stream (GInputStream *input, GCancellable *cancellable, GError **error); -#ifndef OSTREE_ENABLE_EXPERIMENTAL_API -gboolean ostree_validate_collection_id (const char *collection_id, GError **error); -#endif /* !OSTREE_ENABLE_EXPERIMENTAL_API */ - gboolean _ostree_compare_timestamps (const char *current_rev, guint64 current_ts, @@ -218,28 +214,4 @@ _ostree_compare_timestamps (const char *current_rev, guint64 new_ts, GError **error); -#if (defined(OSTREE_COMPILATION) || GLIB_CHECK_VERSION(2, 44, 0)) && !defined(OSTREE_ENABLE_EXPERIMENTAL_API) -#include -#include "ostree-ref.h" -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) -G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) - -#include "ostree-repo-finder.h" -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) -G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) - -#include "ostree-repo-finder-avahi.h" -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderAvahi, g_object_unref) - -#include "ostree-repo-finder-config.h" -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref) - -#include "ostree-repo-finder-mount.h" -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderMount, g_object_unref) - -#include "ostree-repo-finder-override.h" -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderOverride, g_object_unref) -#endif - G_END_DECLS diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 33d6a48b..080137f3 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -302,6 +302,7 @@ ostree_validate_remote_name (const char *remote_name, * * Returns: %TRUE if @collection_id is a valid collection ID, %FALSE if it is invalid * or %NULL + * Since: 2018.6 */ gboolean ostree_validate_collection_id (const char *collection_id, GError **error) diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index b65c9ba9..08b7d451 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -271,11 +271,9 @@ typedef enum { * This is most useful in concert with `OSTREE_COMMIT_META_KEY_REF_BINDING`, * as it more strongly binds the commit to the repository and branch. * - * Since: 2017.9 + * Since: 2018.6 */ -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API #define OSTREE_COMMIT_META_KEY_COLLECTION_BINDING "ostree.collection-binding" -#endif _OSTREE_PUBLIC const GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype); @@ -324,10 +322,8 @@ int ostree_cmp_checksum_bytes (const guchar *a, const guchar *b); _OSTREE_PUBLIC gboolean ostree_validate_rev (const char *rev, GError **error); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API _OSTREE_PUBLIC gboolean ostree_validate_collection_id (const char *collection_id, GError **error); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ _OSTREE_PUBLIC gboolean ostree_validate_remote_name (const char *remote_name, GError **error); diff --git a/src/libostree/ostree-ref.c b/src/libostree/ostree-ref.c index 9a410499..34d67240 100644 --- a/src/libostree/ostree-ref.c +++ b/src/libostree/ostree-ref.c @@ -47,7 +47,7 @@ G_DEFINE_BOXED_TYPE (OstreeCollectionRef, ostree_collection_ref, * operations. * * Returns: (transfer full): a new #OstreeCollectionRef - * Since: 2017.8 + * Since: 2018.6 */ OstreeCollectionRef * ostree_collection_ref_new (const gchar *collection_id, @@ -73,7 +73,7 @@ ostree_collection_ref_new (const gchar *collection_id, * Create a copy of the given @ref. * * Returns: (transfer full): a newly allocated copy of @ref - * Since: 2017.8 + * Since: 2018.6 */ OstreeCollectionRef * ostree_collection_ref_dup (const OstreeCollectionRef *ref) @@ -89,7 +89,7 @@ ostree_collection_ref_dup (const OstreeCollectionRef *ref) * * Free the given @ref. * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_collection_ref_free (OstreeCollectionRef *ref) @@ -109,7 +109,7 @@ ostree_collection_ref_free (OstreeCollectionRef *ref) * @ref must be non-%NULL. * * Returns: hash value for @ref - * Since: 2017.8 + * Since: 2018.6 */ guint ostree_collection_ref_hash (gconstpointer ref) @@ -131,7 +131,7 @@ ostree_collection_ref_hash (gconstpointer ref) * ref name, and %FALSE otherwise. Both @ref1 and @ref2 must be non-%NULL. * * Returns: %TRUE if @ref1 and @ref2 are equal, %FALSE otherwise - * Since: 2017.8 + * Since: 2018.6 */ gboolean ostree_collection_ref_equal (gconstpointer ref1, @@ -152,7 +152,7 @@ ostree_collection_ref_equal (gconstpointer ref1, * %NULL. * * Returns: (transfer full) (array zero-terminated=1): a newly allocated copy of @refs - * Since: 2017.8 + * Since: 2018.6 */ OstreeCollectionRef ** ostree_collection_ref_dupv (const OstreeCollectionRef * const *refs) @@ -178,7 +178,7 @@ ostree_collection_ref_dupv (const OstreeCollectionRef * const *refs) * Free the given array of @refs, including freeing all its elements. @refs * must be %NULL-terminated; it may be empty, but must not be %NULL. * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_collection_ref_freev (OstreeCollectionRef **refs) diff --git a/src/libostree/ostree-ref.h b/src/libostree/ostree-ref.h index 7aa57fde..8df685ed 100644 --- a/src/libostree/ostree-ref.h +++ b/src/libostree/ostree-ref.h @@ -42,7 +42,7 @@ G_BEGIN_DECLS * (@collection_id, @ref_name). For backwards compatibility, @collection_id may be %NULL, * indicating a ref name which is not globally unique. * - * Since: 2017.8 + * Since: 2018.6 */ typedef struct { @@ -82,7 +82,7 @@ void ostree_collection_ref_freev (OstreeCollectionRef **refs); * g_auto(OstreeCollectionRefv) refs = NULL; * ]| * - * Since: 2017.8 + * Since: 2018.6 */ typedef OstreeCollectionRef** OstreeCollectionRefv; diff --git a/src/libostree/ostree-remote-private.h b/src/libostree/ostree-remote-private.h index 5cdc5ff3..061412de 100644 --- a/src/libostree/ostree-remote-private.h +++ b/src/libostree/ostree-remote-private.h @@ -60,8 +60,4 @@ G_GNUC_INTERNAL OstreeRemote *ostree_remote_new_from_keyfile (GKeyFile *keyfile, const gchar *group); -#if (defined(OSTREE_COMPILATION) || GLIB_CHECK_VERSION(2, 44, 0)) && !defined(OSTREE_ENABLE_EXPERIMENTAL_API) -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) -#endif - G_END_DECLS diff --git a/src/libostree/ostree-remote.c b/src/libostree/ostree-remote.c index da325d18..2b068e15 100644 --- a/src/libostree/ostree-remote.c +++ b/src/libostree/ostree-remote.c @@ -50,7 +50,7 @@ * refs are currently on a remote, or the commits they currently point to. Use * #OstreeRepo in combination with an #OstreeRemote to query that information. * - * Since: 2017.6 + * Since: 2018.6 */ OstreeRemote * @@ -119,7 +119,7 @@ ostree_remote_new_from_keyfile (GKeyFile *keyfile, * Increase the reference count on the given @remote. * * Returns: (transfer full): a copy of @remote, for convenience - * Since: 2017.6 + * Since: 2018.6 */ OstreeRemote * ostree_remote_ref (OstreeRemote *remote) @@ -138,7 +138,7 @@ ostree_remote_ref (OstreeRemote *remote) * Decrease the reference count on the given @remote and free it if the * reference count reaches 0. * - * Since: 2017.6 + * Since: 2018.6 */ void ostree_remote_unref (OstreeRemote *remote) @@ -158,11 +158,9 @@ ostree_remote_unref (OstreeRemote *remote) } } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API G_DEFINE_BOXED_TYPE(OstreeRemote, ostree_remote, ostree_remote_ref, ostree_remote_unref); -#endif /** * ostree_remote_get_name: @@ -173,7 +171,7 @@ G_DEFINE_BOXED_TYPE(OstreeRemote, ostree_remote, * arbitrary, string. * * Returns: remote’s name - * Since: 2017.7 + * Since: 2018.6 */ const gchar * ostree_remote_get_name (OstreeRemote *remote) @@ -191,7 +189,7 @@ ostree_remote_get_name (OstreeRemote *remote) * Get the URL from the remote. * * Returns: (transfer full): the remote's URL - * Since: 2017.14 + * Since: 2018.6 */ gchar * ostree_remote_get_url (OstreeRemote *remote) diff --git a/src/libostree/ostree-remote.h b/src/libostree/ostree-remote.h index 285f5900..a785b4e6 100644 --- a/src/libostree/ostree-remote.h +++ b/src/libostree/ostree-remote.h @@ -42,12 +42,8 @@ G_BEGIN_DECLS * remotes can only be passed around as (reference counted) opaque handles. In * future, more API may be added to create and interrogate them. * - * Since: 2017.6 + * Since: 2018.6 */ -#ifndef OSTREE_ENABLE_EXPERIMENTAL_API -/* This is in ostree-types.h otherwise */ -typedef struct OstreeRemote OstreeRemote; -#endif _OSTREE_PUBLIC GType ostree_remote_get_type (void) G_GNUC_CONST; diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 86ee5e30..8285a1a0 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1961,7 +1961,7 @@ ostree_repo_transaction_set_ref (OstreeRepo *self, * * Multithreading: Since v2017.15 this function is MT safe. * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_transaction_set_collection_ref (OstreeRepo *self, @@ -2045,7 +2045,7 @@ ostree_repo_set_alias_ref_immediate (OstreeRepo *self, * case where we're creating or overwriting an existing ref. * * Returns: %TRUE on success, %FALSE otherwise - * Since: 2017.8 + * Since: 2018.6 */ gboolean ostree_repo_set_collection_ref_immediate (OstreeRepo *self, diff --git a/src/libostree/ostree-repo-finder-avahi.c b/src/libostree/ostree-repo-finder-avahi.c index fcf62048..e48b60f4 100644 --- a/src/libostree/ostree-repo-finder-avahi.c +++ b/src/libostree/ostree-repo-finder-avahi.c @@ -98,7 +98,7 @@ * and the resolver is used to retrieve information about services advertised by * each peer, including the services’ TXT records. * - * Since: 2017.8 + * Since: 2018.6 */ #ifdef HAVE_AVAHI @@ -1363,7 +1363,7 @@ ostree_repo_finder_avahi_init (OstreeRepoFinderAvahi *self) * If @context is %NULL, the current thread-default #GMainContext is used. * * Returns: (transfer full): a new #OstreeRepoFinderAvahi - * Since: 2017.8 + * Since: 2018.6 */ OstreeRepoFinderAvahi * ostree_repo_finder_avahi_new (GMainContext *context) @@ -1412,7 +1412,7 @@ ostree_repo_finder_avahi_new (GMainContext *context) * #OstreeRepoFinderAvahi instance, or to call it after * ostree_repo_finder_avahi_stop(). * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_finder_avahi_start (OstreeRepoFinderAvahi *self, @@ -1494,7 +1494,7 @@ static gboolean stop_cb (gpointer user_data); * #OstreeRepoFinderAvahi instance, or to call it before * ostree_repo_finder_avahi_start(). * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_finder_avahi_stop (OstreeRepoFinderAvahi *self) diff --git a/src/libostree/ostree-repo-finder-config.c b/src/libostree/ostree-repo-finder-config.c index 58cf321c..8e617afc 100644 --- a/src/libostree/ostree-repo-finder-config.c +++ b/src/libostree/ostree-repo-finder-config.c @@ -55,7 +55,7 @@ * intersection is non-empty, that remote is returned as a result. Remotes which * do not have their `collection-id` key configured are ignored. * - * Since: 2017.8 + * Since: 2018.6 */ static void ostree_repo_finder_config_iface_init (OstreeRepoFinderInterface *iface); @@ -233,7 +233,7 @@ ostree_repo_finder_config_iface_init (OstreeRepoFinderInterface *iface) * Create a new #OstreeRepoFinderConfig. * * Returns: (transfer full): a new #OstreeRepoFinderConfig - * Since: 2017.8 + * Since: 2018.6 */ OstreeRepoFinderConfig * ostree_repo_finder_config_new (void) diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index 117033df..864510e5 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -67,7 +67,7 @@ * The volume monitor used to find mounted volumes can be overridden by setting * #OstreeRepoFinderMount:monitor. By default, g_volume_monitor_get() is used. * - * Since: 2017.8 + * Since: 2018.6 */ typedef GList/**/ ObjectList; @@ -648,7 +648,7 @@ ostree_repo_finder_mount_class_init (OstreeRepoFinderMountClass *klass) * * Volume monitor to use to look up mounted volumes when queried. * - * Since: 2017.8 + * Since: 2018.6 */ g_object_class_install_property (object_class, PROP_MONITOR, g_param_spec_object ("monitor", @@ -679,7 +679,7 @@ ostree_repo_finder_mount_iface_init (OstreeRepoFinderInterface *iface) * be used. * * Returns: (transfer full): a new #OstreeRepoFinderMount - * Since: 2017.8 + * Since: 2018.6 */ OstreeRepoFinderMount * ostree_repo_finder_mount_new (GVolumeMonitor *monitor) diff --git a/src/libostree/ostree-repo-finder-override.c b/src/libostree/ostree-repo-finder-override.c index 0ee0270d..6917dbe0 100644 --- a/src/libostree/ostree-repo-finder-override.c +++ b/src/libostree/ostree-repo-finder-override.c @@ -60,7 +60,7 @@ * which uses #OstreeRepoFinder. For production use, #OstreeRepoFinderConfig is * recommended instead. * - * Since: 2017.13 + * Since: 2018.6 */ static void ostree_repo_finder_override_iface_init (OstreeRepoFinderInterface *iface); @@ -295,7 +295,7 @@ ostree_repo_finder_override_iface_init (OstreeRepoFinderInterface *iface) * Create a new #OstreeRepoFinderOverride. * * Returns: (transfer full): a new #OstreeRepoFinderOverride - * Since: 2017.13 + * Since: 2018.6 */ OstreeRepoFinderOverride * ostree_repo_finder_override_new (void) @@ -310,7 +310,7 @@ ostree_repo_finder_override_new (void) * Add the given @uri to the set of URIs which the repo finder will search for * matching refs when ostree_repo_finder_resolve_async() is called on it. * - * Since: 2017.13 + * Since: 2018.6 */ void ostree_repo_finder_override_add_uri (OstreeRepoFinderOverride *self, diff --git a/src/libostree/ostree-repo-finder.c b/src/libostree/ostree-repo-finder.c index e7943c3e..32407ff5 100644 --- a/src/libostree/ostree-repo-finder.c +++ b/src/libostree/ostree-repo-finder.c @@ -138,7 +138,7 @@ static void resolve_cb (GObject *obj, * Pass the results to ostree_repo_pull_from_remotes_async() to pull the given * @refs from those remotes. * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_finder_resolve_async (OstreeRepoFinder *self, @@ -196,7 +196,7 @@ resolve_cb (GObject *obj, * * Returns: (transfer full) (element-type OstreeRepoFinderResult): array of zero * or more results - * Since: 2017.8 + * Since: 2018.6 */ GPtrArray * ostree_repo_finder_resolve_finish (OstreeRepoFinder *self, @@ -254,7 +254,7 @@ static void resolve_all_finished_one (GTask *task); * A version of ostree_repo_finder_resolve_async() which queries one or more * @finders in parallel and combines the results. * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_finder_resolve_all_async (OstreeRepoFinder * const *finders, @@ -411,7 +411,7 @@ resolve_all_finished_one (GTask *task) * * Returns: (transfer full) (element-type OstreeRepoFinderResult): array of zero * or more results - * Since: 2017.8 + * Since: 2018.6 */ GPtrArray * ostree_repo_finder_resolve_all_finish (GAsyncResult *result, @@ -446,7 +446,7 @@ G_DEFINE_BOXED_TYPE (OstreeRepoFinderResult, ostree_repo_finder_result, * are as described in the #OstreeRepoFinderResult documentation. * * Returns: (transfer full): a new #OstreeRepoFinderResult - * Since: 2017.8 + * Since: 2018.6 */ OstreeRepoFinderResult * ostree_repo_finder_result_new (OstreeRemote *remote, @@ -480,7 +480,7 @@ ostree_repo_finder_result_new (OstreeRemote *remote, * Copy an #OstreeRepoFinderResult. * * Returns: (transfer full): a newly allocated copy of @result - * Since: 2017.8 + * Since: 2018.6 */ OstreeRepoFinderResult * ostree_repo_finder_result_dup (OstreeRepoFinderResult *result) @@ -502,7 +502,7 @@ ostree_repo_finder_result_dup (OstreeRepoFinderResult *result) * * Returns: <0 if @a is ordered before @b, 0 if they are ordered equally, * >0 if @b is ordered before @a - * Since: 2017.8 + * Since: 2018.6 */ gint ostree_repo_finder_result_compare (const OstreeRepoFinderResult *a, @@ -549,7 +549,7 @@ ostree_repo_finder_result_compare (const OstreeRepoFinderResult *a, * * Free the given @result. * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_finder_result_free (OstreeRepoFinderResult *result) @@ -571,7 +571,7 @@ ostree_repo_finder_result_free (OstreeRepoFinderResult *result) * * Free the given @results array, freeing each element and the container. * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_finder_result_freev (OstreeRepoFinderResult **results) diff --git a/src/libostree/ostree-repo-finder.h b/src/libostree/ostree-repo-finder.h index e622c9a6..7c9be300 100644 --- a/src/libostree/ostree-repo-finder.h +++ b/src/libostree/ostree-repo-finder.h @@ -133,7 +133,7 @@ GPtrArray *ostree_repo_finder_resolve_all_finish (GAsyncResult *result, * ostree_repo_find_remotes_async (2) there was an error in trying to get the * commit metadata (3) the checksum for this ref is %NULL in @ref_to_checksum. * - * Since: 2017.8 + * Since: 2018.6 */ typedef struct { @@ -176,7 +176,7 @@ void ostree_repo_finder_result_free (OstreeRepoFinderResult *result); * g_auto(OstreeRepoFinderResultv) results = NULL; * ]| * - * Since: 2017.8 + * Since: 2018.6 */ typedef OstreeRepoFinderResult** OstreeRepoFinderResultv; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 77203638..63aa451a 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -460,33 +460,4 @@ OstreeRepoAutoLock * _ostree_repo_auto_lock_push (OstreeRepo *self, void _ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock); G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, _ostree_repo_auto_lock_cleanup) -#ifndef OSTREE_ENABLE_EXPERIMENTAL_API - -/* These APIs are duplicated in the public headers when doing an - * experimental-API build. - */ -const gchar * ostree_repo_get_collection_id (OstreeRepo *self); -gboolean ostree_repo_set_collection_id (OstreeRepo *self, - const gchar *collection_id, - GError **error); - -gboolean ostree_repo_list_collection_refs (OstreeRepo *self, - const char *match_collection_id, - GHashTable **out_all_refs, - OstreeRepoListRefsExtFlags flags, - GCancellable *cancellable, - GError **error); - -void ostree_repo_transaction_set_collection_ref (OstreeRepo *self, - const OstreeCollectionRef *ref, - const char *checksum); - -gboolean ostree_repo_set_collection_ref_immediate (OstreeRepo *self, - const OstreeCollectionRef *ref, - const char *checksum, - GCancellable *cancellable, - GError **error); - -#endif /* !OSTREE_ENABLE_EXPERIMENTAL_API */ - G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 550a33de..9553272e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -41,14 +41,12 @@ #include "ostree-remote-private.h" #include "ot-fs-utils.h" -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include "ostree-repo-finder.h" #include "ostree-repo-finder-config.h" #include "ostree-repo-finder-mount.h" #ifdef HAVE_AVAHI #include "ostree-repo-finder-avahi.h" #endif /* HAVE_AVAHI */ -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include #include @@ -904,15 +902,11 @@ fetch_ref_contents (OtPullData *pull_data, if (pull_data->remote_repo_local != NULL && ref->collection_id != NULL) { -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API if (!ostree_repo_resolve_collection_ref (pull_data->remote_repo_local, ref, FALSE, OSTREE_REPO_RESOLVE_REV_EXT_NONE, &ret_contents, cancellable, error)) return FALSE; -#else /* if !OSTREE_ENABLE_EXPERIMENTAL_API */ - g_assert_not_reached (); -#endif /* !OSTREE_ENABLE_EXPERIMENTAL_API */ } else if (pull_data->remote_repo_local != NULL) { @@ -1586,7 +1580,6 @@ _ostree_repo_verify_bindings (const char *collection_id, if (collection_id != NULL) { -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API const char *collection_id_binding; if (!g_variant_lookup (metadata, OSTREE_COMMIT_META_KEY_COLLECTION_BINDING, @@ -1601,14 +1594,11 @@ _ostree_repo_verify_bindings (const char *collection_id, "metadata, while the remote it came from has " "collection ID ‘%s’", collection_id_binding, collection_id); -#endif } return TRUE; } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API - /* Reads the collection-id of a given remote from the repo * configuration. */ @@ -1692,7 +1682,7 @@ check_remote_matches_collection_id (OstreeRepo *repo, * * Returns: (transfer full): #OstreeRemote containing the GPG keyring for * @collection_id - * Since: 2017.8 + * Since: 2018.6 */ OstreeRemote * ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, @@ -1763,8 +1753,6 @@ ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, } } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - #ifdef HAVE_LIBCURL_OR_LIBSOUP /* Look at a commit object, and determine whether there are @@ -1822,9 +1810,7 @@ scan_commit_object (OtPullData *pull_data, * branch, otherwise we requested a commit checksum without specifying a branch. */ g_autofree char *remote_collection_id = NULL; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API remote_collection_id = get_remote_repo_collection_id (pull_data); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ if (!_ostree_repo_verify_bindings (remote_collection_id, (ref != NULL) ? ref->ref_name : NULL, commit, error)) @@ -4619,8 +4605,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, return ret; } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API - /* Structure used in ostree_repo_find_remotes_async() which stores metadata * about a given OSTree commit. This includes the metadata from the commit * #GVariant, plus some working state which is used to work out which remotes @@ -4903,7 +4887,7 @@ static void find_remotes_cb (GObject *obj, * * This will use the thread-default #GMainContext, but will not iterate it. * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_find_remotes_async (OstreeRepo *self, @@ -5639,7 +5623,7 @@ error: * Returns: (transfer full) (array zero-terminated=1): a potentially empty array * of #OstreeRepoFinderResults, followed by a %NULL terminator element; or * %NULL on error - * Since: 2017.8 + * Since: 2018.6 */ OstreeRepoFinderResult ** ostree_repo_find_remotes_finish (OstreeRepo *self, @@ -5719,7 +5703,7 @@ copy_option (GVariantDict *master_options, * milliseconds, if any; only values higher than 0 are valid * * `append-user-agent` (`s`): Additional string to append to the user agent * - * Since: 2017.8 + * Since: 2018.6 */ void ostree_repo_pull_from_remotes_async (OstreeRepo *self, @@ -5920,7 +5904,7 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, * ostree_repo_pull_from_remotes_async(). * * Returns: %TRUE on success, %FALSE otherwise - * Since: 2017.8 + * Since: 2018.6 */ gboolean ostree_repo_pull_from_remotes_finish (OstreeRepo *self, @@ -5935,8 +5919,6 @@ ostree_repo_pull_from_remotes_finish (OstreeRepo *self, return g_task_propagate_boolean (G_TASK (result), error); } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - /** * ostree_repo_remote_fetch_summary_with_options: * @self: Self @@ -6089,8 +6071,6 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, return FALSE; } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API - void ostree_repo_find_remotes_async (OstreeRepo *self, const OstreeCollectionRef * const *refs, @@ -6158,6 +6138,4 @@ ostree_repo_pull_from_remotes_finish (OstreeRepo *self, return FALSE; } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - #endif /* HAVE_LIBCURL_OR_LIBSOUP */ diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index ad0fd57c..d10e302e 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -470,7 +470,6 @@ ostree_repo_resolve_rev_ext (OstreeRepo *self, return _ostree_repo_resolve_rev_internal (self, refspec, allow_noent, FALSE, out_rev, error); } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API /** * ostree_repo_resolve_collection_ref: * @self: an #OstreeRepo @@ -494,7 +493,7 @@ ostree_repo_resolve_rev_ext (OstreeRepo *self, * There are currently no @flags which affect the behaviour of this function. * * Returns: %TRUE on success, %FALSE on failure - * Since: 2017.12 + * Since: 2018.6 */ gboolean ostree_repo_resolve_collection_ref (OstreeRepo *self, @@ -531,7 +530,6 @@ ostree_repo_resolve_collection_ref (OstreeRepo *self, *out_rev = g_strdup (ret_contents); return TRUE; } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ static gboolean enumerate_refs_recurse (OstreeRepo *repo, @@ -876,7 +874,6 @@ ostree_repo_remote_list_refs (OstreeRepo *self, return TRUE; } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API static gboolean remote_list_collection_refs_process_refs (OstreeRepo *self, const gchar *remote_name, @@ -934,7 +931,7 @@ remote_list_collection_refs_process_refs (OstreeRepo *self, * Any refs for other collections stored in the repository will also be returned. * No filtering is performed. * - * Since: 2017.10 + * Since: 2018.6 */ gboolean ostree_repo_remote_list_collection_refs (OstreeRepo *self, @@ -995,7 +992,6 @@ ostree_repo_remote_list_collection_refs (OstreeRepo *self, ot_transfer_out_value (out_all_refs, &ret_all_refs); return TRUE; } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ static char * relative_symlink_to (const char *relpath, @@ -1206,7 +1202,7 @@ _ostree_repo_update_collection_refs (OstreeRepo *self, * %OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES in @flags. * * Returns: %TRUE on success, %FALSE otherwise - * Since: 2017.8 + * Since: 2018.6 */ gboolean ostree_repo_list_collection_refs (OstreeRepo *self, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 43c18767..52784b4c 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5702,7 +5702,7 @@ _ostree_repo_memory_cache_ref_destroy (OstreeRepoMemoryCacheRef *state) * Get the collection ID of this repository. See [collection IDs][collection-ids]. * * Returns: (nullable): collection ID for the repository - * Since: 2017.8 + * Since: 2018.6 */ const gchar * ostree_repo_get_collection_id (OstreeRepo *self) @@ -5723,7 +5723,7 @@ ostree_repo_get_collection_id (OstreeRepo *self) * configuration on disk using ostree_repo_write_config(). * * Returns: %TRUE on success, %FALSE otherwise - * Since: 2017.8 + * Since: 2018.6 */ gboolean ostree_repo_set_collection_id (OstreeRepo *self, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index d86d241e..60fa3d3e 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -28,10 +28,8 @@ #include "ostree-core.h" #include "ostree-types.h" #include "ostree-async-progress.h" -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include "ostree-ref.h" #include "ostree-repo-finder.h" -#endif #include "ostree-sepolicy.h" #include "ostree-gpg-verify-result.h" @@ -107,8 +105,6 @@ OstreeRepo * ostree_repo_create_at (int dfd, GCancellable *cancellable, GError **error); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API - _OSTREE_PUBLIC const gchar * ostree_repo_get_collection_id (OstreeRepo *self); _OSTREE_PUBLIC @@ -116,8 +112,6 @@ gboolean ostree_repo_set_collection_id (OstreeRepo *self, const gchar *collection_id, GError **error); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - _OSTREE_PUBLIC GFile * ostree_repo_get_path (OstreeRepo *self); @@ -339,15 +333,11 @@ void ostree_repo_transaction_set_ref (OstreeRepo *self, const char *ref, const char *checksum); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API - _OSTREE_PUBLIC void ostree_repo_transaction_set_collection_ref (OstreeRepo *self, const OstreeCollectionRef *ref, const char *checksum); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - _OSTREE_PUBLIC gboolean ostree_repo_set_ref_immediate (OstreeRepo *self, const char *remote, @@ -364,8 +354,6 @@ gboolean ostree_repo_set_alias_ref_immediate (OstreeRepo *self, GCancellable *cancellable, GError **error); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API - _OSTREE_PUBLIC gboolean ostree_repo_set_collection_ref_immediate (OstreeRepo *self, const OstreeCollectionRef *ref, @@ -373,8 +361,6 @@ gboolean ostree_repo_set_collection_ref_immediate (OstreeRepo GCancellable *cancellable, GError **error); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - _OSTREE_PUBLIC gboolean ostree_repo_has_object (OstreeRepo *self, OstreeObjectType objtype, @@ -479,7 +465,6 @@ gboolean ostree_repo_resolve_rev_ext (OstreeRepo *self, char **out_rev, GError **error); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API _OSTREE_PUBLIC gboolean ostree_repo_resolve_collection_ref (OstreeRepo *self, const OstreeCollectionRef *ref, @@ -488,7 +473,6 @@ gboolean ostree_repo_resolve_collection_ref (OstreeRepo char **out_rev, GCancellable *cancellable, GError **error); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ _OSTREE_PUBLIC gboolean ostree_repo_list_refs (OstreeRepo *self, @@ -524,14 +508,12 @@ gboolean ostree_repo_remote_list_refs (OstreeRepo *self, GCancellable *cancellable, GError **error); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API _OSTREE_PUBLIC gboolean ostree_repo_remote_list_collection_refs (OstreeRepo *self, const char *remote_name, GHashTable **out_all_refs, GCancellable *cancellable, GError **error); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ _OSTREE_PUBLIC gboolean ostree_repo_load_variant (OstreeRepo *self, @@ -1265,8 +1247,6 @@ gboolean ostree_repo_pull_with_options (OstreeRepo *self, GCancellable *cancellable, GError **error); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API - _OSTREE_PUBLIC void ostree_repo_find_remotes_async (OstreeRepo *self, const OstreeCollectionRef * const *refs, @@ -1308,8 +1288,6 @@ gboolean ostree_repo_list_collection_refs (OstreeRepo *self, GCancellable *cancellable, GError **error); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - _OSTREE_PUBLIC void ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress, gpointer user_data); @@ -1394,8 +1372,6 @@ gboolean ostree_repo_regenerate_summary (OstreeRepo *self, GCancellable *cancellable, GError **error); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API - /** * OSTREE_REPO_METADATA_REF: * @@ -1415,12 +1391,10 @@ gboolean ostree_repo_regenerate_summary (OstreeRepo *self, * keys must be namespaced by product or developer. For example, * `exampleos.end-of-life`. The `ostree.` prefix is reserved. * - * Since: 2017.8 + * Since: 2018.6 */ #define OSTREE_REPO_METADATA_REF "ostree-metadata" -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - G_END_DECLS diff --git a/src/libostree/ostree-types.h b/src/libostree/ostree-types.h index 852eb82b..bbc6ca64 100644 --- a/src/libostree/ostree-types.h +++ b/src/libostree/ostree-types.h @@ -38,9 +38,6 @@ typedef struct OstreeSysroot OstreeSysroot; typedef struct OstreeSysrootUpgrader OstreeSysrootUpgrader; typedef struct OstreeMutableTree OstreeMutableTree; typedef struct OstreeRepoFile OstreeRepoFile; - -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API typedef struct OstreeRemote OstreeRemote; -#endif G_END_DECLS diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index 793d04f0..cbeb99b2 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -25,9 +25,7 @@ #include #include #include -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include -#endif #include #include #include @@ -35,15 +33,11 @@ #include #include #include - -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include #include #include #include #include #include -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ - #include #include diff --git a/src/ostree/main.c b/src/ostree/main.c index 9ed0f880..c5b45012 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -63,14 +63,12 @@ static OstreeCommand commands[] = { { "export", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_export, "Stream COMMIT to stdout in tar format" }, -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API { "find-remotes", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_find_remotes, "Find remotes to serve the given refs" }, { "create-usb", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_create_usb, "Copy the refs to a USB stick" }, -#endif { "fsck", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_fsck, "Check the repository for consistency" }, diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 370f0e5b..c2f78700 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -331,7 +331,6 @@ parse_keyvalue_strings (GVariantBuilder *builder, return TRUE; } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API static void add_collection_binding (OstreeRepo *repo, GVariantBuilder *metadata_builder) @@ -344,7 +343,6 @@ add_collection_binding (OstreeRepo *repo, g_variant_builder_add (metadata_builder, "{s@v}", OSTREE_COMMIT_META_KEY_COLLECTION_BINDING, g_variant_new_variant (g_variant_new_string (collection_id))); } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ static int compare_strings (gconstpointer a, gconstpointer b) @@ -383,13 +381,11 @@ fill_bindings (OstreeRepo *repo, add_ref_binding (metadata_builder); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API /* Allow the collection ID to be overridden using * --add-metadata-string=ostree.collection-binding=blah */ if (metadata == NULL || !g_variant_lookup (metadata, OSTREE_COMMIT_META_KEY_COLLECTION_BINDING, "*", NULL)) add_collection_binding (repo, metadata_builder); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ *out_metadata = g_variant_ref_sink (g_variant_builder_end (metadata_builder)); } diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 83000c6e..a7ecd3d0 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -264,7 +264,6 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation, return FALSE; } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API if (!opt_quiet) g_print ("Validating refs in collections...\n"); @@ -282,7 +281,6 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation, &found_corruption, cancellable, error)) return FALSE; } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ if (!opt_quiet) g_print ("Enumerating objects...\n"); @@ -327,13 +325,11 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation, g_autoptr(GVariant) metadata = g_variant_get_child_value (commit, 0); const char *collection_id = NULL; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API if (!g_variant_lookup (metadata, OSTREE_COMMIT_META_KEY_COLLECTION_BINDING, "&s", &collection_id)) collection_id = NULL; -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ g_autofree const char **refs = NULL; if (g_variant_lookup (metadata, @@ -345,7 +341,6 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation, { g_autofree char *checksum_for_ref = NULL; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API if (collection_id != NULL) { const OstreeCollectionRef collection_ref = { (char *) collection_id, (char *) *iter }; @@ -358,7 +353,6 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation, return FALSE; } else -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ { if (!ostree_repo_resolve_rev (repo, *iter, TRUE, &checksum_for_ref, error)) diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c index a4fb5c51..217bf310 100644 --- a/src/ostree/ot-builtin-init.c +++ b/src/ostree/ot-builtin-init.c @@ -28,9 +28,7 @@ #include "ostree.h" static char *opt_mode = "bare"; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API static char *opt_collection_id = NULL; -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ /* ATTENTION: * Please remember to update the bash-completion script (bash/ostree) and @@ -39,10 +37,8 @@ static char *opt_collection_id = NULL; static GOptionEntry options[] = { { "mode", 0, 0, G_OPTION_ARG_STRING, &opt_mode, "Initialize repository in given mode (bare, bare-user, bare-user-only, archive)", NULL }, -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API { "collection-id", 0, 0, G_OPTION_ARG_STRING, &opt_collection_id, "Globally unique ID for this repository as an collection of refs for redistribution to other repositories", "COLLECTION-ID" }, -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ { NULL } }; @@ -61,10 +57,8 @@ ostree_builtin_init (int argc, char **argv,OstreeCommandInvocation *invocation, if (!ostree_repo_mode_from_string (opt_mode, &mode, error)) goto out; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API if (!ostree_repo_set_collection_id (repo, opt_collection_id, error)) goto out; -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ if (!ostree_repo_create (repo, mode, NULL, error)) goto out; diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c index c34bbf4c..2f560d14 100644 --- a/src/ostree/ot-builtin-prune.c +++ b/src/ostree/ot-builtin-prune.c @@ -59,9 +59,7 @@ static gboolean delete_commit (OstreeRepo *repo, const char *commit_to_delete, GCancellable *cancellable, GError **error) { g_autoptr(GHashTable) refs = NULL; /* (element-type utf8 utf8) */ -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API g_autoptr(GHashTable) collection_refs = NULL; /* (element-type OstreeCollectionRef utf8) */ -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ /* Check refs which are not in a collection. */ if (!ostree_repo_list_refs (repo, NULL, &refs, cancellable, error)) @@ -73,7 +71,6 @@ delete_commit (OstreeRepo *repo, const char *commit_to_delete, GCancellable *can return glnx_throw (error, "Commit '%s' is referenced by '%s'", commit_to_delete, ref); } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API /* And check refs which *are* in a collection. */ if (!ostree_repo_list_collection_refs (repo, NULL, &collection_refs, OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES, @@ -87,7 +84,6 @@ delete_commit (OstreeRepo *repo, const char *commit_to_delete, GCancellable *can return glnx_throw (error, "Commit '%s' is referenced by (%s, %s)", commit_to_delete, ref->collection_id, ref->ref_name); } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ if (!ot_enable_tombstone_commits (repo, error)) return FALSE; diff --git a/src/ostree/ot-builtin-refs.c b/src/ostree/ot-builtin-refs.c index 7c9fa82f..a7f77ac8 100644 --- a/src/ostree/ot-builtin-refs.c +++ b/src/ostree/ot-builtin-refs.c @@ -31,9 +31,7 @@ static gboolean opt_delete; static gboolean opt_list; static gboolean opt_alias; static char *opt_create; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API static gboolean opt_collections; -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ /* ATTENTION: * Please remember to update the bash-completion script (bash/ostree) and @@ -45,13 +43,10 @@ static GOptionEntry options[] = { { "list", 0, 0, G_OPTION_ARG_NONE, &opt_list, "Do not remove the prefix from the refs", NULL }, { "alias", 'A', 0, G_OPTION_ARG_NONE, &opt_alias, "If used with --create, create an alias, otherwise just list aliases", NULL }, { "create", 0, 0, G_OPTION_ARG_STRING, &opt_create, "Create a new ref for an existing commit", "NEWREF" }, -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API { "collections", 'c', 0, G_OPTION_ARG_NONE, &opt_collections, "Enable listing collection IDs for refs", NULL }, -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ { NULL } }; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API static gboolean do_ref_with_collections (OstreeRepo *repo, const char *refspec_prefix, @@ -136,7 +131,6 @@ do_ref_with_collections (OstreeRepo *repo, out: return ret; } -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellable *cancellable, GError **error) { @@ -147,10 +141,8 @@ static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellab gboolean ret = FALSE; gboolean is_list; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API if (opt_collections) return do_ref_with_collections (repo, refspec_prefix, cancellable, error); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ /* If we're doing aliasing, we need the full list of aliases mostly to allow * replacing existing aliases. diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c index 281e0df6..c95c6df0 100644 --- a/src/ostree/ot-builtin-summary.c +++ b/src/ostree/ot-builtin-summary.c @@ -108,16 +108,11 @@ ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocati return FALSE; } -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API const char *collection_id = ostree_repo_get_collection_id (repo); -#else /* if !OSTREE_ENABLE_EXPERIMENTAL_API */ - const char *collection_id = NULL; -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ /* Write out a new metadata commit for the repository. */ if (collection_id != NULL) { -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API OstreeCollectionRef collection_ref = { (gchar *) collection_id, (gchar *) OSTREE_REPO_METADATA_REF }; g_autofree char *old_ostree_metadata_checksum = NULL; g_autofree gchar *new_ostree_metadata_checksum = NULL; @@ -192,10 +187,6 @@ ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocati if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error)) return FALSE; -#else /* if !OSTREE_ENABLE_EXPERIMENTAL_API */ - g_assert_not_reached (); - return FALSE; -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ } /* Regenerate and sign the conventional summary file. */ diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index b79bf589..7a7d41bc 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -39,10 +39,8 @@ BUILTINPROTO(checksum); BUILTINPROTO(commit); BUILTINPROTO(diff); BUILTINPROTO(export); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API BUILTINPROTO(find_remotes); BUILTINPROTO(create_usb); -#endif BUILTINPROTO(gpg_sign); BUILTINPROTO(init); BUILTINPROTO(log); diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index 4d90934c..8b339dbd 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -32,9 +32,7 @@ static gboolean opt_no_gpg_verify; static gboolean opt_if_not_exists; static char *opt_gpg_import; static char *opt_contenturl; -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API static char *opt_collection_id; -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ static char *opt_sysroot; static char *opt_repo; @@ -49,10 +47,8 @@ static GOptionEntry option_entries[] = { { "if-not-exists", 0, 0, G_OPTION_ARG_NONE, &opt_if_not_exists, "Do nothing if the provided remote exists", NULL }, { "gpg-import", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_import, "Import GPG key from FILE", "FILE" }, { "contenturl", 0, 0, G_OPTION_ARG_STRING, &opt_contenturl, "Use URL when fetching content", "URL" }, -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API { "collection-id", 0, 0, G_OPTION_ARG_STRING, &opt_collection_id, "Globally unique ID for this repository as an collection of refs for redistribution to other repositories", "COLLECTION-ID" }, -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ { "repo", 0, 0, G_OPTION_ARG_FILENAME, &opt_repo, "Path to OSTree repository (defaults to /sysroot/ostree/repo)", "PATH" }, { "sysroot", 0, 0, G_OPTION_ARG_FILENAME, &opt_sysroot, "Use sysroot at PATH (overrides --repo)", "PATH" }, { NULL } @@ -133,11 +129,9 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio "gpg-verify", g_variant_new_variant (g_variant_new_boolean (FALSE))); -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API if (opt_collection_id != NULL) g_variant_builder_add (optbuilder, "{s@v}", "collection-id", g_variant_new_variant (g_variant_new_take_string (g_steal_pointer (&opt_collection_id)))); -#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ options = g_variant_ref_sink (g_variant_builder_end (optbuilder)); From 1d830c17922c14b08d208f5d5b5f091851037470 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 6 Jun 2018 13:05:28 -0700 Subject: [PATCH 39/45] Revert "lib: Fix building against old glib versions" This reverts commit f1d9196076d4aea1f64e0d2cbd17bfa2891b8c4c. Since libglnx.h does not get installed, it can't be included in ostree-autocleanups.h, which is included by ostree.h. Closes: #1615 Approved by: jlebon --- src/libostree/ostree-autocleanups.h | 1 - src/libostree/ostree-ref.c | 1 + src/libostree/ostree-repo-finder-config.c | 1 + src/libostree/ostree-repo-finder-mount.c | 1 + src/libostree/ostree-repo-finder-override.c | 1 + src/libostree/ostree-repo.c | 1 + tests/test-repo-finder-mount.c | 1 + 7 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index ae678721..9a54aee7 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -24,7 +24,6 @@ #pragma once #include -#include G_BEGIN_DECLS diff --git a/src/libostree/ostree-ref.c b/src/libostree/ostree-ref.c index 34d67240..35fd5821 100644 --- a/src/libostree/ostree-ref.c +++ b/src/libostree/ostree-ref.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "ostree-autocleanups.h" #include "ostree-core.h" diff --git a/src/libostree/ostree-repo-finder-config.c b/src/libostree/ostree-repo-finder-config.c index 8e617afc..4366d72a 100644 --- a/src/libostree/ostree-repo-finder-config.c +++ b/src/libostree/ostree-repo-finder-config.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "ostree-autocleanups.h" #include "ostree-remote-private.h" diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index 864510e5..c259f3e6 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "ostree-autocleanups.h" diff --git a/src/libostree/ostree-repo-finder-override.c b/src/libostree/ostree-repo-finder-override.c index 6917dbe0..32199546 100644 --- a/src/libostree/ostree-repo-finder-override.c +++ b/src/libostree/ostree-repo-finder-override.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "ostree-autocleanups.h" #include "ostree-remote-private.h" diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 52784b4c..f3da1cae 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -28,6 +28,7 @@ #include #include #include +#include "libglnx.h" #include "otutil.h" #include diff --git a/tests/test-repo-finder-mount.c b/tests/test-repo-finder-mount.c index 948a3245..55eb47fb 100644 --- a/tests/test-repo-finder-mount.c +++ b/tests/test-repo-finder-mount.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "libostreetest.h" From 6ea21696a3373d32ea759a794c9673075539e7c4 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 6 Jun 2018 13:41:37 -0700 Subject: [PATCH 40/45] Fix building against old glib versions We need to include libglnx.h in places where ostree-autocleanups.h is included, so that we get backports of G_DEFINE_AUTOPTR_CLEANUP_FUNC and friends. Closes: #1615 Approved by: jlebon --- src/libostree/ostree-repo-finder-avahi-parser.c | 1 + src/libostree/ostree-repo-finder-avahi.c | 1 + src/libostree/ostree-repo-finder.c | 1 + tests/repo-finder-mount.c | 1 + 4 files changed, 4 insertions(+) diff --git a/src/libostree/ostree-repo-finder-avahi-parser.c b/src/libostree/ostree-repo-finder-avahi-parser.c index 0ca8253f..afc9790c 100644 --- a/src/libostree/ostree-repo-finder-avahi-parser.c +++ b/src/libostree/ostree-repo-finder-avahi-parser.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include "ostree-autocleanups.h" diff --git a/src/libostree/ostree-repo-finder-avahi.c b/src/libostree/ostree-repo-finder-avahi.c index e48b60f4..223d8f0a 100644 --- a/src/libostree/ostree-repo-finder-avahi.c +++ b/src/libostree/ostree-repo-finder-avahi.c @@ -43,6 +43,7 @@ #include #include #include +#include #include "ostree-autocleanups.h" #include "ostree-repo-finder.h" diff --git a/src/libostree/ostree-repo-finder.c b/src/libostree/ostree-repo-finder.c index 32407ff5..ed44ddca 100644 --- a/src/libostree/ostree-repo-finder.c +++ b/src/libostree/ostree-repo-finder.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "ostree-autocleanups.h" #include "ostree-core.h" diff --git a/tests/repo-finder-mount.c b/tests/repo-finder-mount.c index be0e87e3..2cb1d230 100644 --- a/tests/repo-finder-mount.c +++ b/tests/repo-finder-mount.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "ostree-autocleanups.h" From 6f3b5620de29654624eea1c3e6691c46be058dc8 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 6 Jun 2018 14:34:01 -0700 Subject: [PATCH 41/45] tests: Add test for including ostree.h Closes: #1615 Approved by: jlebon --- Makefile-tests.am | 9 ++++++- tests/.gitignore | 1 + tests/test-include-ostree-h.c | 45 +++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 tests/test-include-ostree-h.c diff --git a/Makefile-tests.am b/Makefile-tests.am index 1bdad4c1..97a71d0d 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -242,7 +242,7 @@ endif _installed_or_uninstalled_test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-mutable-tree \ tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \ tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum \ - tests/test-basic-c tests/test-sysroot-c tests/test-pull-c tests/test-repo + tests/test-basic-c tests/test-sysroot-c tests/test-pull-c tests/test-repo tests/test-include-ostree-h if USE_AVAHI test_programs += tests/test-repo-finder-avahi @@ -278,6 +278,13 @@ tests_test_bloom_SOURCES = src/libostree/ostree-bloom.c tests/test-bloom.c tests_test_bloom_CFLAGS = $(TESTS_CFLAGS) tests_test_bloom_LDADD = $(TESTS_LDADD) +tests_test_include_ostree_h_SOURCES = tests/test-include-ostree-h.c +# Don't use TESTS_CFLAGS so we test if the public header can be included by external programs +tests_test_include_ostree_h_CFLAGS = $(AM_CFLAGS) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -I$(srcdir)/src/libostree -I$(builddir)/src/libostree +# Don't define OSTREE_COMPILATION so that we're compiling as if it's an external program +tests_test_include_ostree_h_CPPFLAGS = $(AM_CPPFLAGS) -UOSTREE_COMPILATION +tests_test_include_ostree_h_LDADD = $(TESTS_LDADD) + if USE_AVAHI tests_test_repo_finder_avahi_SOURCES = src/libostree/ostree-repo-finder-avahi-parser.c tests/test-repo-finder-avahi.c tests_test_repo_finder_avahi_CFLAGS = $(TESTS_CFLAGS) diff --git a/tests/.gitignore b/tests/.gitignore index d5b49748..8f03c026 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -11,6 +11,7 @@ test-bloom test-bsdiff test-checksum test-gpg-verify-result +test-include-ostree-h test-keyfile-utils test-mutable-tree test-ot-opt-utils diff --git a/tests/test-include-ostree-h.c b/tests/test-include-ostree-h.c new file mode 100644 index 00000000..af41bb3c --- /dev/null +++ b/tests/test-include-ostree-h.c @@ -0,0 +1,45 @@ +/* + * Copyright © 2018 Endless Mobile, Inc. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Matthew Leeds + */ + +#include "config.h" + +#include +#include +#include + +static void +test_include_ostree_h_compiled (void) +{ +} + +/* Just ensure that we can compile with ostree.h included */ +int main (int argc, char **argv) +{ + setlocale (LC_ALL, ""); + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/include-ostree-h/compiled", test_include_ostree_h_compiled); + + return g_test_run(); +} From ecdebeb20e232e5ec373be25a8cbf106180fcc0a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 7 Jun 2018 14:42:42 -0400 Subject: [PATCH 42/45] switchroot: Allow letting ostree-prepare-root mount /var In some scenarios, it might make sense to let `ostree-prepare-root` do the `/var` mount from the state root as before. For example, one may want to do some system configuration before the switch root. This of course comes at the expense of supporting `/var` as a mount point in `/etc/fstab`. Closes: #1617 Approved by: cgwalters --- src/switchroot/ostree-mount-util.h | 2 ++ src/switchroot/ostree-prepare-root.c | 13 +++++++++---- src/switchroot/ostree-system-generator.c | 10 ++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/switchroot/ostree-mount-util.h b/src/switchroot/ostree-mount-util.h index 08e10f97..0b40bb40 100644 --- a/src/switchroot/ostree-mount-util.h +++ b/src/switchroot/ostree-mount-util.h @@ -31,6 +31,8 @@ #include #include +#define INITRAMFS_MOUNT_VAR "/run/ostree/initramfs-mount-var" + static inline int path_is_on_readonly_fs (char *path) { diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index 5ed9b60f..53df463c 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -151,13 +151,18 @@ main(int argc, char *argv[]) if (chdir (deploy_path) < 0) err (EXIT_FAILURE, "failed to chdir to deploy_path"); - /* In the systemd case, this is handled by ostree-system-generator */ + bool mount_var = true; + /* In the systemd case, this is handled by ostree-system-generator by default */ #ifndef HAVE_SYSTEMD_AND_LIBMOUNT - /* Link to the deployment's /var */ - if (mount ("../../var", "var", NULL, MS_MGC_VAL|MS_BIND, NULL) < 0) - err (EXIT_FAILURE, "failed to bind mount ../../var to var"); + /* file in /run can override that behaviour */ + if (lstat (INITRAMFS_MOUNT_VAR, &stbuf) < 0) + mount_var = false; #endif + /* Link to the deployment's /var */ + if (mount_var && mount ("../../var", "var", NULL, MS_MGC_VAL|MS_BIND, NULL) < 0) + err (EXIT_FAILURE, "failed to bind mount ../../var to var"); + char srcpath[PATH_MAX]; /* If /boot is on the same partition, use a bind mount to make it visible * at /boot inside the deployment. */ diff --git a/src/switchroot/ostree-system-generator.c b/src/switchroot/ostree-system-generator.c index 970ef41e..799a3104 100644 --- a/src/switchroot/ostree-system-generator.c +++ b/src/switchroot/ostree-system-generator.c @@ -49,6 +49,16 @@ main(int argc, char *argv[]) exit (EXIT_SUCCESS); } + /* We conflict with the magic ostree-mount-deployment-var file for ostree-prepare-root */ + { struct stat stbuf; + if (fstatat (AT_FDCWD, INITRAMFS_MOUNT_VAR, &stbuf, 0) == 0) + { + if (unlinkat (AT_FDCWD, INITRAMFS_MOUNT_VAR, 0) < 0) + err (EXIT_FAILURE, "Can't unlink " INITRAMFS_MOUNT_VAR); + exit (EXIT_SUCCESS); + } + } + if (argc > 1 && argc != 4) errx (EXIT_FAILURE, "This program takes three or no arguments"); From fcb8f45f7038223196d53d98924cc82c1a6df543 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 8 Jun 2018 09:10:41 +0100 Subject: [PATCH 43/45] Switch to experimental branch --- debian/changelog | 6 ++++++ debian/control | 2 +- debian/gbp.conf | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index a6109cfd..93cbf766 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +ostree (2018.5-2) UNRELEASED; urgency=medium + + * Switch to experimental branch + + -- Simon McVittie Fri, 08 Jun 2018 09:10:21 +0100 + ostree (2018.5-1) unstable; urgency=medium * New upstream release diff --git a/debian/control b/debian/control index fdd7df90..eabf90fb 100644 --- a/debian/control +++ b/debian/control @@ -46,7 +46,7 @@ Build-Depends-Indep: Rules-Requires-Root: no Standards-Version: 4.1.4 Homepage: https://github.com/ostreedev/ostree/ -Vcs-Git: https://salsa.debian.org/debian/ostree.git +Vcs-Git: https://salsa.debian.org/debian/ostree.git -b debian/experimental Vcs-Browser: https://salsa.debian.org/debian/ostree Package: gir1.2-ostree-1.0 diff --git a/debian/gbp.conf b/debian/gbp.conf index 2bda9c8b..eda2cdd2 100644 --- a/debian/gbp.conf +++ b/debian/gbp.conf @@ -1,7 +1,7 @@ [DEFAULT] pristine-tar = True compression = xz -debian-branch = debian/master +debian-branch = debian/experimental upstream-branch = upstream/latest patch-numbers = False upstream-vcs-tag = v%(version)s From 192e0d9740a6211f478f477e11f83b46c3d0ce60 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 8 Jun 2018 09:31:32 +0100 Subject: [PATCH 44/45] New upstream git snapshot * New upstream git snapshot with support for peer-to-peer software collections, required by Flatpak's peer-to-peer app sharing feature - d/copyright: Update - d/libostree-1-1.symbols: Update - Build-depend on Avahi libraries --- debian/changelog | 7 ++++- debian/control | 2 ++ debian/copyright | 2 +- debian/libostree-1-1.symbols | 51 ++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 93cbf766..0721667f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,11 @@ -ostree (2018.5-2) UNRELEASED; urgency=medium +ostree (2018.5+42+gecdebeb2-1) UNRELEASED; urgency=medium * Switch to experimental branch + * New upstream git snapshot with support for peer-to-peer software + collections, required by Flatpak's peer-to-peer app sharing feature + - d/copyright: Update + - d/libostree-1-1.symbols: Update + - Build-depend on Avahi libraries -- Simon McVittie Fri, 08 Jun 2018 09:10:21 +0100 diff --git a/debian/control b/debian/control index eabf90fb..a71776e8 100644 --- a/debian/control +++ b/debian/control @@ -25,6 +25,8 @@ Build-Depends: gtk-doc-tools , libarchive-dev, libattr1-dev, + libavahi-client-dev (>= 0.6.31), + libavahi-glib-dev (>= 0.6.31), libcap-dev, libfuse-dev, libgirepository1.0-dev, diff --git a/debian/copyright b/debian/copyright index 0426e936..7b262872 100644 --- a/debian/copyright +++ b/debian/copyright @@ -21,7 +21,7 @@ Copyright: © 2014 Owen Taylor © 2015 Dan Nicholson © 2015 Canonical Ltd. - © 2017 Endless Mobile, Inc. + © 2017-2018 Endless Mobile, Inc. © 2018 Matthew Leeds License: LGPL-2+ diff --git a/debian/libostree-1-1.symbols b/debian/libostree-1-1.symbols index 5982819c..0a5b8887 100644 --- a/debian/libostree-1-1.symbols +++ b/debian/libostree-1-1.symbols @@ -25,6 +25,7 @@ libostree-1.so.1 libostree-1-1 #MINVER# LIBOSTREE_2018.2@LIBOSTREE_2018.2 2018.2 LIBOSTREE_2018.3@LIBOSTREE_2018.3 2018.3 LIBOSTREE_2018.5@LIBOSTREE_2018.5 2018.5 + LIBOSTREE_2018.6@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_async_progress_finish@LIBOSTREE_2016.3 2016.4 ostree_async_progress_get@LIBOSTREE_2017.6 2017.6 ostree_async_progress_get_status@LIBOSTREE_2016.3 2016.4 @@ -73,6 +74,14 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_checksum_to_bytes_v@LIBOSTREE_2016.3 2016.4 ostree_cmd__private__@LIBOSTREE_2016.3 2016.4 ostree_cmp_checksum_bytes@LIBOSTREE_2016.3 2016.4 + ostree_collection_ref_dup@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_collection_ref_dupv@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_collection_ref_equal@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_collection_ref_free@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_collection_ref_freev@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_collection_ref_get_type@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_collection_ref_hash@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_collection_ref_new@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_commit_get_content_checksum@LIBOSTREE_2018.2 2018.2 ostree_commit_get_parent@LIBOSTREE_2016.3 2016.4 ostree_commit_get_timestamp@LIBOSTREE_2016.3 2016.4 @@ -145,6 +154,11 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_raw_file_to_archive_z2_stream@LIBOSTREE_2016.6 2016.6 ostree_raw_file_to_archive_z2_stream_with_options@LIBOSTREE_2017.3 2017.3 ostree_raw_file_to_content_stream@LIBOSTREE_2016.3 2016.4 + ostree_remote_get_name@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_remote_get_type@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_remote_get_url@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_remote_ref@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_remote_unref@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_abort_transaction@LIBOSTREE_2016.3 2016.4 ostree_repo_add_gpg_signature_summary@LIBOSTREE_2016.3 2016.4 ostree_repo_append_gpg_signature@LIBOSTREE_2016.3 2016.4 @@ -191,7 +205,32 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_repo_file_tree_get_metadata_checksum@LIBOSTREE_2016.3 2016.4 ostree_repo_file_tree_query_child@LIBOSTREE_2016.3 2016.4 ostree_repo_file_tree_set_metadata@LIBOSTREE_2016.3 2016.4 + ostree_repo_find_remotes_async@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_find_remotes_finish@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_avahi_get_type@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_avahi_new@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_avahi_start@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_avahi_stop@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_config_get_type@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_config_new@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_get_type@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_mount_get_type@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_mount_new@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_override_add_uri@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_override_get_type@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_override_new@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_resolve_all_async@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_resolve_all_finish@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_resolve_async@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_resolve_finish@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_result_compare@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_result_dup@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_result_free@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_result_freev@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_result_get_type@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_finder_result_new@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_fsck_object@LIBOSTREE_2017.15 2017.15 + ostree_repo_get_collection_id@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_get_config@LIBOSTREE_2016.3 2016.4 ostree_repo_get_dfd@LIBOSTREE_2016.4 2016.4 ostree_repo_get_disable_fsync@LIBOSTREE_2016.3 2016.3 @@ -210,6 +249,7 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_repo_import_object_from_with_trust@LIBOSTREE_2016.5 2016.5 ostree_repo_is_system@LIBOSTREE_2016.3 2016.4 ostree_repo_is_writable@LIBOSTREE_2016.3 2016.4 + ostree_repo_list_collection_refs@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_list_commit_objects_starting_with@LIBOSTREE_2016.3 2016.4 ostree_repo_list_objects@LIBOSTREE_2016.3 2016.4 ostree_repo_list_refs@LIBOSTREE_2016.3 2016.4 @@ -233,6 +273,8 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_repo_prune_static_deltas@LIBOSTREE_2016.3 2016.4 ostree_repo_pull@LIBOSTREE_2016.3 2016.4 ostree_repo_pull_default_console_progress_changed@LIBOSTREE_2016.3 2016.4 + ostree_repo_pull_from_remotes_async@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_pull_from_remotes_finish@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_pull_one_dir@LIBOSTREE_2016.3 2016.4 ostree_repo_pull_with_options@LIBOSTREE_2016.3 2016.4 ostree_repo_query_object_storage_size@LIBOSTREE_2016.3 2016.4 @@ -250,18 +292,24 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_repo_remote_get_url@LIBOSTREE_2016.3 2016.4 ostree_repo_remote_gpg_import@LIBOSTREE_2016.3 2016.4 ostree_repo_remote_list@LIBOSTREE_2016.3 2016.4 + ostree_repo_remote_list_collection_refs@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_remote_list_refs@LIBOSTREE_2016.3 2016.4 + ostree_repo_resolve_collection_ref@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_resolve_keyring_for_collection@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_resolve_rev@LIBOSTREE_2016.3 2016.4 ostree_repo_resolve_rev_ext@LIBOSTREE_2016.7 2016.7 ostree_repo_scan_hardlinks@LIBOSTREE_2016.3 2016.4 ostree_repo_set_alias_ref_immediate@LIBOSTREE_2017.10 2017.10 ostree_repo_set_cache_dir@LIBOSTREE_2016.5 2016.5 + ostree_repo_set_collection_id@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 + ostree_repo_set_collection_ref_immediate@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_set_disable_fsync@LIBOSTREE_2016.3 2016.4 ostree_repo_set_ref_immediate@LIBOSTREE_2016.3 2016.4 ostree_repo_sign_commit@LIBOSTREE_2016.3 2016.4 ostree_repo_sign_delta@LIBOSTREE_2016.3 2016.4 ostree_repo_static_delta_execute_offline@LIBOSTREE_2016.3 2016.4 ostree_repo_static_delta_generate@LIBOSTREE_2016.3 2016.4 + ostree_repo_transaction_set_collection_ref@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_transaction_set_ref@LIBOSTREE_2016.3 2016.4 ostree_repo_transaction_set_refspec@LIBOSTREE_2016.3 2016.4 ostree_repo_transaction_stats_get_type@LIBOSTREE_2016.3 2016.4 @@ -271,6 +319,7 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_repo_traverse_new_parents@LIBOSTREE_2018.5 2018.5 ostree_repo_traverse_new_reachable@LIBOSTREE_2016.3 2016.4 ostree_repo_traverse_parents_get_commits@LIBOSTREE_2018.5 2018.5 + ostree_repo_traverse_reachable_refs@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_repo_verify_commit@LIBOSTREE_2016.3 2016.4 ostree_repo_verify_commit_ext@LIBOSTREE_2016.3 2016.4 ostree_repo_verify_commit_for_remote@LIBOSTREE_2016.14 2016.14 @@ -303,6 +352,7 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_sepolicy_restorecon@LIBOSTREE_2016.3 2016.4 ostree_sepolicy_setfscreatecon@LIBOSTREE_2016.3 2016.4 ostree_sysroot_cleanup@LIBOSTREE_2016.3 2016.4 + ostree_sysroot_cleanup_prune_repo@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_sysroot_deploy_tree@LIBOSTREE_2016.3 2016.4 ostree_sysroot_deployment_set_kargs@LIBOSTREE_2016.3 2016.4 ostree_sysroot_deployment_set_mutable@LIBOSTREE_2016.3 2016.4 @@ -356,6 +406,7 @@ libostree-1.so.1 libostree-1-1 #MINVER# ostree_sysroot_write_deployments_with_options@LIBOSTREE_2017.4 2017.4 ostree_sysroot_write_origin_file@LIBOSTREE_2016.3 2016.4 ostree_validate_checksum_string@LIBOSTREE_2016.3 2016.4 + ostree_validate_collection_id@LIBOSTREE_2018.6 2018.5+42+gecdebeb2 ostree_validate_remote_name@LIBOSTREE_2017.8 2017.8 ostree_validate_rev@LIBOSTREE_2016.3 2016.4 ostree_validate_structureof_checksum_string@LIBOSTREE_2016.3 2016.4 From da28448005c2285b9996564ebc5cfe6fae528615 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 8 Jun 2018 10:43:11 +0100 Subject: [PATCH 45/45] d/rules: Explicitly enable various desired libraries --- debian/changelog | 1 + debian/rules | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 0721667f..1dc317a3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,7 @@ ostree (2018.5+42+gecdebeb2-1) UNRELEASED; urgency=medium - d/copyright: Update - d/libostree-1-1.symbols: Update - Build-depend on Avahi libraries + * d/rules: Explicitly enable various desired libraries -- Simon McVittie Fri, 08 Jun 2018 09:10:21 +0100 diff --git a/debian/rules b/debian/rules index d8b3e7ab..3f92293f 100755 --- a/debian/rules +++ b/debian/rules @@ -15,12 +15,18 @@ configure_options = \ GJS=some-nonexistent-gjs-binary \ --enable-installed-tests \ --libexecdir='$${prefix}/lib' \ + --with-avahi \ --with-dracut \ --with-grub2 \ --with-grub2-mkconfig-path=/usr/sbin/grub-mkconfig \ + --with-libarchive \ --with-libsystemd \ + --with-libmount \ + --with-soup \ + --with-selinux \ --with-systemdsystemgeneratordir=/lib/systemd/system-generators \ --with-systemdsystemunitdir=/lib/systemd/system \ + --without-smack \ $(NULL) ifneq ($(filter libostree-doc,$(binaries)),)