From 1ac46828f2cd14e6207b133e2a4158f2a394f653 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Apr 2016 15:20:57 -0400 Subject: [PATCH 001/128] core: Remove ostree_checksum_update_meta that was not implemented If a symbol falls in a git merkle tree forest, but no one hears it, did it ever exist? I'm claiming this isn't an API/ABI break because nothing could have actually used this. `git log -S checksum_update_meta` leads me to 38ef75e6e04f7ec19844f9a1c178e6066682d13d which was before we really had a stable shared library. Closes: #263 Approved by: giuseppe --- apidoc/ostree-sections.txt | 1 - src/libostree/ostree-core.h | 3 --- 2 files changed, 4 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 1eef5da7..d94e3ddd 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -100,7 +100,6 @@ ostree_checksum_bytes_peek_validate ostree_cmp_checksum_bytes ostree_validate_rev ostree_parse_refspec -ostree_checksum_update_meta ostree_object_type_to_string ostree_object_type_from_string ostree_hash_object_name diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 29ef7b28..97d86bc5 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -214,9 +214,6 @@ gboolean ostree_parse_refspec (const char *refspec, char **out_ref, GError **error); -_OSTREE_PUBLIC -void ostree_checksum_update_meta (GChecksum *checksum, GFileInfo *file_info, GVariant *xattrs); - _OSTREE_PUBLIC const char * ostree_object_type_to_string (OstreeObjectType objtype); From 6923c088c5ad42f61fea9526e2919b610c745994 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Apr 2016 15:51:51 -0400 Subject: [PATCH 002/128] lib: Remove ostree_repo_file_make_empty_tree This was removed in 14d682305b9c8d65e46f0aaf31e6851cf2042193 - long before we had a stable library. I again claim removing unimplemented symbols from the header is not an API/ABI break. Closes: #263 Approved by: giuseppe --- apidoc/ostree-sections.txt | 1 - src/libostree/ostree-repo-file.h | 3 --- 2 files changed, 4 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index d94e3ddd..815b6d47 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -338,7 +338,6 @@ ostree_repo_file_ensure_resolved ostree_repo_file_get_xattrs ostree_repo_file_get_repo ostree_repo_file_get_root -ostree_repo_file_make_empty_tree ostree_repo_file_tree_set_metadata ostree_repo_file_tree_get_contents_checksum ostree_repo_file_tree_get_metadata_checksum diff --git a/src/libostree/ostree-repo-file.h b/src/libostree/ostree-repo-file.h index fb651645..6f9b79fe 100644 --- a/src/libostree/ostree-repo-file.h +++ b/src/libostree/ostree-repo-file.h @@ -58,9 +58,6 @@ OstreeRepo * ostree_repo_file_get_repo (OstreeRepoFile *self); _OSTREE_PUBLIC OstreeRepoFile * ostree_repo_file_get_root (OstreeRepoFile *self); -_OSTREE_PUBLIC -void ostree_repo_file_make_empty_tree (OstreeRepoFile *self); - _OSTREE_PUBLIC void ostree_repo_file_tree_set_metadata (OstreeRepoFile *self, const char *checksum, From e9a1809c4d9682efdcb1bb50f62658cbf215bbf0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Apr 2016 15:49:05 -0400 Subject: [PATCH 003/128] Rename test-abi to test-symbols, start verifying symbols are documented We keep forgetting to update `apidoc/ostree-sections.txt`, so let's start enforcing it. Of course it turns out we had some bugs here like symbols marked as public but never implemented, etc. Those are fixed in the prior commits. Closes: #263 Approved by: giuseppe --- Makefile-tests.am | 2 +- apidoc/ostree-sections.txt | 34 ++++++++++++++++++++++++++ src/libostree/libostree.sym | 10 ++++---- tests/{test-abi.sh => test-symbols.sh} | 13 ++++++++-- 4 files changed, 51 insertions(+), 8 deletions(-) rename tests/{test-abi.sh => test-symbols.sh} (69%) diff --git a/Makefile-tests.am b/Makefile-tests.am index b3d75142..47af9e70 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -28,7 +28,7 @@ TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ PATH=$$(cd $(top_builddir) && pwd):$${PATH} \ $(NULL) -uninstalled_test_scripts = tests/test-abi.sh +uninstalled_test_scripts = tests/test-symbols.sh test_scripts = \ tests/test-basic.sh \ diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 815b6d47..3b185ac0 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -27,7 +27,9 @@ OstreeBootconfigParser ostree_bootconfig_parser_new ostree_bootconfig_parser_clone ostree_bootconfig_parser_parse +ostree_bootconfig_parser_parse_at ostree_bootconfig_parser_write +ostree_bootconfig_parser_write_at ostree_bootconfig_parser_set ostree_bootconfig_parser_get @@ -97,6 +99,8 @@ ostree_checksum_inplace_from_bytes ostree_checksum_inplace_to_bytes ostree_checksum_bytes_peek ostree_checksum_bytes_peek_validate +ostree_checksum_b64_inplace_from_bytes +ostree_checksum_b64_inplace_to_bytes ostree_cmp_checksum_bytes ostree_validate_rev ostree_parse_refspec @@ -109,6 +113,7 @@ ostree_object_to_string ostree_object_from_string ostree_content_stream_parse ostree_content_file_parse +ostree_content_file_parse_at ostree_raw_file_to_content_stream ostree_checksum_file_from_input ostree_checksum_file @@ -140,11 +145,14 @@ ostree_deployment_get_bootcsum ostree_deployment_get_bootserial ostree_deployment_get_bootconfig ostree_deployment_get_origin +ostree_deployment_get_origin_relpath +ostree_deployment_get_unlocked ostree_deployment_set_index ostree_deployment_set_bootserial ostree_deployment_set_bootconfig ostree_deployment_set_origin ostree_deployment_clone +ostree_deployment_unlocked_state_to_string OSTREE_DEPLOYMENT OSTREE_IS_DEPLOYMENT @@ -226,6 +234,7 @@ ostree_repo_create ostree_repo_get_path ostree_repo_get_mode ostree_repo_get_config +ostree_repo_get_dfd ostree_repo_copy_config ostree_repo_remote_add ostree_repo_remote_delete @@ -237,6 +246,9 @@ ostree_repo_remote_get_gpg_verify ostree_repo_remote_get_gpg_verify_summary ostree_repo_remote_gpg_import ostree_repo_remote_fetch_summary +ostree_repo_get_remote_boolean_option +ostree_repo_get_remote_list_option +ostree_repo_get_remote_option ostree_repo_get_parent ostree_repo_write_config OstreeRepoTransactionStats @@ -247,6 +259,8 @@ ostree_repo_abort_transaction ostree_repo_transaction_set_refspec ostree_repo_transaction_set_ref ostree_repo_set_ref_immediate +ostree_repo_set_cache_dir +ostree_repo_sign_delta ostree_repo_has_object ostree_repo_write_metadata ostree_repo_write_metadata_async @@ -269,6 +283,8 @@ ostree_repo_load_object_stream ostree_repo_query_object_storage_size ostree_repo_import_object_from ostree_repo_import_object_from_with_trust +ostree_repo_import_archive_to_mtree +ostree_repo_export_tree_to_archive ostree_repo_delete_object OstreeRepoCommitFilterResult OstreeRepoCommitFilter @@ -284,6 +300,7 @@ ostree_repo_commit_modifier_unref ostree_repo_devino_cache_new ostree_repo_devino_cache_ref ostree_repo_devino_cache_unref +ostree_repo_devino_cache_get_type ostree_repo_write_directory_to_mtree ostree_repo_write_dfd_to_mtree ostree_repo_write_archive_to_mtree @@ -309,8 +326,16 @@ ostree_repo_static_delta_execute_offline ostree_repo_traverse_new_reachable ostree_repo_traverse_commit ostree_repo_traverse_commit_union +ostree_repo_commit_traverse_iter_cleanup +ostree_repo_commit_traverse_iter_clear +ostree_repo_commit_traverse_iter_get_dir +ostree_repo_commit_traverse_iter_get_file +ostree_repo_commit_traverse_iter_init_commit +ostree_repo_commit_traverse_iter_init_dirtree +ostree_repo_commit_traverse_iter_next OstreeRepoPruneFlags ostree_repo_prune +ostree_repo_prune_static_deltas OstreeRepoPullFlags ostree_repo_pull ostree_repo_pull_one_dir @@ -318,6 +343,7 @@ ostree_repo_pull_with_options ostree_repo_pull_default_console_progress_changed ostree_repo_sign_commit ostree_repo_append_gpg_signature +ostree_repo_add_gpg_signature_summary ostree_repo_verify_commit ostree_repo_verify_commit_ext ostree_repo_verify_summary @@ -364,8 +390,11 @@ ostree_sepolicy_new ostree_sepolicy_get_path ostree_sepolicy_get_name ostree_sepolicy_get_label +ostree_sepolicy_get_csum OstreeSePolicyRestoreconFlags ostree_sepolicy_restorecon +ostree_sepolicy_setfscreatecon +ostree_sepolicy_fscreatecon_cleanup OSTREE_SEPOLICY OSTREE_IS_SEPOLICY @@ -386,6 +415,7 @@ ostree_sysroot_try_lock ostree_sysroot_lock_async ostree_sysroot_lock_finish ostree_sysroot_unlock +ostree_sysroot_unload ostree_sysroot_get_fd ostree_sysroot_ensure_initialized ostree_sysroot_get_bootversion @@ -393,13 +423,17 @@ ostree_sysroot_get_subbootversion ostree_sysroot_get_deployments ostree_sysroot_get_booted_deployment ostree_sysroot_get_deployment_directory +ostree_sysroot_get_deployment_dirpath ostree_sysroot_get_deployment_origin_path ostree_sysroot_cleanup ostree_sysroot_prepare_cleanup ostree_sysroot_get_repo ostree_sysroot_init_osname ostree_sysroot_deployment_set_kargs +ostree_sysroot_deployment_set_mutable +ostree_sysroot_deployment_unlock ostree_sysroot_write_deployments +ostree_sysroot_write_origin_file ostree_sysroot_deploy_tree ostree_sysroot_get_merge_deployment ostree_sysroot_origin_new_from_refspec diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 89a1457e..8c0d8389 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -323,11 +323,6 @@ global: ostree_deployment_unlocked_state_to_string; } LIBOSTREE_2016.3; -/* NOTE NOTE NOTE - * Versions above here are released. Only add symbols below this line. - * NOTE NOTE NOTE - */ - LIBOSTREE_2016.5 { global: ostree_repo_import_object_from_with_trust; @@ -337,3 +332,8 @@ global: ostree_repo_get_remote_boolean_option; ostree_repo_set_cache_dir; } LIBOSTREE_2016.4; + +/* NOTE NOTE NOTE + * Versions above here are released. Only add symbols below this line. + * NOTE NOTE NOTE + */ diff --git a/tests/test-abi.sh b/tests/test-symbols.sh similarity index 69% rename from tests/test-abi.sh rename to tests/test-symbols.sh index 62af3652..55157c07 100755 --- a/tests/test-abi.sh +++ b/tests/test-symbols.sh @@ -19,10 +19,19 @@ set -euo pipefail -echo '1..1' +echo '1..2' +echo "Verifying all expected symbols are actually exported..." grep ' ostree_[A-Za-z0-9_]*;' ${G_TEST_SRCDIR}/src/libostree/libostree.sym | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt eu-readelf -a ${G_TEST_BUILDDIR}/.libs/libostree-1.so | grep 'FUNC.*GLOBAL.*DEFAULT.*@@LIBOSTREE_' | sed -e 's,^.* \(ostree_[A-Za-z0-9_]*\)@@LIBOSTREE_[0-9_.]*,\1,' |sort -u > found-symbols.txt diff -u expected-symbols.txt found-symbols.txt +echo "ok exports" -echo 'ok' +# cmd__private__ is private. The fetcher symbol should not have been made public. +grep -E -v '(ostree_cmd__private__)|(ostree_fetcher_config_flags_get_type)' found-symbols.txt > expected-documented.txt + +echo "Verifying all public symbols are documented:" +grep '^ostree_' ${G_TEST_SRCDIR}/apidoc/ostree-sections.txt |sort -u > found-documented.txt +diff -u expected-documented.txt found-documented.txt + +echo 'ok documented symbols' From 24cab773a2c849a8153529e3eb5f967302339f68 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 17 Apr 2016 09:57:06 +0100 Subject: [PATCH 004/128] libtest: show files' contents when assertions about them fail I've seen an intermittent test failure in an autobuilder (sbuild) environment where logs from failed builds cannot be retrieved, but I can no longer reproduce it. Put the contents of the offending file in the test's failing output so that if the failure comes back, it can be debugged. Signed-off-by: Simon McVittie Closes: #264 Approved by: cgwalters --- tests/libtest.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 572f023c..afdf3649 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -100,13 +100,17 @@ assert_has_dir () { assert_not_has_file () { if test -f "$1"; then - echo 1>&2 "File '$1' exists"; exit 1 + sed -e 's/^/# /' < "$1" >&2 + echo 1>&2 "File '$1' exists" + exit 1 fi } assert_not_file_has_content () { if grep -q -e "$2" "$1"; then - echo 1>&2 "File '$1' incorrectly matches regexp '$2'"; exit 1 + sed -e 's/^/# /' < "$1" >&2 + echo 1>&2 "File '$1' incorrectly matches regexp '$2'" + exit 1 fi } @@ -118,13 +122,17 @@ assert_not_has_dir () { assert_file_has_content () { if ! grep -q -e "$2" "$1"; then - echo 1>&2 "File '$1' doesn't match regexp '$2'"; exit 1 + sed -e 's/^/# /' < "$1" >&2 + echo 1>&2 "File '$1' doesn't match regexp '$2'" + exit 1 fi } assert_file_empty() { if test -s "$1"; then - echo 1>&2 "File '$1' is not empty"; exit 1 + sed -e 's/^/# /' < "$1" >&2 + echo 1>&2 "File '$1' is not empty" + exit 1 fi } From d5498a8b1672e761652b2f2b511f42961b4feed3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 18 Apr 2016 10:25:32 -0400 Subject: [PATCH 005/128] libtest: Skip FUSE if CAP_SYS_ADMIN not in bounding set I'm not sure if something changed or there was a race condition , or if extraterrestrials intervened to make the tests pass before, but in the Docker container `/dev/fuse` does exist and since we appear to be running as uid 0 it'll be writable. We previously discussed that another requirement for FUSE (and most setuid binaries) is having `CAP_SYS_ADMIN`, so let's check for that in the bounding set too. Closes: #266 Approved by: jlebon --- tests/libtest.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/libtest.sh b/tests/libtest.sh index afdf3649..0ce10a62 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -378,6 +378,11 @@ skip_without_fuse () { exit 0 fi + if ! capsh --print | grep -q 'Bounding set.*[^a-z]cap_sys_admin'; then + echo "1..0 # SKIP No cap_sys_admin in bounding set, can't use FUSE" + exit 0 + fi + if ! [ -w /dev/fuse ]; then echo "1..0 # SKIP no write access to /dev/fuse" exit 0 From 925faca10880d6742cb1168235688f8c46db3bb8 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 19 Apr 2016 10:42:51 +0200 Subject: [PATCH 006/128] lib: Fix a typo Closes: #267 Approved by: cgwalters --- src/libostree/ostree-repo-static-delta-compilation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index b3989234..2a28e7db 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -1226,7 +1226,7 @@ get_fallback_headers (OstreeRepo *self, * * The @params argument should be an a{sv}. The following attributes * are known: - * - min-fallback-size: u: Minimume uncompressed size in megabytes to use fallback, 0 to disable fallbacks + * - min-fallback-size: u: Minimum uncompressed size in megabytes to use fallback, 0 to disable fallbacks * - max-chunk-size: u: Maximum size in megabytes of a delta part * - max-bsdiff-size: u: Maximum size in megabytes to consider bsdiff compression * for input files From 90400d3a6253e709f7210d6658662aecd8a09b45 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 19 Apr 2016 10:43:32 +0200 Subject: [PATCH 007/128] trivial-httpd: Remove useless code Closes: #267 Approved by: cgwalters --- src/ostree/ot-builtin-trivial-httpd.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ostree/ot-builtin-trivial-httpd.c b/src/ostree/ot-builtin-trivial-httpd.c index 038559bb..07d02fb9 100644 --- a/src/ostree/ot-builtin-trivial-httpd.c +++ b/src/ostree/ot-builtin-trivial-httpd.c @@ -305,9 +305,6 @@ httpd_callback (SoupServer *server, SoupMessage *msg, SoupClientContext *context, gpointer data) { OtTrivialHttpd *self = data; - SoupMessageHeadersIter iter; - - soup_message_headers_iter_init (&iter, msg->request_headers); if (msg->method == SOUP_METHOD_GET || msg->method == SOUP_METHOD_HEAD) do_get (self, server, msg, path, context); From 6ff7c876dc15209e0c9b323c43d4b271ddacdd57 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 19 Apr 2016 10:56:13 +0200 Subject: [PATCH 008/128] lib: Fix an introspection annotation The "allow-none" annotation does not make sense for a plain boolean parameter. Closes: #267 Approved by: cgwalters --- src/libostree/ostree-repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 08e6a48f..f5c1b00a 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -426,7 +426,7 @@ ostree_repo_get_remote_list_option (OstreeRepo *self, * @self: A OstreeRepo * @remote_name: Name * @option_name: Option - * @default_value: (allow-none): Value returned if @option_name is not present + * @default_value: Value returned if @option_name is not present * @out_value: (out) : location to store the result. * @error: Error * From 9db2f43b144d130d8b6cc4aaca76eee2f4bfcacf Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 19 Apr 2016 10:57:27 +0200 Subject: [PATCH 009/128] tests, lib: Remove unused variables Closes: #267 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 2 -- tests/readdir-rand.c | 1 - 2 files changed, 3 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index eef5f039..968b5951 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1889,8 +1889,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, GHashTableIter hash_iter; gpointer key, value; g_autoptr(GBytes) bytes_summary = NULL; - g_autofree char *remote_key = NULL; - g_autofree char *path = NULL; g_autofree char *metalink_url_str = NULL; g_autoptr(GHashTable) requested_refs_to_fetch = NULL; g_autoptr(GHashTable) commits_to_fetch = NULL; diff --git a/tests/readdir-rand.c b/tests/readdir-rand.c index 527d4a3a..ee43756f 100644 --- a/tests/readdir-rand.c +++ b/tests/readdir-rand.c @@ -97,7 +97,6 @@ readdir (DIR *dirp) while (cache_another) { DirEntries *de; - GSList *l; errno = 0; ret = real_readdir (dirp); From 5079f70ec07d695dc636b34728dc504426db61ea Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Apr 2016 14:17:08 +0200 Subject: [PATCH 010/128] ostree export: Add --subpath support This lets you export a subset of a commit. Closes: #265 Approved by: cgwalters --- src/libostree/ostree-repo-libarchive.c | 2 +- src/ostree/ot-builtin-export.c | 10 +++++++++- tests/test-export.sh | 14 ++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 7a30192c..6e6e9806 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -480,7 +480,7 @@ file_to_archive_entry_common (GFile *root, g_autoptr(GVariant) xattrs = NULL; time_t ts = (time_t) opts->timestamp_secs; - if (pathstr && !pathstr[0]) + if (pathstr == NULL || !pathstr[0]) { g_free (pathstr); pathstr = g_strdup ("."); diff --git a/src/ostree/ot-builtin-export.c b/src/ostree/ot-builtin-export.c index cccb50e7..9a7842ab 100644 --- a/src/ostree/ot-builtin-export.c +++ b/src/ostree/ot-builtin-export.c @@ -32,10 +32,12 @@ #endif static char *opt_output_path; +static char *opt_subpath; static gboolean opt_no_xattrs; static GOptionEntry options[] = { { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, + { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, { "output", 'o', 0, G_OPTION_ARG_STRING, &opt_output_path, "Output to PATH ", "PATH" }, { NULL } }; @@ -60,6 +62,7 @@ ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError gboolean ret = FALSE; const char *rev; g_autoptr(GFile) root = NULL; + g_autoptr(GFile) subtree = NULL; g_autofree char *commit = NULL; g_autoptr(GVariant) commit_data = NULL; struct archive *a; @@ -124,7 +127,12 @@ ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError opts.timestamp_secs = ostree_commit_get_timestamp (commit_data); - if (!ostree_repo_export_tree_to_archive (repo, &opts, (OstreeRepoFile*)root, a, + if (opt_subpath) + subtree = g_file_resolve_relative_path (root, opt_subpath); + else + subtree = g_object_ref (root); + + if (!ostree_repo_export_tree_to_archive (repo, &opts, (OstreeRepoFile*)subtree, a, cancellable, error)) goto out; diff --git a/tests/test-export.sh b/tests/test-export.sh index 1e81a2e9..8666e177 100755 --- a/tests/test-export.sh +++ b/tests/test-export.sh @@ -23,7 +23,7 @@ set -euo pipefail setup_test_repository "archive-z2" -echo '1..2' +echo '1..3' $OSTREE checkout test2 test2-co $OSTREE commit --no-xattrs -b test2-noxattrs -s "test2 without xattrs" --tree=dir=test2-co @@ -35,10 +35,20 @@ mkdir t (cd t && tar xf ../test2.tar) ${CMD_PREFIX} ostree --repo=repo diff --no-xattrs test2-noxattrs ./t > diff.txt assert_file_empty diff.txt -rm test2.tar diff.txt t -rf echo 'ok export gnutar diff (no xattrs)' +cd ${test_tmpdir} +${OSTREE} 'export' test2-noxattrs --subpath=baz -o test2-subpath.tar +mkdir t2 +(cd t2 && tar xf ../test2-subpath.tar) +${CMD_PREFIX} ostree --repo=repo diff --no-xattrs ./t2 ./t/baz > diff.txt +assert_file_empty diff.txt + +echo 'ok export --subpath gnutar diff (no xattrs)' + +rm test2.tar test2-subpath.tar diff.txt t t2 -rf + cd ${test_tmpdir} ${OSTREE} 'export' test2 -o test2.tar ${OSTREE} commit -b test2-from-tar -s 'Import from tar' --tree=tar=test2.tar From 5595664e47a483512a39a643adb54c523b77de97 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 18 Apr 2016 14:26:02 +0200 Subject: [PATCH 011/128] ostree export: Add --prefix option This lets you set a prefix for the resulting archive patsh. Especially useful in combination with --subpath, for instance --subpath=subdir --prefix=subdir to extract just subdir. Closes: #265 Approved by: cgwalters --- src/libostree/ostree-repo-libarchive.c | 6 ++++++ src/libostree/ostree-repo.h | 5 ++++- src/ostree/ot-builtin-export.c | 4 ++++ tests/test-export.sh | 23 +++++++++++++++++++++-- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 6e6e9806..6856e94b 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -480,6 +480,12 @@ file_to_archive_entry_common (GFile *root, g_autoptr(GVariant) xattrs = NULL; time_t ts = (time_t) opts->timestamp_secs; + if (opts->path_prefix && opts->path_prefix[0]) + { + g_autofree char *old_pathstr = pathstr; + pathstr = g_strconcat (opts->path_prefix, old_pathstr, NULL); + } + if (pathstr == NULL || !pathstr[0]) { g_free (pathstr); diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 8a04e8e5..20873a05 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -610,7 +610,10 @@ typedef struct { guint64 timestamp_secs; guint unused_uint[8]; - gpointer unused_ptrs[8]; + + char *path_prefix; + + gpointer unused_ptrs[7]; } OstreeRepoExportArchiveOptions; _OSTREE_PUBLIC diff --git a/src/ostree/ot-builtin-export.c b/src/ostree/ot-builtin-export.c index 9a7842ab..5b84d1ab 100644 --- a/src/ostree/ot-builtin-export.c +++ b/src/ostree/ot-builtin-export.c @@ -33,11 +33,13 @@ static char *opt_output_path; static char *opt_subpath; +static char *opt_prefix; static gboolean opt_no_xattrs; static GOptionEntry options[] = { { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, + { "prefix", 0, 0, G_OPTION_ARG_STRING, &opt_prefix, "Add PATH as prefix to archive pathnames", "PATH" }, { "output", 'o', 0, G_OPTION_ARG_STRING, &opt_output_path, "Output to PATH ", "PATH" }, { NULL } }; @@ -132,6 +134,8 @@ ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError else subtree = g_object_ref (root); + opts.path_prefix = opt_prefix; + if (!ostree_repo_export_tree_to_archive (repo, &opts, (OstreeRepoFile*)subtree, a, cancellable, error)) goto out; diff --git a/tests/test-export.sh b/tests/test-export.sh index 8666e177..856c4073 100755 --- a/tests/test-export.sh +++ b/tests/test-export.sh @@ -23,7 +23,7 @@ set -euo pipefail setup_test_repository "archive-z2" -echo '1..3' +echo '1..5' $OSTREE checkout test2 test2-co $OSTREE commit --no-xattrs -b test2-noxattrs -s "test2 without xattrs" --tree=dir=test2-co @@ -47,7 +47,26 @@ assert_file_empty diff.txt echo 'ok export --subpath gnutar diff (no xattrs)' -rm test2.tar test2-subpath.tar diff.txt t t2 -rf +cd ${test_tmpdir} +${OSTREE} 'export' test2-noxattrs --prefix=the-prefix/ -o test2-prefix.tar +mkdir t3 +(cd t3 && tar xf ../test2-prefix.tar) +${CMD_PREFIX} ostree --repo=repo diff --no-xattrs test2-noxattrs ./t3/the-prefix > diff.txt +assert_file_empty diff.txt + +echo 'ok export --prefix gnutar diff (no xattrs)' + +cd ${test_tmpdir} +${OSTREE} 'export' test2-noxattrs --subpath=baz --prefix=the-prefix/ -o test2-prefix-subpath.tar +mkdir t4 +(cd t4 && tar xf ../test2-prefix-subpath.tar) +${CMD_PREFIX} ostree --repo=repo diff --no-xattrs ./t4/the-prefix ./t/baz > diff.txt +${CMD_PREFIX} ostree --repo=repo diff --no-xattrs test2-noxattrs ./t3/the-prefix > diff.txt +assert_file_empty diff.txt + +echo 'ok export --prefix --subpath gnutar diff (no xattrs)' + +rm test2.tar test2-subpath.tar diff.txt t t2 t3 t4 -rf cd ${test_tmpdir} ${OSTREE} 'export' test2 -o test2.tar From bd3ad8cc5c4275a144f5fd32b1fe931be91eddb1 Mon Sep 17 00:00:00 2001 From: Gatis Paeglis Date: Wed, 20 Apr 2016 16:22:36 +0200 Subject: [PATCH 012/128] Remove unused variables from install_deployment_kernel Closes: #270 Approved by: cgwalters --- src/libostree/ostree-sysroot-deploy.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 77ecb9af..88772365 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1277,7 +1277,6 @@ install_deployment_kernel (OstreeSysroot *sysroot, struct stat stbuf; const char *osname = ostree_deployment_get_osname (deployment); const char *bootcsum = ostree_deployment_get_bootcsum (deployment); - g_autoptr(GFile) bootdir = NULL; g_autofree char *bootcsumdir = NULL; g_autofree char *bootconfdir = NULL; g_autofree char *bootconf_name = NULL; @@ -1293,10 +1292,6 @@ install_deployment_kernel (OstreeSysroot *sysroot, g_autofree char *contents = NULL; g_autofree char *deployment_version = NULL; g_autoptr(GHashTable) osrelease_values = NULL; - g_autofree char *linux_relpath = NULL; - g_autofree char *linux_key = NULL; - g_autofree char *initramfs_relpath = NULL; - g_autofree char *initrd_key = NULL; g_autofree char *version_key = NULL; g_autofree char *ostree_kernel_arg = NULL; g_autofree char *options_key = NULL; From 8f8ab56211802633d39a67f60b62535fd9a9b573 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 20 Apr 2016 15:33:44 +0200 Subject: [PATCH 013/128] repo: Allow loading files staged in the transaction Currently we can load metadata from the stage dir, but not file data, which makes no sense. Closes: #269 Approved by: cgwalters --- src/libostree/ostree-repo.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index f5c1b00a..3c37d342 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3066,6 +3066,13 @@ ostree_repo_load_file (OstreeRepo *self, error)) goto out; + if (fd < 0 && self->commit_stagedir_fd != -1) + { + if (!ot_openat_ignore_enoent (self->commit_stagedir_fd, loose_path_buf, &fd, + error)) + goto out; + } + if (fd != -1) { tmp_stream = g_unix_input_stream_new (fd, TRUE); From 8609cb036b935ce942214e9fdee6d90de0a210af Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 21 Apr 2016 15:14:51 -0400 Subject: [PATCH 014/128] repo: Simplify internal has_object() lookup code There was some leftover intermediate cruft here I noticed while reviewing another patch: - We had an output `GFile*` for that was never used - We required the caller to allocate the loose pathbuf, but none of them ever reused it - We had an extra intermediate function Also while looking at this, I'm now uncertain whether some of the callers of `_ostree_repo_has_loose_object` should really be invoking `ostree_repo_has_object()`, but let's leave that aside for now. Closes: #272 Approved by: alexlarsson --- src/libostree/ostree-repo-commit.c | 14 ++++------- src/libostree/ostree-repo-private.h | 2 -- src/libostree/ostree-repo.c | 37 ++++------------------------- 3 files changed, 9 insertions(+), 44 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 7f03e11d..cd365da9 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -573,11 +573,8 @@ _ostree_repo_open_trusted_content_bare (OstreeRepo *self, g_autofree char *temp_filename = NULL; g_autoptr(GOutputStream) ret_stream = NULL; gboolean have_obj; - char loose_objpath[_OSTREE_LOOSE_PATH_MAX]; - if (!_ostree_repo_has_loose_object (self, checksum, OSTREE_OBJECT_TYPE_FILE, - &have_obj, loose_objpath, - NULL, + if (!_ostree_repo_has_loose_object (self, checksum, OSTREE_OBJECT_TYPE_FILE, &have_obj, cancellable, error)) goto out; @@ -662,7 +659,6 @@ write_object (OstreeRepo *self, gboolean temp_file_is_regular; gboolean temp_file_is_symlink; gboolean object_is_symlink = FALSE; - char loose_objpath[_OSTREE_LOOSE_PATH_MAX]; gssize unpacked_size = 0; gboolean indexable = FALSE; @@ -673,9 +669,8 @@ write_object (OstreeRepo *self, if (expected_checksum) { - if (!_ostree_repo_has_loose_object (self, expected_checksum, objtype, - &have_obj, loose_objpath, - NULL, cancellable, error)) + if (!_ostree_repo_has_loose_object (self, expected_checksum, objtype, &have_obj, + cancellable, error)) goto out; if (have_obj) { @@ -852,8 +847,7 @@ write_object (OstreeRepo *self, repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); } - if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype, - &have_obj, loose_objpath, NULL, + if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype, &have_obj, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 6a9092e9..54127be0 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -135,8 +135,6 @@ _ostree_repo_has_loose_object (OstreeRepo *self, const char *checksum, OstreeObjectType objtype, gboolean *out_is_stored, - char *loose_path_buf, - GFile **out_stored_path, GCancellable *cancellable, GError **error); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 3c37d342..875c9480 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3292,15 +3292,13 @@ _ostree_repo_has_loose_object (OstreeRepo *self, const char *checksum, OstreeObjectType objtype, gboolean *out_is_stored, - char *loose_path_buf, - GFile **out_stored_path, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; struct stat stbuf; int res = -1; - gboolean tmp_file = FALSE; + char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; _ostree_loose_path (loose_path_buf, checksum, objtype, self->mode); @@ -3316,9 +3314,7 @@ _ostree_repo_has_loose_object (OstreeRepo *self, } } - if (res == 0) - tmp_file = TRUE; - else + if (res < 0) { do res = fstatat (self->objects_dir_fd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW); @@ -3332,32 +3328,10 @@ _ostree_repo_has_loose_object (OstreeRepo *self, ret = TRUE; *out_is_stored = (res != -1); - - if (out_stored_path) - { - if (res != -1) - *out_stored_path = g_file_resolve_relative_path (tmp_file ? self->tmp_dir : self->objects_dir, loose_path_buf); - else - *out_stored_path = NULL; - } out: return ret; } -gboolean -_ostree_repo_find_object (OstreeRepo *self, - OstreeObjectType objtype, - const char *checksum, - GFile **out_stored_path, - GCancellable *cancellable, - GError **error) -{ - gboolean has_object; - char loose_path[_OSTREE_LOOSE_PATH_MAX]; - return _ostree_repo_has_loose_object (self, checksum, objtype, &has_object, loose_path, - out_stored_path, cancellable, error); -} - /** * ostree_repo_has_object: * @self: Repo @@ -3382,13 +3356,12 @@ ostree_repo_has_object (OstreeRepo *self, { gboolean ret = FALSE; gboolean ret_have_object; - g_autoptr(GFile) loose_path = NULL; - if (!_ostree_repo_find_object (self, objtype, checksum, &loose_path, - cancellable, error)) + if (!_ostree_repo_has_loose_object (self, checksum, objtype, &ret_have_object, + cancellable, error)) goto out; - ret_have_object = (loose_path != NULL); + /* In the future, here is where we would also look up in metadata pack files */ if (!ret_have_object && self->parent_repo) { From ad019def9a43c0cf10526f575a32f84c0c71ee78 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 22 Apr 2016 09:29:59 +0200 Subject: [PATCH 015/128] repo: Fix temporary directory allocation We looked for and locked old temporary directories so we can reuse them if not in use. However, once we found one that we can reuse we didn't stop iterating, and eventually we reached the end. This means we can lock multiple dirs. Closes: #273 Approved by: giuseppe --- src/libostree/ostree-repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 875c9480..35bc3a5d 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5012,7 +5012,7 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, if (!glnx_dirfd_iterator_init_at (tmpdir_dfd, ".", FALSE, &dfd_iter, error)) return FALSE; - while (TRUE) + while (tmpdir_name == NULL) { gs_dirfd_iterator_cleanup GSDirFdIterator child_dfd_iter = { 0, }; struct dirent *dent; From 08d2b8ab7bb4cadd0ad41fcfa9c7cc7762316861 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 19 Apr 2016 11:32:14 +0200 Subject: [PATCH 016/128] lib: Add URL override pull option The "override-url" option allows to use the other URL while still using some options from the passed remote. Closes: #271 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 6 +++++- src/libostree/ostree-repo.c | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 968b5951..708df5b6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1909,6 +1909,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean require_static_deltas = FALSE; gboolean opt_gpg_verify = FALSE; gboolean opt_gpg_verify_summary = FALSE; + const char *url_override = NULL; if (options) { @@ -1926,6 +1927,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas); (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run); + (void) g_variant_lookup (options, "override-url", "&s", &url_override); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); @@ -2021,7 +2023,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, { g_autofree char *baseurl = NULL; - if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error)) + if (url_override != NULL) + baseurl = g_strdup (url_override); + else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error)) goto out; pull_data->base_uri = soup_uri_new (baseurl); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 35bc3a5d..97233d3d 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4077,6 +4077,7 @@ ostree_repo_pull_one_dir (OstreeRepo *self, * * require-static-deltas (b): Require static deltas * * override-commit-ids (as): Array of specific commit IDs to fetch for refs * * dry-run (b): Only print information on what will be downloaded (requires static deltas) + * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, From 560cc49234b111257ffe1fdc50fe7cc30d8d9fdd Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 20 Apr 2016 10:34:04 +0200 Subject: [PATCH 017/128] pull: Add a --url option This allows ostree to pull the objects from a different URL without modifying the repo's config. Closes: #271 Approved by: cgwalters --- src/ostree/ot-builtin-pull.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 734f7440..f69d276c 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -37,7 +37,8 @@ static gboolean opt_untrusted; static char* opt_subpath; static char* opt_cache_dir; static int opt_depth = 0; - +static char* opt_url; + static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, { "cache-dir", 0, 0, G_OPTION_ARG_STRING, &opt_cache_dir, "Use custom cache dir", NULL }, @@ -49,6 +50,7 @@ static GOptionEntry options[] = { { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Do not trust (local) sources", NULL }, { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, + { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL }, { NULL } }; @@ -228,6 +230,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** GVariantBuilder builder; g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); + if (opt_url) + g_variant_builder_add (&builder, "{s@v}", "override-url", + g_variant_new_variant (g_variant_new_string (opt_url))); if (opt_subpath) g_variant_builder_add (&builder, "{s@v}", "subdir", g_variant_new_variant (g_variant_new_string (opt_subpath))); From a6ce16223119948f3ad5793e73d982b6f4d8d3c0 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 22 Apr 2016 14:40:05 +0200 Subject: [PATCH 018/128] trivial-httpd: Add a --log-file option Might be useful to see what files the client wants to fetch. Closes: #271 Approved by: cgwalters --- src/ostree/ot-builtin-trivial-httpd.c | 73 ++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-builtin-trivial-httpd.c b/src/ostree/ot-builtin-trivial-httpd.c index 07d02fb9..0300b128 100644 --- a/src/ostree/ot-builtin-trivial-httpd.c +++ b/src/ostree/ot-builtin-trivial-httpd.c @@ -22,6 +22,8 @@ #include +#include + #include "ot-main.h" #include "ot-builtins.h" #include "ostree.h" @@ -32,6 +34,7 @@ #include static char *opt_port_file = NULL; +static char *opt_log = NULL; static gboolean opt_daemonize; static gboolean opt_autoexit; static gboolean opt_force_ranges; @@ -40,6 +43,7 @@ static gint opt_port = 0; typedef struct { GFile *root; gboolean running; + GOutputStream *log; } OtTrivialHttpd; static GOptionEntry options[] = { @@ -48,9 +52,28 @@ static GOptionEntry options[] = { { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", NULL }, { "port-file", 'p', 0, G_OPTION_ARG_FILENAME, &opt_port_file, "Write port number to PATH (- for standard output)", "PATH" }, { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL }, + { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here", "PATH" }, { NULL } }; +static void +httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...) +{ + g_autoptr(GString) str = NULL; + va_list args; + gsize written; + + if (!httpd->log) + return; + + str = g_string_new (NULL); + va_start (args, format); + g_string_vprintf (str, format, args); + va_end (args); + + g_output_stream_write_all (httpd->log, str->str, str->len, &written, NULL, NULL); +} + static int compare_strings (gconstpointer a, gconstpointer b) { @@ -154,6 +177,7 @@ do_get (OtTrivialHttpd *self, struct stat stbuf; g_autofree char *safepath = NULL; + httpd_log (self, "serving %s\n", path); if (strstr (path, "../") != NULL) { soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN); @@ -296,6 +320,16 @@ do_get (OtTrivialHttpd *self, soup_message_set_status (msg, SOUP_STATUS_OK); } out: + { + guint status = 0; + g_autofree gchar *reason = NULL; + + g_object_get (msg, + "status-code", &status, + "reason-phrase", &reason, + NULL); + httpd_log (self, " status: %s (%u)\n", reason, status); + } return; } @@ -351,6 +385,37 @@ ostree_builtin_trivial_httpd (int argc, char **argv, GCancellable *cancellable, app->root = g_file_new_for_path (dirpath); + if (opt_log) + { + GOutputStream *stream = NULL; + + if (g_strcmp0 (opt_log, "-") == 0) + { + if (opt_daemonize) + { + ot_util_usage_error (context, "Cannot use --log-file=- and --daemonize at the same time", error); + goto out; + } + stream = G_OUTPUT_STREAM (g_unix_output_stream_new (STDOUT_FILENO, FALSE)); + } + else + { + g_autoptr(GFile) log_file; + GFileOutputStream* log_stream; + + log_file = g_file_new_for_path (opt_log); + log_stream = g_file_create (log_file, + G_FILE_CREATE_PRIVATE, + cancellable, + error); + if (!log_stream) + goto out; + stream = G_OUTPUT_STREAM (log_stream); + } + + app->log = stream; + } + #if SOUP_CHECK_VERSION(2, 48, 0) server = soup_server_new (SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL); if (!soup_server_listen_all (server, opt_port, 0, error)) @@ -456,13 +521,17 @@ ostree_builtin_trivial_httpd (int argc, char **argv, GCancellable *cancellable, goto out; g_signal_connect (dirmon, "changed", G_CALLBACK (on_dir_changed), app); } - + { + g_autofree gchar *path = g_file_get_path (app->root); + httpd_log (app, "serving at root %s\n", path); + } while (app->running) g_main_context_iteration (NULL, TRUE); - + ret = TRUE; out: g_clear_object (&app->root); + g_clear_object (&app->log); if (context) g_option_context_free (context); return ret; From c2931583d3df106358d23080e49b9a6572b3b1fb Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 22 Apr 2016 14:41:04 +0200 Subject: [PATCH 019/128] tests: Add a test for ostree pull --url Closes: #271 Approved by: cgwalters --- Makefile-tests.am | 1 + tests/test-pull-override-url.sh | 80 +++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100755 tests/test-pull-override-url.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 47af9e70..9f24ea1d 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -51,6 +51,7 @@ test_scripts = \ tests/test-pull-summary-sigs.sh \ tests/test-pull-resume.sh \ tests/test-pull-untrusted.sh \ + tests/test-pull-override-url.sh \ tests/test-local-pull.sh \ tests/test-local-pull-depth.sh \ tests/test-gpg-signed-commit.sh \ diff --git a/tests/test-pull-override-url.sh b/tests/test-pull-override-url.sh new file mode 100755 index 00000000..d81b3454 --- /dev/null +++ b/tests/test-pull-override-url.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Copyright (C) 2016 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive-z2" + +echo '1..1' + +cd ${test_tmpdir} +# get a list of XX/XXXXXXX...XX.commit +find ostree-srv/gnomerepo/objects -name '*.commit' | cut -d/ -f4- | sort >${test_tmpdir}/original_commits + + +gnomerepo_url="$(cat httpd-address)/ostree/gnomerepo" +mkdir mirror-srv +cd mirror-srv +mkdir gnomerepo +${CMD_PREFIX} ostree --repo=gnomerepo init --mode "archive-z2" +${CMD_PREFIX} ostree --repo=gnomerepo remote add --set=gpg-verify=false origin ${gnomerepo_url} +${CMD_PREFIX} ostree --repo=gnomerepo pull --mirror --depth=-1 origin main + +find gnomerepo/objects -name '*.commit' | cut -d/ -f3- | sort >${test_tmpdir}/mirror_commits + +if ! cmp --quiet ${test_tmpdir}/original_commits ${test_tmpdir}/mirror_commits +then + assert_not_reached 'original repository and its mirror should have the same commits' +fi + +cd ${test_tmpdir} +mkdir mirror-httpd +cd mirror-httpd +ln -s ${test_tmpdir}/mirror-srv ostree +mirror_log="${test_tmpdir}/mirror_log" +${CMD_PREFIX} ostree trivial-httpd --log-file=${mirror_log} --autoexit --daemonize -p ${test_tmpdir}/mirror-httpd-port +port=$(cat ${test_tmpdir}/mirror-httpd-port) +echo "http://127.0.0.1:${port}" > ${test_tmpdir}/mirror-httpd-address + +cd ${test_tmpdir} +mirrorrepo_url="$(cat mirror-httpd-address)/ostree/gnomerepo" +mkdir repo +${CMD_PREFIX} ostree --repo=repo init +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin ${gnomerepo_url} +rm -rf ${test_tmpdir}/ostree-srv/gnomerepo +if ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 origin main +then + assert_not_reached 'pulling from removed remote should have failed' +fi + +if ! ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 --url=${mirrorrepo_url} origin main +then + cat ${mirror_log} + assert_not_reached 'fetching from the overridden URL should succeed' +fi +find repo/objects -name '*.commit' | cut -d/ -f3- | sort >${test_tmpdir}/repo_commits + +if ! cmp --quiet ${test_tmpdir}/original_commits ${test_tmpdir}/repo_commits +then + assert_not_reached 'original repository and its mirror should have the same commits' +fi + +echo "ok pull url override" From f2fd1f50e23da45860ef57acdf3a77198a1c3760 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 22 Apr 2016 10:28:53 +0200 Subject: [PATCH 020/128] repo: Handle parent repos with different remote configuration In the case we have a repo with a parent, and the child repo has a remote called "foo", but some option is unset. Then when we look up the parent repo for a value before using the default we will fail due to the parent not having the "foo" remote. As soon as we find the requested remote at some point in the hierarchy we need to ignore further errors and use the default value. Closes: #274 Approved by: giuseppe --- src/libostree/ostree-repo.c | 47 +++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 97233d3d..b4d702a1 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -322,12 +322,16 @@ ostree_repo_get_remote_option (OstreeRepo *self, { if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) { - if (self->parent_repo != NULL) - return ostree_repo_get_remote_option (self->parent_repo, - remote_name, option_name, - default_value, - out_value, - error); + /* Note: We ignore errors on the parent because the parent config may not + specify this remote, causing a "remote not found" error, but we found + the remote at some point, so we need to instead return the default */ + if (self->parent_repo != NULL && + ostree_repo_get_remote_option (self->parent_repo, + remote_name, option_name, + default_value, + out_value, + NULL)) + return TRUE; value = g_strdup (default_value); ret = TRUE; @@ -397,11 +401,16 @@ ostree_repo_get_remote_list_option (OstreeRepo *self, /* Default value if key not found is always NULL. */ if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) { - if (self->parent_repo != NULL) - return ostree_repo_get_remote_list_option (self->parent_repo, - remote_name, option_name, - out_value, - error); + /* Note: We ignore errors on the parent because the parent config may not + specify this remote, causing a "remote not found" error, but we found + the remote at some point, so we need to instead return the default */ + if (self->parent_repo != NULL && + ostree_repo_get_remote_list_option (self->parent_repo, + remote_name, option_name, + out_value, + NULL)) + return TRUE; + ret = TRUE; } else if (temp_error) @@ -464,12 +473,16 @@ ostree_repo_get_remote_boolean_option (OstreeRepo *self, { if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) { - if (self->parent_repo != NULL) - return ostree_repo_get_remote_boolean_option (self->parent_repo, - remote_name, option_name, - default_value, - out_value, - error); + /* Note: We ignore errors on the parent because the parent config may not + specify this remote, causing a "remote not found" error, but we found + the remote at some point, so we need to instead return the default */ + if (self->parent_repo != NULL && + ostree_repo_get_remote_boolean_option (self->parent_repo, + remote_name, option_name, + default_value, + out_value, + NULL)) + return TRUE; value = default_value; ret = TRUE; From ad0de186bb968cb25987d0118aa94552ca9dfc33 Mon Sep 17 00:00:00 2001 From: Gatis Paeglis Date: Mon, 25 Apr 2016 13:57:03 +0200 Subject: [PATCH 021/128] Fix --enable_rofiles_fuse=no build Closes: #276 Approved by: cgwalters --- Makefile.am | 2 ++ configure.ac | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 488d4b6d..e49b7c59 100644 --- a/Makefile.am +++ b/Makefile.am @@ -71,7 +71,9 @@ include Makefile-otutil.am include Makefile-libostree.am include Makefile-ostree.am include Makefile-switchroot.am +if BUILDOPT_FUSE include src/rofiles-fuse/Makefile-inc.am +endif include Makefile-tests.am include Makefile-boot.am include Makefile-man.am diff --git a/configure.ac b/configure.ac index dca9f536..6af60e8c 100644 --- a/configure.ac +++ b/configure.ac @@ -222,7 +222,7 @@ AC_ARG_ENABLE(rofiles-fuse, [AS_HELP_STRING([--enable-rofiles-fuse], [generate rofiles-fuse helper [default=yes]])],, enable_rofiles_fuse=yes) -AS_IF([ test $enable_rofiles_fuse != xno ], [ +AS_IF([ test x$enable_rofiles_fuse != xno ], [ PKG_CHECK_MODULES(BUILDOPT_FUSE, $FUSE_DEPENDENCY) ], [enable_rofiles_fuse=no]) AM_CONDITIONAL(BUILDOPT_FUSE, test x$enable_rofiles_fuse = xyes) From 542df553e2c51a9d2f9cfc4212dfd996f54b3f8b Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Mon, 25 Apr 2016 12:50:32 +0100 Subject: [PATCH 022/128] test-parent: this test requires user xattrs Signed-off-by: Simon McVittie Closes: #278 Approved by: cgwalters --- tests/test-parent.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test-parent.sh b/tests/test-parent.sh index 05b102a2..d1f5d7c9 100755 --- a/tests/test-parent.sh +++ b/tests/test-parent.sh @@ -21,6 +21,8 @@ set -euo pipefail . $(dirname $0)/libtest.sh +skip_without_user_xattrs + echo '1..2' setup_test_repository "archive-z2" From 72abe6237093e6db2f34a434696c9b53cdd52827 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 25 Apr 2016 15:59:36 -0400 Subject: [PATCH 023/128] deploy: Start hardlinking kernel/initramfs on single-part again Commit https://github.com/ostreedev/ostree/commit/1810de2b51680dbf35fdbd33d0a8d7e65eadc91f lost an optimization where we would try hardlinks for the kernel/initramfs in `/boot`. This would be a noticeable space savings on single-partition systems. Closes: #277 Approved by: gatispaeglis --- src/libostree/ostree-sysroot-deploy.c | 45 +++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 88772365..d1d624b6 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -87,6 +87,39 @@ symlink_at_replace (const char *oldpath, return ret; } +/* Try a hardlink if we can, otherwise fall back to copying. Used + * right now for kernels/initramfs in /boot, where we can just + * hardlink if we're on the same partition. + */ +static gboolean +hardlink_or_copy_at (int src_dfd, + const char *src_subpath, + int dest_dfd, + const char *dest_subpath, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + + if (linkat (src_dfd, src_subpath, dest_dfd, dest_subpath, 0) != 0) + { + if (errno == EMLINK || errno == EXDEV) + { + return glnx_file_copy_at (src_dfd, src_subpath, NULL, dest_dfd, dest_subpath, 0, + cancellable, error); + } + else + { + glnx_set_error_from_errno (error); + goto out; + } + } + + ret = TRUE; + out: + return ret; +} + static gboolean dirfd_copy_attributes_and_xattrs (int src_parent_dfd, const char *src_name, @@ -1337,9 +1370,9 @@ install_deployment_kernel (OstreeSysroot *sysroot, glnx_set_prefix_error_from_errno (error, "fstat %s", dest_kernel_name); goto out; } - if (!glnx_file_copy_at (tree_boot_dfd, tree_kernel_name, NULL, - bootcsum_dfd, dest_kernel_name, 0, - cancellable, error)) + if (!hardlink_or_copy_at (tree_boot_dfd, tree_kernel_name, + bootcsum_dfd, dest_kernel_name, + cancellable, error)) goto out; } @@ -1354,9 +1387,9 @@ install_deployment_kernel (OstreeSysroot *sysroot, glnx_set_prefix_error_from_errno (error, "fstat %s", dest_initramfs_name); goto out; } - if (!glnx_file_copy_at (tree_boot_dfd, tree_initramfs_name, NULL, - bootcsum_dfd, dest_initramfs_name, 0, - cancellable, error)) + if (!hardlink_or_copy_at (tree_boot_dfd, tree_initramfs_name, + bootcsum_dfd, dest_initramfs_name, + cancellable, error)) goto out; } } From 831e9dcdeac28b56c29e84f58ae90fb70e73626c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 28 Apr 2016 11:34:27 -0400 Subject: [PATCH 024/128] build: Introduce --with-dracut=yesbutnoconf It's actually just easier for build systems (e.g. rpm-ostree) using dracut to use `--add ostree` rather than indirecting through the conf file. This makes it easier for yum-managed systems to install ostree without side effects. Closes: #279 Approved by: gatispaeglis --- Makefile-boot.am | 3 ++- configure.ac | 11 +++++++++-- packaging/ostree.spec.in | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile-boot.am b/Makefile-boot.am index 7ec54fa3..8b62d1ba 100644 --- a/Makefile-boot.am +++ b/Makefile-boot.am @@ -21,7 +21,8 @@ if BUILDOPT_DRACUT # Not using $(libdir) here is intentional, dracut modules go in prefix/lib dracutmoddir = $(prefix)/lib/dracut/modules.d/98ostree dracutmod_SCRIPTS = src/boot/dracut/module-setup.sh - +endif +if BUILDOPT_DRACUT_CONF dracutconfdir = $(sysconfdir)/dracut.conf.d dracutconf_DATA = src/boot/dracut/ostree.conf endif diff --git a/configure.ac b/configure.ac index 6af60e8c..54d30e99 100644 --- a/configure.ac +++ b/configure.ac @@ -231,7 +231,14 @@ AC_ARG_WITH(dracut, AS_HELP_STRING([--with-dracut], [Install dracut module (default: no)]),, [with_dracut=no]) -AM_CONDITIONAL(BUILDOPT_DRACUT, test x$with_dracut = xyes) +case x$with_dracut in + xno) ;; + xyes) ;; + xyesbutnoconf) ;; + *) AC_MSG_ERROR([Unknown --with-dracut value $with_dracut]) +esac +AM_CONDITIONAL(BUILDOPT_DRACUT, test x$with_dracut = xyes || test x$with_dracut = xyesbutnoconf) +AM_CONDITIONAL(BUILDOPT_DRACUT_CONF, test x$with_dracut = xyes) AC_ARG_WITH(mkinitcpio, AS_HELP_STRING([--with-mkinitcpio], @@ -239,7 +246,7 @@ AC_ARG_WITH(mkinitcpio, [with_mkinitcpio=no]) AM_CONDITIONAL(BUILDOPT_MKINITCPIO, test x$with_mkinitcpio = xyes) -AS_IF([test "x$with_dracut" = "xyes" || test "x$with_mkinitcpio" = "xyes"], [ +AS_IF([test "x$with_dracut" = "xyes" || test "x$with_dracut" = "xyesbutnoconf" || test "x$with_mkinitcpio" = "xyes"], [ with_systemd=yes AC_ARG_WITH([systemdsystemunitdir], AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files]), diff --git a/packaging/ostree.spec.in b/packaging/ostree.spec.in index 2d6a2660..2f07f0a8 100644 --- a/packaging/ostree.spec.in +++ b/packaging/ostree.spec.in @@ -72,7 +72,7 @@ env NOCONFIGURE=1 ./autogen.sh %configure --disable-silent-rules \ --enable-gtk-doc \ --with-selinux \ - --with-dracut + --with-dracut=yesbutnoconf make %{?_smp_mflags} %install @@ -94,7 +94,6 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/ostree %{_sbindir}/ostree-prepare-root %{_sbindir}/ostree-remount -%{_sysconfdir}/dracut.conf.d/ostree.conf %dir %{_prefix}/lib/dracut/modules.d/98ostree %{_prefix}/lib/systemd/system/ostree*.service %{_prefix}/lib/dracut/modules.d/98ostree/* From a6c731f6e7d1014aab5989c98a6432d28080905b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 1 May 2016 14:22:52 -0400 Subject: [PATCH 025/128] libglnx porting: Migrate from GSConsole To GLnxConsoleRef. There were some subtleties here, for example we used to reference `GSConsole` inside the progress changed function, which at first seems like an ABI hazard, because e.g. rpm-ostree or xdg-app could still be passing a `GSConsole` instance there. Luckily, it turns out to be compatible to just start calling libglnx here. Another issue was that due to libglnx's use of the cleanup function, we needed to ensure we always called `ostree_async_progress_finish()` *before* the cleanup function was invoked. Closes: #280 Approved by: giuseppe --- libglnx | 2 +- src/libostree/ostree-repo.c | 11 +++-- src/ostree/ot-admin-builtin-switch.c | 40 +++++++--------- src/ostree/ot-admin-builtin-upgrade.c | 36 ++++++-------- src/ostree/ot-builtin-pull-local.c | 20 ++++---- src/ostree/ot-builtin-pull.c | 68 +++++++++++++-------------- 6 files changed, 81 insertions(+), 96 deletions(-) diff --git a/libglnx b/libglnx index 76952275..47ddbfa5 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 769522753c25537e520adc322fa62e5390272add +Subproject commit 47ddbfa56341df3a9453854e1101e1c2f2359ddb diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index b4d702a1..75bc5c2a 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -27,6 +27,7 @@ #include #include #include "otutil.h" +#include #include "ostree-core-private.h" #include "ostree-repo-private.h" @@ -4120,12 +4121,15 @@ ostree_repo_pull_with_options (OstreeRepo *self, * custom status message, or else outstanding fetch progress in bytes/sec, * or else outstanding content or metadata writes to the repository in * number of objects. + * + * Compatibility note: this function previously assumed that @user_data + * was a pointer to a #GSConsole instance. This is no longer the case, + * and @user_data is ignored. **/ void ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress, gpointer user_data) { - GSConsole *console = user_data; GString *buf; g_autofree char *status = NULL; guint outstanding_fetches; @@ -4135,7 +4139,8 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress guint fetched_delta_parts; guint total_delta_parts; - if (!console) + /* Historical note; we used to treat this as a GSConsole instance */ + if (user_data == NULL) return; buf = g_string_new (""); @@ -4202,7 +4207,7 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress g_string_append_printf (buf, "Scanning metadata: %u", n_scanned_metadata); } - gs_console_begin_status_line (console, buf->str, NULL, NULL); + glnx_console_text (buf->str); g_string_free (buf, TRUE); } diff --git a/src/ostree/ot-admin-builtin-switch.c b/src/ostree/ot-admin-builtin-switch.c index 0e0101a9..b313e830 100644 --- a/src/ostree/ot-admin-builtin-switch.c +++ b/src/ostree/ot-admin-builtin-switch.c @@ -61,8 +61,6 @@ ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GErro glnx_unref_object OstreeSysrootUpgrader *upgrader = NULL; glnx_unref_object OstreeAsyncProgress *progress = NULL; gboolean changed; - GSConsole *console = NULL; - gboolean in_status_line = FALSE; GKeyFile *old_origin; GKeyFile *new_origin = NULL; @@ -125,28 +123,24 @@ ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GErro if (!ostree_sysroot_upgrader_set_origin (upgrader, new_origin, cancellable, error)) goto out; - console = gs_console_get (); - if (console) - { - gs_console_begin_status_line (console, "", NULL, NULL); - in_status_line = TRUE; - progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console); - } + { g_auto(GLnxConsoleRef) console = { 0, }; + glnx_console_lock (&console); - /* Always allow older...there's not going to be a chronological - * relationship necessarily. - */ - if (!ostree_sysroot_upgrader_pull (upgrader, 0, - OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER, - progress, &changed, - cancellable, error)) - goto out; + if (console.is_tty) + progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); - if (in_status_line) - { - gs_console_end_status_line (console, NULL, NULL); - in_status_line = FALSE; - } + /* Always allow older...there's not going to be a chronological + * relationship necessarily. + */ + if (!ostree_sysroot_upgrader_pull (upgrader, 0, + OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER, + progress, &changed, + cancellable, error)) + goto out; + + if (progress) + ostree_async_progress_finish (progress); + } if (!ostree_sysroot_upgrader_deploy (upgrader, cancellable, error)) goto out; @@ -171,8 +165,6 @@ ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GErro ret = TRUE; out: - if (in_status_line) - gs_console_end_status_line (console, NULL, NULL); if (new_origin) g_key_file_unref (new_origin); if (context) diff --git a/src/ostree/ot-admin-builtin-upgrade.c b/src/ostree/ot-admin-builtin-upgrade.c index b3b531c0..81f9bb6f 100644 --- a/src/ostree/ot-admin-builtin-upgrade.c +++ b/src/ostree/ot-admin-builtin-upgrade.c @@ -55,8 +55,6 @@ ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GErr g_autoptr(GFile) deployment_path = NULL; g_autoptr(GFile) deployment_origin_path = NULL; g_autoptr(GKeyFile) origin = NULL; - GSConsole *console = NULL; - gboolean in_status_line = FALSE; glnx_unref_object OstreeAsyncProgress *progress = NULL; gboolean changed; OstreeSysrootUpgraderPullFlags upgraderpullflags = 0; @@ -108,27 +106,23 @@ ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GErr } } - console = gs_console_get (); - if (console) - { - gs_console_begin_status_line (console, "", NULL, NULL); - in_status_line = TRUE; - progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console); - } + { g_auto(GLnxConsoleRef) console = { 0, }; + glnx_console_lock (&console); - if (opt_allow_downgrade) - upgraderpullflags |= OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER; + if (console.is_tty) + progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); - if (!ostree_sysroot_upgrader_pull (upgrader, 0, upgraderpullflags, - progress, &changed, - cancellable, error)) - goto out; + if (opt_allow_downgrade) + upgraderpullflags |= OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER; + + if (!ostree_sysroot_upgrader_pull (upgrader, 0, upgraderpullflags, + progress, &changed, + cancellable, error)) + goto out; - if (in_status_line) - { - gs_console_end_status_line (console, NULL, NULL); - in_status_line = FALSE; - } + if (progress) + ostree_async_progress_finish (progress); + } if (!changed) { @@ -148,8 +142,6 @@ ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GErr ret = TRUE; out: - if (in_status_line) - gs_console_end_status_line (console, NULL, NULL); if (context) g_option_context_free (context); return ret; diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index 36057ec6..7fb6f03a 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -55,7 +55,6 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr glnx_unref_object OstreeRepo *repo = NULL; int i; const char *src_repo_arg; - GSConsole *console = NULL; g_autofree char *src_repo_uri = NULL; glnx_unref_object OstreeAsyncProgress *progress = NULL; g_autoptr(GPtrArray) refs_to_fetch = NULL; @@ -132,14 +131,11 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr g_ptr_array_add (refs_to_fetch, NULL); } - console = gs_console_get (); - if (console) - { - gs_console_begin_status_line (console, "", NULL, NULL); - progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console); - } - { GVariantBuilder builder; + g_auto(GLnxConsoleRef) console = { 0, }; + + glnx_console_lock (&console); + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); g_variant_builder_add (&builder, "{s@v}", "flags", @@ -158,17 +154,21 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr g_variant_builder_add (&builder, "{s@v}", "depth", g_variant_new_variant (g_variant_new_int32 (opt_depth))); + if (console.is_tty) + progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); + if (!ostree_repo_pull_with_options (repo, src_repo_uri, g_variant_builder_end (&builder), progress, cancellable, error)) goto out; + + if (progress) + ostree_async_progress_finish (progress); } ret = TRUE; out: - if (progress) - ostree_async_progress_finish (progress); if (context) g_option_context_free (context); if (repo) diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index f69d276c..99b25937 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -58,16 +58,17 @@ static void gpg_verify_result_cb (OstreeRepo *repo, const char *checksum, OstreeGpgVerifyResult *result, - GSConsole *console) + GLnxConsoleRef *console) { - /* Temporarily place the GSConsole stream (which is just stdout) - * back in normal mode before printing GPG verification results. */ - gs_console_end_status_line (console, NULL, NULL); + /* Temporarily place the tty back in normal mode before printing GPG + * verification results. + */ + glnx_console_unlock (console); g_print ("\n"); ostree_print_gpg_verify_result (result); - gs_console_begin_status_line (console, "", NULL, NULL); + glnx_console_lock (console); } static gboolean printed_console_progress; @@ -111,7 +112,6 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** gboolean ret = FALSE; g_autofree char *remote = NULL; OstreeRepoPullFlags pullflags = 0; - GSConsole *console = NULL; g_autoptr(GPtrArray) refs_to_fetch = NULL; g_autoptr(GPtrArray) override_commit_ids = NULL; glnx_unref_object OstreeAsyncProgress *progress = NULL; @@ -206,30 +206,13 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_ptr_array_add (refs_to_fetch, NULL); } - if (!opt_dry_run) - { - console = gs_console_get (); - if (console) - { - gs_console_begin_status_line (console, "", NULL, NULL); - progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console); - signal_handler_id = g_signal_connect (repo, "gpg-verify-result", - G_CALLBACK (gpg_verify_result_cb), - console); - } - } - else - { - progress = ostree_async_progress_new_and_connect (dry_run_console_progress_changed, console); - signal_handler_id = g_signal_connect (repo, "gpg-verify-result", - G_CALLBACK (gpg_verify_result_cb), - console); - } - { GVariantBuilder builder; + g_auto(GLnxConsoleRef) console = { 0, }; g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); + glnx_console_lock (&console); + if (opt_url) g_variant_builder_add (&builder, "{s@v}", "override-url", g_variant_new_variant (g_variant_new_string (opt_url))); @@ -257,25 +240,38 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); + if (!opt_dry_run) + { + if (console.is_tty) + progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); + } + else + { + progress = ostree_async_progress_new_and_connect (dry_run_console_progress_changed, NULL); + } + + if (console.is_tty) + { + signal_handler_id = g_signal_connect (repo, "gpg-verify-result", + G_CALLBACK (gpg_verify_result_cb), + &console); + } + if (!ostree_repo_pull_with_options (repo, remote, g_variant_builder_end (&builder), progress, cancellable, error)) goto out; + + if (progress) + ostree_async_progress_finish (progress); + + if (opt_dry_run) + g_assert (printed_console_progress); } - if (progress) - ostree_async_progress_finish (progress); - - if (opt_dry_run) - g_assert (printed_console_progress); - ret = TRUE; out: if (signal_handler_id > 0) g_signal_handler_disconnect (repo, signal_handler_id); - - if (console) - gs_console_end_status_line (console, NULL, NULL); - if (context) g_option_context_free (context); return ret; From 3111248373c16a3fd6ecb0566bc6f5f42fd4b54c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 2 May 2016 10:20:46 -0400 Subject: [PATCH 026/128] repo: Fix reading repos on read-only media I have a cache drive I often mount read-only, and the previous commit for opening `tmp/cache` broke since `errno == EROFS`, not `EPERM`. It turns out we already had the concept of a "writable" repo, so just piggy back off that. Closes: #281 Approved by: giuseppe --- src/libostree/ostree-repo.c | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 75bc5c2a..420e2afb 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2416,7 +2416,6 @@ ostree_repo_open (OstreeRepo *self, g_autofree char *version = NULL; g_autofree char *mode = NULL; g_autofree char *parent_repo_path = NULL; - g_autoptr(GError) temp_error = NULL; g_return_val_if_fail (error == NULL || *error == NULL, FALSE); @@ -2550,32 +2549,13 @@ ostree_repo_open (OstreeRepo *self, if (!glnx_opendirat (self->repo_dir_fd, "tmp", TRUE, &self->tmp_dir_fd, error)) goto out; - if (!glnx_shutil_mkdir_p_at (self->tmp_dir_fd, _OSTREE_CACHE_DIR, 0775, cancellable, &temp_error)) + if (self->writable) { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED)) - { - g_clear_error (&temp_error); - g_debug ("No permissions to create cache dir"); - } - else - { - g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; - } - } + if (!glnx_shutil_mkdir_p_at (self->tmp_dir_fd, _OSTREE_CACHE_DIR, 0775, cancellable, error)) + goto out; - if (!glnx_opendirat (self->tmp_dir_fd, _OSTREE_CACHE_DIR, TRUE, &self->cache_dir_fd, &temp_error)) - { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&temp_error); - g_debug ("No cache dir"); - } - else - { - g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; - } + if (!glnx_opendirat (self->tmp_dir_fd, _OSTREE_CACHE_DIR, TRUE, &self->cache_dir_fd, error)) + goto out; } if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && self->enable_uncompressed_cache) From 15b3cab65eb005574f84c9f1df2c8af14bf53129 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 19 Apr 2016 12:11:01 -0400 Subject: [PATCH 027/128] repo: Add OSTREE_REPO_TEST_ERROR=pre-commit env var Setting this causes commit to error out. There are other ways we could do this in a more sophisticated fashion, such as via SystemTap etc. But this has low-tech applicablity, works as non-root. The reason I'm adding this is so that we can add test cases for cleanup of the `tmp/staging-` directory. Closes: #170 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 7 +++++++ src/libostree/ostree-repo-private.h | 6 ++++++ src/libostree/ostree-repo.c | 6 ++++++ tests/basic-test.sh | 11 ++++++++++- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index cd365da9..8e00646c 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1442,6 +1442,13 @@ ostree_repo_commit_transaction (OstreeRepo *self, g_return_val_if_fail (self->in_transaction == TRUE, FALSE); + if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_PRE_COMMIT) > 0) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "OSTREE_REPO_TEST_ERROR_PRE_COMMIT specified"); + goto out; + } + if (syncfs (self->tmp_dir_fd) < 0) { glnx_set_error_from_errno (error); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 54127be0..70600f4e 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -36,6 +36,10 @@ G_BEGIN_DECLS #define _OSTREE_SUMMARY_CACHE_DIR "summaries" #define _OSTREE_CACHE_DIR "cache" +typedef enum { + OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0) +} OstreeRepoTestErrorFlags; + /** * OstreeRepo: * @@ -86,6 +90,8 @@ struct OstreeRepo { uid_t target_owner_uid; gid_t target_owner_gid; + guint test_error_flags; /* OstreeRepoTestErrorFlags */ + GKeyFile *config; GHashTable *remotes; GMutex remotes_lock; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 420e2afb..d44aeabe 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -798,6 +798,9 @@ ostree_repo_init (OstreeRepo *self) { static gsize gpgme_initialized; GLnxLockFile empty_lockfile = GLNX_LOCK_FILE_INIT; + const GDebugKey test_error_keys[] = { + { "pre-commit", OSTREE_REPO_TEST_ERROR_PRE_COMMIT }, + }; if (g_once_init_enter (&gpgme_initialized)) { @@ -806,6 +809,9 @@ ostree_repo_init (OstreeRepo *self) g_once_init_leave (&gpgme_initialized, 1); } + self->test_error_flags = g_parse_debug_string (g_getenv ("OSTREE_REPO_TEST_ERROR"), + test_error_keys, G_N_ELEMENTS (test_error_keys)); + g_mutex_init (&self->cache_lock); g_mutex_init (&self->txn_stats_lock); diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 5519764a..2edb15d2 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..53" +echo "1..54" $OSTREE checkout test2 checkout-test2 echo "ok checkout" @@ -397,6 +397,15 @@ assert_file_has_content test2-checkout/baz/cow moo assert_has_dir repo2/uncompressed-objects-cache echo "ok disable cache checkout" +cd ${test_tmpdir} +rm checkout-test2 -rf +$OSTREE checkout test2 checkout-test2 +if env OSTREE_REPO_TEST_ERROR=pre-commit $OSTREE commit -b test2 -s '' $test_tmpdir/checkout-test2 2>err.txt; then + assert_not_reached "Should have hit OSTREE_REPO_TEST_ERROR_PRE_COMMIT" +fi +assert_file_has_content err.txt OSTREE_REPO_TEST_ERROR_PRE_COMMIT +echo "ok test error pre commit" + # Whiteouts cd ${test_tmpdir} mkdir -p overlay/baz/ From e3ec83a934159d16dacf940b22e80446a79c6595 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 19 Apr 2016 15:19:46 -0400 Subject: [PATCH 028/128] repo: Add OSTREE_BOOTID override envvar for debugging It's useful for test cases to be able to influence this. Conflicts: src/libostree/ostree-repo.c Closes: #170 Approved by: jlebon --- src/libostree/ostree-repo.c | 20 ++++++++++++++------ tests/basic-test.sh | 17 +++++++++++++++-- tests/libtest.sh | 6 ++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index d44aeabe..a34e2ec1 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2431,12 +2431,20 @@ ostree_repo_open (OstreeRepo *self, /* We use a per-boot identifier to keep track of which file contents * possibly haven't been sync'd to disk. */ - if (!g_file_get_contents ("/proc/sys/kernel/random/boot_id", - &self->boot_id, - NULL, - error)) - goto out; - g_strdelimit (self->boot_id, "\n", '\0'); + { const char *env_bootid = getenv ("OSTREE_BOOTID"); + + if (env_bootid != NULL) + self->boot_id = g_strdup (env_bootid); + else + { + if (!g_file_get_contents ("/proc/sys/kernel/random/boot_id", + &self->boot_id, + NULL, + error)) + goto out; + g_strdelimit (self->boot_id, "\n", '\0'); + } + } if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (self->repodir), TRUE, &self->repo_dir_fd, error)) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 2edb15d2..b2477955 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -400,11 +400,24 @@ echo "ok disable cache checkout" cd ${test_tmpdir} rm checkout-test2 -rf $OSTREE checkout test2 checkout-test2 -if env OSTREE_REPO_TEST_ERROR=pre-commit $OSTREE commit -b test2 -s '' $test_tmpdir/checkout-test2 2>err.txt; then +date > checkout-test2/date.txt +rm repo/tmp/* -rf +export TEST_BOOTID=3072029c-8b10-60d1-d31b-8422eeff9b42 +if env OSTREE_REPO_TEST_ERROR=pre-commit OSTREE_BOOTID=${TEST_BOOTID} \ + $OSTREE commit -b test2 -s '' $test_tmpdir/checkout-test2 2>err.txt; then assert_not_reached "Should have hit OSTREE_REPO_TEST_ERROR_PRE_COMMIT" fi assert_file_has_content err.txt OSTREE_REPO_TEST_ERROR_PRE_COMMIT -echo "ok test error pre commit" +found_staging=0 +for d in $(find repo/tmp/ -maxdepth 1 -type d); do + bn=$(basename $d) + if test ${bn##staging-} != ${bn}; then + assert_str_match "${bn}" "^staging-${TEST_BOOTID}-" + found_staging=1 + fi +done +assert_streq "${found_staging}" 1 +echo "ok test error pre commit/bootid" # Whiteouts cd ${test_tmpdir} diff --git a/tests/libtest.sh b/tests/libtest.sh index 0ce10a62..4d403a93 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -86,6 +86,12 @@ assert_streq () { test "$1" = "$2" || (echo 1>&2 "$1 != $2"; exit 1) } +assert_str_match () { + if ! echo "$1" | grep -E -q "$2"; then + (echo 1>&2 "$1 does not match regexp $2"; exit 1) + fi +} + assert_not_streq () { (! test "$1" = "$2") || (echo 1>&2 "$1 == $2"; exit 1) } From a56ba6081a465a3b97bd689459d6260c447d21c0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 13 Jan 2016 11:33:54 -0500 Subject: [PATCH 029/128] repo: Clean up staging directory for previous boot IDs We had a policy of cleaning up all files in `$repo/tmp` older than one day, but we should really clean up previous bootid staging directories too, as they can potentially take up a lot of disk space. https://bugzilla.gnome.org/show_bug.cgi?id=760531 Closes: #170 Approved by: jlebon --- Makefile-tests.am | 1 + src/libostree/ostree-fetcher.c | 2 +- src/libostree/ostree-repo-commit.c | 97 +++++++++++++++------- src/libostree/ostree-repo-private.h | 15 +++- src/libostree/ostree-repo.c | 118 ++++++++++++++++++--------- tests/test-admin-deploy-bootid-gc.sh | 58 +++++++++++++ 6 files changed, 220 insertions(+), 71 deletions(-) create mode 100755 tests/test-admin-deploy-bootid-gc.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 9f24ea1d..4986b427 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -63,6 +63,7 @@ test_scripts = \ tests/test-admin-deploy-etcmerge-cornercases.sh \ tests/test-admin-deploy-uboot.sh \ tests/test-admin-deploy-grub2.sh \ + tests/test-admin-deploy-bootid-gc.sh \ tests/test-admin-instutil-set-kargs.sh \ tests/test-admin-upgrade-not-backwards.sh \ tests/test-admin-pull-deploy-commit.sh \ diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index d7915ba6..3606e8fb 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -383,7 +383,7 @@ session_thread_request_uri (ThreadClosure *thread_closure, if (thread_closure->tmpdir_name == NULL) { if (!_ostree_repo_allocate_tmpdir (thread_closure->base_tmpdir_dfd, - "fetcher-", + OSTREE_REPO_TMPDIR_FETCHER, &thread_closure->tmpdir_name, &thread_closure->tmpdir_dfd, &thread_closure->tmpdir_lock, diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8e00646c..0d7f9bee 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1161,7 +1161,6 @@ ostree_repo_prepare_transaction (OstreeRepo *self, { gboolean ret = FALSE; gboolean ret_transaction_resume = FALSE; - g_autofree char *stagedir_boot_id_prefix = NULL; g_autofree char *stagedir_name = NULL; glnx_fd_close int stagedir_fd = -1; g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; @@ -1172,10 +1171,8 @@ ostree_repo_prepare_transaction (OstreeRepo *self, self->in_transaction = TRUE; - stagedir_boot_id_prefix = g_strconcat ("staging-", self->boot_id, "-", NULL); - if (!_ostree_repo_allocate_tmpdir (self->tmp_dir_fd, - stagedir_boot_id_prefix, + self->stagedir_prefix, &self->commit_stagedir_name, &self->commit_stagedir_fd, &self->commit_stagedir_lock, @@ -1281,44 +1278,86 @@ cleanup_tmpdir (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - g_autoptr(GFileEnumerator) enumerator = NULL; + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; guint64 curtime_secs; - enumerator = g_file_enumerate_children (self->tmp_dir, "standard::name,time::modified", - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); - if (!enumerator) - goto out; - curtime_secs = g_get_real_time () / 1000000; + if (!glnx_dirfd_iterator_init_at (self->tmp_dir_fd, ".", TRUE, &dfd_iter, error)) + goto out; + while (TRUE) { - GFileInfo *file_info; - GFile *path; - guint64 mtime; guint64 delta; + struct dirent *dent; + struct stat stbuf; + g_auto(GLnxLockFile) lockfile = GLNX_LOCK_FILE_INIT; + gboolean did_lock; - if (!gs_file_enumerator_iterate (enumerator, &file_info, &path, - cancellable, error)) + if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error)) goto out; - if (file_info == NULL) + + if (dent == NULL) break; - mtime = g_file_info_get_attribute_uint64 (file_info, "time::modified"); - if (mtime > curtime_secs) - continue; - /* Only delete files older than a day. To do better, we would - * need to coordinate between multiple processes in a reliable - * fashion. See - * https://bugzilla.gnome.org/show_bug.cgi?id=709115 - */ - delta = curtime_secs - mtime; - if (delta > 60*60*24) + if (TEMP_FAILURE_RETRY (fstatat (dfd_iter.fd, dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW)) < 0) { - if (!glnx_shutil_rm_rf_at (AT_FDCWD, gs_file_get_path_cached (path), cancellable, error)) + if (errno == ENOENT) /* Did another cleanup win? */ + continue; + glnx_set_error_from_errno (error); + goto out; + } + + /* First, if it's a directory which needs locking, but it's + * busy, skip it. + */ + if (_ostree_repo_is_locked_tmpdir (dent->d_name)) + { + if (!_ostree_repo_try_lock_tmpdir (dfd_iter.fd, dent->d_name, + &lockfile, &did_lock, error)) goto out; + if (!did_lock) + continue; + } + + /* If however this is the staging directory for the *current* + * boot, then don't delete it now - we may end up reusing it, as + * is the point. + */ + if (g_str_has_prefix (dent->d_name, self->stagedir_prefix)) + continue; + else if (g_str_has_prefix (dent->d_name, OSTREE_REPO_TMPDIR_STAGING)) + { + /* But, crucially we can now clean up staging directories + * from *other* boots + */ + if (!glnx_shutil_rm_rf_at (dfd_iter.fd, dent->d_name, cancellable, error)) + goto out; + } + /* FIXME - move OSTREE_REPO_TMPDIR_FETCHER underneath the + * staging/boot-id scheme as well, since all of the "did it get + * fsync'd" concerns apply to that as well. Then we can skip + * this special case. + */ + else if (g_str_has_prefix (dent->d_name, OSTREE_REPO_TMPDIR_FETCHER)) + continue; + else + { + /* Now we do time-based cleanup. Ignore it if it's somehow + * in the future... + */ + if (stbuf.st_mtime > curtime_secs) + continue; + + /* Now, we arbitrarily delete files/directories older than a + * day, since that's what we were doing before we had locking. + */ + delta = curtime_secs - stbuf.st_mtime; + if (delta > 60*60*24) + { + if (!glnx_shutil_rm_rf_at (dfd_iter.fd, dent->d_name, cancellable, error)) + goto out; + } } } diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 70600f4e..273edd1a 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -48,7 +48,7 @@ typedef enum { struct OstreeRepo { GObject parent; - char *boot_id; + char *stagedir_prefix; int commit_stagedir_fd; char *commit_stagedir_name; GLnxLockFile commit_stagedir_lock; @@ -108,6 +108,9 @@ typedef struct { char checksum[65]; } OstreeDevIno; +#define OSTREE_REPO_TMPDIR_STAGING "staging-" +#define OSTREE_REPO_TMPDIR_FETCHER "fetcher-" + gboolean _ostree_repo_allocate_tmpdir (int tmpdir_dfd, const char *tmpdir_prefix, @@ -118,6 +121,16 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, GCancellable *cancellable, GError **error); +gboolean +_ostree_repo_is_locked_tmpdir (const char *filename); + +gboolean +_ostree_repo_try_lock_tmpdir (int tmpdir_dfd, + const char *tmpdir_name, + GLnxLockFile *file_lock_out, + gboolean *out_did_lock, + GError **error); + gboolean _ostree_repo_ensure_loose_objdir_at (int dfd, const char *loose_path, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index a34e2ec1..2c9a7fd3 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -617,7 +617,7 @@ ostree_repo_finalize (GObject *object) g_clear_object (&self->parent_repo); - g_free (self->boot_id); + g_free (self->stagedir_prefix); g_clear_object (&self->repodir); if (self->repo_dir_fd != -1) (void) close (self->repo_dir_fd); @@ -2428,22 +2428,26 @@ ostree_repo_open (OstreeRepo *self, if (self->inited) return TRUE; - /* We use a per-boot identifier to keep track of which file contents - * possibly haven't been sync'd to disk. + /* We use a directory of the form `staging-${BOOT_ID}-${RANDOM}` + * where if the ${BOOT_ID} doesn't match, we know file contents + * possibly haven't been sync'd to disk and need to be discarded. */ { const char *env_bootid = getenv ("OSTREE_BOOTID"); + g_autofree char *boot_id = NULL; if (env_bootid != NULL) - self->boot_id = g_strdup (env_bootid); + boot_id = g_strdup (env_bootid); else { if (!g_file_get_contents ("/proc/sys/kernel/random/boot_id", - &self->boot_id, + &boot_id, NULL, error)) goto out; - g_strdelimit (self->boot_id, "\n", '\0'); + g_strdelimit (boot_id, "\n", '\0'); } + + self->stagedir_prefix = g_strconcat (OSTREE_REPO_TMPDIR_STAGING, boot_id, "-", NULL); } if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (self->repodir), TRUE, @@ -5003,6 +5007,50 @@ ostree_repo_regenerate_summary (OstreeRepo *self, return ret; } +gboolean +_ostree_repo_is_locked_tmpdir (const char *filename) +{ + return g_str_has_prefix (filename, OSTREE_REPO_TMPDIR_STAGING) || + g_str_has_prefix (filename, OSTREE_REPO_TMPDIR_FETCHER); +} + +gboolean +_ostree_repo_try_lock_tmpdir (int tmpdir_dfd, + const char *tmpdir_name, + GLnxLockFile *file_lock_out, + gboolean *out_did_lock, + GError **error) +{ + gboolean ret = FALSE; + g_autofree char *lock_name = g_strconcat (tmpdir_name, "-lock", NULL); + gboolean did_lock = FALSE; + g_autoptr(GError) local_error = NULL; + + /* We put the lock outside the dir, so we can hold the lock + * until the directory is fully removed */ + if (!glnx_make_lock_file (tmpdir_dfd, lock_name, LOCK_EX | LOCK_NB, + file_lock_out, &local_error)) + { + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_WOULD_BLOCK)) + { + did_lock = FALSE; + } + else + { + g_propagate_error (error, g_steal_pointer (&local_error)); + goto out; + } + } + else + { + did_lock = TRUE; + } + + ret = TRUE; + *out_did_lock = did_lock; + out: + return ret; +} /* This allocates and locks a subdir of the repo tmp dir, using an existing * one with the same prefix if it is not in use already. */ @@ -5016,14 +5064,18 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, GCancellable *cancellable, GError **error) { + gboolean ret = FALSE; gboolean reusing_dir = FALSE; + gboolean did_lock; g_autofree char *tmpdir_name = NULL; glnx_fd_close int tmpdir_fd = -1; g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + g_return_val_if_fail (_ostree_repo_is_locked_tmpdir (tmpdir_prefix), FALSE); + /* Look for existing tmpdir (with same prefix) to reuse */ if (!glnx_dirfd_iterator_init_at (tmpdir_dfd, ".", FALSE, &dfd_iter, error)) - return FALSE; + goto out; while (tmpdir_name == NULL) { @@ -5031,10 +5083,9 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, struct dirent *dent; glnx_fd_close int existing_tmpdir_fd = -1; g_autoptr(GError) local_error = NULL; - g_autofree char *lock_name = NULL; if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error)) - return FALSE; + goto out; if (dent == NULL) break; @@ -5055,25 +5106,18 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, else { g_propagate_error (error, g_steal_pointer (&local_error)); - return FALSE; + goto out; } } - lock_name = g_strconcat (dent->d_name, "-lock", NULL); - /* We put the lock outside the dir, so we can hold the lock * until the directory is fully removed */ - if (!glnx_make_lock_file (dfd_iter.fd, lock_name, LOCK_EX | LOCK_NB, - file_lock_out, &local_error)) - { - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_WOULD_BLOCK)) - continue; - else - { - g_propagate_error (error, g_steal_pointer (&local_error)); - return FALSE; - } - } + if (!_ostree_repo_try_lock_tmpdir (dfd_iter.fd, dent->d_name, + file_lock_out, &did_lock, + error)) + goto out; + if (!did_lock) + continue; /* Touch the reused directory so that we don't accidentally * remove it due to being old when cleaning up the tmpdir @@ -5091,32 +5135,24 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, g_autofree char *tmpdir_name_template = g_strconcat (tmpdir_prefix, "XXXXXX", NULL); glnx_fd_close int new_tmpdir_fd = -1; g_autoptr(GError) local_error = NULL; - g_autofree char *lock_name = NULL; /* No existing tmpdir found, create a new */ if (!glnx_mkdtempat (tmpdir_dfd, tmpdir_name_template, 0777, error)) - return FALSE; + goto out; if (!glnx_opendirat (tmpdir_dfd, tmpdir_name_template, FALSE, &new_tmpdir_fd, error)) - return FALSE; - - lock_name = g_strconcat (tmpdir_name_template, "-lock", NULL); + goto out; /* Note, at this point we can race with another process that picks up this * new directory. If that happens we need to retry, making a new directory. */ - if (!glnx_make_lock_file (tmpdir_dfd, lock_name, LOCK_EX | LOCK_NB, - file_lock_out, &local_error)) - { - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_WOULD_BLOCK)) - continue; - else - { - g_propagate_error (error, g_steal_pointer (&local_error)); - return FALSE; - } - } + if (!_ostree_repo_try_lock_tmpdir (tmpdir_dfd, tmpdir_name_template, + file_lock_out, &did_lock, + error)) + goto out; + if (!did_lock) + continue; tmpdir_name = g_steal_pointer (&tmpdir_name_template); tmpdir_fd = glnx_steal_fd (&new_tmpdir_fd); @@ -5131,5 +5167,7 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, if (reusing_dir_out) *reusing_dir_out = reusing_dir; - return TRUE; + ret = TRUE; + out: + return ret; } diff --git a/tests/test-admin-deploy-bootid-gc.sh b/tests/test-admin-deploy-bootid-gc.sh new file mode 100755 index 00000000..ba16f336 --- /dev/null +++ b/tests/test-admin-deploy-bootid-gc.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +# Exports OSTREE_SYSROOT so --sysroot not needed. +setup_os_repository "archive-z2" "syslinux" + +echo "1..1" + +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --remote=testos testos-repo testos/buildmaster/x86_64-runtime +rev=$(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse testos/buildmaster/x86_64-runtime) +export rev +${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=MOO --karg=quiet --os=testos testos:testos/buildmaster/x86_64-runtime +os_repository_new_commit + +rm sysroot/ostree/repo/tmp/* -rf +export TEST_BOOTID=4072029c-8b10-60d1-d31b-8422eeff9b42 +if env OSTREE_REPO_TEST_ERROR=pre-commit OSTREE_BOOTID=${TEST_BOOTID} \ + ${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=MOO --karg=quiet --os=testos testos:testos/buildmaster/x86_64-runtime 2>err.txt; then + assert_not_reached "Should have hit OSTREE_REPO_TEST_ERROR_PRE_COMMIT" +fi +stagepath=$(ls -d sysroot/ostree/repo/tmp/staging-${TEST_BOOTID}-*) +assert_has_dir "${stagepath}" + +# We have an older failed stage, now use a new boot id + +export NEW_TEST_BOOTID=5072029c-8b10-60d1-d31b-8422eeff9b42 +if env OSTREE_REPO_TEST_ERROR=pre-commit OSTREE_BOOTID=${NEW_TEST_BOOTID} \ + ${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=FOO --karg=quiet --os=testos testos:testos/buildmaster/x86_64-runtime 2>err.txt; then + assert_not_reached "Should have hit OSTREE_REPO_TEST_ERROR_PRE_COMMIT" +fi +newstagepath=$(ls -d sysroot/ostree/repo/tmp/staging-${NEW_TEST_BOOTID}-*) +assert_has_dir "${newstagepath}" +env OSTREE_BOOTID=${NEW_TEST_BOOTID} ${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=MOO --karg=quiet --os=testos testos:testos/buildmaster/x86_64-runtime +newstagepath=$(ls -d sysroot/ostree/repo/tmp/staging-${NEW_TEST_BOOTID}-*) +assert_not_has_dir "${stagepath}" +assert_not_has_dir "${newstagepath}" + +echo "ok admin bootid GC" From 7021c4f87658018b876719a57b4efdeb8edfc5e6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 21 Mar 2016 17:54:45 -0400 Subject: [PATCH 030/128] repo: Make repo/tmp expiry configurable via tmp-expiry-seconds We were arbitrarily only deleting content after exactly one day. Some use cases may want something else; make it configurable. Closes: #170 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 7 ++++--- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 10 ++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 0d7f9bee..60eb6260 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1349,11 +1349,12 @@ cleanup_tmpdir (OstreeRepo *self, if (stbuf.st_mtime > curtime_secs) continue; - /* Now, we arbitrarily delete files/directories older than a - * day, since that's what we were doing before we had locking. + /* Now, we're pruning content based on the expiry, which + * defaults to a day. That's what we were doing before we + * had locking...but in future we can be smarter here. */ delta = curtime_secs - stbuf.st_mtime; - if (delta > 60*60*24) + if (delta > self->tmp_expiry_seconds) { if (!glnx_shutil_rm_rf_at (dfd_iter.fd, dent->d_name, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 273edd1a..3d59d911 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -98,6 +98,7 @@ struct OstreeRepo { OstreeRepoMode mode; gboolean enable_uncompressed_cache; gboolean generate_sizes; + guint64 tmp_expiry_seconds; OstreeRepo *parent_repo; }; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 2c9a7fd3..15452c43 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2561,6 +2561,16 @@ ostree_repo_open (OstreeRepo *self, ostree_repo_set_disable_fsync (self, TRUE); } + { g_autofree char *tmp_expiry_seconds = NULL; + + /* 86400 secs = one day */ + if (!ot_keyfile_get_value_with_default (self->config, "core", "tmp-expiry-secs", "86400", + &tmp_expiry_seconds, error)) + goto out; + + self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10); + } + if (!append_remotes_d (self, cancellable, error)) goto out; From 5a90781cd867ea3ff38af5587e2aad188e4c5170 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 5 May 2016 17:20:04 -0400 Subject: [PATCH 031/128] lib: Add more filename validations (no ., .. or /) in commit logic The filesystem commit code will never give us potentially hostile filenames, and when importing from archives, we do some validation. However, we should be extra paranoid and also add error messages in the mtree in case someone tries to import a hostile libarchive-supported format. Closes: #283 Approved by: jlebon --- src/libostree/ostree-mutable-tree.c | 8 ++++++++ src/libostree/ostree-repo-commit.c | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/libostree/ostree-mutable-tree.c b/src/libostree/ostree-mutable-tree.c index bc4f4250..d0f21f37 100644 --- a/src/libostree/ostree-mutable-tree.c +++ b/src/libostree/ostree-mutable-tree.c @@ -159,6 +159,11 @@ ostree_mutable_tree_replace_file (OstreeMutableTree *self, { gboolean ret = FALSE; + g_return_val_if_fail (name != NULL, FALSE); + + if (!ot_util_filename_validate (name, error)) + goto out; + if (g_hash_table_lookup (self->subdirs, name)) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, @@ -187,6 +192,9 @@ ostree_mutable_tree_ensure_dir (OstreeMutableTree *self, g_return_val_if_fail (name != NULL, FALSE); + if (!ot_util_filename_validate (name, error)) + goto out; + if (g_hash_table_lookup (self->files, name)) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 60eb6260..19040a45 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2225,6 +2225,10 @@ create_tree_variant_from_hashes (GHashTable *file_checksums, while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *name = key; + + /* Should have been validated earlier, but be paranoid */ + g_assert (ot_util_filename_validate (name, NULL)); + sorted_filenames = g_slist_prepend (sorted_filenames, (char*)name); } From 6724519080a8b42ef63565a7458821d3bbea8303 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 3 May 2016 17:24:25 -0400 Subject: [PATCH 032/128] libglnx porting: Migrate to glnx_stream_fstat() I ended up deciding to move this one into libglnx, seems like something other libglnx-using software might want to do, even though xdg-app doesn't right now. Closes: #282 Approved by: jlebon --- libglnx | 2 +- src/libostree/ostree-core.c | 2 +- src/libostree/ostree-fetcher.c | 2 +- src/libostree/ostree-repo.c | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libglnx b/libglnx index 47ddbfa5..3d162e77 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 47ddbfa56341df3a9453854e1101e1c2f2359ddb +Subproject commit 3d162e772db80f6664a78583268150d2e1d1d29e diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index ded5e976..55ff4d63 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -613,7 +613,7 @@ ostree_content_file_parse_at (gboolean compressed, cancellable, error)) goto out; - if (!gs_stream_fstat ((GFileDescriptorBased*)file_input, &stbuf, cancellable, error)) + if (!glnx_stream_fstat ((GFileDescriptorBased*)file_input, &stbuf, error)) goto out; if (!ostree_content_stream_parse (compressed, file_input, stbuf.st_size, trusted, diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index 3606e8fb..6a75ad59 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -1067,7 +1067,7 @@ _ostree_fetcher_bytes_transferred (OstreeFetcher *self) if (G_IS_FILE_DESCRIPTOR_BASED (stream)) { - if (gs_stream_fstat ((GFileDescriptorBased*)stream, &stbuf, NULL, NULL)) + if (glnx_stream_fstat ((GFileDescriptorBased*)stream, &stbuf, NULL)) ret += stbuf.st_size; } } diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 15452c43..c395b1dc 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2925,7 +2925,7 @@ load_metadata_internal (OstreeRepo *self, { struct stat stbuf; - if (!gs_stream_fstat ((GFileDescriptorBased*)ret_stream, &stbuf, cancellable, error)) + if (!glnx_stream_fstat ((GFileDescriptorBased*)ret_stream, &stbuf, error)) goto out; *out_size = stbuf.st_size; } @@ -3100,8 +3100,8 @@ ostree_repo_load_file (OstreeRepo *self, tmp_stream = g_unix_input_stream_new (fd, TRUE); fd = -1; /* Transfer ownership */ - if (!gs_stream_fstat ((GFileDescriptorBased*) tmp_stream, &stbuf, - cancellable, error)) + if (!glnx_stream_fstat ((GFileDescriptorBased*) tmp_stream, &stbuf, + error)) goto out; if (!ostree_content_stream_parse (TRUE, tmp_stream, stbuf.st_size, TRUE, From b1d3dd151ce7280a4992cb0033a58623876a0563 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 22 Apr 2016 12:24:04 -0400 Subject: [PATCH 033/128] ostree-repo-libarchive.c: major refactor - Make hardlink handling more generic. The previous strategy worked for tar archives, but not for cpio. It now works for both. - Add support for SEL labeling (through the OstreeRepoCommitModifier) - Add support for xattr_callback (through the OstreeRepoCommitModifier) - Add support for filter (through the OstreeRepoCommitModifier) - Add a use_ostree_convention option Closes: #275 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 16 - src/libostree/ostree-repo-libarchive.c | 1009 ++++++++++++++++++------ src/libostree/ostree-repo-private.h | 16 + src/libostree/ostree-repo.h | 4 +- 4 files changed, 767 insertions(+), 278 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 19040a45..170b7b87 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -36,22 +36,6 @@ #include #include -struct OstreeRepoCommitModifier { - volatile gint refcount; - - OstreeRepoCommitModifierFlags flags; - OstreeRepoCommitFilter filter; - gpointer user_data; - GDestroyNotify destroy_notify; - - OstreeRepoCommitModifierXattrCallback xattr_callback; - GDestroyNotify xattr_destroy; - gpointer xattr_user_data; - - OstreeSePolicy *sepolicy; - GHashTable *devino_cache; -}; - gboolean _ostree_repo_ensure_loose_objdir_at (int dfd, const char *loose_path, diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 6856e94b..57da41d4 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -37,6 +37,8 @@ #ifdef HAVE_LIBARCHIVE +#define DEFAULT_DIRMODE (0755 | S_IFDIR) + static void propagate_libarchive_error (GError **error, struct archive *a) @@ -45,269 +47,759 @@ propagate_libarchive_error (GError **error, "%s", archive_error_string (a)); } +static const char * +path_relative (const char *src, + GError **error) +{ + /* One issue here is that some archives almost record the pathname as just a + * string and don't need to actually encode parent/child relationships in the + * archive. For us however, this will be important. So we do our best to deal + * with non-conventional paths. We also validate the path at the end to make + * sure there are no illegal components. Also important, we relativize the + * path. */ + + /* relativize first (and make /../../ --> /) */ + while (src[0] == '/') + { + src += 1; + if (src[0] == '.' && src[1] == '.' && src[2] == '/') + src += 2; /* keep trailing / so we continue */ + } + + /* now let's skip . and empty components */ + while (TRUE) + { + if (src[0] == '.' && src[1] == '/') + src += 2; + else if (src[0] == '/') + src += 1; + else + break; + } + + /* assume a single '.' means the root dir itself, which we handle as the empty + * string in our code */ + if (src[0] == '.' && src[1] == '\0') + src += 1; + + /* make sure that the final path is valid (no . or ..) */ + if (!ot_util_path_split_validate (src, NULL, error)) + { + g_prefix_error (error, "While making relative path \"%s\":", src); + return NULL; + } + + return src; +} + +static char * +path_relative_ostree (const char *path, + GError **error) +{ + path = path_relative (path, error); + if (path == NULL) + return NULL; + if (g_str_has_prefix (path, "etc/")) + return g_strconcat ("usr/", path, NULL); + else if (strcmp (path, "etc") == 0) + return g_strdup ("usr/etc"); + return g_strdup (path); +} + +static void +append_path_component (char **path_builder, + const char *component) +{ + g_autofree char *s = g_steal_pointer (path_builder); + *path_builder = g_build_filename (s ?: "/", component, NULL); +} + +/* inplace trailing slash squashing */ +static void +squash_trailing_slashes (char *path) +{ + char *endp = path + strlen (path) - 1; + for (; endp > path && *endp == '/'; endp--) + *endp = '\0'; +} + static GFileInfo * -file_info_from_archive_entry_and_modifier (OstreeRepo *repo, - struct archive_entry *entry, - OstreeRepoCommitModifier *modifier) +file_info_from_archive_entry (struct archive_entry *entry) { g_autoptr(GFileInfo) info = NULL; - GFileInfo *modified_info = NULL; - const struct stat *st; + const struct stat *st = NULL; guint32 file_type; + mode_t mode; st = archive_entry_stat (entry); + mode = st->st_mode; - info = _ostree_header_gfile_info_new (st->st_mode, st->st_uid, st->st_gid); - file_type = ot_gfile_type_for_mode (st->st_mode); + /* Some archives only store the permission mode bits in hardlink entries, so + * let's just make it into a regular file. Yes, this hack will work even if + * it's a hardlink to a symlink. */ + if (archive_entry_hardlink (entry)) + mode |= S_IFREG; + info = _ostree_header_gfile_info_new (mode, st->st_uid, st->st_gid); + + file_type = ot_gfile_type_for_mode (mode); if (file_type == G_FILE_TYPE_REGULAR) { g_file_info_set_attribute_uint64 (info, "standard::size", st->st_size); } else if (file_type == G_FILE_TYPE_SYMBOLIC_LINK) { - g_file_info_set_attribute_byte_string (info, "standard::symlink-target", archive_entry_symlink (entry)); + g_file_info_set_attribute_byte_string (info, "standard::symlink-target", + archive_entry_symlink (entry)); } - _ostree_repo_commit_modifier_apply (repo, modifier, - archive_entry_pathname (entry), - info, &modified_info); - - return modified_info; + return g_steal_pointer (&info); } static gboolean -import_libarchive_entry_file (OstreeRepo *self, - OstreeRepoImportArchiveOptions *opts, - struct archive *a, - struct archive_entry *entry, - GFileInfo *file_info, - guchar **out_csum, - GCancellable *cancellable, - GError **error) +builder_add_label (GVariantBuilder *builder, + OstreeSePolicy *sepolicy, + const char *path, + mode_t mode, + GCancellable *cancellable, + GError **error) { - gboolean ret = FALSE; - g_autoptr(GInputStream) file_object_input = NULL; - g_autoptr(GInputStream) archive_stream = NULL; - guint64 length; - - if (g_cancellable_set_error_if_cancelled (cancellable, error)) + g_autofree char *label = NULL; + + if (!sepolicy) + return TRUE; + + if (!ostree_sepolicy_get_label (sepolicy, path, mode, &label, + cancellable, error)) return FALSE; - switch (g_file_info_get_file_type (file_info)) - { - case G_FILE_TYPE_REGULAR: - archive_stream = _ostree_libarchive_input_stream_new (a); - break; - case G_FILE_TYPE_SYMBOLIC_LINK: - break; - default: - if (opts->ignore_unsupported_content) - { - ret = TRUE; - goto out; - } - else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unable to import non-regular/non-symlink file '%s'", - archive_entry_pathname (entry)); - goto out; - } - } - - if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL, - &file_object_input, &length, cancellable, error)) - goto out; - - if (!ostree_repo_write_content (self, NULL, file_object_input, length, out_csum, - cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; + if (label) + g_variant_builder_add (builder, "(@ay@ay)", + g_variant_new_bytestring ("security.selinux"), + g_variant_new_bytestring (label)); + return TRUE; } + +/* Like ostree_mutable_tree_ensure_dir(), but also creates and sets dirmeta if + * the dir has to be created. */ static gboolean -write_libarchive_entry_to_mtree (OstreeRepo *self, - OstreeRepoImportArchiveOptions *opts, - OstreeMutableTree *root, - struct archive *a, - struct archive_entry *entry, - OstreeRepoCommitModifier *modifier, - const guchar *tmp_dir_csum, - GCancellable *cancellable, - GError **error) +mtree_ensure_dir_with_meta (OstreeRepo *repo, + OstreeMutableTree *parent, + const char *name, + GFileInfo *file_info, + GVariant *xattrs, + gboolean error_if_exist, /* XXX: remove if not needed */ + OstreeMutableTree **out_dir, + GCancellable *cancellable, + GError **error) { - gboolean ret = FALSE; - const char *pathname; - const char *hardlink; - const char *basename; + glnx_unref_object OstreeMutableTree *dir = NULL; + g_autofree guchar *csum_raw = NULL; + g_autofree char *csum = NULL; + + if (name[0] == '\0') /* root? */ + dir = g_object_ref (parent); + else if (ostree_mutable_tree_lookup (parent, name, NULL, &dir, error)) + { + if (error_if_exist) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Directory \"%s\" already exists", name); + return FALSE; + } + } + + if (dir == NULL) + { + if (!g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + return FALSE; + + g_clear_error (error); + + if (!ostree_mutable_tree_ensure_dir (parent, name, &dir, error)) + return FALSE; + } + + if (!_ostree_repo_write_directory_meta (repo, file_info, xattrs, + &csum_raw, cancellable, error)) + return FALSE; + + csum = ostree_checksum_from_bytes (csum_raw); + + ostree_mutable_tree_set_metadata_checksum (dir, csum); + + if (out_dir) + *out_dir = g_steal_pointer (&dir); + + return TRUE; +} + +typedef struct { + OstreeRepo *repo; + OstreeRepoImportArchiveOptions *opts; + OstreeMutableTree *root; + struct archive *archive; + struct archive_entry *entry; + GHashTable *deferred_hardlinks; + OstreeRepoCommitModifier *modifier; +} OstreeRepoArchiveImportContext; + +typedef struct { + OstreeMutableTree *parent; + char *path; + guint64 size; +} DeferredHardlink; + +static inline char* +aic_get_final_path (OstreeRepoArchiveImportContext *ctx, + const char *path, + GError **error) +{ + if (ctx->opts->use_ostree_convention) + return path_relative_ostree (path, error); + return g_strdup (path_relative (path, error)); +} + +static inline char* +aic_get_final_entry_pathname (OstreeRepoArchiveImportContext *ctx, + GError **error) +{ + const char *pathname = archive_entry_pathname (ctx->entry); + g_autofree char *final = aic_get_final_path (ctx, pathname, error); + + if (final == NULL) + return NULL; + + /* get rid of trailing slashes some archives put on dirs */ + squash_trailing_slashes (final); + return g_steal_pointer (&final); +} + +static inline char* +aic_get_final_entry_hardlink (OstreeRepoArchiveImportContext *ctx) +{ + GError *local_error = NULL; + const char *hardlink = archive_entry_hardlink (ctx->entry); + g_autofree char *final = NULL; + + if (hardlink != NULL) + { + final = aic_get_final_path (ctx, hardlink, &local_error); + + /* hardlinks always point to a preceding entry, so if there were an error + * it would have failed then */ + g_assert_no_error (local_error); + } + + return g_steal_pointer (&final); +} + +static OstreeRepoCommitFilterResult +aic_apply_modifier_filter (OstreeRepoArchiveImportContext *ctx, + const char *relpath, + GFileInfo **out_file_info) +{ + g_autofree char *hardlink = aic_get_final_entry_hardlink (ctx); g_autoptr(GFileInfo) file_info = NULL; - g_autoptr(GPtrArray) split_path = NULL; - g_autoptr(GPtrArray) hardlink_split_path = NULL; - glnx_unref_object OstreeMutableTree *subdir = NULL; - glnx_unref_object OstreeMutableTree *parent = NULL; - glnx_unref_object OstreeMutableTree *hardlink_source_parent = NULL; - g_autofree char *hardlink_source_checksum = NULL; - glnx_unref_object OstreeMutableTree *hardlink_source_subdir = NULL; - g_autofree guchar *tmp_csum = NULL; - g_autofree char *tmp_checksum = NULL; + g_autofree char *abspath = NULL; + const char *cb_path = NULL; - pathname = archive_entry_pathname (entry); - - if (!ot_util_path_split_validate (pathname, &split_path, error)) - goto out; - - if (split_path->len == 0) - { - parent = NULL; - basename = NULL; - } + if (ctx->opts->callback_with_entry_pathname) + cb_path = archive_entry_pathname (ctx->entry); else { - if (tmp_dir_csum) - { - g_free (tmp_checksum); - tmp_checksum = ostree_checksum_from_bytes (tmp_dir_csum); - if (!ostree_mutable_tree_ensure_parent_dirs (root, split_path, - tmp_checksum, - &parent, - error)) - goto out; - } - else - { - if (!ostree_mutable_tree_walk (root, split_path, 0, &parent, error)) - goto out; - } - basename = (char*)split_path->pdata[split_path->len-1]; + /* the user expects an abspath (where the dir to commit represents /) */ + abspath = g_build_filename ("/", relpath, NULL); + cb_path = abspath; } - hardlink = archive_entry_hardlink (entry); - if (hardlink) - { - const char *hardlink_basename; - - g_assert (parent != NULL); + file_info = file_info_from_archive_entry (ctx->entry); - if (!ot_util_path_split_validate (hardlink, &hardlink_split_path, error)) - goto out; - if (hardlink_split_path->len == 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid hardlink path %s", hardlink); - goto out; - } - - hardlink_basename = hardlink_split_path->pdata[hardlink_split_path->len - 1]; - - if (!ostree_mutable_tree_walk (root, hardlink_split_path, 0, &hardlink_source_parent, error)) - goto out; - - if (!ostree_mutable_tree_lookup (hardlink_source_parent, hardlink_basename, - &hardlink_source_checksum, - &hardlink_source_subdir, - error)) - { - g_prefix_error (error, "While resolving hardlink target: "); - goto out; - } - - if (hardlink_source_subdir) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Hardlink %s refers to directory %s", - pathname, hardlink); - goto out; - } - g_assert (hardlink_source_checksum); - - if (!ostree_mutable_tree_replace_file (parent, - basename, - hardlink_source_checksum, - error)) - goto out; - } - else - { - file_info = file_info_from_archive_entry_and_modifier (self, entry, modifier); - - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_UNKNOWN) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unsupported file for import: %s", pathname); - goto out; - } - - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) - { - - if (!_ostree_repo_write_directory_meta (self, file_info, NULL, &tmp_csum, cancellable, error)) - goto out; - - if (parent == NULL) - { - subdir = g_object_ref (root); - } - else - { - if (!ostree_mutable_tree_ensure_dir (parent, basename, &subdir, error)) - goto out; - } - - g_free (tmp_checksum); - tmp_checksum = ostree_checksum_from_bytes (tmp_csum); - ostree_mutable_tree_set_metadata_checksum (subdir, tmp_checksum); - } - else - { - if (parent == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't import file as root"); - goto out; - } - - if (!import_libarchive_entry_file (self, opts, a, entry, file_info, &tmp_csum, - cancellable, error)) - goto out; - - if (tmp_csum) - { - g_free (tmp_checksum); - tmp_checksum = ostree_checksum_from_bytes (tmp_csum); - if (!ostree_mutable_tree_replace_file (parent, basename, - tmp_checksum, - error)) - goto out; - } - } - } - - ret = TRUE; - out: - return ret; + return _ostree_repo_commit_modifier_apply (ctx->repo, ctx->modifier, cb_path, + file_info, out_file_info); } static gboolean -create_empty_dir_with_uidgid (OstreeRepo *self, - guint32 uid, - guint32 gid, - guint8 **out_csum, - GCancellable *cancellable, - GError **error) +aic_ensure_parent_dir_with_file_info (OstreeRepoArchiveImportContext *ctx, + OstreeMutableTree *parent, + const char *fullpath, + GFileInfo *file_info, + OstreeMutableTree **out_dir, + GCancellable *cancellable, + GError **error) { - g_autoptr(GFileInfo) tmp_dir_info = g_file_info_new (); - - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", uid); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", gid); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); - - return _ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, out_csum, cancellable, error); + const char *name = glnx_basename (fullpath); + g_auto(GVariantBuilder) xattrs_builder; + + /* is this the root directory itself? transform into empty string */ + if (name[0] == '/' && name[1] == '\0') + name++; + + g_variant_builder_init (&xattrs_builder, (GVariantType*)"a(ayay)"); + + if (ctx->modifier && ctx->modifier->sepolicy) + if (!builder_add_label (&xattrs_builder, ctx->modifier->sepolicy, fullpath, + DEFAULT_DIRMODE, cancellable, error)) + return FALSE; + + return mtree_ensure_dir_with_meta (ctx->repo, parent, name, file_info, + g_variant_builder_end (&xattrs_builder), + FALSE /* error_if_exist */, out_dir, + cancellable, error); } -#endif + +static gboolean +aic_ensure_parent_dir (OstreeRepoArchiveImportContext *ctx, + OstreeMutableTree *parent, + const char *fullpath, + OstreeMutableTree **out_dir, + GCancellable *cancellable, + GError **error) +{ + /* Who should own the parent dir? Since it's not in the archive, it's up to + * us. Here, we use the heuristic of simply creating it as the same user as + * the owner of the archive entry for which we're creating the dir. This is OK + * since any nontrivial dir perms should have explicit archive entries. */ + + guint32 uid = archive_entry_uid (ctx->entry); + guint32 gid = archive_entry_gid (ctx->entry); + glnx_unref_object GFileInfo *file_info = g_file_info_new (); + + g_file_info_set_attribute_uint32 (file_info, "unix::uid", uid); + g_file_info_set_attribute_uint32 (file_info, "unix::gid", gid); + g_file_info_set_attribute_uint32 (file_info, "unix::mode", DEFAULT_DIRMODE); + + return aic_ensure_parent_dir_with_file_info (ctx, parent, fullpath, file_info, + out_dir, cancellable, error); +} + +static gboolean +aic_create_parent_dirs (OstreeRepoArchiveImportContext *ctx, + GPtrArray *components, + OstreeMutableTree **out_subdir, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *fullpath = NULL; + glnx_unref_object OstreeMutableTree *dir = NULL; + + /* start with the root itself */ + if (!aic_ensure_parent_dir (ctx, ctx->root, "/", &dir, cancellable, error)) + return FALSE; + + for (guint i = 0; i < components->len-1; i++) + { + glnx_unref_object OstreeMutableTree *subdir = NULL; + append_path_component (&fullpath, components->pdata[i]); + + if (!aic_ensure_parent_dir (ctx, dir, fullpath, &subdir, + cancellable, error)) + return FALSE; + + g_set_object (&dir, subdir); + } + + *out_subdir = g_steal_pointer (&dir); + return TRUE; +} + +static gboolean +aic_get_parent_dir (OstreeRepoArchiveImportContext *ctx, + const char *path, + OstreeMutableTree **out_dir, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GPtrArray) components = NULL; + if (!ot_util_path_split_validate (path, &components, error)) + return FALSE; + + if (components->len == 0) /* root dir? */ + { + *out_dir = g_object_ref (ctx->root); + return TRUE; + } + + if (ostree_mutable_tree_walk (ctx->root, components, 0, out_dir, error)) + return TRUE; /* already exists, nice! */ + + if (!g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + return FALSE; /* some other error occurred */ + + if (ctx->opts->autocreate_parents) + { + g_clear_error (error); + return aic_create_parent_dirs (ctx, components, out_dir, + cancellable, error); + } + + return FALSE; +} + +static gboolean +aic_get_xattrs (OstreeRepoArchiveImportContext *ctx, + const char *path, + GFileInfo *file_info, + GVariant **out_xattrs, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *abspath = g_build_filename ("/", path, NULL); + g_autoptr(GVariant) xattrs = NULL; + const char *cb_path = abspath; + + if (ctx->opts->callback_with_entry_pathname) + cb_path = archive_entry_pathname (ctx->entry); + + if (ctx->modifier && ctx->modifier->xattr_callback) + xattrs = ctx->modifier->xattr_callback (ctx->repo, cb_path, file_info, + ctx->modifier->xattr_user_data); + + if (ctx->modifier && ctx->modifier->sepolicy) + { + mode_t mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); + g_autoptr(GVariantBuilder) builder = + ot_util_variant_builder_from_variant (xattrs, G_VARIANT_TYPE + ("a(ayay)")); + + if (!builder_add_label (builder, ctx->modifier->sepolicy, abspath, mode, + cancellable, error)) + return FALSE; + + if (xattrs) + g_variant_unref (xattrs); + + xattrs = g_variant_builder_end (builder); + g_variant_ref_sink (xattrs); + } + + *out_xattrs = g_steal_pointer (&xattrs); + return TRUE; +} + +/* XXX: add option in ctx->opts to disallow already existing dirs? see + * error_if_exist */ +static gboolean +aic_handle_dir (OstreeRepoArchiveImportContext *ctx, + OstreeMutableTree *parent, + const char *path, + GFileInfo *fi, + GCancellable *cancellable, + GError **error) +{ + const char *name = glnx_basename (path); + g_autoptr(GVariant) xattrs = NULL; + + if (!aic_get_xattrs (ctx, path, fi, &xattrs, cancellable, error)) + return FALSE; + + return mtree_ensure_dir_with_meta (ctx->repo, parent, name, fi, xattrs, + FALSE /* error_if_exist */, NULL, + cancellable, error); +} + +static gboolean +aic_write_file (OstreeRepoArchiveImportContext *ctx, + GFileInfo *fi, + GVariant *xattrs, + char **out_csum, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GInputStream) archive_stream = NULL; + g_autoptr(GInputStream) file_object_input = NULL; + guint64 length; + + g_autofree guchar *csum_raw = NULL; + + if (g_file_info_get_file_type (fi) == G_FILE_TYPE_REGULAR) + archive_stream = _ostree_libarchive_input_stream_new (ctx->archive); + + if (!ostree_raw_file_to_content_stream (archive_stream, fi, xattrs, + &file_object_input, &length, + cancellable, error)) + return FALSE; + + if (!ostree_repo_write_content (ctx->repo, NULL, file_object_input, length, + &csum_raw, cancellable, error)) + return FALSE; + + *out_csum = ostree_checksum_from_bytes (csum_raw); + return TRUE; +} + +static gboolean +aic_import_file (OstreeRepoArchiveImportContext *ctx, + OstreeMutableTree *parent, + const char *path, + GFileInfo *fi, + GCancellable *cancellable, + GError **error) +{ + const char *name = glnx_basename (path); + g_autoptr(GVariant) xattrs = NULL; + g_autofree char *csum = NULL; + + if (!aic_get_xattrs (ctx, path, fi, &xattrs, cancellable, error)) + return FALSE; + + if (!aic_write_file (ctx, fi, xattrs, &csum, cancellable, error)) + return FALSE; + + if (!ostree_mutable_tree_replace_file (parent, name, csum, error)) + return FALSE; + + return TRUE; +} + +static void +aic_add_deferred_hardlink (OstreeRepoArchiveImportContext *ctx, + const char *hardlink, + DeferredHardlink *dh) +{ + gboolean new_slist; + GSList *slist; + + slist = g_hash_table_lookup (ctx->deferred_hardlinks, hardlink); + new_slist = (slist == NULL); + + slist = g_slist_append (slist, dh); + + if (new_slist) + g_hash_table_insert (ctx->deferred_hardlinks, g_strdup (hardlink), slist); +} + +static void +aic_defer_hardlink (OstreeRepoArchiveImportContext *ctx, + OstreeMutableTree *parent, + const char *path, + guint64 size, + const char *hardlink) +{ + DeferredHardlink *dh = g_slice_new (DeferredHardlink); + dh->parent = g_object_ref (parent); + dh->path = g_strdup (path); + dh->size = size; + + aic_add_deferred_hardlink (ctx, hardlink, dh); +} + +static gboolean +aic_handle_file (OstreeRepoArchiveImportContext *ctx, + OstreeMutableTree *parent, + const char *path, + GFileInfo *fi, + GCancellable *cancellable, + GError **error) +{ + /* The wonderful world of hardlinks and archives. We have to be very careful + * here. Do not assume that if a file is a hardlink, it will have size 0 (e.g. + * cpio). Do not assume that if a file will have hardlinks to it, it will have + * size > 0. Also do not assume that its nlink param is present (tar) or even + * accurate (cpio). Also do not assume that hardlinks follow each other in + * order of entries. + * + * These archives were made to be extracted onto a filesystem, not directly + * hashed into an object store. So to be careful, we defer all hardlink + * imports until the very end. Nonzero files have to be imported, hardlink or + * not, since we can't easily seek back to this position later on. + * */ + + g_autofree char *hardlink = aic_get_final_entry_hardlink (ctx); + guint64 size = g_file_info_get_attribute_uint64 (fi, "standard::size"); + + if (hardlink == NULL || size > 0) + if (!aic_import_file (ctx, parent, path, fi, cancellable, error)) + return FALSE; + + if (hardlink) + aic_defer_hardlink (ctx, parent, path, size, hardlink); + + return TRUE; +} + +static gboolean +aic_handle_entry (OstreeRepoArchiveImportContext *ctx, + OstreeMutableTree *parent, + const char *path, + GFileInfo *fi, + GCancellable *cancellable, + GError **error) +{ + switch (g_file_info_get_file_type (fi)) + { + case G_FILE_TYPE_DIRECTORY: + return aic_handle_dir (ctx, parent, path, fi, cancellable, error); + case G_FILE_TYPE_REGULAR: + case G_FILE_TYPE_SYMBOLIC_LINK: + return aic_handle_file (ctx, parent, path, fi, cancellable, error); + default: + if (ctx->opts->ignore_unsupported_content) + return TRUE; + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unsupported file type for path \"%s\"", path); + return FALSE; + } + } +} + +static gboolean +aic_import_entry (OstreeRepoArchiveImportContext *ctx, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFileInfo) fi = NULL; + glnx_unref_object OstreeMutableTree *parent = NULL; + g_autofree char *path = aic_get_final_entry_pathname (ctx, error); + + if (path == NULL) + return FALSE; + + if (aic_apply_modifier_filter (ctx, path, &fi) + == OSTREE_REPO_COMMIT_FILTER_SKIP) + return TRUE; + + if (!aic_get_parent_dir (ctx, path, &parent, cancellable, error)) + return FALSE; + + return aic_handle_entry (ctx, parent, path, fi, cancellable, error); +} + +static gboolean +aic_import_from_hardlink (OstreeRepoArchiveImportContext *ctx, + const char *target, + DeferredHardlink *dh, + GError **error) +{ + g_autofree char *csum = NULL; + const char *name = glnx_basename (target); + const char *name_dh = glnx_basename (dh->path); + g_autoptr(GPtrArray) components = NULL; + glnx_unref_object OstreeMutableTree *parent = NULL; + + if (!ostree_mutable_tree_lookup (dh->parent, name_dh, &csum, NULL, error)) + return FALSE; + + g_assert (csum); + + if (!ot_util_path_split_validate (target, &components, error)) + return FALSE; + + if (!ostree_mutable_tree_walk (ctx->root, components, 0, &parent, error)) + return FALSE; + + if (!ostree_mutable_tree_replace_file (parent, name, csum, error)) + return FALSE; + + return TRUE; +} + +static gboolean +aic_lookup_file_csum (OstreeRepoArchiveImportContext *ctx, + const char *target, + char **out_csum, + GError **error) +{ + g_autofree char *csum = NULL; + const char *name = glnx_basename (target); + glnx_unref_object OstreeMutableTree *parent = NULL; + glnx_unref_object OstreeMutableTree *subdir = NULL; + g_autoptr(GPtrArray) components = NULL; + + if (!ot_util_path_split_validate (target, &components, error)) + return FALSE; + + if (!ostree_mutable_tree_walk (ctx->root, components, 0, &parent, error)) + return FALSE; + + if (!ostree_mutable_tree_lookup (parent, name, &csum, &subdir, error)) + return FALSE; + + if (subdir != NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Expected hardlink file target at \"%s\" but found a " + "directory", target); + return FALSE; + } + + *out_csum = g_steal_pointer (&csum); + return TRUE; +} + +static gboolean +aic_import_deferred_hardlinks_for (OstreeRepoArchiveImportContext *ctx, + const char *target, + GSList *hardlinks, + GError **error) +{ + GSList *payload = hardlinks; + g_autofree char *csum = NULL; + + /* find node with the payload, if any (if none, then they're all hardlinks to + * a zero sized target, and there's no rewrite required) */ + while (payload && ((DeferredHardlink*)payload->data)->size == 0) + payload = g_slist_next (payload); + + /* rewrite the target so it points to the csum of the payload hardlink */ + if (payload) + if (!aic_import_from_hardlink (ctx, target, payload->data, error)) + return FALSE; + + if (!aic_lookup_file_csum (ctx, target, &csum, error)) + return FALSE; + + /* import all the hardlinks */ + for (GSList *hl = hardlinks; hl != NULL; hl = g_slist_next (hl)) + { + DeferredHardlink *df = hl->data; + const char *name = glnx_basename (df->path); + + if (hl == payload) + continue; /* small optimization; no need to redo this one */ + + if (!ostree_mutable_tree_replace_file (df->parent, name, csum, error)) + return FALSE; + } + + return TRUE; +} + +static gboolean +aic_import_deferred_hardlinks (OstreeRepoArchiveImportContext *ctx, + GCancellable *cancellable, + GError **error) +{ + GHashTableIter iter; + gpointer key, value; + + g_hash_table_iter_init (&iter, ctx->deferred_hardlinks); + while (g_hash_table_iter_next (&iter, &key, &value)) + if (!aic_import_deferred_hardlinks_for (ctx, key, value, error)) + return FALSE; + + return TRUE; +} + +static void +deferred_hardlink_free (void *data) +{ + DeferredHardlink *dh = data; + g_object_unref (dh->parent); + g_free (dh->path); + g_slice_free (DeferredHardlink, dh); +} + +static void +deferred_hardlinks_list_free (void *data) +{ + GSList *slist = data; + g_slist_free_full (slist, deferred_hardlink_free); +} +#endif /* HAVE_LIBARCHIVE */ /** * ostree_repo_import_archive_to_mtree: @@ -334,61 +826,56 @@ ostree_repo_import_archive_to_mtree (OstreeRepo *self, #ifdef HAVE_LIBARCHIVE gboolean ret = FALSE; struct archive *a = archive; - struct archive_entry *entry; - g_autofree guchar *tmp_csum = NULL; - int r; + g_autoptr(GHashTable) deferred_hardlinks = + g_hash_table_new_full (g_str_hash, g_str_equal, g_free, + deferred_hardlinks_list_free); + OstreeRepoArchiveImportContext aictx = { + .repo = self, + .opts = opts, + .root = mtree, + .archive = archive, + .deferred_hardlinks = deferred_hardlinks, + .modifier = modifier + }; while (TRUE) { - r = archive_read_next_header (a, &entry); + int r = archive_read_next_header (a, &aictx.entry); if (r == ARCHIVE_EOF) break; - else if (r != ARCHIVE_OK) + if (r != ARCHIVE_OK) { propagate_libarchive_error (error, a); goto out; } - /* TODO - refactor this to only create the metadata on demand - * (i.e. if there is a missing parent dir) - */ - if (opts->autocreate_parents && !tmp_csum) - { - /* Here, we auto-pick the first uid/gid we find in the - * archive. Realistically this is probably always going to - * be root, but eh, at least we try to match. - */ - if (!create_empty_dir_with_uidgid (self, archive_entry_uid (entry), - archive_entry_gid (entry), - &tmp_csum, cancellable, error)) - goto out; - } + if (g_cancellable_set_error_if_cancelled (cancellable, error)) + goto out; - if (!write_libarchive_entry_to_mtree (self, opts, mtree, a, - entry, modifier, tmp_csum, - cancellable, error)) + if (!aic_import_entry (&aictx, cancellable, error)) goto out; } + if (!aic_import_deferred_hardlinks (&aictx, cancellable, error)) + goto out; + /* If we didn't import anything at all, and autocreation of parents * is enabled, automatically create a root directory. This is * useful primarily when importing Docker image layers, which can * just be metadata. */ - if (!ostree_mutable_tree_get_metadata_checksum (mtree) && opts->autocreate_parents) + if (opts->autocreate_parents && + ostree_mutable_tree_get_metadata_checksum (mtree) == NULL) { - char tmp_checksum[65]; + glnx_unref_object GFileInfo *fi = g_file_info_new (); + g_file_info_set_attribute_uint32 (fi, "unix::uid", 0); + g_file_info_set_attribute_uint32 (fi, "unix::gid", 0); + g_file_info_set_attribute_uint32 (fi, "unix::mode", DEFAULT_DIRMODE); - if (!tmp_csum) - { - /* We didn't have any archive entries to match, so pick uid 0, gid 0. */ - if (!create_empty_dir_with_uidgid (self, 0, 0, &tmp_csum, cancellable, error)) - goto out; - } - - ostree_checksum_inplace_from_bytes (tmp_csum, tmp_checksum); - ostree_mutable_tree_set_metadata_checksum (mtree, tmp_checksum); + if (!aic_ensure_parent_dir_with_file_info (&aictx, mtree, "/", fi, NULL, + cancellable, error)) + goto out; } ret = TRUE; @@ -400,7 +887,7 @@ ostree_repo_import_archive_to_mtree (OstreeRepo *self, return FALSE; #endif } - + /** * ostree_repo_write_archive_to_mtree: * @self: An #OstreeRepo @@ -420,8 +907,8 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, OstreeMutableTree *mtree, OstreeRepoCommitModifier *modifier, gboolean autocreate_parents, - GCancellable *cancellable, - GError **error) + GCancellable *cancellable, + GError **error) { #ifdef HAVE_LIBARCHIVE gboolean ret = FALSE; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 3d59d911..62a3c8f6 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -40,6 +40,22 @@ typedef enum { OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0) } OstreeRepoTestErrorFlags; +struct OstreeRepoCommitModifier { + volatile gint refcount; + + OstreeRepoCommitModifierFlags flags; + OstreeRepoCommitFilter filter; + gpointer user_data; + GDestroyNotify destroy_notify; + + OstreeRepoCommitModifierXattrCallback xattr_callback; + GDestroyNotify xattr_destroy; + gpointer xattr_user_data; + + OstreeSePolicy *sepolicy; + GHashTable *devino_cache; +}; + /** * OstreeRepo: * diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 20873a05..d9c61ebc 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -582,7 +582,9 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo * typedef struct { guint ignore_unsupported_content : 1; guint autocreate_parents : 1; - guint reserved : 30; + guint use_ostree_convention : 1; + guint callback_with_entry_pathname : 1; + guint reserved : 28; guint unused_uint[8]; gpointer unused_ptrs[8]; From b717fd2c18ef71002826012a876b7457ef8fd802 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 22 Apr 2016 12:35:48 -0400 Subject: [PATCH 034/128] ot-builtin-commit.c: add --skip-list option This was already supported by the commit modifier API, just needed to expose it. This will also be used to test the libarchive API in a future test. Closes: #275 Approved by: cgwalters --- Makefile-tests.am | 1 - man/ostree-commit.xml | 10 ++- src/ostree/ot-builtin-commit.c | 140 +++++++++++++++++++++++---------- tests/basic-test.sh | 26 +++++- tests/test-setuid.sh | 40 ---------- 5 files changed, 131 insertions(+), 86 deletions(-) delete mode 100755 tests/test-setuid.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 4986b427..bb1cee41 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -72,7 +72,6 @@ test_scripts = \ tests/test-repo-checkout-subpath.sh \ tests/test-reset-nonlinear.sh \ tests/test-oldstyle-partial.sh \ - tests/test-setuid.sh \ tests/test-delta.sh \ tests/test-xattrs.sh \ tests/test-auto-summary.sh \ diff --git a/man/ostree-commit.xml b/man/ostree-commit.xml index 4a86c1d2..c4842584 100644 --- a/man/ostree-commit.xml +++ b/man/ostree-commit.xml @@ -163,7 +163,15 @@ Boston, MA 02111-1307, USA. ="PATH" - File containing list of modifications to make permissions. + File containing list of modifications to make permissions (file mode, followed by space, followed by file path). + + + + + ="PATH" + + + File containing list of file paths to skip (one path per line). diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 1d96855d..a021fbfa 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -36,6 +36,7 @@ static char *opt_parent; static gboolean opt_orphan; static char *opt_branch; static char *opt_statoverride_file; +static char *opt_skiplist_file; static char **opt_metadata_strings; static char **opt_detached_metadata_strings; static gboolean opt_link_checkout_speedup; @@ -84,6 +85,7 @@ static GOptionEntry options[] = { { "tar-autocreate-parents", 0, 0, G_OPTION_ARG_NONE, &opt_tar_autocreate_parents, "When loading tar archives, automatically create parent directories as needed", NULL }, { "skip-if-unchanged", 0, 0, G_OPTION_ARG_NONE, &opt_skip_if_unchanged, "If the contents are unchanged from previous commit, do nothing", NULL }, { "statoverride", 0, 0, G_OPTION_ARG_FILENAME, &opt_statoverride_file, "File containing list of modifications to make to permissions", "PATH" }, + { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" }, { "table-output", 0, 0, G_OPTION_ARG_NONE, &opt_table_output, "Output more information in a KEY: VALUE format", NULL }, { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the commit with", "KEY-ID"}, { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, @@ -95,65 +97,85 @@ static GOptionEntry options[] = { }; static gboolean -parse_statoverride_file (GHashTable **out_mode_add, - GCancellable *cancellable, - GError **error) +parse_file_by_line (const char *path, + gboolean (*cb)(const char*, void*, GError**), + void *cbdata, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; - gsize len; - char **iter = NULL; /* nofree */ - g_autoptr(GHashTable) ret_hash = NULL; - g_autoptr(GFile) path = NULL; g_autofree char *contents = NULL; + g_autoptr(GFile) file = NULL; char **lines = NULL; - path = g_file_new_for_path (opt_statoverride_file); - - if (!g_file_load_contents (path, cancellable, &contents, &len, NULL, - error)) + file = g_file_new_for_path (path); + if (!g_file_load_contents (file, cancellable, &contents, NULL, NULL, error)) goto out; - - ret_hash = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + lines = g_strsplit (contents, "\n", -1); - - for (iter = lines; iter && *iter; iter++) + for (char **iter = lines; iter && *iter; iter++) { - const char *line = *iter; + /* skip empty lines at least */ + if (**iter == '\0') + continue; - if (*line == '+') - { - const char *spc; - guint mode_add; - - spc = strchr (line + 1, ' '); - if (!spc) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Malformed statoverride file"); - goto out; - } - - mode_add = (guint32)(gint32)g_ascii_strtod (line + 1, NULL); - g_hash_table_insert (ret_hash, - g_strdup (spc + 1), - GUINT_TO_POINTER((gint32)mode_add)); - } + if (!cb (*iter, cbdata, error)) + goto out; } ret = TRUE; - ot_transfer_out_value (out_mode_add, &ret_hash); - out: +out: g_strfreev (lines); return ret; } +static gboolean +handle_statoverride_line (const char *line, + void *data, + GError **error) +{ + GHashTable *files = data; + const char *spc; + guint mode_add; + + spc = strchr (line, ' '); + if (spc == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Malformed statoverride file (no space found)"); + return FALSE; + } + + mode_add = (guint32)(gint32)g_ascii_strtod (line, NULL); + g_hash_table_insert (files, g_strdup (spc + 1), + GUINT_TO_POINTER((gint32)mode_add)); + return TRUE; +} + +static gboolean +handle_skiplist_line (const char *line, + void *data, + GError **error) +{ + GHashTable *files = data; + g_hash_table_add (files, g_strdup (line)); + return TRUE; +} + +struct CommitFilterData { + GHashTable *mode_adds; + GHashTable *skip_list; +}; + static OstreeRepoCommitFilterResult commit_filter (OstreeRepo *self, const char *path, GFileInfo *file_info, gpointer user_data) { - GHashTable *mode_adds = user_data; + struct CommitFilterData *data = user_data; + GHashTable *mode_adds = data->mode_adds; + GHashTable *skip_list = data->skip_list; gpointer value; if (opt_owner_uid >= 0) @@ -169,7 +191,13 @@ commit_filter (OstreeRepo *self, current_mode | mode_add); g_hash_table_remove (mode_adds, path); } - + + if (skip_list && g_hash_table_contains (skip_list, path)) + { + g_hash_table_remove (skip_list, path); + return OSTREE_REPO_COMMIT_FILTER_SKIP; + } + return OSTREE_REPO_COMMIT_FILTER_ALLOW; } @@ -310,9 +338,11 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError glnx_unref_object OstreeMutableTree *mtree = NULL; g_autofree char *tree_type = NULL; g_autoptr(GHashTable) mode_adds = NULL; + g_autoptr(GHashTable) skip_list = NULL; OstreeRepoCommitModifierFlags flags = 0; OstreeRepoCommitModifier *modifier = NULL; OstreeRepoTransactionStats stats; + struct CommitFilterData filter_data = { 0, }; context = g_option_context_new ("[PATH] - Commit a new revision"); @@ -324,7 +354,17 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError if (opt_statoverride_file) { - if (!parse_statoverride_file (&mode_adds, cancellable, error)) + mode_adds = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + if (!parse_file_by_line (opt_statoverride_file, handle_statoverride_line, + mode_adds, cancellable, error)) + goto out; + } + + if (opt_skiplist_file) + { + skip_list = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + if (!parse_file_by_line (opt_skiplist_file, handle_skiplist_line, + skip_list, cancellable, error)) goto out; } @@ -359,9 +399,13 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError || opt_owner_uid >= 0 || opt_owner_gid >= 0 || opt_statoverride_file != NULL + || opt_skiplist_file != NULL || opt_no_xattrs) { - modifier = ostree_repo_commit_modifier_new (flags, commit_filter, mode_adds, NULL); + filter_data.mode_adds = mode_adds; + filter_data.skip_list = skip_list; + modifier = ostree_repo_commit_modifier_new (flags, commit_filter, + &filter_data, NULL); } if (opt_parent) @@ -491,6 +535,22 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError goto out; } + if (skip_list && g_hash_table_size (skip_list) > 0) + { + GHashTableIter hash_iter; + gpointer key; + + g_hash_table_iter_init (&hash_iter, skip_list); + + while (g_hash_table_iter_next (&hash_iter, &key, NULL)) + { + g_printerr ("Unmatched skip-list path: %s\n", (char*)key); + } + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unmatched skip-list paths"); + goto out; + } + if (!ostree_repo_write_mtree (repo, mtree, &root, cancellable, error)) goto out; diff --git a/tests/basic-test.sh b/tests/basic-test.sh index b2477955..a44ee630 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..54" +echo "1..55" $OSTREE checkout test2 checkout-test2 echo "ok checkout" @@ -179,13 +179,31 @@ cd ${test_tmpdir}/checkout-test2-4 $OSTREE commit -b test2 -s "no xattrs" --no-xattrs echo "ok commit with no xattrs" +# NB: The + is optional, but we need to make sure we support it cd ${test_tmpdir} cat > test-statoverride.txt < test-skiplist.txt < -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -set -euo pipefail - -echo "1..1" - -. $(dirname $0)/libtest.sh - -setup_test_repository "bare" - -cd ${test_tmpdir} -cat > test-statoverride.txt < Date: Fri, 22 Apr 2016 12:39:54 -0400 Subject: [PATCH 035/128] tests/test-libarchive.sh: add more test - Test both tar and cpio archives - Test more hardlink corner cases - Test symlinks more rigorously - Test stat override - Test skip list Closes: #275 Approved by: cgwalters --- tests/libtest.sh | 21 +++++++ tests/test-libarchive.sh | 117 ++++++++++++++++++++++++++------------- 2 files changed, 99 insertions(+), 39 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 4d403a93..34976e23 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -134,6 +134,18 @@ assert_file_has_content () { fi } +assert_symlink_has_content () { + if ! test -L "$1"; then + echo 1>&2 "File '$1' is not a symbolic link" + exit 1 + fi + if ! readlink "$1" | grep -q -e "$2"; then + sed -e 's/^/# /' < "$1" >&2 + echo 1>&2 "Symbolic link '$1' doesn't match regexp '$2'" + exit 1 + fi +} + assert_file_empty() { if test -s "$1"; then sed -e 's/^/# /' < "$1" >&2 @@ -142,6 +154,15 @@ assert_file_empty() { fi } +assert_files_hardlinked() { + f1=$(stat -c %i $1) + f2=$(stat -c %i $2) + if [ "$f1" != "$f2" ]; then + echo 1>&2 "Files '$1' and '$2' are not hardlinked" + exit 1 + fi +} + setup_test_repository () { mode=$1 shift diff --git a/tests/test-libarchive.sh b/tests/test-libarchive.sh index 7309ffd2..0c579459 100755 --- a/tests/test-libarchive.sh +++ b/tests/test-libarchive.sh @@ -26,57 +26,95 @@ fi . $(dirname $0)/libtest.sh -echo "1..7" +echo "1..20" setup_test_repository "bare" + cd ${test_tmpdir} mkdir foo cd foo -echo hi > hi -ln -s hi hello -mkdir subdir -echo contents > subdir/more -mkdir subdir/1 -touch subdir/1/empty -mkdir subdir/2 -touch subdir/2/empty -echo not > subdir/2/notempty +mkdir -p usr/bin +echo contents > usr/bin/foo +touch usr/bin/foo0 +ln usr/bin/foo usr/bin/bar +ln usr/bin/foo0 usr/bin/bar0 +ln -s foo usr/bin/sl +mkdir -p usr/local/bin +ln usr/bin/foo usr/local/bin/baz +ln usr/bin/foo0 usr/local/bin/baz0 +ln usr/bin/sl usr/local/bin/slhl +touch usr/bin/setuidme +touch usr/bin/skipme tar -c -z -f ../foo.tar.gz . +find . | cpio -o -H newc > ../foo.cpio + cd .. -$OSTREE commit -s "from tar" -b test-tar --tree=tar=foo.tar.gz + +cat > statoverride.txt < skiplist.txt < otherfile -echo foo1 > foo -ln foo bar -tar czf ${test_tmpdir}/hardlinktest.tar.gz . -cd ${test_tmpdir} -$OSTREE commit -s 'hardlinks' -b test-hardlinks --tree=tar=hardlinktest.tar.gz -rm -rf hardlinktest -echo "ok hardlink commit" + # basic content check + assert_file_has_content usr/bin/foo contents + assert_file_has_content usr/bin/bar contents + assert_file_has_content usr/local/bin/baz contents + assert_file_empty usr/bin/foo0 + assert_file_empty usr/bin/bar0 + assert_file_empty usr/local/bin/baz0 + echo "ok $1 contents" -cd ${test_tmpdir} -$OSTREE checkout test-hardlinks test-hardlinks-checkout -cd test-hardlinks-checkout -assert_file_has_content foo foo1 -assert_file_has_content bar foo1 -echo "ok hardlink contents" + # hardlinks + assert_files_hardlinked usr/bin/foo usr/bin/bar + assert_files_hardlinked usr/bin/foo usr/local/bin/baz + echo "ok $1 hardlink" + assert_files_hardlinked usr/bin/foo0 usr/bin/bar0 + assert_files_hardlinked usr/bin/foo0 usr/local/bin/baz0 + echo "ok $1 hardlink to empty files" + + # symlinks + assert_symlink_has_content usr/bin/sl foo + assert_file_has_content usr/bin/sl contents + echo "ok $1 symlink" + # ostree checkout doesn't care if two symlinks are actually hardlinked + # together (which is fine). checking that it's also a symlink is good enough. + assert_symlink_has_content usr/local/bin/slhl foo + echo "ok $1 hardlink to symlink" + + # stat override + test -u usr/bin/setuidme + echo "ok $1 setuid" + + # skip list + test ! -f usr/bin/skipme + echo "ok $1 file skip" + + cd ${test_tmpdir} + rm -rf test-$1-checkout +} + +assert_valid_checkout tar +assert_valid_checkout cpio cd ${test_tmpdir} mkdir multicommit-files @@ -115,3 +153,4 @@ cd ${test_tmpdir} $OSTREE checkout partial partial-checkout cd partial-checkout assert_file_has_content subdir/original "original" +echo "ok tar partial commit contents" From dfbb61270a144dc1aae049b6d1c43927e4a0f4d4 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 22 Apr 2016 12:42:05 -0400 Subject: [PATCH 036/128] tests/test-libarchive-import.c: add tests - Do a bit of refactoring - Add test for use_ostree_convention - Add test for xattr_callback - Add test for SELinux labeling Closes: #275 Approved by: cgwalters --- tests/test-libarchive-import.c | 392 +++++++++++++++++++++++++++++---- 1 file changed, 346 insertions(+), 46 deletions(-) diff --git a/tests/test-libarchive-import.c b/tests/test-libarchive-import.c index 877fa77c..aaa3c37b 100644 --- a/tests/test-libarchive-import.c +++ b/tests/test-libarchive-import.c @@ -42,6 +42,8 @@ test_data_init (TestData *td) GError *error = NULL; struct archive *a = archive_write_new (); struct archive_entry *ae; + uid_t uid = getuid (); + gid_t gid = getgid (); td->tmpd = g_mkdtemp (g_strdup ("/var/tmp/test-libarchive-import-XXXXXX")); g_assert_cmpint (0, ==, chdir (td->tmpd)); @@ -59,12 +61,16 @@ test_data_init (TestData *td) ae = archive_entry_new (); archive_entry_set_pathname (ae, "/"); archive_entry_set_mode (ae, S_IFDIR | 0755); + archive_entry_set_uid (ae, uid); + archive_entry_set_gid (ae, gid); g_assert_cmpint (0, ==, archive_write_header (a, ae)); archive_entry_free (ae); ae = archive_entry_new (); archive_entry_set_pathname (ae, "/file"); archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_uid (ae, uid); + archive_entry_set_gid (ae, gid); archive_entry_set_size (ae, 4); g_assert_cmpint (0, ==, archive_write_header (a, ae)); g_assert_cmpint (4, ==, archive_write_data (a, "foo\n", 4)); @@ -73,6 +79,8 @@ test_data_init (TestData *td) ae = archive_entry_new (); archive_entry_set_pathname (ae, "/devnull"); archive_entry_set_mode (ae, S_IFCHR | 0777); + archive_entry_set_uid (ae, uid); + archive_entry_set_gid (ae, gid); archive_entry_set_devmajor (ae, 1); archive_entry_set_devminor (ae, 3); g_assert_cmpint (0, ==, archive_write_header (a, ae)); @@ -81,6 +89,26 @@ test_data_init (TestData *td) ae = archive_entry_new (); archive_entry_set_pathname (ae, "/anotherfile"); archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_uid (ae, uid); + archive_entry_set_gid (ae, gid); + archive_entry_set_size (ae, 4); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + g_assert_cmpint (4, ==, archive_write_data (a, "bar\n", 4)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/etc"); + archive_entry_set_mode (ae, S_IFDIR | 0755); + archive_entry_set_uid (ae, uid); + archive_entry_set_gid (ae, gid); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/etc/file"); + archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_uid (ae, uid); + archive_entry_set_gid (ae, gid); archive_entry_set_size (ae, 4); g_assert_cmpint (0, ==, archive_write_header (a, ae)); g_assert_cmpint (4, ==, archive_write_data (a, "bar\n", 4)); @@ -123,6 +151,15 @@ spawn_cmdline (const char *cmd, GError **error) return TRUE; } +static void +test_archive_setup (int fd, struct archive *a) +{ + g_assert_cmpint (0, ==, lseek (fd, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, fd, 8192)); +} + static void test_libarchive_noautocreate_empty (gconstpointer data) { @@ -132,10 +169,7 @@ test_libarchive_noautocreate_empty (gconstpointer data) OstreeRepoImportArchiveOptions opts = { 0, }; glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); - g_assert_cmpint (0, ==, lseek (td->fd_empty, 0, SEEK_SET)); - g_assert_cmpint (0, ==, archive_read_support_format_all (a)); - g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); - g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd_empty, 8192)); + test_archive_setup (td->fd_empty, a); (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); g_assert_no_error (error); @@ -153,10 +187,7 @@ test_libarchive_autocreate_empty (gconstpointer data) opts.autocreate_parents = 1; - g_assert_cmpint (0, ==, lseek (td->fd_empty, 0, SEEK_SET)); - g_assert_cmpint (0, ==, archive_read_support_format_all (a)); - g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); - g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd_empty, 8192)); + test_archive_setup (td->fd_empty, a); (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); g_assert_no_error (error); @@ -172,68 +203,95 @@ test_libarchive_error_device_file (gconstpointer data) OstreeRepoImportArchiveOptions opts = { 0, }; glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); - g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET)); - g_assert_cmpint (0, ==, archive_read_support_format_all (a)); - g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); - g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192)); + test_archive_setup (td->fd, a); (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); g_assert (error != NULL); } +static gboolean +skip_if_no_xattr (TestData *td) +{ + /* /var/tmp might actually be a tmpfs */ + if (setxattr (td->tmpd, "user.test-xattr-support", "yes", 4, 0) != 0) + { + int saved_errno = errno; + g_autofree gchar *message + = g_strdup_printf ("unable to setxattr on \"%s\": %s", + td->tmpd, g_strerror (saved_errno)); + g_test_skip (message); + return TRUE; + } + + return FALSE; +} + +static gboolean +import_write_and_ref (OstreeRepo *repo, + OstreeRepoImportArchiveOptions *opts, + struct archive *a, + const char *ref, + OstreeRepoCommitModifier *modifier, + GError **error) +{ + gboolean ret = FALSE; + glnx_unref_object GFile *root = NULL; + g_autofree char *commit_checksum = NULL; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + + if (!ostree_repo_prepare_transaction (repo, NULL, NULL, error)) + goto out; + + if (!ostree_repo_import_archive_to_mtree (repo, opts, a, mtree, modifier, + NULL, error)) + goto out; + + if (!ostree_repo_write_mtree (repo, mtree, &root, NULL, error)) + goto out; + + if (!ostree_repo_write_commit (repo, NULL, "", "", NULL, + OSTREE_REPO_FILE (root), + &commit_checksum, NULL, error)) + goto out; + + ostree_repo_transaction_set_ref (repo, NULL, ref, commit_checksum); + + if (!ostree_repo_commit_transaction (repo, NULL, NULL, error)) + goto out; + + ret = TRUE; +out: + return ret; +} + static void test_libarchive_ignore_device_file (gconstpointer data) { TestData *td = (void*)data; GError *error = NULL; - GCancellable *cancellable = NULL; struct archive *a = archive_read_new (); OstreeRepoImportArchiveOptions opts = { 0, }; - glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); - glnx_unref_object GFile *root = NULL; - g_autofree char *commit_checksum = NULL; - if (setxattr (td->tmpd, "user.test-xattr-support", "yes", 4, 0) != 0) - { - int saved_errno = errno; - g_autofree gchar *message = g_strdup_printf ("unable to setxattr on \"%s\": %s", td->tmpd, g_strerror (saved_errno)); + if (skip_if_no_xattr (td)) + goto out; - g_test_skip (message); - goto out; - } - - g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET)); - g_assert_cmpint (0, ==, archive_read_support_format_all (a)); - g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); - g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192)); + test_archive_setup (td->fd, a); opts.ignore_unsupported_content = TRUE; - if (!ostree_repo_prepare_transaction (td->repo, NULL, cancellable, &error)) - goto out; - - if (!ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error)) - goto out; - - if (!ostree_repo_write_mtree (td->repo, mtree, &root, cancellable, &error)) - goto out; - - if (!ostree_repo_write_commit (td->repo, NULL, "", "", NULL, - OSTREE_REPO_FILE (root), - &commit_checksum, cancellable, &error)) - goto out; - - ostree_repo_transaction_set_ref (td->repo, NULL, "foo", commit_checksum); - - if (!ostree_repo_commit_transaction (td->repo, NULL, cancellable, &error)) + if (!import_write_and_ref (td->repo, &opts, a, "foo", NULL, &error)) goto out; + /* check contents */ if (!spawn_cmdline ("ostree --repo=repo ls foo file", &error)) goto out; if (!spawn_cmdline ("ostree --repo=repo ls foo anotherfile", &error)) goto out; + if (!spawn_cmdline ("ostree --repo=repo ls foo /etc/file", &error)) + goto out; + if (spawn_cmdline ("ostree --repo=repo ls foo devnull", &error)) g_assert_not_reached (); g_assert (error != NULL); @@ -243,6 +301,243 @@ test_libarchive_ignore_device_file (gconstpointer data) g_assert_no_error (error); } +static gboolean +check_ostree_convention (GError *error) +{ + if (!spawn_cmdline ("ostree --repo=repo ls bar file", &error)) + return FALSE; + + if (!spawn_cmdline ("ostree --repo=repo ls bar anotherfile", &error)) + return FALSE; + + if (!spawn_cmdline ("ostree --repo=repo ls bar /usr/etc/file", &error)) + return FALSE; + + if (spawn_cmdline ("ostree --repo=repo ls bar /etc/file", &error)) + g_assert_not_reached (); + g_assert (error != NULL); + g_clear_error (&error); + + if (spawn_cmdline ("ostree --repo=repo ls bar devnull", &error)) + g_assert_not_reached (); + g_assert (error != NULL); + g_clear_error (&error); + + return TRUE; +} + +static void +test_libarchive_ostree_convention (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + + if (skip_if_no_xattr (td)) + goto out; + + test_archive_setup (td->fd, a); + + opts.autocreate_parents = TRUE; + opts.use_ostree_convention = TRUE; + opts.ignore_unsupported_content = TRUE; + + if (!import_write_and_ref (td->repo, &opts, a, "bar", NULL, &error)) + goto out; + + if (!check_ostree_convention (error)) + goto out; + + out: + g_assert_no_error (error); +} + +static GVariant* +xattr_cb (OstreeRepo *repo, + const char *path, + GFileInfo *file_info, + gpointer user_data) +{ + g_auto(GVariantBuilder) builder; + g_variant_builder_init (&builder, (GVariantType*)"a(ayay)"); + if (strcmp (path, "/anotherfile") == 0) + g_variant_builder_add (&builder, "(@ay@ay)", + g_variant_new_bytestring ("user.data"), + g_variant_new_bytestring ("mydata")); + return g_variant_ref_sink (g_variant_builder_end (&builder)); +} + +static void +test_libarchive_xattr_callback (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0 }; + OstreeRepoCommitModifier *modifier = NULL; + char buf[7] = { 0 }; + + if (skip_if_no_xattr (td)) + goto out; + + modifier = ostree_repo_commit_modifier_new (0, NULL, NULL, NULL); + ostree_repo_commit_modifier_set_xattr_callback (modifier, xattr_cb, + NULL, NULL); + + test_archive_setup (td->fd, a); + + opts.ignore_unsupported_content = TRUE; + + if (!import_write_and_ref (td->repo, &opts, a, "baz", modifier, &error)) + goto out; + + /* check contents */ + if (!spawn_cmdline ("ostree --repo=repo checkout baz baz-checkout", &error)) + goto out; + + g_assert_cmpint (0, >, getxattr ("baz-checkout/file", "user.data", NULL, 0)); + g_assert_cmpint (ENODATA, ==, errno); + + if (getxattr ("baz-checkout/anotherfile", "user.data", buf, sizeof buf) < 0) + { + glnx_set_prefix_error_from_errno (&error, "%s", "getxattr"); + goto out; + } + + g_assert_cmpstr (buf, ==, "mydata"); + + out: + if (modifier) + ostree_repo_commit_modifier_unref (modifier); + g_assert_no_error (error); +} + +static GVariant* +path_cb (OstreeRepo *repo, + const char *path, + GFileInfo *file_info, + gpointer user_data) +{ + if (strcmp (path, "/etc/file") == 0) + *(gboolean*)user_data = TRUE; + return NULL; +} + +static void +entry_pathname_test_helper (gconstpointer data, gboolean on) +{ + TestData *td = (void*)data; GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + OstreeRepoCommitModifier *modifier = NULL; + gboolean met_etc_file = FALSE; + + modifier = ostree_repo_commit_modifier_new (0, NULL, NULL, NULL); + ostree_repo_commit_modifier_set_xattr_callback (modifier, path_cb, + NULL, &met_etc_file); + + test_archive_setup (td->fd, a); + + opts.autocreate_parents = TRUE; + opts.use_ostree_convention = TRUE; + opts.ignore_unsupported_content = TRUE; + opts.callback_with_entry_pathname = on; + + if (!import_write_and_ref (td->repo, &opts, a, "bar", modifier, &error)) + goto out; + + /* the flag shouldn't have any effect on the final tree */ + if (!check_ostree_convention (error)) + goto out; + + if (!on && met_etc_file) + { + g_set_error_literal (&error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Received callback with /etc/file"); + goto out; + } + + if (on && !met_etc_file) + { + g_set_error_literal (&error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Did not receive callback with /etc/file"); + goto out; + } + + out: + g_assert_no_error (error); +} + +static void +test_libarchive_no_use_entry_pathname (gconstpointer data) +{ + entry_pathname_test_helper (data, FALSE); +} + +static void +test_libarchive_use_entry_pathname (gconstpointer data) +{ + entry_pathname_test_helper (data, TRUE); +} + +static void +test_libarchive_selinux (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0 }; + glnx_unref_object OstreeSePolicy *sepol = NULL; + OstreeRepoCommitModifier *modifier = NULL; + char buf[64] = { 0 }; + + if (skip_if_no_xattr (td)) + goto out; + + { + glnx_unref_object GFile *root = g_file_new_for_path ("/"); + + /* creation should always succeed */ + sepol = ostree_sepolicy_new (root, NULL, &error); + g_assert (sepol != NULL); + } + + if (ostree_sepolicy_get_name (sepol) == NULL) + { + g_test_skip ("SELinux disabled"); + goto out; + } + + modifier = ostree_repo_commit_modifier_new (0, NULL, NULL, NULL); + ostree_repo_commit_modifier_set_sepolicy (modifier, sepol); + + test_archive_setup (td->fd, a); + + opts.ignore_unsupported_content = TRUE; + + if (!import_write_and_ref (td->repo, &opts, a, "bob", modifier, &error)) + goto out; + + /* check contents */ + if (!spawn_cmdline ("ostree --repo=repo checkout bob bob-checkout", &error)) + goto out; + + if (getxattr ("bob-checkout/etc", "security.selinux", buf, sizeof buf) < 0) + { + glnx_set_prefix_error_from_errno (&error, "%s", "getxattr"); + goto out; + } + + buf[(sizeof buf) - 1] = '\0'; + g_assert_cmpstr (buf, ==, "system_u:object_r:etc_t:s0"); + + out: + if (modifier) + ostree_repo_commit_modifier_unref (modifier); + g_assert_no_error (error); +} + int main (int argc, char **argv) { TestData td = {NULL,}; @@ -256,10 +551,15 @@ int main (int argc, char **argv) g_test_add_data_func ("/libarchive/autocreate-empty", &td, test_libarchive_autocreate_empty); g_test_add_data_func ("/libarchive/error-device-file", &td, test_libarchive_error_device_file); g_test_add_data_func ("/libarchive/ignore-device-file", &td, test_libarchive_ignore_device_file); + g_test_add_data_func ("/libarchive/ostree-convention", &td, test_libarchive_ostree_convention); + g_test_add_data_func ("/libarchive/xattr-callback", &td, test_libarchive_xattr_callback); + g_test_add_data_func ("/libarchive/no-use-entry-pathname", &td, test_libarchive_no_use_entry_pathname); + g_test_add_data_func ("/libarchive/use-entry-pathname", &td, test_libarchive_use_entry_pathname); + g_test_add_data_func ("/libarchive/selinux", &td, test_libarchive_selinux); r = g_test_run(); - if (td.tmpd) + if (td.tmpd && g_getenv ("TEST_SKIP_CLEANUP") == NULL) (void) glnx_shutil_rm_rf_at (AT_FDCWD, td.tmpd, NULL, NULL); return r; } From 49322a3f53fa2e4b98663963d6b4cc030f32e484 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 6 May 2016 12:25:27 -0400 Subject: [PATCH 037/128] build: Use --std=gnu99 It's actually very nice to be able to declare loop variables inside the initializer. Ideally we could turn off nested functions though. Closes: #284 Approved by: jlebon --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index e49b7c59..cf67c4ba 100644 --- a/Makefile.am +++ b/Makefile.am @@ -27,7 +27,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DG_LOG_DOMAIN=\"OSTree\" \ -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40 \ -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 -DSOUP_VERSION_MAX_ALLOWED=SOUP_VERSION_2_48 -AM_CFLAGS += $(WARN_CFLAGS) +AM_CFLAGS += -std=gnu99 $(WARN_CFLAGS) DISTCHECK_CONFIGURE_FLAGS += --enable-gtk-doc --disable-maintainer-mode GITIGNOREFILES = aclocal.m4 build-aux/ buildutil/*.m4 config.h.in gtk-doc.make From d270085ef33a08a20ef3c17f886c3f60f50ce1c6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 7 May 2016 11:22:47 -0400 Subject: [PATCH 038/128] trivial-httpd: Add __attribute__((format)) I was briefly looking at building with clang mostly since it detects unused variables with cleanup attributes, but then I hit this fatal error. It's a hard compile error with `-Wformat-nonliteral` since clang doesn't know it's a format string. --- src/ostree/ot-builtin-trivial-httpd.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ostree/ot-builtin-trivial-httpd.c b/src/ostree/ot-builtin-trivial-httpd.c index 0300b128..811e8924 100644 --- a/src/ostree/ot-builtin-trivial-httpd.c +++ b/src/ostree/ot-builtin-trivial-httpd.c @@ -56,6 +56,9 @@ static GOptionEntry options[] = { { NULL } }; +static void +httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...) __attribute__ ((format(printf, 2, 3))); + static void httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...) { From 58532e591731ef5b613c59906e5df1c4ffc7ad69 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 9 May 2016 15:11:18 +0200 Subject: [PATCH 039/128] sysroot: Plug some leaks The deployments and booted deployment were never unreffed. Also, remote (void) from the call to the unload function - it returns nothing, so there is no need to fool the compiler. Closes: #287 Approved by: cgwalters --- src/libostree/ostree-sysroot.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index b114a901..e05f5c29 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -70,10 +70,12 @@ ostree_sysroot_finalize (GObject *object) g_clear_object (&self->path); g_clear_object (&self->sepolicy); g_clear_object (&self->repo); + g_clear_pointer (&self->deployments, g_ptr_array_unref); + g_clear_object (&self->booted_deployment); glnx_release_lock_file (&self->lock); - (void) ostree_sysroot_unload (self); + ostree_sysroot_unload (self); G_OBJECT_CLASS (ostree_sysroot_parent_class)->finalize (object); } @@ -873,7 +875,7 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, } g_clear_pointer (&self->deployments, g_ptr_array_unref); - g_clear_pointer (&self->booted_deployment, g_object_unref); + g_clear_object (&self->booted_deployment); self->bootversion = -1; self->subbootversion = -1; From 65e7969b0bd023ccbd532a8e843dbf54341fb2cb Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 13 Apr 2016 16:44:21 -0400 Subject: [PATCH 040/128] OstreeRepo: add extensions directory It's very useful for third-party applications to have someplace to store their data guaranteed to be on the same device as the repo (thus ensuring hardlinks) while still being shielded away from any of OSTree's timely garbage collections. We create a new "extensions/" subdirectory where apps can include whatever they wish in "extensions/myapp/". This subdirectory is completely unmanaged by ostree. NB: I didn't bother making it a member of the OstreeRepo proper since we don't really use it for anything else yet. Closes: #286 Approved by: cgwalters --- src/libostree/ostree-repo.c | 7 +++++++ tests/basic-test.sh | 8 ++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index c395b1dc..7caf528e 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2210,6 +2210,13 @@ ostree_repo_create (OstreeRepo *self, if (!g_file_make_directory (self->tmp_dir, cancellable, error)) goto out; + { + g_autoptr(GFile) extensions_dir = + g_file_resolve_relative_path (self->repodir, "extensions"); + if (!g_file_make_directory (extensions_dir, cancellable, error)) + goto out; + } + g_clear_object (&child); child = g_file_get_child (self->repodir, "refs"); if (!g_file_make_directory (child, cancellable, error)) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index a44ee630..0373fcbc 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..55" +echo "1..56" $OSTREE checkout test2 checkout-test2 echo "ok checkout" @@ -508,5 +508,9 @@ cd .. if cmp timestamp-{orig,new}.txt; then assert_not_reached "failed to update mtime on repo" fi - echo "ok mtime updated" + +cd ${test_tmpdir} +$OSTREE init --mode=bare --repo=repo-extensions +assert_has_dir repo-extensions/extensions +echo "ok extensions dir" From 3ae87e3f9e88bb1ce614e8ade1c82c60d0175627 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 10 May 2016 11:59:12 +0200 Subject: [PATCH 041/128] core: Improve variant format docs Use markdown to actually format the tuple member descriptions as a list. Currently they are hardly legible as gtk-doc formats them into a single sentence. Also update the additional metadata docs in the summary objects. Closes: #289 Approved by: cgwalters --- src/libostree/ostree-core.h | 41 +++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 97d86bc5..29b5a1c0 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -90,10 +90,10 @@ typedef enum { /** * OSTREE_DIRMETA_GVARIANT_FORMAT: * - * u - uid - * u - gid - * u - mode - * a(ayay) - xattrs + * - u - uid + * - u - gid + * - u - mode + * - a(ayay) - xattrs */ #define OSTREE_DIRMETA_GVARIANT_STRING "(uuua(ayay))" #define OSTREE_DIRMETA_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_DIRMETA_GVARIANT_STRING) @@ -106,10 +106,10 @@ typedef enum { * can't store in the real filesystem but we can still use a regular .file object * that we can hardlink to in the case of a user-mode checkout. * - * u - uid - * u - gid - * u - mode - * a(ayay) - xattrs + * - u - uid + * - u - gid + * - u - mode + * - a(ayay) - xattrs */ #define OSTREE_FILEMETA_GVARIANT_STRING "(uuua(ayay))" #define OSTREE_FILEMETA_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_FILEMETA_GVARIANT_STRING) @@ -117,8 +117,8 @@ typedef enum { /** * OSTREE_TREE_GVARIANT_FORMAT: * - * a(say) - array of (filename, checksum) for files - * a(sayay) - array of (dirname, tree_checksum, meta_checksum) for directories + * - a(say) - array of (filename, checksum) for files + * - a(sayay) - array of (dirname, tree_checksum, meta_checksum) for directories */ #define OSTREE_TREE_GVARIANT_STRING "(a(say)a(sayay))" #define OSTREE_TREE_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_TREE_GVARIANT_STRING) @@ -126,14 +126,14 @@ typedef enum { /** * OSTREE_COMMIT_GVARIANT_FORMAT: * - * a{sv} - Metadata - * ay - parent checksum (empty string for initial) - * a(say) - Related objects - * s - subject - * s - body - * t - Timestamp in seconds since the epoch (UTC) - * ay - Root tree contents - * ay - Root tree metadata + * - a{sv} - Metadata + * - ay - parent checksum (empty string for initial) + * - a(say) - Related objects + * - s - subject + * - s - body + * - t - Timestamp in seconds since the epoch (UTC) + * - ay - Root tree contents + * - ay - Root tree metadata */ #define OSTREE_COMMIT_GVARIANT_STRING "(a{sv}aya(say)sstayay)" #define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_COMMIT_GVARIANT_STRING) @@ -141,8 +141,9 @@ typedef enum { /** * OSTREE_SUMMARY_GVARIANT_FORMAT: * - * refs: a(s(taya{sv})) - Map of ref name -> (latest commit size, latest commit checksum, additional metadata), sorted by ref name - * extensions: a{sv} - Additional metadata, none defined at the current time + * - a(s(taya{sv})) - Map of ref name -> (latest commit size, latest commit checksum, additional metadata), sorted by ref name + * - a{sv} - Additional metadata, at the current time the following are defined: + * - key: "ostree.static-deltas", value: a{sv}, static delta name -> 32 bytes of checksum */ #define OSTREE_SUMMARY_GVARIANT_STRING "(a(s(taya{sv}))a{sv})" #define OSTREE_SUMMARY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_SUMMARY_GVARIANT_STRING) From bdd73913fb29ff8cd78efe83580b8d4dc134dde3 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 10 May 2016 12:23:03 +0200 Subject: [PATCH 042/128] repo: Plug a leak The base URI created in this line was always (sans erroneous situations) overwritten in the code block below without freeing it previously, so it leaked. Closes: #290 Approved by: cgwalters --- src/libostree/ostree-repo.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 7caf528e..243b7ae6 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1913,8 +1913,6 @@ repo_remote_fetch_summary (OstreeRepo *self, if (fetcher == NULL) goto out; - base_uri = soup_uri_new (metalink_url_string); - { g_autofree char *url_string = NULL; if (metalink_url_string) From aa946cc13636bc45f45f8850e47c456a6019d363 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 10 May 2016 12:45:49 +0200 Subject: [PATCH 043/128] repo: Allow using options for fetching summary This adds a _with_options variant of the ostree_repo_remote_fetch_summary function, so we can tell the fetcher to use a specific URL instead taking it from the remote config. Closes: #290 Approved by: cgwalters --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 6 ++++ src/libostree/ostree-repo.c | 55 ++++++++++++++++++++++++++++++++----- src/libostree/ostree-repo.h | 9 ++++++ 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 3b185ac0..b9b292c7 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -246,6 +246,7 @@ ostree_repo_remote_get_gpg_verify ostree_repo_remote_get_gpg_verify_summary ostree_repo_remote_gpg_import ostree_repo_remote_fetch_summary +ostree_repo_remote_fetch_summary_with_options ostree_repo_get_remote_boolean_option ostree_repo_get_remote_list_option ostree_repo_get_remote_option diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 8c0d8389..2aa72512 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -337,3 +337,9 @@ global: * Versions above here are released. Only add symbols below this line. * NOTE NOTE NOTE */ + +LIBOSTREE_2016.6 { +global: + ostree_repo_remote_fetch_summary_with_options; + +} LIBOSTREE_2016.5; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 243b7ae6..42813e9c 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1895,6 +1895,7 @@ static gboolean repo_remote_fetch_summary (OstreeRepo *self, const char *name, const char *metalink_url_string, + GVariant *options, GBytes **out_summary, GBytes **out_signatures, GCancellable *cancellable, @@ -1905,6 +1906,10 @@ repo_remote_fetch_summary (OstreeRepo *self, gboolean ret = FALSE; SoupURI *base_uri = NULL; gboolean from_cache = FALSE; + g_autofree char *url_override = NULL; + + if (options) + (void) g_variant_lookup (options, "override-url", "&s", &url_override); mainctx = g_main_context_new (); g_main_context_push_thread_default (mainctx); @@ -1917,11 +1922,10 @@ repo_remote_fetch_summary (OstreeRepo *self, g_autofree char *url_string = NULL; if (metalink_url_string) url_string = g_strdup (metalink_url_string); - else - { - if (!ostree_repo_remote_get_url (self, name, &url_string, error)) - goto out; - } + else if (url_override) + url_string = g_strdup (url_override); + else if (!ostree_repo_remote_get_url (self, name, &url_string, error)) + goto out; base_uri = soup_uri_new (url_string); if (base_uri == NULL) @@ -2004,8 +2008,8 @@ repo_remote_fetch_summary (OstreeRepo *self, * ostree_repo_remote_fetch_summary: * @self: Self * @name: name of a remote - * @out_summary: (allow-none): return location for raw summary data, or %NULL - * @out_signatures: (allow-none): return location for raw summary signature + * @out_summary: (nullable): return location for raw summary data, or %NULL + * @out_signatures: (nullable): return location for raw summary signature * data, or %NULL * @cancellable: a #GCancellable * @error: a #GError @@ -2030,6 +2034,42 @@ ostree_repo_remote_fetch_summary (OstreeRepo *self, GBytes **out_signatures, GCancellable *cancellable, GError **error) +{ + return ostree_repo_remote_fetch_summary_with_options (self, + name, + NULL, + out_summary, + out_signatures, + cancellable, + error); +} + +/** + * ostree_repo_remote_fetch_summary_with_options: + * @self: Self + * @name: name of a remote + * @options: (nullable): A GVariant a{sv} with an extensible set of flags + * @out_summary: (nullable): return location for raw summary data, or %NULL + * @out_signatures: (nullable): return location for raw summary signature + * data, or %NULL + * @cancellable: a #GCancellable + * @error: a #GError + * + * Like ostree_repo_remote_fetch_summary(), but supports an extensible set of flags. + * The following are currently defined: + * + * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options + * + * Returns: %TRUE on success, %FALSE on failure + */ +gboolean +ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, + const char *name, + GVariant *options, + GBytes **out_summary, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error) { #ifdef HAVE_LIBSOUP g_autofree char *metalink_url_string = NULL; @@ -2048,6 +2088,7 @@ ostree_repo_remote_fetch_summary (OstreeRepo *self, if (!repo_remote_fetch_summary (self, name, metalink_url_string, + options, &summary, &signatures, cancellable, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index d9c61ebc..f0fa53a3 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -194,6 +194,15 @@ gboolean ostree_repo_remote_fetch_summary (OstreeRepo *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, + const char *name, + GVariant *options, + GBytes **out_summary, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC OstreeRepo * ostree_repo_get_parent (OstreeRepo *self); From 862e6ecdcc58f025696b1394adfc0fcf7322df23 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 11 May 2016 11:04:04 +0200 Subject: [PATCH 044/128] libostree: Variant-related leak plugs and fixes This tries to avoid leaking GVariantBuilders and GVariants in some situations. The leaks were usually happening when some error occurred or because of unclear variant ownership situation. The former is mostly about making sure that g_variant_builder_clear is called on builders that didn't finish their variant building process. The latter is surely more work - sometimes the result of g_variant_builder_end() should not be passed directly to a function, but rather stored in a g_autoptr(GVariant), sunk and then passed to a function. IMO, with an advent of g_autoptr, GVariants should be always sunk instead of relying on some receiver function sinking it. This would make an easy-to-follow policy of always sinking your variants. Functions could then assume that the passed variant is already sunk. These leaks are still happenning in commands, but they are less harmful, since that code will not be used by some daemon as a library routine. Closes: #291 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 2 +- src/libostree/ostree-repo-libarchive.c | 4 +++- src/libostree/ostree-repo-pull.c | 5 ++++- .../ostree-repo-static-delta-compilation.c | 14 ++++++++------ src/libostree/ostree-repo.c | 16 +++++++++------- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 170b7b87..f384624c 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2384,7 +2384,7 @@ get_modified_xattrs (OstreeRepo *self, if (label) { - GVariantBuilder *builder; + g_autoptr(GVariantBuilder) builder = NULL; /* ret_xattrs may be NULL */ builder = ot_util_variant_builder_from_variant (ret_xattrs, diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 57da41d4..0d62124d 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -331,6 +331,7 @@ aic_ensure_parent_dir_with_file_info (OstreeRepoArchiveImportContext *ctx, { const char *name = glnx_basename (fullpath); g_auto(GVariantBuilder) xattrs_builder; + g_autoptr(GVariant) xattrs = NULL; /* is this the root directory itself? transform into empty string */ if (name[0] == '/' && name[1] == '\0') @@ -343,8 +344,9 @@ aic_ensure_parent_dir_with_file_info (OstreeRepoArchiveImportContext *ctx, DEFAULT_DIRMODE, cancellable, error)) return FALSE; + xattrs = g_variant_ref_sink (g_variant_builder_end (&xattrs_builder)); return mtree_ensure_dir_with_meta (ctx->repo, parent, name, file_info, - g_variant_builder_end (&xattrs_builder), + xattrs, FALSE /* error_if_exist */, out_dir, cancellable, error); } diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 708df5b6..619657c7 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1759,6 +1759,8 @@ ostree_repo_pull_one_dir (OstreeRepo *self, GError **error) { GVariantBuilder builder; + g_autoptr(GVariant) options = NULL; + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); if (dir_to_pull) @@ -1770,7 +1772,8 @@ ostree_repo_pull_one_dir (OstreeRepo *self, g_variant_builder_add (&builder, "{s@v}", "refs", g_variant_new_variant (g_variant_new_strv ((const char *const*) refs_to_fetch, -1))); - return ostree_repo_pull_with_options (self, remote_name, g_variant_builder_end (&builder), + options = g_variant_ref_sink (g_variant_builder_end (&builder)); + return ostree_repo_pull_with_options (self, remote_name, options, progress, cancellable, error); } diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 2a28e7db..218bbcc8 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -1253,7 +1253,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, guint min_fallback_size; guint max_bsdiff_size; guint max_chunk_size; - GVariantBuilder metadata_builder; + g_auto(GVariantBuilder) metadata_builder = {0,}; DeltaOpts delta_opts = DELTAOPT_FLAG_NONE; guint64 total_compressed_size = 0; guint64 total_uncompressed_size = 0; @@ -1384,16 +1384,18 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_autoptr(GVariant) delta_part_content = NULL; g_autoptr(GVariant) delta_part = NULL; g_autoptr(GVariant) delta_part_header = NULL; - GVariantBuilder *mode_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(uuu)")); - GVariantBuilder *xattr_builder = g_variant_builder_new (G_VARIANT_TYPE ("aa(ayay)")); + g_auto(GVariantBuilder) mode_builder = {0,}; + g_auto(GVariantBuilder) xattr_builder = {0,}; guint8 compression_type_char; + g_variant_builder_init (&mode_builder, G_VARIANT_TYPE ("a(uuu)")); + g_variant_builder_init (&xattr_builder, G_VARIANT_TYPE ("aa(ayay)")); { guint j; for (j = 0; j < part_builder->modes->len; j++) - g_variant_builder_add_value (mode_builder, part_builder->modes->pdata[j]); + g_variant_builder_add_value (&mode_builder, part_builder->modes->pdata[j]); for (j = 0; j < part_builder->xattrs->len; j++) - g_variant_builder_add_value (xattr_builder, part_builder->xattrs->pdata[j]); + g_variant_builder_add_value (&xattr_builder, part_builder->xattrs->pdata[j]); } payload_b = g_string_free_to_bytes (part_builder->payload); @@ -1403,7 +1405,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, part_builder->operations = NULL; /* FIXME - avoid duplicating memory here */ delta_part_content = g_variant_new ("(a(uuu)aa(ayay)@ay@ay)", - mode_builder, xattr_builder, + &mode_builder, &xattr_builder, ot_gvariant_new_ay_bytes (payload_b), ot_gvariant_new_ay_bytes (operations_b)); g_variant_ref_sink (delta_part_content); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 42813e9c..91f9615e 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3510,13 +3510,16 @@ ostree_repo_delete_object (OstreeRepo *self, if (tombstone_commits) { - g_autoptr(GVariantBuilder) builder = NULL; - builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - g_variant_builder_add (builder, "{sv}", "commit", g_variant_new_bytestring (sha256)); + g_auto(GVariantBuilder) builder = {0,}; + g_autoptr(GVariant) variant = NULL; + + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); + g_variant_builder_add (&builder, "{sv}", "commit", g_variant_new_bytestring (sha256)); + variant = g_variant_ref_sink (g_variant_builder_end (&builder)); if (!ostree_repo_write_metadata_trusted (self, OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT, sha256, - g_variant_builder_end (builder), + variant, cancellable, error)) goto out; @@ -4286,7 +4289,6 @@ ostree_repo_append_gpg_signature (OstreeRepo *self, gboolean ret = FALSE; g_autoptr(GVariant) metadata = NULL; g_autoptr(GVariant) new_metadata = NULL; - g_autoptr(GVariantBuilder) builder = NULL; if (!ostree_repo_read_commit_detached_metadata (self, commit_checksum, @@ -4954,7 +4956,7 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_autoptr(GVariant) summary = NULL; GList *ordered_keys = NULL; GList *iter = NULL; - GVariantDict additional_metadata_builder; + g_auto(GVariantDict) additional_metadata_builder = {0,}; if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error)) goto out; @@ -4987,7 +4989,7 @@ ostree_repo_regenerate_summary (OstreeRepo *self, { guint i; g_autoptr(GPtrArray) delta_names = NULL; - GVariantDict deltas_builder; + g_auto(GVariantDict) deltas_builder = {0,}; g_autoptr(GVariant) deltas = NULL; if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error)) From f98b8bd0d7e4f98951e8d460c1f1b30538da697f Mon Sep 17 00:00:00 2001 From: Sam Spilsbury Date: Tue, 21 Jul 2015 08:16:14 +0800 Subject: [PATCH 045/128] Makefile: Move to AM_DISTCHECK_CONFIGURE_FLAGS Since automake 1.11.2 it is recommended that packages use AM_DISTCHECK_CONFIGURE_FLAGS instead of DISTCHECK_CONFIGURE_FLAGS as the latter is intended to be a user variable. https://bugzilla.gnome.org/show_bug.cgi?id=766298 Closes: #293 Approved by: cgwalters --- Makefile-decls.am | 2 +- Makefile.am | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile-decls.am b/Makefile-decls.am index d8ec5ab4..f7ebd422 100644 --- a/Makefile-decls.am +++ b/Makefile-decls.am @@ -18,7 +18,7 @@ # Common variables AM_CPPFLAGS = AM_CFLAGS = -DISTCHECK_CONFIGURE_FLAGS = +AM_DISTCHECK_CONFIGURE_FLAGS = SUBDIRS = NULL = BUILT_SOURCES = diff --git a/Makefile.am b/Makefile.am index cf67c4ba..a003af25 100644 --- a/Makefile.am +++ b/Makefile.am @@ -28,7 +28,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40 \ -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 -DSOUP_VERSION_MAX_ALLOWED=SOUP_VERSION_2_48 AM_CFLAGS += -std=gnu99 $(WARN_CFLAGS) -DISTCHECK_CONFIGURE_FLAGS += --enable-gtk-doc --disable-maintainer-mode +AM_DISTCHECK_CONFIGURE_FLAGS += --enable-gtk-doc --disable-maintainer-mode GITIGNOREFILES = aclocal.m4 build-aux/ buildutil/*.m4 config.h.in gtk-doc.make From a5f703799f8da2b20eba949d2d053d4b307490c9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 12 May 2016 10:32:17 -0400 Subject: [PATCH 046/128] Make enum generation private by default, export stub symbol When we added enum type generation, the generated symbols used the `ostree_` prefix, and at the time that implied they were public. So we started (if built with libsoup) exporting `ostree_fetcher_config_flags_get_type`. I think it's not worth confusing ABI checkers, so let's export the dummy symbol forever, and switch enums to be private by default. We should revisit this and also export some auto-generated enum types for public enums, but that's a separate patch. Closes: #296 Approved by: gatispaeglis --- Makefile-libostree-defines.am | 1 + Makefile-libostree.am | 1 + src/libostree/ostree-dummy-enumtypes.c | 32 +++++++++++++++++++++++ src/libostree/ostree-dummy-enumtypes.h | 29 ++++++++++++++++++++ src/libostree/ostree-enumtypes.c.template | 2 +- src/libostree/ostree-enumtypes.h.template | 5 ++-- 6 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 src/libostree/ostree-dummy-enumtypes.c create mode 100644 src/libostree/ostree-dummy-enumtypes.h diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index f623900c..1db75e0c 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -22,6 +22,7 @@ libostree_public_headers = \ src/libostree/ostree.h \ src/libostree/ostree-async-progress.h \ src/libostree/ostree-core.h \ + src/libostree/ostree-dummy-enumtypes.h \ src/libostree/ostree-mutable-tree.h \ src/libostree/ostree-repo.h \ src/libostree/ostree-types.h \ diff --git a/Makefile-libostree.am b/Makefile-libostree.am index a50b2b9d..affdd745 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -69,6 +69,7 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-cmdprivate.c \ src/libostree/ostree-core-private.h \ src/libostree/ostree-core.c \ + src/libostree/ostree-dummy-enumtypes.c \ src/libostree/ostree-checksum-input-stream.c \ src/libostree/ostree-checksum-input-stream.h \ src/libostree/ostree-chain-input-stream.c \ diff --git a/src/libostree/ostree-dummy-enumtypes.c b/src/libostree/ostree-dummy-enumtypes.c new file mode 100644 index 00000000..27fc7b4d --- /dev/null +++ b/src/libostree/ostree-dummy-enumtypes.c @@ -0,0 +1,32 @@ +/* This file declares a stub function that is only exported + * to pacify ABI checkers - no one could really have used it. + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "ostree-dummy-enumtypes.h" + +/* Exported for backwards compat - see + * https://bugzilla.gnome.org/show_bug.cgi?id=764131 + * https://github.com/ostreedev/ostree/pull/294 + */ +GType +ostree_fetcher_config_flags_get_type (void) +{ + return G_TYPE_INVALID; +} diff --git a/src/libostree/ostree-dummy-enumtypes.h b/src/libostree/ostree-dummy-enumtypes.h new file mode 100644 index 00000000..e58e1884 --- /dev/null +++ b/src/libostree/ostree-dummy-enumtypes.h @@ -0,0 +1,29 @@ +/* This file declares a stub function that is only exported + * to pacify ABI checkers - no one could really have used it. + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#pragma once + +#include + +#ifndef __GI_SCANNER__ +_OSTREE_PUBLIC GType +ostree_fetcher_config_flags_get_type (void); +#endif diff --git a/src/libostree/ostree-enumtypes.c.template b/src/libostree/ostree-enumtypes.c.template index fe8807ae..ab1b2aec 100644 --- a/src/libostree/ostree-enumtypes.c.template +++ b/src/libostree/ostree-enumtypes.c.template @@ -28,7 +28,7 @@ /*** BEGIN value-header ***/ GType -@enum_name@_get_type (void) +_@enum_name@_get_type (void) { static volatile gsize the_type__volatile = 0; diff --git a/src/libostree/ostree-enumtypes.h.template b/src/libostree/ostree-enumtypes.h.template index 40899d7f..ec20fe0d 100644 --- a/src/libostree/ostree-enumtypes.h.template +++ b/src/libostree/ostree-enumtypes.h.template @@ -32,9 +32,8 @@ G_BEGIN_DECLS /*** END file-production ***/ /*** BEGIN enumeration-production ***/ -#define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type ()) -_OSTREE_PUBLIC -GType @enum_name@_get_type (void) G_GNUC_CONST; +#define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (_@enum_name@_get_type ()) +GType _@enum_name@_get_type (void) G_GNUC_CONST; /*** END enumeration-production ***/ From 817060e7722e3515e9e5f5e6474a2166347dbefe Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 12 May 2016 10:15:24 -0400 Subject: [PATCH 047/128] build: Don't run glib-mkenums if there's nothing to do This fixes the build without libsoup. Closes: #296 Approved by: gatispaeglis --- Makefile-libostree.am | 10 +++++++--- src/libostree/ostree-dummy-enumtypes.c | 3 +-- src/libostree/ostree-dummy-enumtypes.h | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index affdd745..a619cbcf 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -33,9 +33,11 @@ lib_LTLIBRARIES += libostree-1.la libostreeincludedir = $(includedir)/ostree-1 libostreeinclude_HEADERS = $(libostree_public_headers) -ENUM_TYPES = \ - $(srcdir)/src/libostree/ostree-fetcher.h \ - $(NULL) +ENUM_TYPES = $(NULL) + +if USE_LIBSOUP +ENUM_TYPES += $(srcdir)/src/libostree/ostree-fetcher.h +endif src/libostree/ostree-enumtypes.h: src/libostree/ostree-enumtypes.h.template $(ENUM_TYPES) $(AM_V_GEN) $(GLIB_MKENUMS) \ @@ -48,12 +50,14 @@ src/libostree/ostree-enumtypes.c: src/libostree/ostree-enumtypes.c.template $(EN --fhead "#include \"ostree-enumtypes.h\"" \ $(ENUM_TYPES) > $@.tmp && mv $@.tmp $@ +if USE_LIBSOUP ENUM_GENERATED = \ src/libostree/ostree-enumtypes.h \ src/libostree/ostree-enumtypes.c \ $(NULL) BUILT_SOURCES += $(ENUM_GENERATED) +endif CLEANFILES += $(BUILT_SOURCES) diff --git a/src/libostree/ostree-dummy-enumtypes.c b/src/libostree/ostree-dummy-enumtypes.c index 27fc7b4d..259273b5 100644 --- a/src/libostree/ostree-dummy-enumtypes.c +++ b/src/libostree/ostree-dummy-enumtypes.c @@ -1,7 +1,7 @@ /* This file declares a stub function that is only exported * to pacify ABI checkers - no one could really have used it. * - * Copyright (C) 2015 Red Hat, Inc. + * Copyright (C) 2016 Red Hat, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -23,7 +23,6 @@ /* Exported for backwards compat - see * https://bugzilla.gnome.org/show_bug.cgi?id=764131 - * https://github.com/ostreedev/ostree/pull/294 */ GType ostree_fetcher_config_flags_get_type (void) diff --git a/src/libostree/ostree-dummy-enumtypes.h b/src/libostree/ostree-dummy-enumtypes.h index e58e1884..017ed28e 100644 --- a/src/libostree/ostree-dummy-enumtypes.h +++ b/src/libostree/ostree-dummy-enumtypes.h @@ -1,7 +1,7 @@ /* This file declares a stub function that is only exported * to pacify ABI checkers - no one could really have used it. * - * Copyright (C) 2015 Red Hat, Inc. + * Copyright (C) 2016 Red Hat, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public From d451b87fbeb44f1e385c68d40e77b8f8112331ce Mon Sep 17 00:00:00 2001 From: Micah Abbott Date: Thu, 12 May 2016 16:54:06 -0400 Subject: [PATCH 048/128] docs: fix swapped link syntax A few links in the docs had the Markdown syntax swapped like: (link title)[link url] Just cleaned up those. Verified via `mkdocs serve` Closes: #297 Approved by: cgwalters --- docs/manual/atomic-upgrades.md | 4 ++-- docs/manual/formats.md | 3 +-- docs/manual/repo.md | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/manual/atomic-upgrades.md b/docs/manual/atomic-upgrades.md index b5f398d6..40515b83 100644 --- a/docs/manual/atomic-upgrades.md +++ b/docs/manual/atomic-upgrades.md @@ -45,7 +45,7 @@ operate "live" on the currently booted filesystem. The way they could work with OSTree is instead to take the list of installed packages in the currently booted tree, and compute a new filesystem from that. A later chapter describes in more details how this could work: -[adapting-existing.md](Adapting Existing Systems). +[Adapting Existing Systems](adapting-existing.md). For the purposes of this section, let's assume that we have a newly generated filesystem tree stored in the repo (which shares @@ -56,7 +56,7 @@ checking it back out of the repo into a deployment. Given a commit to deploy, OSTree first allocates a directory for it. This is of the form `/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. -The `$serial` is normally 0, but if a +The `$serial` is normally `0`, but if a given commit is deployed more than once, it will be incremented. This is supported because the previous deployment may have configuration in `/etc` that we do not want to use or overwrite. diff --git a/docs/manual/formats.md b/docs/manual/formats.md index e689f8a8..87d0005f 100644 --- a/docs/manual/formats.md +++ b/docs/manual/formats.md @@ -125,8 +125,7 @@ the client executes. This "updates as code" model allows for multiple content generation strategies. The design of this was inspired by that of Chromium: -[http://dev.chromium.org/chromium-os/chromiumos-design-docs/filesystem-autoupdate](ChromiumOS -autoupdate). +[ChromiumOS Autoupdate](http://dev.chromium.org/chromium-os/chromiumos-design-docs/filesystem-autoupdate). ### The delta superblock diff --git a/docs/manual/repo.md b/docs/manual/repo.md index d3be549c..d6f67090 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -98,7 +98,7 @@ that. A later addition to OSTree is the concept of a "summary" file, created via the `ostree summary -u` command. This was introduced for a few reasons. A primary use case is to be a target a -(Metalink)[https://en.wikipedia.org/wiki/Metalink], which requires a +[Metalink](https://en.wikipedia.org/wiki/Metalink), which requires a single file with a known checksum as a target. The summary file primarily contains two mappings: From cb4545670826a8bb157395206473a51a3c59181e Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 13 May 2016 13:34:57 +0200 Subject: [PATCH 049/128] libostree: Fix bracket-missing warnings Apparently I got the bracketing wrong in 862e6ecdcc58f025696b1394adfc0fcf7322df23: src/libostree/ostree-repo.c: In function 'ostree_repo_delete_object': src/libostree/ostree-repo.c:3538:11: warning: missing braces around initializer [-Wmissing-braces] g_auto(GVariantBuilder) builder = {0,}; Closes: #298 Approved by: cgwalters --- src/libostree/ostree-repo-static-delta-compilation.c | 6 +++--- src/libostree/ostree-repo.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 218bbcc8..eecaa8bc 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -1253,7 +1253,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, guint min_fallback_size; guint max_bsdiff_size; guint max_chunk_size; - g_auto(GVariantBuilder) metadata_builder = {0,}; + g_auto(GVariantBuilder) metadata_builder = {{0,}}; DeltaOpts delta_opts = DELTAOPT_FLAG_NONE; guint64 total_compressed_size = 0; guint64 total_uncompressed_size = 0; @@ -1384,8 +1384,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_autoptr(GVariant) delta_part_content = NULL; g_autoptr(GVariant) delta_part = NULL; g_autoptr(GVariant) delta_part_header = NULL; - g_auto(GVariantBuilder) mode_builder = {0,}; - g_auto(GVariantBuilder) xattr_builder = {0,}; + g_auto(GVariantBuilder) mode_builder = {{0,}}; + g_auto(GVariantBuilder) xattr_builder = {{0,}}; guint8 compression_type_char; g_variant_builder_init (&mode_builder, G_VARIANT_TYPE ("a(uuu)")); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 91f9615e..65b955ac 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3510,7 +3510,7 @@ ostree_repo_delete_object (OstreeRepo *self, if (tombstone_commits) { - g_auto(GVariantBuilder) builder = {0,}; + g_auto(GVariantBuilder) builder = {{0,}}; g_autoptr(GVariant) variant = NULL; g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); @@ -4956,7 +4956,7 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_autoptr(GVariant) summary = NULL; GList *ordered_keys = NULL; GList *iter = NULL; - g_auto(GVariantDict) additional_metadata_builder = {0,}; + g_auto(GVariantDict) additional_metadata_builder = {{0,}}; if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error)) goto out; @@ -4989,7 +4989,7 @@ ostree_repo_regenerate_summary (OstreeRepo *self, { guint i; g_autoptr(GPtrArray) delta_names = NULL; - g_auto(GVariantDict) deltas_builder = {0,}; + g_auto(GVariantDict) deltas_builder = {{0,}}; g_autoptr(GVariant) deltas = NULL; if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error)) From 574805b0f38ef32bd3766d1f62e857b51170d388 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 13 May 2016 09:59:26 -0400 Subject: [PATCH 050/128] README: x/xdg-app/flatpak/ I'm mostly just testing some Homu updates. Closes: #299 Approved by: cgwalters --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15cba7ab..caa8503d 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ provide a minimal host for Docker formatted Linux containers. Replicating a base immutable OS, then using Docker for applications meshes together two different tools with different tradeoffs. -[xdg-app](https://github.com/alexlarsson/xdg-app) uses OSTree +[flatpak](https://github.com/alexlarsson/xdg-app) uses OSTree for desktop application containers. [GNOME Continuous](https://wiki.gnome.org/Projects/GnomeContinuous) is From 441c03ba9ee86d025c7db38c05d22b8ad41d1459 Mon Sep 17 00:00:00 2001 From: Gatis Paeglis Date: Thu, 12 May 2016 14:42:38 +0200 Subject: [PATCH 051/128] Fix build when have_libsoup_client_certs=no This fixes a build failure with older libsoup versions that do not have the client certificates feature. Closes: #294 Approved by: cgwalters --- src/libostree/ostree-fetcher.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index 6a75ad59..91b419a5 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -273,6 +273,7 @@ session_thread_set_proxy_cb (ThreadClosure *thread_closure, proxy_uri, NULL); } +#ifdef HAVE_LIBSOUP_CLIENT_CERTS static void session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure, gpointer data) @@ -288,6 +289,7 @@ session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure, SOUP_SESSION_TLS_INTERACTION, interaction, NULL); } +#endif static void session_thread_set_tls_database_cb (ThreadClosure *thread_closure, From 51e4e3b3647daf702cfcc06438163b34e6879ae3 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 12 May 2016 14:42:26 +0200 Subject: [PATCH 052/128] libostree: Add g_autoptr() support for the public types To avoid breaking the ostree users we hide the autoptr setup behind an OSTREE_WITH_AUTOCLEANUPS variable. The variable is set to 0 by default, so users that want to start using them, have to define the variable themselves to 1. Probably after some transition period, the variable will be set 1 by default. This is conditional on GLib >= 2.44.0 being available. It does not bump ostree's dependency on GLib. Closes: #295 Approved by: cgwalters --- Makefile-libostree.am | 1 + src/libostree/ostree-autocleanups.h | 65 +++++++++++++++++++++++++++++ src/libostree/ostree.h | 2 + 3 files changed, 68 insertions(+) create mode 100644 src/libostree/ostree-autocleanups.h diff --git a/Makefile-libostree.am b/Makefile-libostree.am index a619cbcf..700d372f 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -128,6 +128,7 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-gpg-verifier.h \ src/libostree/ostree-gpg-verify-result.c \ src/libostree/ostree-gpg-verify-result-private.h \ + src/libostree/ostree-autocleanups.h \ $(NULL) if USE_LIBARCHIVE libostree_1_la_SOURCES += src/libostree/ostree-libarchive-input-stream.h \ diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h new file mode 100644 index 00000000..d04ba9aa --- /dev/null +++ b/src/libostree/ostree-autocleanups.h @@ -0,0 +1,65 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2016 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Author: Krzesimir Nowak + */ + +#pragma once + +#include + +G_BEGIN_DECLS + +#ifndef OSTREE_WITH_AUTOCLEANUPS +#define OSTREE_WITH_AUTOCLEANUPS 0 +#endif + +#if OSTREE_WITH_AUTOCLEANUPS && GLIB_CHECK_VERSION(2, 44, 0) + +/* + * The following types have no specific clear/free/unref functions, so + * they can be used as the stack-allocated variables or as the + * g_autofree heap-allocated variables. + * + * OstreeRepoTransactionStats + * OstreeRepoImportArchiveOptions + * OstreeRepoExportArchiveOptions + * OstreeRepoCheckoutOptions + */ + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeDiffItem, ostree_diff_item_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoCommitModifier, ostree_repo_commit_modifier_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoDevInoCache, ostree_repo_devino_cache_unref) + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeAsyncProgress, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeBootconfigParser, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeDeployment, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeGpgVerifyResult, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeMutableTree, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepo, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFile, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSePolicy, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSysroot, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSysrootUpgrader, g_object_unref) + +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_commit_traverse_iter_clear) + +#endif + +G_END_DECLS diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index f6bb6adb..9846a7e3 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -31,3 +31,5 @@ #include #include #include + +#include From 24e3ccb7807cee22ab43af49b32fd24457793cc2 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 13 May 2016 07:59:10 +0200 Subject: [PATCH 053/128] build: Remind about bumping related versions Closes: #295 Approved by: cgwalters --- configure.ac | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index 54d30e99..e5bd8c7e 100644 --- a/configure.ac +++ b/configure.ac @@ -45,6 +45,9 @@ PKG_PROG_PKG_CONFIG AM_PATH_GLIB_2_0 +dnl When bumping the gio-unix-2.0 dependency (or glib-2.0 in general), +dnl remember to bump GLIB_VERSION_MIN_REQUIRED and +dnl GLIB_VERSION_MAX_ALLOWED in Makefile.am GIO_DEPENDENCY="gio-unix-2.0 >= 2.40.0 libgsystem >= 2015.1" PKG_CHECK_MODULES(OT_DEP_GIO_UNIX, $GIO_DEPENDENCY) @@ -57,6 +60,9 @@ PKG_CHECK_MODULES(OT_DEP_ZLIB, zlib) dnl We're not actually linking to this, just using the header PKG_CHECK_MODULES(OT_DEP_E2P, e2p) +dnl When bumping the libsoup-2.4 dependency, remember to bump +dnl SOUP_VERSION_MIN_REQUIRED and SOUP_VERSION_MAX_ALLOWED in +dnl Makefile.am SOUP_DEPENDENCY="libsoup-2.4 >= 2.39.1" AC_ARG_WITH(soup, AS_HELP_STRING([--with-soup], [Use libsoup @<:@default=yes@:>@]), From d5f61043351ff1c0bea650873ec2541e1ede1f0a Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 13 May 2016 08:08:49 +0200 Subject: [PATCH 054/128] build: Allow ostree to use g_autoptr on its types internally As an example, I g_autoptred one OstreeRepo in the "ostree init" command. Closes: #295 Approved by: cgwalters --- Makefile.am | 1 + src/libostree/ostree-autocleanups.h | 5 ++++- src/ostree/ot-builtin-init.c | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index a003af25..1de31544 100644 --- a/Makefile.am +++ b/Makefile.am @@ -24,6 +24,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DLOCALEDIR=\"$(datadir)/locale\" -DSYSCONFDIR=\"$(sysconfdir)\" \ -DSHORTENED_SYSCONFDIR=\"$(shortened_sysconfdir)\" \ -DOSTREE_FEATURES='"$(OSTREE_FEATURES)"' \ + -DOSTREE_COMPILATION \ -DG_LOG_DOMAIN=\"OSTree\" \ -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40 \ -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 -DSOUP_VERSION_MAX_ALLOWED=SOUP_VERSION_2_48 diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index d04ba9aa..7301ef1d 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -30,7 +30,10 @@ G_BEGIN_DECLS #define OSTREE_WITH_AUTOCLEANUPS 0 #endif -#if OSTREE_WITH_AUTOCLEANUPS && GLIB_CHECK_VERSION(2, 44, 0) +/* ostree can use g_autoptr backports from libglnx when glib is too + * old, but still avoid exposing them to users that also have an old + * glib */ +#if defined(OSTREE_COMPILATION) || (OSTREE_WITH_AUTOCLEANUPS && GLIB_CHECK_VERSION(2, 44, 0)) /* * The following types have no specific clear/free/unref functions, so diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c index 9e7c8a2b..a250b793 100644 --- a/src/ostree/ot-builtin-init.c +++ b/src/ostree/ot-builtin-init.c @@ -37,7 +37,7 @@ gboolean ostree_builtin_init (int argc, char **argv, GCancellable *cancellable, GError **error) { GOptionContext *context = NULL; - glnx_unref_object OstreeRepo *repo = NULL; + g_autoptr(OstreeRepo) repo = NULL; gboolean ret = FALSE; OstreeRepoMode mode; From 506902f8a0e750008240541dae22562598eaa445 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 19 May 2016 14:28:25 +0200 Subject: [PATCH 055/128] build: Install new autocleanups header Haven't noticed before that there is a separate variable for a list of OSTree public headers. This fixes an embarrassing error that prohibits building a project that includes ostree.h. Closes: #302 Approved by: cgwalters --- Makefile-libostree-defines.am | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index 1db75e0c..2d478bb6 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -21,6 +21,7 @@ libostree_public_headers = \ src/libostree/ostree.h \ src/libostree/ostree-async-progress.h \ + src/libostree/ostree-autocleanups.h \ src/libostree/ostree-core.h \ src/libostree/ostree-dummy-enumtypes.h \ src/libostree/ostree-mutable-tree.h \ From 86ac8062f5311fe94ef4096f4e136ce993d5e9b2 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 20 May 2016 10:59:59 +0200 Subject: [PATCH 056/128] tests: Fix build with old glib test-checksum.c was using g_autofree without including libglnx.h which has the backport for that. Closes: #304 Approved by: cgwalters --- tests/test-checksum.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test-checksum.c b/tests/test-checksum.c index 25d3c37e..e7fd7be4 100644 --- a/tests/test-checksum.c +++ b/tests/test-checksum.c @@ -20,6 +20,7 @@ #include "config.h" +#include "libglnx.h" #include "libgsystem.h" #include #include From e3e82c54c16392a9712ed9288fe7998a575fec56 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Mon, 23 May 2016 08:53:34 +0200 Subject: [PATCH 057/128] man: add missing options for commit Signed-off-by: Giuseppe Scrivano Closes: #306 Approved by: cgwalters --- man/ostree-commit.xml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/man/ostree-commit.xml b/man/ostree-commit.xml index c4842584..00e20428 100644 --- a/man/ostree-commit.xml +++ b/man/ostree-commit.xml @@ -214,6 +214,22 @@ Boston, MA 02111-1307, USA. Override the timestamp of the commit to TIMESTAMP. + + + + + + Create a commit without writing a ref + + + + + ="POLICY" + + + POLICY is a boolean which specifies whether fsync should be used or not. Default to true. + + From 2f2af9252db84b2a0c91324788722c4335411c8b Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 24 May 2016 10:36:38 +0200 Subject: [PATCH 058/128] libotutil: Plug a leak There's no need to allocate the variant builder on a heap, so allocate it on the stack and avoid a memory leak at the same time. Closes: #307 Approved by: cgwalters --- src/libotutil/ot-variant-utils.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index 2dc07582..7b54542e 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -33,7 +33,10 @@ GVariant * ot_gvariant_new_empty_string_dict (void) { - return g_variant_builder_end (g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"))); + g_auto(GVariantBuilder) builder = {{0,}}; + + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); + return g_variant_builder_end (&builder); } GVariant * From d36862479860df5b133dca51dbf0607bbe849a5b Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 25 May 2016 11:00:19 +0200 Subject: [PATCH 059/128] Build on older versions of glib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Various places need to include libglnx.h for the autoptr backport fallbacks to be there before ostree-autocleanups.h is included. This fixes the build on centos7· Closes: #309 Approved by: giuseppe --- src/libostree/ostree-core.c | 2 +- src/libostree/ostree-diff.c | 1 + src/libostree/ostree-fetcher.c | 1 + src/libostree/ostree-repo-pull.c | 1 + src/libostree/ostree-repo-traverse.c | 1 + src/ostree/ot-admin-builtin-instutil.c | 2 +- src/ostree/ot-admin-functions.c | 1 + src/ostree/ot-builtin-admin.c | 2 +- src/ostree/ot-editor.c | 2 +- src/ostree/ot-main.c | 2 +- src/ostree/ot-main.h | 2 +- 11 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 55ff4d63..00f767b3 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -26,11 +26,11 @@ #include #include #include +#include "libglnx.h" #include "ostree.h" #include "ostree-core-private.h" #include "ostree-chain-input-stream.h" #include "otutil.h" -#include "libglnx.h" #define ALIGN_VALUE(this, boundary) \ (( ((unsigned long)(this)) + (((unsigned long)(boundary)) -1)) & (~(((unsigned long)(boundary))-1))) diff --git a/src/libostree/ostree-diff.c b/src/libostree/ostree-diff.c index 4b733461..84961028 100644 --- a/src/libostree/ostree-diff.c +++ b/src/libostree/ostree-diff.c @@ -22,6 +22,7 @@ #include "config.h" +#include "libglnx.h" #include "ostree.h" #include "otutil.h" diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index 91b419a5..d956d959 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -25,6 +25,7 @@ #include #include +#include "libglnx.h" #include "ostree-fetcher.h" #ifdef HAVE_LIBSOUP_CLIENT_CERTS #include "ostree-tls-cert-interaction.h" diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 619657c7..2f382856 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -22,6 +22,7 @@ #include "config.h" +#include "libglnx.h" #include "ostree.h" #include "ostree-core-private.h" #include "ostree-repo-private.h" diff --git a/src/libostree/ostree-repo-traverse.c b/src/libostree/ostree-repo-traverse.c index bb437c38..503ab329 100644 --- a/src/libostree/ostree-repo-traverse.c +++ b/src/libostree/ostree-repo-traverse.c @@ -22,6 +22,7 @@ #include "config.h" +#include "libglnx.h" #include "ostree.h" #include "otutil.h" diff --git a/src/ostree/ot-admin-builtin-instutil.c b/src/ostree/ot-admin-builtin-instutil.c index 1087b381..88894a6a 100644 --- a/src/ostree/ot-admin-builtin-instutil.c +++ b/src/ostree/ot-admin-builtin-instutil.c @@ -20,11 +20,11 @@ #include "config.h" +#include "ot-main.h" #include "ot-builtins.h" #include "ot-admin-instutil-builtins.h" #include "ot-admin-builtins.h" #include "ot-admin-functions.h" -#include "ot-main.h" #include "ostree.h" #include diff --git a/src/ostree/ot-admin-functions.c b/src/ostree/ot-admin-functions.c index bc9034e1..ed4dfdfb 100644 --- a/src/ostree/ot-admin-functions.c +++ b/src/ostree/ot-admin-functions.c @@ -22,6 +22,7 @@ #include "config.h" +#include "libglnx.h" #include "ot-admin-functions.h" #include "otutil.h" #include "ostree.h" diff --git a/src/ostree/ot-builtin-admin.c b/src/ostree/ot-builtin-admin.c index 8b866170..7e7b04b5 100644 --- a/src/ostree/ot-builtin-admin.c +++ b/src/ostree/ot-builtin-admin.c @@ -22,10 +22,10 @@ #include "config.h" +#include "ot-main.h" #include "ot-builtins.h" #include "ot-admin-builtins.h" #include "ot-admin-functions.h" -#include "ot-main.h" #include "ostree.h" #include "ostree-repo-file.h" diff --git a/src/ostree/ot-editor.c b/src/ostree/ot-editor.c index 4c29c81e..2e045844 100644 --- a/src/ostree/ot-editor.c +++ b/src/ostree/ot-editor.c @@ -22,9 +22,9 @@ #include "config.h" +#include "libglnx.h" #include "ot-editor.h" #include "libgsystem.h" -#include "libglnx.h" #include #include diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 93f841dc..18c13239 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -27,9 +27,9 @@ #include #include +#include "ot-main.h" #include "ostree.h" #include "ot-admin-functions.h" -#include "ot-main.h" #include "otutil.h" static char *opt_repo; diff --git a/src/ostree/ot-main.h b/src/ostree/ot-main.h index 32620c52..3bb75243 100644 --- a/src/ostree/ot-main.h +++ b/src/ostree/ot-main.h @@ -22,8 +22,8 @@ #pragma once -#include "ostree.h" #include "libglnx.h" +#include "ostree.h" typedef enum { OSTREE_BUILTIN_FLAG_NONE = 0, From dfa1d190b6f647ca5f4f4e8c93f37c7cb4fe0a82 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Mon, 23 May 2016 12:42:17 -0600 Subject: [PATCH 060/128] commit: accept NULL subject argument When given a NULL subject, use "" instead, like for the body argument Closes: #305 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index f384624c..8f5067c0 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1962,7 +1962,7 @@ create_empty_gvariant_dict (void) * ostree_repo_write_commit: * @self: Repo * @parent: (allow-none): ASCII SHA256 checksum for parent, or %NULL for none - * @subject: Subject + * @subject: (allow-none): Subject * @body: (allow-none): Body * @metadata: (allow-none): GVariant of type a{sv}, or %NULL for none * @root: The tree to point the commit to @@ -2006,7 +2006,7 @@ ostree_repo_write_commit (OstreeRepo *self, * ostree_repo_write_commit_with_time: * @self: Repo * @parent: (allow-none): ASCII SHA256 checksum for parent, or %NULL for none - * @subject: Subject + * @subject: (allow-none): Subject * @body: (allow-none): Body * @metadata: (allow-none): GVariant of type a{sv}, or %NULL for none * @root: The tree to point the commit to @@ -2036,8 +2036,6 @@ ostree_repo_write_commit_with_time (OstreeRepo *self, g_autofree guchar *commit_csum = NULL; OstreeRepoFile *repo_root = OSTREE_REPO_FILE (root); - g_return_val_if_fail (subject != NULL, FALSE); - /* Add sizes information to our metadata object */ if (!add_size_index_to_metadata (self, metadata, &new_metadata, cancellable, error)) @@ -2047,7 +2045,7 @@ ostree_repo_write_commit_with_time (OstreeRepo *self, new_metadata ? new_metadata : create_empty_gvariant_dict (), parent ? ostree_checksum_to_bytes_v (parent) : ot_gvariant_new_bytearray (NULL, 0), g_variant_new_array (G_VARIANT_TYPE ("(say)"), NULL, 0), - subject, body ? body : "", + subject ? subject : "", body ? body : "", GUINT64_TO_BE (time), ostree_checksum_to_bytes_v (ostree_repo_file_tree_get_contents_checksum (repo_root)), ostree_checksum_to_bytes_v (ostree_repo_file_tree_get_metadata_checksum (repo_root))); From b5bd28adae5909b5b7e0159e0f2dde268ddf7907 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Mon, 23 May 2016 12:44:36 -0600 Subject: [PATCH 061/128] commit: Add --editor / -e command line argument Previously, the behavior was to run the editor if no subject or body was specified. This led to using commit -s '' in scripts, to avoid running the editor. In practice, manually editing commit messages is rare, whereas automated scripting is common, so instead default to not running the editor and only run it when -e is given. Closes: #305 Approved by: cgwalters --- src/ostree/ot-builtin-commit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index a021fbfa..5acd0243 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -32,6 +32,7 @@ static char *opt_subject; static char *opt_body; +static gboolean opt_editor; static char *opt_parent; static gboolean opt_orphan; static char *opt_branch; @@ -73,6 +74,7 @@ static GOptionEntry options[] = { { "parent", 0, 0, G_OPTION_ARG_STRING, &opt_parent, "Parent ref, or \"none\"", "REF" }, { "subject", 's', 0, G_OPTION_ARG_STRING, &opt_subject, "One line subject", "SUBJECT" }, { "body", 'm', 0, G_OPTION_ARG_STRING, &opt_body, "Full description", "BODY" }, + { "editor", 'e', 0, G_OPTION_ARG_NONE, &opt_editor, "Use an editor to write the commit message", NULL }, { "branch", 'b', 0, G_OPTION_ARG_STRING, &opt_branch, "Branch", "BRANCH" }, { "orphan", 0, 0, G_OPTION_ARG_NONE, &opt_orphan, "Create a commit without writing a ref", NULL }, { "tree", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_trees, "Overlay the given argument as a tree", "dir=PATH or tar=TARFILE or ref=COMMIT" }, @@ -425,7 +427,7 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError goto out; } - if (!opt_subject && !opt_body) + if (opt_editor) { if (!commit_editor (repo, opt_branch, &opt_subject, &opt_body, cancellable, error)) goto out; From 5e336333f38fe95f238cb122257776714ba5ed17 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Mon, 23 May 2016 12:47:22 -0600 Subject: [PATCH 062/128] commit: Display subject/body in commit editor This allows automation to prefill the subject/body for the editor. It could also be useful if some kind of "amend commit" functionality was implemented. Closes: #305 Approved by: cgwalters --- src/ostree/ot-builtin-commit.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 5acd0243..99687c52 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -218,14 +218,18 @@ commit_editor (OstreeRepo *repo, char **lines = NULL; int i; - *subject = NULL; - *body = NULL; - input = g_strdup_printf ("\n" "# Please enter the commit message for your changes. The first line will\n" "# become the subject, and the remainder the body. Lines starting\n" "# with '#' will be ignored, and an empty message aborts the commit." - "%s%s\n", branch ? "\n#\n# Branch: " : "", branch ?: ""); + "%s%s%s%s%s%s\n" + , branch ? "\n#\n# Branch: " : "", branch ? branch : "" + , *subject ? "\n" : "", *subject ? *subject : "" + , *body ? "\n" : "", *body ? *body : "" + ); + + *subject = NULL; + *body = NULL; output = ot_editor_prompt (repo, input, cancellable, error); if (output == NULL) From 80774680e8df8d0fa9da295b981be3d8ba86bc8c Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Mon, 23 May 2016 12:49:37 -0600 Subject: [PATCH 063/128] commit: Don't require a subject In practice, a lot of subjects are empty, because the commit date and branch are sufficient identification. For example, rpm-ostree does not use subjects. It also doesn't use the command-line ostree commit tool, so this was not a problem there, but this makes the behavior consistent. Also adds a test that empty subjects and omitting the subject are equivalent. The --timestamp is so that the commits do not have different timestamps. Closes: #305 Approved by: cgwalters --- src/ostree/ot-builtin-commit.c | 7 ------- tests/basic-test.sh | 8 +++++++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 99687c52..85c4c65d 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -437,13 +437,6 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError goto out; } - if (!opt_subject) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "A subject must be specified with --subject"); - goto out; - } - if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error)) goto out; diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 0373fcbc..93f67f17 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..56" +echo "1..57" $OSTREE checkout test2 checkout-test2 echo "ok checkout" @@ -103,6 +103,12 @@ $OSTREE commit -b test2-no-parent -s '' --parent=none $test_tmpdir/checkout-test assert_streq $($OSTREE log test2-no-parent |grep '^commit' | wc -l) "1" echo "ok commit no parent" +cd ${test_tmpdir} +empty_rev=$($OSTREE commit -b test2-no-subject -s '' --timestamp="2005-10-29 12:43:29 +0000" $test_tmpdir/checkout-test2-4) +omitted_rev=$($OSTREE commit -b test2-no-subject-2 --timestamp="2005-10-29 12:43:29 +0000" $test_tmpdir/checkout-test2-4) +assert_streq $empty_rev $omitted_rev +echo "ok commit no subject" + cd ${test_tmpdir} $OSTREE commit -b test2-custom-parent -s '' $test_tmpdir/checkout-test2-4 $OSTREE commit -b test2-custom-parent -s '' $test_tmpdir/checkout-test2-4 From c82f15201a51d6ff802139839cffea1a158ec0f9 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Mon, 23 May 2016 12:52:56 -0600 Subject: [PATCH 064/128] log: Display "(no subject)" for commits without a subject This resolves the ambiguity for a message with a body but no subject, and also hints that OSTree is using two bytes to store the empty commit subject/body (when instead they could be stored, or rather, not stored, as part of the metadata) Closes: #305 Approved by: cgwalters --- src/ostree/ot-dump.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index 670ccd6a..48b087af 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -118,8 +118,15 @@ dump_commit (GVariant *variant, g_print ("Version: %s\n", version); } - g_print ("\n"); - dump_indented_lines (subject); + if (subject[0]) + { + g_print ("\n"); + dump_indented_lines (subject); + } + else + { + g_print ("(no subject)\n"); + } if (body[0]) { From 6bf94ec2335cd34e6c0b729bc836388732621906 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Mon, 23 May 2016 13:32:36 -0600 Subject: [PATCH 065/128] commit: Update manpage to reflect --editor Closes: #305 Approved by: cgwalters --- man/ostree-commit.xml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/man/ostree-commit.xml b/man/ostree-commit.xml index 00e20428..8f0037f6 100644 --- a/man/ostree-commit.xml +++ b/man/ostree-commit.xml @@ -57,7 +57,7 @@ Boston, MA 02111-1307, USA. Description - This allows you to commit changes to a branch. The specification of the branch is required. If no commit message is specified with then a text editor will be opened. The commit will be aborted if the commit subject is left empty. The command will print the checksum of a successful commit. + This allows you to commit changes to a branch. The specification of the branch is required. The command will print the checksum of a successful commit. @@ -68,7 +68,7 @@ Boston, MA 02111-1307, USA. , ="SUBJECT" - One line subject. + One line subject. (optional) @@ -76,7 +76,15 @@ Boston, MA 02111-1307, USA. , ="BODY" - Full description. + Full description. (optional) + + + + + , + + + Open a text editor for the commit description. It will use OSTREE_EDITOR, VISUAL, EDITOR, or vi, in descending order of preference. The commit will be aborted if the message is left empty. @@ -84,7 +92,7 @@ Boston, MA 02111-1307, USA. , ="BRANCH" - Branch. + Branch. Required, unless --orphan is given. @@ -219,7 +227,7 @@ Boston, MA 02111-1307, USA. - Create a commit without writing a ref + Create a commit without writing to a ref (branch) From 569e43c280d6b09b5a683c5b4ea41a448f1f7ce2 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 23 May 2016 13:28:04 +0200 Subject: [PATCH 066/128] core: Add a function creating an archive-z2 content stream It is quite similar to the already existing ostree_raw_file_to_content_stream function, so I factored the common part to a separate function. The difference is that we cannot report the size of the resulting stream. Can be useful for serving a "bare" repository as a faked "archive-z2" repository. Closes: #308 Approved by: cgwalters --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 2 +- src/libostree/ostree-core.c | 134 +++++++++++++++++++++++++--------- src/libostree/ostree-core.h | 9 +++ tests/test-basic-c.c | 138 ++++++++++++++++++++++++++++++++++++ 5 files changed, 250 insertions(+), 34 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index b9b292c7..105783f4 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -114,6 +114,7 @@ ostree_object_from_string ostree_content_stream_parse ostree_content_file_parse ostree_content_file_parse_at +ostree_raw_file_to_archive_z2_stream ostree_raw_file_to_content_stream ostree_checksum_file_from_input ostree_checksum_file diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 2aa72512..71f4bc9b 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -341,5 +341,5 @@ global: LIBOSTREE_2016.6 { global: ostree_repo_remote_fetch_summary_with_options; - + ostree_raw_file_to_archive_z2_stream; } LIBOSTREE_2016.5; diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 00f767b3..d393c496 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -400,6 +400,98 @@ write_file_header_update_checksum (GOutputStream *out, return ret; } +/* + * header_and_input_to_stream: + * @file_header: A file header + * @input: File raw content stream + * @out_input: (out): Serialized object stream + * @out_header_size: (out): Length of the header + * @cancellable: Cancellable + * @error: Error + * + * Combines @file_header and @input into a single stream. + */ +static gboolean +header_and_input_to_stream (GVariant *file_header, + GInputStream *input, + GInputStream **out_input, + guint64 *out_header_size, + GCancellable *cancellable, + GError **error) +{ + gpointer header_data; + gsize header_size; + g_autoptr(GInputStream) ret_input = NULL; + g_autoptr(GPtrArray) streams = NULL; + g_autoptr(GOutputStream) header_out_stream = NULL; + g_autoptr(GInputStream) header_in_stream = NULL; + + header_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free); + + if (!_ostree_write_variant_with_size (header_out_stream, file_header, 0, NULL, NULL, + cancellable, error)) + return FALSE; + + if (!g_output_stream_close (header_out_stream, cancellable, error)) + return FALSE; + + header_size = g_memory_output_stream_get_data_size ((GMemoryOutputStream*) header_out_stream); + header_data = g_memory_output_stream_steal_data ((GMemoryOutputStream*) header_out_stream); + header_in_stream = g_memory_input_stream_new_from_data (header_data, header_size, g_free); + + streams = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); + + g_ptr_array_add (streams, g_object_ref (header_in_stream)); + if (input) + g_ptr_array_add (streams, g_object_ref (input)); + + ret_input = (GInputStream*)ostree_chain_input_stream_new (streams); + ot_transfer_out_value (out_input, &ret_input); + if (out_header_size) + *out_header_size = header_size; + + return TRUE; +} + +/** + * ostree_raw_file_to_archive_z2_stream: + * @input: File raw content stream + * @file_info: A file info + * @xattrs: (allow-none): Optional extended attributes + * @out_input: (out): Serialized object stream + * @cancellable: Cancellable + * @error: Error + * + * Convert from a "bare" file representation into an + * OSTREE_OBJECT_TYPE_FILE stream suitable for ostree pull. + */ +gboolean +ostree_raw_file_to_archive_z2_stream (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + GInputStream **out_input, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariant) file_header = NULL; + g_autoptr(GInputStream) zlib_input = NULL; + + file_header = _ostree_zlib_file_header_new (file_info, xattrs); + if (input != NULL) + { + g_autoptr(GConverter) zlib_compressor = NULL; + + zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9)); + zlib_input = g_converter_input_stream_new (input, zlib_compressor); + } + return header_and_input_to_stream (file_header, + zlib_input, + out_input, + NULL, + cancellable, + error); +} + /** * ostree_raw_file_to_content_stream: * @input: File raw content stream @@ -423,44 +515,20 @@ ostree_raw_file_to_content_stream (GInputStream *input, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - gpointer header_data; - gsize header_size; - g_autoptr(GInputStream) ret_input = NULL; g_autoptr(GVariant) file_header = NULL; - g_autoptr(GPtrArray) streams = NULL; - g_autoptr(GOutputStream) header_out_stream = NULL; - g_autoptr(GInputStream) header_in_stream = NULL; + guint64 header_size; file_header = file_header_new (file_info, xattrs); - - header_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free); - - if (!_ostree_write_variant_with_size (header_out_stream, file_header, 0, NULL, NULL, - cancellable, error)) - goto out; - - if (!g_output_stream_close (header_out_stream, cancellable, error)) - goto out; - - header_size = g_memory_output_stream_get_data_size ((GMemoryOutputStream*) header_out_stream); - header_data = g_memory_output_stream_steal_data ((GMemoryOutputStream*) header_out_stream); - header_in_stream = g_memory_input_stream_new_from_data (header_data, header_size, g_free); - - streams = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - - g_ptr_array_add (streams, g_object_ref (header_in_stream)); - if (input) - g_ptr_array_add (streams, g_object_ref (input)); - - ret_input = (GInputStream*)ostree_chain_input_stream_new (streams); - - ret = TRUE; - ot_transfer_out_value (out_input, &ret_input); + if (!header_and_input_to_stream (file_header, + input, + out_input, + &header_size, + cancellable, + error)) + return FALSE; if (out_length) *out_length = header_size + g_file_info_get_size (file_info); - out: - return ret; + return TRUE; } /** diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 29b5a1c0..4d788b20 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -274,6 +274,15 @@ gboolean ostree_content_file_parse_at (gboolean compressed, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean +ostree_raw_file_to_archive_z2_stream (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + GInputStream **out_input, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_raw_file_to_content_stream (GInputStream *input, GFileInfo *file_info, diff --git a/tests/test-basic-c.c b/tests/test-basic-c.c index d5dcc811..447c46ea 100644 --- a/tests/test-basic-c.c +++ b/tests/test-basic-c.c @@ -34,6 +34,143 @@ test_repo_is_not_system (gconstpointer data) g_assert (!ostree_repo_is_system (repo)); } +static GBytes * +input_stream_to_bytes (GInputStream *input) +{ + g_autoptr(GOutputStream) mem_out_stream = NULL; + g_autoptr(GError) error = NULL; + + if (input == NULL) + return g_bytes_new (NULL, 0); + + mem_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free); + g_output_stream_splice (mem_out_stream, + input, + G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, + NULL, + &error); + g_assert_no_error (error); + + return g_memory_output_stream_steal_as_bytes (G_MEMORY_OUTPUT_STREAM (mem_out_stream)); +} + +static void +test_raw_file_to_archive_z2_stream (gconstpointer data) +{ + OstreeRepo *repo = OSTREE_REPO (data); + g_autofree gchar *commit_checksum = NULL; + g_autoptr(GHashTable) reachable = NULL; + g_autoptr(GError) error = NULL; + /* branch name of the test repository, see setup_test_repository in libtest.sh */ + const gchar *rev = "test2"; + GHashTableIter iter; + GVariant *serialized_object; + guint checks = 0; + + ostree_repo_resolve_rev (repo, + rev, + FALSE, + &commit_checksum, + &error); + g_assert_no_error (error); + reachable = ostree_repo_traverse_new_reachable (); + ostree_repo_traverse_commit (repo, + commit_checksum, + -1, + &reachable, + NULL, + &error); + g_assert_no_error (error); + g_hash_table_iter_init (&iter, reachable); + while (g_hash_table_iter_next (&iter, (gpointer*)&serialized_object, NULL)) + { + const gchar *object_checksum; + OstreeObjectType object_type; + g_autoptr(GInputStream) input = NULL; + g_autoptr(GFileInfo) info = NULL; + g_autoptr(GVariant) xattrs = NULL; + g_autoptr(GBytes) input_bytes = NULL; + g_autoptr(GInputStream) mem_input = NULL; + g_autoptr(GInputStream) zlib_stream = NULL; + g_autoptr(GBytes) zlib_bytes = NULL; + g_autoptr(GInputStream) mem_zlib = NULL; + g_autoptr(GInputStream) input2 = NULL; + g_autoptr(GFileInfo) info2 = NULL; + g_autoptr(GVariant) xattrs2 = NULL; + g_autoptr(GBytes) input2_bytes = NULL; + + ostree_object_name_deserialize (serialized_object, &object_checksum, &object_type); + if (object_type != OSTREE_OBJECT_TYPE_FILE) + continue; + + ostree_repo_load_file (repo, + object_checksum, + &input, + &info, + &xattrs, + NULL, + &error); + g_assert_no_error (error); + + input_bytes = input_stream_to_bytes (input); + /* This is to simulate NULL input received from + * ostree_repo_load_file. Instead of creating the mem_input + * variable, I could also rewind the input stream and pass it to + * the function below, but this would assume that the input + * stream implements either the GSeekable or + * GFileDescriptorBased interface. */ + if (input != NULL) + mem_input = g_memory_input_stream_new_from_bytes (input_bytes); + ostree_raw_file_to_archive_z2_stream (mem_input, + info, + xattrs, + &zlib_stream, + NULL, + &error); + g_assert_no_error (error); + + zlib_bytes = input_stream_to_bytes (zlib_stream); + mem_zlib = g_memory_input_stream_new_from_bytes (zlib_bytes); + ostree_content_stream_parse (FALSE, + mem_zlib, + g_bytes_get_size (zlib_bytes), + FALSE, + &input2, + &info2, + &xattrs2, + NULL, + &error); + g_assert_error (error, G_IO_ERROR, G_IO_ERROR_FAILED); + g_clear_error (&error); + + g_seekable_seek (G_SEEKABLE (mem_zlib), + 0, + G_SEEK_SET, + NULL, + &error); + g_assert_no_error (error); + + ostree_content_stream_parse (TRUE, + mem_zlib, + g_bytes_get_size (zlib_bytes), + FALSE, + &input2, + &info2, + &xattrs2, + NULL, + &error); + g_assert_no_error (error); + + input2_bytes = input_stream_to_bytes (input2); + g_assert_true (g_bytes_equal (input_bytes, input2_bytes)); + g_assert_true (g_variant_equal (xattrs, xattrs2)); + /* TODO: Not sure how to compare fileinfos */ + ++checks; + } + /* to make sure we really tested the function */ + g_assert_cmpint (checks, >, 0); +} + int main (int argc, char **argv) { g_autoptr(GError) error = NULL; @@ -46,6 +183,7 @@ int main (int argc, char **argv) goto out; g_test_add_data_func ("/repo-not-system", repo, test_repo_is_not_system); + g_test_add_data_func ("/raw-file-to-archive-z2-stream", repo, test_raw_file_to_archive_z2_stream); return g_test_run(); out: From 89bfb1d503bc7c8bf7f7de85dd0797f8cde5f515 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 26 May 2016 11:48:21 +0200 Subject: [PATCH 067/128] repo: Factor out the check of gpg result to a separate function I plan to add a function for verifying any data which may return the error about lack of trusted signatures, so let's avoid the redundancy and put the check in the separate function. Closes: #310 Approved by: cgwalters --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 3 ++- src/libostree/ostree-gpg-verify-result.c | 30 ++++++++++++++++++++++++ src/libostree/ostree-gpg-verify-result.h | 4 ++++ src/libostree/ostree-repo.c | 24 ++----------------- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 105783f4..9ce28ee5 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -185,6 +185,7 @@ ostree_gpg_verify_result_get_all OstreeGpgSignatureFormatFlags ostree_gpg_verify_result_describe ostree_gpg_verify_result_describe_variant +ostree_gpg_verify_result_require_valid_signature OSTREE_GPG_VERIFY_RESULT OSTREE_IS_GPG_VERIFY_RESULT diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 71f4bc9b..24f6723b 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -340,6 +340,7 @@ global: LIBOSTREE_2016.6 { global: - ostree_repo_remote_fetch_summary_with_options; + ostree_gpg_verify_result_require_valid_signature; ostree_raw_file_to_archive_z2_stream; + ostree_repo_remote_fetch_summary_with_options; } LIBOSTREE_2016.5; diff --git a/src/libostree/ostree-gpg-verify-result.c b/src/libostree/ostree-gpg-verify-result.c index 37fbfb5c..d72856c9 100644 --- a/src/libostree/ostree-gpg-verify-result.c +++ b/src/libostree/ostree-gpg-verify-result.c @@ -622,3 +622,33 @@ ostree_gpg_verify_result_describe_variant (GVariant *variant, } } } + +/** + * ostree_gpg_verify_result_require_valid_signature: + * @result: (nullable): an #OstreeGpgVerifyResult + * @error: A #GError + * + * Checks if the result contains at least one signature from the + * trusted keyring. You can call this function immediately after + * ostree_repo_verify_summary() or ostree_repo_verify_commit_ext() - + * it will handle the %NULL @result and filled @error too. + * + * Returns: %TRUE if @result was not %NULL and had at least one + * signature from trusted keyring, otherwise %FALSE + */ +gboolean +ostree_gpg_verify_result_require_valid_signature (OstreeGpgVerifyResult *result, + GError **error) +{ + if (result == NULL) + return FALSE; + + if (ostree_gpg_verify_result_count_valid (result) == 0) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "GPG signatures found, but none are in trusted keyring"); + return FALSE; + } + + return TRUE; +} diff --git a/src/libostree/ostree-gpg-verify-result.h b/src/libostree/ostree-gpg-verify-result.h index 8894afdf..f9512538 100644 --- a/src/libostree/ostree-gpg-verify-result.h +++ b/src/libostree/ostree-gpg-verify-result.h @@ -133,4 +133,8 @@ void ostree_gpg_verify_result_describe_variant (GVariant *variant, const gchar *line_prefix, OstreeGpgSignatureFormatFlags flags); +_OSTREE_PUBLIC +gboolean ostree_gpg_verify_result_require_valid_signature (OstreeGpgVerifyResult *result, + GError **error); + G_END_DECLS diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 65b955ac..1b08162d 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2116,15 +2116,8 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, signatures, cancellable, error); - if (result == NULL) + if (!ostree_gpg_verify_result_require_valid_signature (result, error)) goto out; - - if (ostree_gpg_verify_result_count_valid (result) == 0) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "GPG signatures found, but none are in trusted keyring"); - goto out; - } } if (out_summary != NULL) @@ -4838,25 +4831,12 @@ ostree_repo_verify_commit (OstreeRepo *self, GError **error) { glnx_unref_object OstreeGpgVerifyResult *result = NULL; - gboolean ret = FALSE; result = ostree_repo_verify_commit_ext (self, commit_checksum, keyringdir, extra_keyring, cancellable, error); - if (result == NULL) - goto out; - if (ostree_gpg_verify_result_count_valid (result) == 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "GPG signatures found, but none are in trusted keyring"); - goto out; - } - - ret = TRUE; - - out: - return ret; + return ostree_gpg_verify_result_require_valid_signature (result, error); } /** From 582169f83e7170228008ff24c159e15b7ac69565 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 26 May 2016 11:51:13 +0200 Subject: [PATCH 068/128] repo: Add a missing Returns paragraph Closes: #310 Approved by: cgwalters --- src/libostree/ostree-repo.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1b08162d..446d69a8 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4821,6 +4821,8 @@ out: * * Check for a valid GPG signature on commit named by the ASCII * checksum @commit_checksum. + * + * Returns: %TRUE if there was a GPG signature from a trusted keyring, otherwise %FALSE */ gboolean ostree_repo_verify_commit (OstreeRepo *self, From 27f0c6980af23485ee007d1b5ae0ecac6d6736df Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 26 May 2016 11:53:11 +0200 Subject: [PATCH 069/128] repo: Factor out setting up a GPG verifier Moved out setting up a GPG verifier to a separate function, as I would like to use it for the any data verification function in the following commit. Closes: #310 Approved by: cgwalters --- src/libostree/ostree-gpg-verifier.h | 5 ++ src/libostree/ostree-repo.c | 73 +++++++++++++++++++---------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/libostree/ostree-gpg-verifier.h b/src/libostree/ostree-gpg-verifier.h index 209f7342..2db39f3b 100644 --- a/src/libostree/ostree-gpg-verifier.h +++ b/src/libostree/ostree-gpg-verifier.h @@ -35,6 +35,11 @@ G_BEGIN_DECLS typedef struct OstreeGpgVerifier OstreeGpgVerifier; +/* If this type becomes public in future, move this autoptr cleanup + * definition to the ostree-autocleanups.h header file. Right now it + * relies on glnx's fallback definition of the macro. */ +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeGpgVerifier, g_object_unref) + GType _ostree_gpg_verifier_get_type (void); OstreeGpgVerifier *_ostree_gpg_verifier_new (void); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 446d69a8..0f231c0c 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -26,6 +26,7 @@ #include #include #include +#include "libglnx.h" #include "otutil.h" #include @@ -36,6 +37,7 @@ #include "ostree-gpg-verifier.h" #include "ostree-repo-static-delta-private.h" #include "ot-fs-utils.h" +#include "ostree-autocleanups.h" #ifdef HAVE_LIBSOUP #include "ostree-metalink.h" @@ -4647,23 +4649,17 @@ find_keyring (OstreeRepo *self, return NULL; } -OstreeGpgVerifyResult * -_ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, - GBytes *signed_data, - GVariant *metadata, - const char *remote_name, - GFile *keyringdir, - GFile *extra_keyring, - GCancellable *cancellable, - GError **error) +static OstreeGpgVerifyResult * +_ostree_repo_gpg_verify_data_internal (OstreeRepo *self, + const gchar *remote_name, + GBytes *data, + GBytes *signatures, + GFile *keyringdir, + GFile *extra_keyring, + GCancellable *cancellable, + GError **error) { - OstreeGpgVerifyResult *result = NULL; glnx_unref_object OstreeGpgVerifier *verifier = NULL; - g_autoptr(GVariant) signaturedata = NULL; - GByteArray *buffer; - GVariantIter iter; - GVariant *child; - g_autoptr (GBytes) signatures = NULL; gboolean add_global_keyring_dir = TRUE; verifier = _ostree_gpg_verifier_new (); @@ -4674,7 +4670,7 @@ _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, if (!_ostree_gpg_verifier_add_keyring_dir (verifier, self->repodir, cancellable, error)) - goto out; + return NULL; } else if (remote_name != NULL) { @@ -4685,7 +4681,7 @@ _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, remote = ost_repo_get_remote_inherited (self, remote_name, error); if (remote == NULL) - goto out; + return NULL; file = find_keyring (self, remote, cancellable); @@ -4702,20 +4698,43 @@ _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, { /* Use the deprecated global keyring directory. */ if (!_ostree_gpg_verifier_add_global_keyring_dir (verifier, cancellable, error)) - goto out; + return NULL; } if (keyringdir) { if (!_ostree_gpg_verifier_add_keyring_dir (verifier, keyringdir, cancellable, error)) - goto out; + return NULL; } if (extra_keyring != NULL) { _ostree_gpg_verifier_add_keyring (verifier, extra_keyring); } + return _ostree_gpg_verifier_check_signature (verifier, + data, + signatures, + cancellable, + error); +} + +OstreeGpgVerifyResult * +_ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, + GBytes *signed_data, + GVariant *metadata, + const char *remote_name, + GFile *keyringdir, + GFile *extra_keyring, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariant) signaturedata = NULL; + GByteArray *buffer; + GVariantIter iter; + GVariant *child; + g_autoptr (GBytes) signatures = NULL; + if (metadata) signaturedata = g_variant_lookup_value (metadata, _OSTREE_METADATA_GPGSIGS_NAME, @@ -4724,7 +4743,7 @@ _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, { g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, "GPG verification enabled, but no signatures found (use gpg-verify=false in remote config to disable)"); - goto out; + return NULL; } /* OpenPGP data is organized into binary records called packets. RFC 4880 @@ -4746,12 +4765,14 @@ _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, } signatures = g_byte_array_free_to_bytes (buffer); - result = _ostree_gpg_verifier_check_signature (verifier, - signed_data, signatures, - cancellable, error); - - out: - return result; + return _ostree_repo_gpg_verify_data_internal (self, + remote_name, + signed_data, + signatures, + keyringdir, + extra_keyring, + cancellable, + error); } /* Needed an internal version for the remote_name parameter. */ From 4929ab4033894d0aaf1987ebc07d7ccbe23323e1 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 26 May 2016 11:54:28 +0200 Subject: [PATCH 070/128] repo: Add functions for verifying any data in repository This can be useful for validating the 3rd party data that is put in the extensions directory and is signed with the same keys as commits or the summary file. Closes: #310 Approved by: cgwalters --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 1 + src/libostree/ostree-repo.c | 43 +++++++++++++++++++++++++++++++++++++ src/libostree/ostree-repo.h | 10 +++++++++ 4 files changed, 55 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 9ce28ee5..3003da2b 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -347,6 +347,7 @@ ostree_repo_pull_default_console_progress_changed ostree_repo_sign_commit ostree_repo_append_gpg_signature ostree_repo_add_gpg_signature_summary +ostree_repo_gpg_verify_data ostree_repo_verify_commit ostree_repo_verify_commit_ext ostree_repo_verify_summary diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 24f6723b..3e17828a 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -342,5 +342,6 @@ LIBOSTREE_2016.6 { global: ostree_gpg_verify_result_require_valid_signature; ostree_raw_file_to_archive_z2_stream; + ostree_repo_gpg_verify_data; ostree_repo_remote_fetch_summary_with_options; } LIBOSTREE_2016.5; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 0f231c0c..f2538f51 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4893,6 +4893,49 @@ ostree_repo_verify_commit_ext (OstreeRepo *self, error); } +/** + * ostree_repo_gpg_verify_data: + * @self: Repository + * @remote_name: (nullable): Name of remote + * @data: Data as a #GBytes + * @signatures: Signatures as a #GBytes + * @keyringdir: (nullable): Path to directory GPG keyrings; overrides built-in default if given + * @extra_keyring: (nullable): Path to additional keyring file (not a directory) + * @cancellable: Cancellable + * @error: Error + * + * Verify @signatures for @data using GPG keys in the keyring for + * @remote_name, and return an #OstreeGpgVerifyResult. + * + * The @remote_name parameter can be %NULL. In that case it will do + * the verifications using GPG keys in the keyrings of all remotes. + * + * Returns: (transfer full): an #OstreeGpgVerifyResult, or %NULL on error + */ +OstreeGpgVerifyResult * +ostree_repo_gpg_verify_data (OstreeRepo *self, + const gchar *remote_name, + GBytes *data, + GBytes *signatures, + GFile *keyringdir, + GFile *extra_keyring, + GCancellable *cancellable, + GError **error) +{ + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + g_return_val_if_fail (data != NULL, NULL); + g_return_val_if_fail (signatures != NULL, NULL); + + return _ostree_repo_gpg_verify_data_internal (self, + (remote_name != NULL) ? remote_name : OSTREE_ALL_REMOTES, + data, + signatures, + keyringdir, + extra_keyring, + cancellable, + error); +} + /** * ostree_repo_verify_summary: * @self: Repo diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index f0fa53a3..b81ccc93 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1030,6 +1030,16 @@ OstreeGpgVerifyResult * ostree_repo_verify_commit_ext (OstreeRepo *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +OstreeGpgVerifyResult * ostree_repo_gpg_verify_data (OstreeRepo *self, + const gchar *remote_name, + GBytes *data, + GBytes *signatures, + GFile *keyringdir, + GFile *extra_keyring, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC OstreeGpgVerifyResult * ostree_repo_verify_summary (OstreeRepo *self, const char *remote_name, From 33047d5d4fd135f3976484be30ed7dbb9c692c67 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 29 May 2016 13:23:04 -0400 Subject: [PATCH 071/128] libglnx porting: Port "load utf8 file" contents API callers Closes: #312 Approved by: giuseppe --- src/libostree/ostree-bootloader-syslinux.c | 3 ++- src/libostree/ostree-bootloader-uboot.c | 3 ++- src/libotutil/ot-gio-utils.c | 3 ++- src/ostree/ot-editor.c | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-bootloader-syslinux.c b/src/libostree/ostree-bootloader-syslinux.c index 16951629..05cb173e 100644 --- a/src/libostree/ostree-bootloader-syslinux.c +++ b/src/libostree/ostree-bootloader-syslinux.c @@ -138,7 +138,8 @@ _ostree_bootloader_syslinux_write_config (OstreeBootloader *bootloader, bootversion); /* This should follow the symbolic link to the current bootversion. */ - config_contents = gs_file_load_contents_utf8 (self->config_path, cancellable, error); + config_contents = glnx_file_get_contents_utf8_at (AT_FDCWD, gs_file_get_path_cached (self->config_path), NULL, + cancellable, error); if (!config_contents) goto out; diff --git a/src/libostree/ostree-bootloader-uboot.c b/src/libostree/ostree-bootloader-uboot.c index f67e9bdb..f95ea843 100644 --- a/src/libostree/ostree-bootloader-uboot.c +++ b/src/libostree/ostree-bootloader-uboot.c @@ -113,7 +113,8 @@ _ostree_bootloader_uboot_write_config (OstreeBootloader *bootloader, g_autoptr(GPtrArray) new_lines = NULL; /* This should follow the symbolic link to the current bootversion. */ - config_contents = gs_file_load_contents_utf8 (self->config_path, cancellable, error); + config_contents = glnx_file_get_contents_utf8_at (AT_FDCWD, gs_file_get_path_cached (self->config_path), NULL, + cancellable, error); if (!config_contents) return FALSE; diff --git a/src/libotutil/ot-gio-utils.c b/src/libotutil/ot-gio-utils.c index 9b8b0dce..103f6edd 100644 --- a/src/libotutil/ot-gio-utils.c +++ b/src/libotutil/ot-gio-utils.c @@ -254,7 +254,8 @@ ot_gfile_load_contents_utf8_allow_noent (GFile *path, GError *temp_error = NULL; g_autofree char *ret_contents = NULL; - ret_contents = gs_file_load_contents_utf8 (path, cancellable, &temp_error); + ret_contents = glnx_file_get_contents_utf8_at (AT_FDCWD, gs_file_get_path_cached (path), NULL, + cancellable, &temp_error); if (!ret_contents) { if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) diff --git a/src/ostree/ot-editor.c b/src/ostree/ot-editor.c index 2e045844..66c6e7be 100644 --- a/src/ostree/ot-editor.c +++ b/src/ostree/ot-editor.c @@ -101,7 +101,8 @@ ot_editor_prompt (OstreeRepo *repo, goto out; } - ret = gs_file_load_contents_utf8 (file, cancellable, error); + ret = glnx_file_get_contents_utf8_at (AT_FDCWD, gs_file_get_path_cached (file), NULL, + cancellable, error); out: if (file) From 9d39d3af855c83ccb77f1d6a07309aea56e402cb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 29 May 2016 13:29:18 -0400 Subject: [PATCH 072/128] repo: Port uncompressed cache GC to libglnx - Kills a user of `gs_file_unlink` - Is fd-relative - Is way less malloc-y. Closes: #312 Approved by: giuseppe --- src/libostree/ostree-repo-checkout.c | 46 +++++++++++++--------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 0a77da02..95d3747b 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -957,38 +957,36 @@ ostree_repo_checkout_gc (OstreeRepo *self, g_hash_table_iter_init (&iter, to_clean_dirs); while (to_clean_dirs && g_hash_table_iter_next (&iter, &key, &value)) { - g_autoptr(GFile) objdir = NULL; - g_autoptr(GFileEnumerator) enumerator = NULL; - g_autofree char *objdir_name = NULL; + g_autofree char *objdir_name = g_strdup_printf ("%02x", GPOINTER_TO_UINT (key)); + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - objdir_name = g_strdup_printf ("%02x", GPOINTER_TO_UINT (key)); - objdir = g_file_get_child (self->uncompressed_objects_dir, objdir_name); - - enumerator = g_file_enumerate_children (objdir, "standard::name,standard::type,unix::inode,unix::nlink", - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); - if (!enumerator) + if (!glnx_dirfd_iterator_init_at (self->uncompressed_objects_dir_fd, objdir_name, FALSE, + &dfd_iter, error)) goto out; - + while (TRUE) { - GFileInfo *file_info; - guint32 nlinks; + struct dirent *dent; + struct stat stbuf; - if (!gs_file_enumerator_iterate (enumerator, &file_info, NULL, - cancellable, error)) + if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error)) goto out; - if (file_info == NULL) + if (dent == NULL) break; - - nlinks = g_file_info_get_attribute_uint32 (file_info, "unix::nlink"); - if (nlinks == 1) + + if (fstatat (dfd_iter.fd, dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW) != 0) { - g_autoptr(GFile) objpath = NULL; - objpath = g_file_get_child (objdir, g_file_info_get_name (file_info)); - if (!gs_file_unlink (objpath, cancellable, error)) - goto out; + glnx_set_error_from_errno (error); + goto out; + } + + if (stbuf.st_nlink == 1) + { + if (unlinkat (dfd_iter.fd, dent->d_name, 0) != 0) + { + glnx_set_error_from_errno (error); + goto out; + } } } } From e5eb6f3a220cddc01fd4d8a6db66935bddd3b3b2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 29 May 2016 13:33:25 -0400 Subject: [PATCH 073/128] libglnx porting: Port callers of gs_file_unlink to just unlink() In the second case it's actually less code too. Closes: #312 Approved by: giuseppe --- src/libostree/ostree-repo.c | 7 +++++-- src/libotutil/ot-gio-utils.c | 21 ++++----------------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index f2538f51..1d0b2f93 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1255,8 +1255,11 @@ impl_repo_remote_delete (OstreeRepo *self, if (remote->file != NULL) { - if (!gs_file_unlink (remote->file, cancellable, error)) - goto out; + if (unlink (gs_file_get_path_cached (remote->file)) != 0) + { + glnx_set_error_from_errno (error); + goto out; + } } else { diff --git a/src/libotutil/ot-gio-utils.c b/src/libotutil/ot-gio-utils.c index 103f6edd..6541a321 100644 --- a/src/libotutil/ot-gio-utils.c +++ b/src/libotutil/ot-gio-utils.c @@ -390,25 +390,12 @@ ot_gfile_ensure_unlinked (GFile *path, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - GError *temp_error = NULL; - - if (!gs_file_unlink (path, cancellable, &temp_error)) + if (unlink (gs_file_get_path_cached (path)) != 0) { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&temp_error); - } - else - { - g_propagate_error (error, temp_error); - goto out; - } + if (errno != ENOENT) + return FALSE; } - - ret = TRUE; - out: - return ret; + return TRUE; } /** From 6e57987c7c64e870ee748978825a672bd10acab8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 29 May 2016 13:50:34 -0400 Subject: [PATCH 074/128] libglnx porting: Port away from gs_file_ensure_directory() The init-fs code in particular gets (IMO) a lot cleaner being fd relative. Closes: #312 Approved by: giuseppe --- .../ostree-repo-static-delta-compilation.c | 3 +- src/libostree/ostree-repo.c | 13 ++++-- src/libostree/ostree-sysroot.c | 37 ++++++++-------- src/ostree/ot-admin-builtin-init-fs.c | 42 ++++++++----------- 4 files changed, 49 insertions(+), 46 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index eecaa8bc..4e4e7f9c 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -1483,7 +1483,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self, descriptor_dir = g_file_get_parent (descriptor_path); - if (!gs_file_ensure_directory (descriptor_dir, TRUE, cancellable, error)) + if (!glnx_shutil_mkdir_p_at (AT_FDCWD, gs_file_get_path_cached (descriptor_dir), 0755, + cancellable, error)) goto out; for (i = 0; i < part_tempfiles->len; i++) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1d0b2f93..84907c45 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2228,8 +2228,14 @@ ostree_repo_create (OstreeRepo *self, if (!ostree_repo_mode_to_string (mode, &mode_str, error)) goto out; - if (!gs_file_ensure_directory (self->repodir, FALSE, cancellable, error)) - goto out; + if (mkdir (gs_file_get_path_cached (self->repodir), 0755) != 0) + { + if (errno != EEXIST) + { + glnx_set_error_from_errno (error); + goto out; + } + } config_data = g_string_new (DEFAULT_CONFIG_CONTENTS); g_string_append_printf (config_data, "mode=%s\n", mode_str); @@ -2632,7 +2638,8 @@ ostree_repo_open (OstreeRepo *self, if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && self->enable_uncompressed_cache) { - if (!gs_file_ensure_directory (self->uncompressed_objects_dir, TRUE, cancellable, error)) + if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, "uncompressed-objects-cache", 0755, + cancellable, error)) goto out; if (!glnx_opendirat (self->repo_dir_fd, "uncompressed-objects-cache", TRUE, &self->uncompressed_objects_dir_fd, diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index e05f5c29..bc8dde66 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -278,28 +278,29 @@ ostree_sysroot_ensure_initialized (OstreeSysroot *self, GError **error) { gboolean ret = FALSE; - g_autoptr(GFile) dir = NULL; - g_autoptr(GFile) ostree_dir = NULL; - g_autoptr(GFile) repo_dir = NULL; + struct stat stbuf; - ostree_dir = g_file_get_child (self->path, "ostree"); - repo_dir = g_file_get_child (ostree_dir, "repo"); - if (!gs_file_ensure_directory (repo_dir, TRUE, cancellable, error)) + if (!ensure_sysroot_fd (self, error)) goto out; - g_clear_object (&dir); - dir = g_file_get_child (ostree_dir, "deploy"); - if (!gs_file_ensure_directory (dir, TRUE, cancellable, error)) - goto out; - - g_clear_object (&dir); - dir = ot_gfile_get_child_build_path (ostree_dir, "repo", "objects", NULL); - if (!g_file_query_exists (dir, NULL)) - { - glnx_unref_object OstreeRepo *repo = ostree_repo_new (repo_dir); - if (!ostree_repo_create (repo, OSTREE_REPO_MODE_BARE, + if (!glnx_shutil_mkdir_p_at (self->sysroot_fd, "ostree/repo", 0755, cancellable, error)) - goto out; + goto out; + + if (!glnx_shutil_mkdir_p_at (self->sysroot_fd, "ostree/deploy", 0755, + cancellable, error)) + goto out; + + if (fstatat (self->sysroot_fd, "ostree/repo/objects", &stbuf, 0) != 0) + { + if (errno == ENOENT) + { + g_autoptr(GFile) repo_dir = g_file_resolve_relative_path (self->path, "ostree/repo"); + glnx_unref_object OstreeRepo *repo = ostree_repo_new (repo_dir); + if (!ostree_repo_create (repo, OSTREE_REPO_MODE_BARE, + cancellable, error)) + goto out; + } } ret = TRUE; diff --git a/src/ostree/ot-admin-builtin-init-fs.c b/src/ostree/ot-admin-builtin-init-fs.c index 0172f01c..1f122553 100644 --- a/src/ostree/ot-admin-builtin-init-fs.c +++ b/src/ostree/ot-admin-builtin-init-fs.c @@ -39,8 +39,7 @@ ot_admin_builtin_init_fs (int argc, char **argv, GCancellable *cancellable, GErr GOptionContext *context; glnx_unref_object OstreeSysroot *sysroot = NULL; gboolean ret = FALSE; - g_autoptr(GFile) dir = NULL; - g_autoptr(GFile) child = NULL; + glnx_fd_close int root_dfd = -1; glnx_unref_object OstreeSysroot *target_sysroot = NULL; guint i; const char *normal_toplevels[] = {"boot", "dev", "home", "proc", "run", "sys"}; @@ -58,36 +57,31 @@ ot_admin_builtin_init_fs (int argc, char **argv, GCancellable *cancellable, GErr goto out; } - dir = g_file_new_for_path (argv[1]); - target_sysroot = ostree_sysroot_new (dir); + if (!glnx_opendirat (AT_FDCWD, argv[1], TRUE, &root_dfd, error)) + goto out; + { g_autoptr(GFile) dir = g_file_new_for_path (argv[1]); + target_sysroot = ostree_sysroot_new (dir); + } for (i = 0; i < G_N_ELEMENTS(normal_toplevels); i++) { - child = g_file_get_child (dir, normal_toplevels[i]); - if (!gs_file_ensure_directory_mode (child, 0755, cancellable, error)) + if (!glnx_shutil_mkdir_p_at (root_dfd, normal_toplevels[i], 0755, + cancellable, error)) goto out; - g_clear_object (&child); } - - child = g_file_get_child (dir, "root"); - if (!gs_file_ensure_directory_mode (child, 0700, cancellable, error)) + + if (!glnx_shutil_mkdir_p_at (root_dfd, "root", 0700, + cancellable, error)) goto out; - g_clear_object (&child); - child = g_file_get_child (dir, "tmp"); - if (!gs_file_ensure_directory_mode (child, 01777, cancellable, error)) + if (!glnx_shutil_mkdir_p_at (root_dfd, "tmp", 01777, + cancellable, error)) goto out; - /* FIXME - we should be using an API that explicitly ignores umask; - */ - { - const char *path = gs_file_get_path_cached (child); - if (chmod (path, 01777) == -1) - { - gs_set_prefix_error_from_errno (error, errno, "chmod"); - goto out; - } - } - g_clear_object (&child); + if (fchmodat (root_dfd, "tmp", 01777, 0) == -1) + { + glnx_set_prefix_error_from_errno (error, "chmod: %s", "tmp"); + goto out; + } if (!ostree_sysroot_ensure_initialized (target_sysroot, cancellable, error)) goto out; From 7748c361ef1ca2aaca1e20316b60f28a94821a32 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 May 2016 11:20:18 -0400 Subject: [PATCH 075/128] sysroot: Correct error handling path in previous libglnx port I happened to have the github page open with my commit after it was merged, and then noticed a bug. We still need to throw if we hit a non-`ENOENT` error. Closes: #315 Approved by: giuseppe --- src/libostree/ostree-sysroot.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index bc8dde66..8949f8f5 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -293,7 +293,12 @@ ostree_sysroot_ensure_initialized (OstreeSysroot *self, if (fstatat (self->sysroot_fd, "ostree/repo/objects", &stbuf, 0) != 0) { - if (errno == ENOENT) + if (errno != ENOENT) + { + glnx_set_prefix_error_from_errno (error, "stat %s", "ostree/repo/objects"); + goto out; + } + else { g_autoptr(GFile) repo_dir = g_file_resolve_relative_path (self->path, "ostree/repo"); glnx_unref_object OstreeRepo *repo = ostree_repo_new (repo_dir); From 24cb0ffc6a3711179b34f95a54c12e839b88d43b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 29 May 2016 13:06:36 -0400 Subject: [PATCH 076/128] core: Port to new libglnx tempname API Drops another libgsystem use, and as bonus we malloc less too. Closes: #311 Approved by: jlebon --- libglnx | 2 +- src/libostree/ostree-core.c | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libglnx b/libglnx index 3d162e77..afe3c3a8 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 3d162e772db80f6664a78583268150d2e1d1d29e +Subproject commit afe3c3a86178c29ceaa3a5e46397ab2fa97202b3 diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index d393c496..135d15d4 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -989,14 +989,13 @@ _ostree_make_temporary_symlink_at (int tmp_dirfd, GError **error) { gboolean ret = FALSE; - g_autofree char *tmpname = NULL; + char *tmpname = g_strdup ("tmplink.XXXXXX"); guint i; const int max_attempts = 128; for (i = 0; i < max_attempts; i++) { - g_free (tmpname); - tmpname = gs_fileutil_gen_tmp_name (NULL, NULL); + glnx_gen_temp_name (tmpname); if (symlinkat (target, tmp_dirfd, tmpname) < 0) { if (errno == EEXIST) From a189b19fd000f83b2bbd065e0dd47ee18f5c3f50 Mon Sep 17 00:00:00 2001 From: Sam Spilsbury Date: Wed, 1 Jun 2016 08:15:00 +0000 Subject: [PATCH 077/128] repo: Display estimated time remaining when pulling Bug 765429 said that not having a time estimate can be annoying when working with large pulls. There isn't any complex time estimation logic here - we just take the number of bytes remaining and do a linear projection of the bytes per second rate at the current point in time. Closes: #318 Approved by: cgwalters --- src/libostree/ostree-repo.c | 52 ++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 84907c45..36c2ac44 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4170,6 +4170,38 @@ ostree_repo_pull_with_options (OstreeRepo *self, #endif +/** + * _formatted_time_remaining_from_seconds + * @seconds_remaining: Estimated number of seconds remaining. + * + * Returns a strings showing the number of days, hours, minutes + * and seconds remaining. + **/ +static char * +_formatted_time_remaining_from_seconds (guint64 seconds_remaining) +{ + guint64 minutes_remaining = seconds_remaining / 60; + guint64 hours_remaining = minutes_remaining / 60; + guint64 days_remaining = hours_remaining / 24; + + GString *description = g_string_new (NULL); + + if (days_remaining) + g_string_append_printf (description, "%lu days ", days_remaining); + + if (hours_remaining) + g_string_append_printf (description, "%lu hours ", hours_remaining % 24); + + if (minutes_remaining) + g_string_append_printf (description, "%lu minutes ", minutes_remaining % 60); + + if (seconds_remaining) + g_string_append_printf (description, "%lu seconds ", seconds_remaining % 60); + + return g_string_free (description, FALSE); +} + + /** * ostree_repo_pull_default_console_progress_changed: * @progress: Async progress @@ -4225,28 +4257,36 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress guint fetched = ostree_async_progress_get_uint (progress, "fetched"); guint metadata_fetched = ostree_async_progress_get_uint (progress, "metadata-fetched"); guint requested = ostree_async_progress_get_uint (progress, "requested"); - guint64 bytes_sec = (g_get_monotonic_time () - ostree_async_progress_get_uint64 (progress, "start-time")) / G_USEC_PER_SEC; + guint64 start_time = ostree_async_progress_get_uint64 (progress, "start-time"); + guint64 total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); + guint64 current_time = g_get_monotonic_time (); + guint64 bytes_sec = bytes_transferred / ((current_time - start_time) / G_USEC_PER_SEC); + guint64 est_time_remaining = (total_delta_part_size - bytes_transferred) / bytes_sec; g_autofree char *formatted_bytes_transferred = g_format_size_full (bytes_transferred, 0); g_autofree char *formatted_bytes_sec = NULL; + g_autofree char *formatted_est_time_remaining = NULL; if (!bytes_sec) // Ignore first second - formatted_bytes_sec = g_strdup ("-"); + { + formatted_bytes_sec = g_strdup ("-"); + formatted_est_time_remaining = g_strdup ("- "); + } else { - bytes_sec = bytes_transferred / bytes_sec; formatted_bytes_sec = g_format_size (bytes_sec); + formatted_est_time_remaining = _formatted_time_remaining_from_seconds (est_time_remaining); } if (total_delta_parts > 0) { - guint64 total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); g_autofree char *formatted_total = g_format_size (total_delta_part_size); - g_string_append_printf (buf, "Receiving delta parts: %u/%u %s/s %s/%s", + /* No space between %s and remaining, since formatted_est_time_remaining has a trailing space */ + g_string_append_printf (buf, "Receiving delta parts: %u/%u %s/s %s/%s %sremaining", fetched_delta_parts, total_delta_parts, formatted_bytes_sec, formatted_bytes_transferred, - formatted_total); + formatted_total, formatted_est_time_remaining); } else if (outstanding_metadata_fetches) { From f725d39f111bd0d362a583c3a4902e3194fd74d8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 1 Jun 2016 09:35:47 -0400 Subject: [PATCH 078/128] lib: Change ot_util_variant_map helpers to consistently sink refs This is similar to changes Krzesimir has been doing recently - we really don't need the ergonomics of floating refs since we have autocleanups. We should continue to change most of our code to sink refs. Specifically here it was pretty broken that the `_map()` API was sinking but the other two weren't, and this broke some refactoring I was trying to do later. Closes: #317 Approved by: jlebon --- src/libostree/ostree-repo-static-delta-core.c | 1 + src/libotutil/ot-variant-utils.c | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 9e3ed09f..01383712 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -512,6 +512,7 @@ _ostree_static_delta_part_open (GInputStream *part_in, g_bytes_get_size (inline_part_bytes) - 1); ret_part = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), content_bytes, trusted); + g_variant_ref_sink (ret_part); } if (!skip_checksum) diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index 7b54542e..b9fe94b9 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -166,7 +166,6 @@ ot_util_variant_map_at (int dfd, GError **error) { glnx_fd_close int fd = -1; - g_autoptr(GVariant) ret_variant = NULL; fd = openat (dfd, path, O_RDONLY | O_CLOEXEC); if (fd < 0) @@ -224,8 +223,8 @@ ot_util_variant_map_fd (int fd, mdata->len = len; ret = TRUE; - *out_variant = g_variant_new_from_data (type, map, len, trusted, - variant_map_data_destroy, mdata); + *out_variant = g_variant_ref_sink (g_variant_new_from_data (type, map, len, trusted, + variant_map_data_destroy, mdata)); out: return ret; } From c148631a98407647bb989bb73d2b707e309344f9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 31 May 2016 11:53:27 -0400 Subject: [PATCH 079/128] lib: Drop GFile variant mapping API for fd-relative In addition to generic fd relative porting, this is a necessary preparatory step for libglnx porting, because when I tried to use `g_mapped_file_new` I hit an issue with it using a different error domain from GIO. Thankfully libglnx consistently uses the GIO error domain, and here we're now using it for the `open()` call. Closes: #317 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 5 ++-- src/libostree/ostree-repo.c | 5 ++-- src/libotutil/ot-variant-utils.c | 40 ------------------------------ src/libotutil/ot-variant-utils.h | 6 ----- src/ostree/ot-builtin-show.c | 5 +--- 5 files changed, 7 insertions(+), 54 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8f5067c0..96403d61 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2098,8 +2098,9 @@ ostree_repo_read_commit_detached_metadata (OstreeRepo *self, g_autoptr(GVariant) ret_metadata = NULL; GError *temp_error = NULL; - if (!ot_util_variant_map (metadata_path, G_VARIANT_TYPE ("a{sv}"), - TRUE, &ret_metadata, &temp_error)) + if (!ot_util_variant_map_at (AT_FDCWD, gs_file_get_path_cached (metadata_path), + G_VARIANT_TYPE ("a{sv}"), + TRUE, &ret_metadata, &temp_error)) { if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) { diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 36c2ac44..2332d2bb 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4627,8 +4627,9 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, if (!summary_data) goto out; - if (!ot_util_variant_map (signature_path, G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), - TRUE, &existing_signatures, &temp_error)) + if (!ot_util_variant_map_at (AT_FDCWD, gs_file_get_path_cached (signature_path), + G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), + TRUE, &existing_signatures, &temp_error)) { if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) { diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index b9fe94b9..26054459 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -117,46 +117,6 @@ ot_util_variant_take_ref (GVariant *variant) return g_variant_take_ref (variant); } -/** - * ot_util_variant_map: - * @src: a #GFile - * @type: Use this for variant - * @trusted: See documentation of g_variant_new_from_data() - * @out_variant: (out): Return location for new variant - * @error: - * - * Memory-map @src, and store a new #GVariant referring to this memory - * in @out_variant. Note the returned @out_variant is not floating. - */ -gboolean -ot_util_variant_map (GFile *src, - const GVariantType *type, - gboolean trusted, - GVariant **out_variant, - GError **error) -{ - gboolean ret = FALSE; - g_autoptr(GVariant) ret_variant = NULL; - GMappedFile *mfile = NULL; - - mfile = gs_file_map_noatime (src, NULL, error); - if (!mfile) - goto out; - - ret_variant = g_variant_new_from_data (type, - g_mapped_file_get_contents (mfile), - g_mapped_file_get_length (mfile), - trusted, - (GDestroyNotify) g_mapped_file_unref, - mfile); - g_variant_ref_sink (ret_variant); - - ret = TRUE; - ot_transfer_out_value(out_variant, &ret_variant); - out: - return ret; -} - gboolean ot_util_variant_map_at (int dfd, const char *path, diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h index 1a7abe0e..f185b4fd 100644 --- a/src/libotutil/ot-variant-utils.h +++ b/src/libotutil/ot-variant-utils.h @@ -42,12 +42,6 @@ gboolean ot_util_variant_save (GFile *dest, GCancellable *cancellable, GError **error); -gboolean ot_util_variant_map (GFile *src, - const GVariantType *type, - gboolean trusted, - GVariant **out_variant, - GError **error); - gboolean ot_util_variant_map_at (int dfd, const char *path, const GVariantType *type, diff --git a/src/ostree/ot-builtin-show.c b/src/ostree/ot-builtin-show.c index a1b4db5f..ef541c2a 100644 --- a/src/ostree/ot-builtin-show.c +++ b/src/ostree/ot-builtin-show.c @@ -51,12 +51,9 @@ do_print_variant_generic (const GVariantType *type, GError **error) { gboolean ret = FALSE; - g_autoptr(GFile) f = NULL; g_autoptr(GVariant) variant = NULL; - f = g_file_new_for_path (filename); - - if (!ot_util_variant_map (f, type, TRUE, &variant, error)) + if (!ot_util_variant_map_at (AT_FDCWD, filename, type, TRUE, &variant, error)) goto out; ot_dump_variant (variant); From 900c085f216cd11eb1249fe7b657089a1f0d09a4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 31 May 2016 11:27:19 -0400 Subject: [PATCH 080/128] libglnx porting: Port away from gs_file_map_noatime() The "no atime" thing was mostly useful only before "relative atime" updates landed. Users who care about performance will turn it off entirely anyways. Closes: #316 Approved by: jlebon --- src/libostree/ostree-repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 2332d2bb..0d7ba116 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4459,7 +4459,7 @@ sign_data (OstreeRepo *self, if (!g_output_stream_close (tmp_signature_output, cancellable, error)) goto out; - signature_file = gs_file_map_noatime (tmp_signature_file, cancellable, error); + signature_file = g_mapped_file_new (gs_file_get_path_cached (tmp_signature_file), FALSE, error); if (!signature_file) goto out; ret_signature = g_mapped_file_get_bytes (signature_file); From eaea07fe432aa7f66764fdb4eeab0259c14aa2cf Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 31 May 2016 11:35:35 -0400 Subject: [PATCH 081/128] glnx porting: Port away from gs_file_get_basename_cached() In some cases we use glnx_basename(), in others we already had a `GFileInfo` around with the name. Closes: #316 Approved by: jlebon --- src/libostree/ostree-repo-static-delta-core.c | 4 ++-- src/libostree/ostree-sysroot-cleanup.c | 4 ++-- src/libostree/ostree-sysroot-deploy.c | 2 +- src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 01383712..3284c2ae 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -126,8 +126,8 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, if (g_file_info_get_file_type (file_info2) != G_FILE_TYPE_DIRECTORY) continue; - name1 = gs_file_get_basename_cached (child); - name2 = gs_file_get_basename_cached (child2); + name1 = g_file_info_get_name (file_info); + name2 = g_file_info_get_name (file_info2); { g_autoptr(GFile) meta_path = g_file_get_child (child2, "superblock"); diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index 5c370507..64c1389e 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -32,7 +32,7 @@ _ostree_sysroot_list_deployment_dirs_for_os (GFile *osdir, GError **error) { gboolean ret = FALSE; - const char *osname = gs_file_get_basename_cached (osdir); + const char *osname = glnx_basename (gs_file_get_path_cached (osdir)); g_autoptr(GFileEnumerator) dir_enum = NULL; g_autoptr(GFile) osdeploy_dir = NULL; GError *temp_error = NULL; @@ -370,7 +370,7 @@ cleanup_old_deployments (OstreeSysroot *self, g_autofree char *osname = NULL; g_autofree char *bootcsum = NULL; - if (!parse_bootdir_name (gs_file_get_basename_cached (bootdir), + if (!parse_bootdir_name (glnx_basename (gs_file_get_path_cached (bootdir)), &osname, &bootcsum)) g_assert_not_reached (); diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index d1d624b6..0740b2e1 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -684,7 +684,7 @@ relabel_recursively (OstreeSysroot *sysroot, if (file_info == NULL) break; - g_ptr_array_add (path_parts, (char*)gs_file_get_basename_cached (child)); + g_ptr_array_add (path_parts, (char*)g_file_info_get_name (file_info)); ftype = g_file_info_get_file_type (file_info); if (ftype == G_FILE_TYPE_DIRECTORY) diff --git a/src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c b/src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c index 1f0e91e3..776ab54f 100644 --- a/src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c +++ b/src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c @@ -120,7 +120,7 @@ relabel_recursively (OstreeSePolicy *sepolicy, if (file_info == NULL) break; - g_ptr_array_add (path_parts, (char*)gs_file_get_basename_cached (child)); + g_ptr_array_add (path_parts, (char*)g_file_info_get_name (file_info)); ftype = g_file_info_get_file_type (file_info); if (ftype == G_FILE_TYPE_DIRECTORY) From d78e12fc39075ba6ff87f5c42f602af08f75a2d7 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 3 Jun 2016 21:35:52 +0200 Subject: [PATCH 082/128] summary: Fix a typo The --gpg-sign parameter will be used for signing the summary, not the commit. This probably was copy-pasted from the ostree commit command. Closes: #320 Approved by: cgwalters --- src/ostree/ot-builtin-summary.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c index 4622ceef..2ff1c965 100644 --- a/src/ostree/ot-builtin-summary.c +++ b/src/ostree/ot-builtin-summary.c @@ -31,7 +31,7 @@ static char *opt_gpg_homedir; static GOptionEntry options[] = { { "update", 'u', 0, G_OPTION_ARG_NONE, &opt_update, "Update the summary", NULL }, - { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the commit with", "KEY-ID"}, + { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the summary with", "KEY-ID"}, { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, { NULL } }; From 085801185125893a53d7a65732c6832f875ed5d6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 6 Jun 2016 15:00:13 -0400 Subject: [PATCH 083/128] tests: Use strict mode by default for C tests I was extending the C-based test suite to cover more, and noticed that we weren't aborting if a command failed. That made the tests somewhat less useful, so let's fix it. Closes: #321 Approved by: jlebon --- tests/libostreetest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/libostreetest.c b/tests/libostreetest.c index 58283368..a557ee37 100644 --- a/tests/libostreetest.c +++ b/tests/libostreetest.c @@ -40,7 +40,7 @@ run_libtest (const char *cmd, GError **error) g_ptr_array_add (argv, "bash"); g_ptr_array_add (argv, "-c"); - g_string_append (cmdstr, ". "); + g_string_append (cmdstr, "set -xeuo pipefail; . "); g_string_append (cmdstr, builddir); g_string_append (cmdstr, "/tests/libtest.sh; "); g_string_append (cmdstr, cmd); @@ -68,7 +68,7 @@ ot_test_setup_repo (GCancellable *cancellable, g_autoptr(GFile) repo_path = g_file_new_for_path ("repo"); glnx_unref_object OstreeRepo* ret_repo = NULL; - if (!run_libtest ("setup_test_repository", error)) + if (!run_libtest ("setup_test_repository archive-z2", error)) goto out; ret_repo = ostree_repo_new (repo_path); From 7fb49037ab6033bc53fa2eb3b516d7709b558d90 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 6 Jun 2016 14:49:11 -0400 Subject: [PATCH 084/128] tests/test-pull-c: New test that runs through the pull API via C We have had in the past issues with running `ostree_repo_pull()` multiple times in the same process, embarassingly enough. Nothing in the current test suite covers this, so let's start. Closes: #322 Approved by: jlebon --- Makefile-tests.am | 5 +- tests/libostreetest.c | 8 +-- tests/libostreetest.h | 1 + tests/test-pull-c.c | 132 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 tests/test-pull-c.c diff --git a/Makefile-tests.am b/Makefile-tests.am index bb1cee41..58397d37 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -140,7 +140,7 @@ endif test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-mutable-tree \ tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \ tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum \ - tests/test-basic-c tests/test-sysroot-c + tests/test-basic-c tests/test-sysroot-c tests/test-pull-c # An interactive tool noinst_PROGRAMS += tests/test-rollsum-cli @@ -177,6 +177,9 @@ tests_test_basic_c_LDADD = $(TESTS_LDADD) tests_test_sysroot_c_CFLAGS = $(TESTS_CFLAGS) tests_test_sysroot_c_LDADD = $(TESTS_LDADD) +tests_test_pull_c_CFLAGS = $(TESTS_CFLAGS) +tests_test_pull_c_LDADD = $(TESTS_LDADD) + tests_test_ot_unix_utils_CFLAGS = $(TESTS_CFLAGS) tests_test_ot_unix_utils_LDADD = $(TESTS_LDADD) diff --git a/tests/libostreetest.c b/tests/libostreetest.c index a557ee37..aff6c1e8 100644 --- a/tests/libostreetest.c +++ b/tests/libostreetest.c @@ -28,8 +28,8 @@ /* This function hovers in a quantum superposition of horrifying and * beautiful. Future generations may interpret it as modern art. */ -static gboolean -run_libtest (const char *cmd, GError **error) +gboolean +ot_test_run_libtest (const char *cmd, GError **error) { gboolean ret = FALSE; const char *builddir = g_getenv ("G_TEST_BUILDDIR"); @@ -68,7 +68,7 @@ ot_test_setup_repo (GCancellable *cancellable, g_autoptr(GFile) repo_path = g_file_new_for_path ("repo"); glnx_unref_object OstreeRepo* ret_repo = NULL; - if (!run_libtest ("setup_test_repository archive-z2", error)) + if (!ot_test_run_libtest ("setup_test_repository archive-z2", error)) goto out; ret_repo = ostree_repo_new (repo_path); @@ -91,7 +91,7 @@ ot_test_setup_sysroot (GCancellable *cancellable, g_autoptr(GFile) sysroot_path = g_file_new_for_path ("sysroot"); glnx_unref_object OstreeSysroot *ret_sysroot = NULL; - if (!run_libtest ("setup_os_repository \"archive-z2\" \"syslinux\"", error)) + if (!ot_test_run_libtest ("setup_os_repository \"archive-z2\" \"syslinux\"", error)) goto out; ret_sysroot = ostree_sysroot_new (sysroot_path); diff --git a/tests/libostreetest.h b/tests/libostreetest.h index eb9bb0b2..f777545e 100644 --- a/tests/libostreetest.h +++ b/tests/libostreetest.h @@ -27,6 +27,7 @@ G_BEGIN_DECLS +gboolean ot_test_run_libtest (const char *cmd, GError **error); OstreeRepo *ot_test_setup_repo (GCancellable *cancellable, GError **error); diff --git a/tests/test-pull-c.c b/tests/test-pull-c.c new file mode 100644 index 00000000..d784312f --- /dev/null +++ b/tests/test-pull-c.c @@ -0,0 +1,132 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2016 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" +#include "libglnx.h" +#include +#include +#include +#include + +#include "libostreetest.h" + +typedef struct { + OstreeRepo *repo; +} TestData; + +static void +test_data_init (TestData *td) +{ + GError *local_error = NULL; + GError **error = &local_error; + g_autofree char *http_address = NULL; + g_autofree char *repo_url = NULL; + + td->repo = ot_test_setup_repo (NULL, error); + if (!td->repo) + goto out; + + if (!ot_test_run_libtest ("setup_fake_remote_repo1 archive-z2", error)) + goto out; + + if (!g_file_get_contents ("httpd-address", &http_address, NULL, error)) + goto out; + + repo_url = g_strconcat (http_address, "/ostree/gnomerepo", NULL); + + { g_autoptr(GVariantBuilder) builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + g_autoptr(GVariant) opts = NULL; + + g_variant_builder_add (builder, "{s@v}", "gpg-verify", g_variant_new_variant (g_variant_new_boolean (FALSE))); + opts = g_variant_ref_sink (g_variant_builder_end (builder)); + + if (!ostree_repo_remote_change (td->repo, NULL, OSTREE_REPO_REMOTE_CHANGE_ADD, + "origin", repo_url, opts, NULL, error)) + goto out; + } + + out: + g_assert_no_error (local_error); +} + +static void +test_pull_multi_nochange (gconstpointer data) +{ + GError *local_error = NULL; + GError **error = &local_error; + TestData *td = (void*)data; + char *refs[] = { "main", NULL }; + + if (!ostree_repo_pull (td->repo, "origin", (char**)&refs, 0, NULL, NULL, error)) + goto out; + if (!ostree_repo_pull (td->repo, "origin", (char**)&refs, 0, NULL, NULL, error)) + goto out; + if (!ostree_repo_pull (td->repo, "origin", (char**)&refs, 0, NULL, NULL, error)) + goto out; + + out: + g_assert_no_error (local_error); +} + +static void +test_pull_multi_error_then_ok (gconstpointer data) +{ + GError *local_error = NULL; + GError **error = &local_error; + + TestData *td = (void*)data; + char *ok_refs[] = { "main", NULL }; + char *bad_refs[] = { "nosuchbranch", NULL }; + + for (guint i = 0; i < 3; i++) + { + g_autoptr(GError) tmp_error = NULL; + if (!ostree_repo_pull (td->repo, "origin", (char**)&ok_refs, 0, NULL, NULL, error)) + goto out; + if (ostree_repo_pull (td->repo, "origin", (char**)&bad_refs, 0, NULL, NULL, &tmp_error)) + g_assert_not_reached (); + g_clear_error (&tmp_error); + if (ostree_repo_pull (td->repo, "origin", (char**)&bad_refs, 0, NULL, NULL, &tmp_error)) + g_assert_not_reached (); + g_clear_error (&tmp_error); + if (!ostree_repo_pull (td->repo, "origin", (char**)&ok_refs, 0, NULL, NULL, error)) + goto out; + } + + out: + g_assert_no_error (local_error); +} + +int main (int argc, char **argv) +{ + TestData td = {NULL,}; + int r; + + test_data_init (&td); + + g_test_init (&argc, &argv, NULL); + + g_test_add_data_func ("/test-pull-c/multi-nochange", &td, test_pull_multi_nochange); + g_test_add_data_func ("/test-pull-c/multi-ok-error-repeat", &td, test_pull_multi_error_then_ok); + + r = g_test_run(); + + return r; +} From 2240d1108ee357e4c2120b48e9c1f10e4042042a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 7 Jun 2016 14:44:55 -0400 Subject: [PATCH 085/128] ostree_repo_write_commit: add missing docstring arg Closes: #325 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 96403d61..b8f4824a 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2010,6 +2010,7 @@ ostree_repo_write_commit (OstreeRepo *self, * @body: (allow-none): Body * @metadata: (allow-none): GVariant of type a{sv}, or %NULL for none * @root: The tree to point the commit to + * @time: The time to use to stamp the commit * @out_commit: (out): Resulting ASCII SHA256 checksum for commit * @cancellable: Cancellable * @error: Error From b4c49f5acf030552bcde564eb53d04ea521c7641 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 7 Jun 2016 15:08:16 -0400 Subject: [PATCH 086/128] configure.ac: properly check for bison The AC_PROG_YACC already does the checking for us, so we don't need to use AC_CHECK_PROGS (which wasn't working anyway because autoconf didn't run the check thinking it was cached already from the earlier AC_PROC_YACC, so we didn't get the default ":" value) Closes: #325 Approved by: cgwalters --- configure.ac | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index e5bd8c7e..d1b732f7 100644 --- a/configure.ac +++ b/configure.ac @@ -38,8 +38,7 @@ GLIB_TESTS AC_CHECK_HEADER([sys/xattr.h],,[AC_MSG_ERROR([You must have sys/xattr.h from glibc])]) -AC_CHECK_PROGS(YACC, 'bison -y', :) -AS_IF([test "$YACC" = :], [AC_MSG_ERROR([bison not found but required])]) +AS_IF([test "$YACC" != "bison -y"], [AC_MSG_ERROR([bison not found but required])]) PKG_PROG_PKG_CONFIG From b5da2f524c3b92eb9364d2a00d07c078111541a4 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 9 Jun 2016 12:42:28 +0200 Subject: [PATCH 087/128] repo: Fix build on 32-bit systems On 32-bit systems the modifier for printing 64bit values should be %llu instead of %lu. Just use appriopriate macros that do the right thing. Closes: #329 Approved by: giuseppe --- src/libostree/ostree-repo.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 0d7ba116..22b874b3 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4187,16 +4187,16 @@ _formatted_time_remaining_from_seconds (guint64 seconds_remaining) GString *description = g_string_new (NULL); if (days_remaining) - g_string_append_printf (description, "%lu days ", days_remaining); + g_string_append_printf (description, "%" G_GUINT64_FORMAT " days ", days_remaining); if (hours_remaining) - g_string_append_printf (description, "%lu hours ", hours_remaining % 24); + g_string_append_printf (description, "%" G_GUINT64_FORMAT " hours ", hours_remaining % 24); if (minutes_remaining) - g_string_append_printf (description, "%lu minutes ", minutes_remaining % 60); + g_string_append_printf (description, "%" G_GUINT64_FORMAT " minutes ", minutes_remaining % 60); if (seconds_remaining) - g_string_append_printf (description, "%lu seconds ", seconds_remaining % 60); + g_string_append_printf (description, "%" G_GUINT64_FORMAT " seconds ", seconds_remaining % 60); return g_string_free (description, FALSE); } From a79c47415ac504210d2820eb2d57c0eccb9d1c40 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 3 Jun 2016 10:18:57 -0400 Subject: [PATCH 088/128] glnx porting: Drop last uses of gs_file_get_basename_cached() I apparently missed a few uses in the previous porting. Closes: #319 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 4 ++-- src/libotutil/ot-gio-utils.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 0740b2e1..f0e7ecca 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -756,12 +756,12 @@ selinux_relabel_file (OstreeSysroot *sysroot, goto out; g_ptr_array_add (path_parts, (char*)prefix); - g_ptr_array_add (path_parts, (char*)gs_file_get_basename_cached (path)); + g_ptr_array_add (path_parts, (char*)g_file_info_get_name (file_info)); if (!relabel_one_path (sysroot, sepolicy, path, file_info, path_parts, cancellable, error)) { g_prefix_error (error, "Relabeling /%s/%s: ", prefix, - gs_file_get_basename_cached (path)); + g_file_info_get_name (file_info)); goto out; } diff --git a/src/libotutil/ot-gio-utils.c b/src/libotutil/ot-gio-utils.c index 6541a321..a9da4504 100644 --- a/src/libotutil/ot-gio-utils.c +++ b/src/libotutil/ot-gio-utils.c @@ -358,7 +358,7 @@ ot_gfile_replace_contents_fsync (GFile *path, { gboolean ret = FALSE; int parent_dfd; - const char *target_basename = gs_file_get_basename_cached (path); + const char *target_basename = glnx_basename (gs_file_get_path_cached (path)); g_autoptr(GFile) parent = NULL; parent = g_file_get_parent (path); @@ -447,7 +447,7 @@ ot_util_ensure_directory_and_fsync (GFile *dir, { gboolean ret = FALSE; int parentfd = -1; - const char *basename = gs_file_get_basename_cached (dir); + const char *basename = glnx_basename (gs_file_get_path_cached (dir)); g_autoptr(GFile) parent = g_file_get_parent (dir); again: From 882561b01c799882befe285d875d2ee3dbb6a28c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 3 Jun 2016 10:16:25 -0400 Subject: [PATCH 089/128] libglnx porting: Drop last use of gs_transfer_out_value() Closes: #319 Approved by: jlebon --- src/libostree/ostree-repo-static-delta-core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 3284c2ae..2b1338b5 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -159,7 +159,8 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, } ret = TRUE; - gs_transfer_out_value (out_deltas, &ret_deltas); + if (out_deltas) + *out_deltas = g_steal_pointer (&ret_deltas); out: return ret; } From 5a996c04defaed1c3ae396aa4e352a853de36b58 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 3 Jun 2016 10:22:05 -0400 Subject: [PATCH 090/128] glnx porting: Drop use of gs_file_openat_noatime As noted previously the "noatime" thing is mostly obsoleted by relatime. Closes: #319 Approved by: jlebon --- src/libostree/ostree-repo.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 22b874b3..38dac761 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3091,8 +3091,13 @@ _ostree_repo_read_bare_fd (OstreeRepo *self, _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode); - return gs_file_openat_noatime (self->objects_dir_fd, loose_path_buf, out_fd, - cancellable, error); + *out_fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); + if (*out_fd < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + return TRUE; } /** From 3a03a35071f9ac56de2345f5f496c662e04d8029 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 3 Jun 2016 10:40:56 -0400 Subject: [PATCH 091/128] lib: Add `_ALLOW_NOENT` flag to internal variant mapping API We have a lot of "allow_noent" type wrapper functions since a common pattern is to allow files to not exist, but still throw cleanly on other issues. This is another instance of that, and cleans up duplicated error handling code. Part of this is prep for moving away from `GFile` consumers. Closes: #319 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 15 +++------------ src/libostree/ostree-repo-static-delta-core.c | 2 +- src/libostree/ostree-repo.c | 15 ++------------- src/libotutil/ot-variant-utils.c | 17 +++++++++++++---- src/libotutil/ot-variant-utils.h | 7 ++++++- 5 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index b8f4824a..6cb46ca9 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2097,22 +2097,13 @@ ostree_repo_read_commit_detached_metadata (OstreeRepo *self, g_autoptr(GFile) metadata_path = _ostree_repo_get_commit_metadata_loose_path (self, checksum); g_autoptr(GVariant) ret_metadata = NULL; - GError *temp_error = NULL; if (!ot_util_variant_map_at (AT_FDCWD, gs_file_get_path_cached (metadata_path), G_VARIANT_TYPE ("a{sv}"), - TRUE, &ret_metadata, &temp_error)) + OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error)) { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&temp_error); - } - else - { - g_prefix_error (error, "Unable to read existing detached metadata: "); - g_propagate_error (error, temp_error); - goto out; - } + g_prefix_error (error, "Unable to read existing detached metadata: "); + goto out; } ret = TRUE; diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 2b1338b5..de9e6c7a 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -842,7 +842,7 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, if (!ot_util_variant_map_at (self->repo_dir_fd, superblock_path, (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, - TRUE, &delta_superblock, error)) + OT_VARIANT_MAP_TRUSTED, &delta_superblock, error)) goto out; g_print ("%s\n", g_variant_print (delta_superblock, 1)); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 38dac761..2a26ffed 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4620,7 +4620,6 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, g_autoptr(GBytes) summary_data = NULL; g_autoptr(GFile) summary_file = NULL; g_autoptr(GFile) signature_path = NULL; - GError *temp_error = NULL; g_autoptr(GVariant) existing_signatures = NULL; g_autoptr(GVariant) new_metadata = NULL; g_autoptr(GVariant) normalized = NULL; @@ -4634,18 +4633,8 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, if (!ot_util_variant_map_at (AT_FDCWD, gs_file_get_path_cached (signature_path), G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), - TRUE, &existing_signatures, &temp_error)) - { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&temp_error); - } - else - { - g_propagate_error (error, temp_error); - goto out; - } - } + OT_VARIANT_MAP_ALLOW_NOENT, &existing_signatures, error)) + goto out; for (i = 0; key_id[i]; i++) { diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index 26054459..9d4a62d2 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -121,18 +121,27 @@ gboolean ot_util_variant_map_at (int dfd, const char *path, const GVariantType *type, - gboolean trusted, + OtVariantMapFlags flags, GVariant **out_variant, GError **error) { glnx_fd_close int fd = -1; + const gboolean trusted = (flags & OT_VARIANT_MAP_TRUSTED) > 0; fd = openat (dfd, path, O_RDONLY | O_CLOEXEC); if (fd < 0) { - glnx_set_error_from_errno (error); - g_prefix_error (error, "Opening %s: ", path); - return FALSE; + if (errno == ENOENT && (flags & OT_VARIANT_MAP_ALLOW_NOENT) > 0) + { + *out_variant = NULL; + return TRUE; + } + else + { + glnx_set_error_from_errno (error); + g_prefix_error (error, "Opening %s: ", path); + return FALSE; + } } return ot_util_variant_map_fd (fd, 0, type, trusted, out_variant, error); diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h index f185b4fd..8a33cf60 100644 --- a/src/libotutil/ot-variant-utils.h +++ b/src/libotutil/ot-variant-utils.h @@ -42,10 +42,15 @@ gboolean ot_util_variant_save (GFile *dest, GCancellable *cancellable, GError **error); +typedef enum { + OT_VARIANT_MAP_TRUSTED = (1 << 0), + OT_VARIANT_MAP_ALLOW_NOENT = (1 << 1) +} OtVariantMapFlags; + gboolean ot_util_variant_map_at (int dfd, const char *path, const GVariantType *type, - gboolean trusted, + OtVariantMapFlags flags, GVariant **out_variant, GError **error); From 70e548925809f3727e9e6ecff32a9cdf5bdb7185 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 3 Jun 2016 11:00:10 -0400 Subject: [PATCH 092/128] lib: Add a helper for mmap->bytes with openat(), use it in repo This kills another GSystem consumer...I think down the line I'd like to do something like "detect whether file is > 1k if so, mmap, otherwise just readall()" so we can use this helper in more places. Closes: #319 Approved by: jlebon --- src/libostree/ostree-repo.c | 4 +--- src/libotutil/ot-fs-utils.c | 21 +++++++++++++++++++++ src/libotutil/ot-fs-utils.h | 4 ++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 2a26ffed..5c4e941e 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4618,7 +4618,6 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, { gboolean ret = FALSE; g_autoptr(GBytes) summary_data = NULL; - g_autoptr(GFile) summary_file = NULL; g_autoptr(GFile) signature_path = NULL; g_autoptr(GVariant) existing_signatures = NULL; g_autoptr(GVariant) new_metadata = NULL; @@ -4626,8 +4625,7 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, guint i; signature_path = g_file_resolve_relative_path (self->repodir, "summary.sig"); - summary_file = g_file_resolve_relative_path (self->repodir, "summary"); - summary_data = gs_file_map_readonly (summary_file, cancellable, error); + summary_data = ot_file_mapat_bytes (self->repo_dir_fd, "summary", error); if (!summary_data) goto out; diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c index 4d45cd06..46a0405b 100644 --- a/src/libotutil/ot-fs-utils.c +++ b/src/libotutil/ot-fs-utils.c @@ -230,3 +230,24 @@ ot_openat_ignore_enoent (int dfd, out: return ret; } + +GBytes * +ot_file_mapat_bytes (int dfd, + const char *path, + GError **error) +{ + glnx_fd_close int fd = openat (dfd, path, O_RDONLY | O_CLOEXEC); + g_autoptr(GMappedFile) mfile = NULL; + + if (fd < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + + mfile = g_mapped_file_new_from_fd (fd, FALSE, error); + if (!mfile) + return FALSE; + + return g_mapped_file_get_bytes (mfile); +} diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h index cfeea74d..27f0f38e 100644 --- a/src/libotutil/ot-fs-utils.h +++ b/src/libotutil/ot-fs-utils.h @@ -66,4 +66,8 @@ gboolean ot_openat_ignore_enoent (int dfd, int *out_fd, GError **error); +GBytes *ot_file_mapat_bytes (int dfd, + const char *path, + GError **error); + G_END_DECLS From f94b191ae5e3eab191bc7e9773d780725da896d9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 3 Jun 2016 11:04:17 -0400 Subject: [PATCH 093/128] repo: (minor) Drop a GFile usage in signature creation Just one user of hundreds but, the rest of the function was ported now. Closes: #319 Approved by: jlebon --- src/libostree/ostree-repo.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 5c4e941e..6af5d8e6 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4618,18 +4618,16 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, { gboolean ret = FALSE; g_autoptr(GBytes) summary_data = NULL; - g_autoptr(GFile) signature_path = NULL; g_autoptr(GVariant) existing_signatures = NULL; g_autoptr(GVariant) new_metadata = NULL; g_autoptr(GVariant) normalized = NULL; guint i; - signature_path = g_file_resolve_relative_path (self->repodir, "summary.sig"); summary_data = ot_file_mapat_bytes (self->repo_dir_fd, "summary", error); if (!summary_data) goto out; - if (!ot_util_variant_map_at (AT_FDCWD, gs_file_get_path_cached (signature_path), + if (!ot_util_variant_map_at (self->repo_dir_fd, "summary.sig", G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), OT_VARIANT_MAP_ALLOW_NOENT, &existing_signatures, error)) goto out; From 0e9a875393ec038de5fdfa2aa0934ee03ff75334 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Wed, 8 Jun 2016 15:39:06 -0600 Subject: [PATCH 094/128] repo: use OSTREE_TIMESTAMP (=1) for checked-out files 1 is a better choice than 0 because some programs use 0 as a special value; for example, GNU Tar warns of an "implausibly old timestamp" with 0. Closes: #330 Approved by: cgwalters --- docs/manual/repo.md | 16 ++++++++++++++++ src/libostree/ostree-repo-checkout.c | 4 ++-- src/libostree/ostree-repo-commit.c | 8 ++------ src/libostree/ostree-repo-libarchive.c | 6 +++--- src/libostree/ostree-repo-private.h | 2 ++ src/libostree/ostree-repo.c | 8 ++++++++ tests/basic-test.sh | 4 ++-- 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/docs/manual/repo.md b/docs/manual/repo.md index d6f67090..bce7e0c9 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -47,6 +47,22 @@ payload sections. The header contains uid, gid, mode, and symbolic link target (for symlinks), as well as extended attributes. After the header, for regular files, the content follows. +The OSTree data format intentionally does not contain timestamps. The reasoning +is that data files may be downloaded at different times, and by different build +systems, and so will have different timestamps but identical physical content. +These files may be large, so most users would like them to be shared, both in +the repository and between the repository and deployments. + +This could cause problems with programs that check if files are out-of-date by +comparing timestamps. For Git, the logical choice is to not mess with +timestamps, because unnecessary rebuilding is better than a broken tree. +However, OSTree has to hardlink files to check them out, and commits are assumed +to be internally consistent with no build steps needed. For this reason, OSTree +acts as though all timestamps are set to time_t 1, so that comparisons will be +considered up-to-date. 1 is a better choice than 0 because some programs use 0 +as a special value; for example, GNU Tar warns of an "implausibly old time +stamp" with 0. + # Repository types and locations Also unlike git, an OSTree repository can be in one of three separate diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 95d3747b..227227b6 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -752,12 +752,12 @@ checkout_tree_at (OstreeRepo *self, } } - /* Set directory mtime to 0, so that it is constant for all checkouts. + /* Set directory mtime to OSTREE_TIMESTAMP, so that it is constant for all checkouts. * Must be done after setting permissions and creating all children. */ if (!did_exist) { - const struct timespec times[2] = { { 0, UTIME_OMIT }, { 0, } }; + const struct timespec times[2] = { { OSTREE_TIMESTAMP, UTIME_OMIT }, { OSTREE_TIMESTAMP, 0} }; do res = futimens (destination_dfd, times); while (G_UNLIKELY (res == -1 && errno == EINTR)); diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 6cb46ca9..6166c652 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -210,7 +210,6 @@ commit_loose_object_trusted (OstreeRepo *self, else { int res; - struct timespec times[2]; if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE) { @@ -266,12 +265,9 @@ commit_loose_object_trusted (OstreeRepo *self, { /* To satisfy tools such as guile which compare mtimes * to determine whether or not source files need to be compiled, - * set the modification time to 0. + * set the modification time to OSTREE_TIMESTAMP. */ - times[0].tv_sec = 0; /* atime */ - times[0].tv_nsec = UTIME_OMIT; - times[1].tv_sec = 0; /* mtime */ - times[1].tv_nsec = 0; + const struct timespec times[2] = { { OSTREE_TIMESTAMP, UTIME_OMIT }, { OSTREE_TIMESTAMP, 0} }; do res = futimens (fd, times); while (G_UNLIKELY (res == -1 && errno == EINTR)); diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 0d62124d..45427ef7 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -982,9 +982,9 @@ file_to_archive_entry_common (GFile *root, } archive_entry_update_pathname_utf8 (entry, pathstr); - archive_entry_set_ctime (entry, ts, 0); - archive_entry_set_mtime (entry, ts, 0); - archive_entry_set_atime (entry, ts, 0); + archive_entry_set_ctime (entry, ts, OSTREE_TIMESTAMP); + archive_entry_set_mtime (entry, ts, OSTREE_TIMESTAMP); + archive_entry_set_atime (entry, ts, OSTREE_TIMESTAMP); archive_entry_set_uid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::uid")); archive_entry_set_gid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::gid")); archive_entry_set_mode (entry, g_file_info_get_attribute_uint32 (file_info, "unix::mode")); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 62a3c8f6..35b05c76 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -36,6 +36,8 @@ G_BEGIN_DECLS #define _OSTREE_SUMMARY_CACHE_DIR "summaries" #define _OSTREE_CACHE_DIR "cache" +#define OSTREE_TIMESTAMP (1) + typedef enum { OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0) } OstreeRepoTestErrorFlags; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 6af5d8e6..fd02833a 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -942,6 +942,14 @@ ostree_repo_is_writable (OstreeRepo *self, return self->writable; } +/** + * _ostree_repo_update_mtime: + * @self: Repo + * @error: a #GError + * + * Bump the mtime of the repository so that programs + * can detect that the refs have updated. + */ gboolean _ostree_repo_update_mtime (OstreeRepo *self, GError **error) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 93f67f17..0fe372dc 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -386,9 +386,9 @@ else $OSTREE checkout test2 test2-checkout fi stat '--format=%Y' test2-checkout/baz/cow > cow-mtime -assert_file_has_content cow-mtime 0 +assert_file_has_content cow-mtime 1 stat '--format=%Y' test2-checkout/baz/deeper > deeper-mtime -assert_file_has_content deeper-mtime 0 +assert_file_has_content deeper-mtime 1 echo "ok content mtime" cd ${test_tmpdir} From da5cb099a6941c2752fa9c09c9537602f81324a2 Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Wed, 8 Jun 2016 12:48:44 -0600 Subject: [PATCH 095/128] pull: Move libsoup-related code from ostree-repo.c to ostree-repo-pull.c This centralizes the ifdef's in one file, which will make it easier to write new pull backends. ostree-repo-pull.c is now built unconditionally Closes: #327 Approved by: cgwalters --- Makefile-libostree.am | 2 +- src/libostree/ostree-repo-private.h | 29 -- src/libostree/ostree-repo-pull.c | 439 +++++++++++++++++++++++++++- src/libostree/ostree-repo.c | 414 -------------------------- 4 files changed, 435 insertions(+), 449 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 700d372f..13013c1f 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -95,6 +95,7 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-repo.c \ src/libostree/ostree-repo-checkout.c \ src/libostree/ostree-repo-commit.c \ + src/libostree/ostree-repo-pull.c \ src/libostree/ostree-repo-libarchive.c \ src/libostree/ostree-repo-prune.c \ src/libostree/ostree-repo-refs.c \ @@ -159,7 +160,6 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-fetcher.c \ src/libostree/ostree-metalink.h \ src/libostree/ostree-metalink.c \ - src/libostree/ostree-repo-pull.c \ $(NULL) libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) libostree_1_la_LIBADD += $(OT_INTERNAL_SOUP_LIBS) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 35b05c76..f330d169 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -23,10 +23,6 @@ #include "ostree-repo.h" #include "libglnx.h" -#ifdef HAVE_LIBSOUP -#include "ostree-fetcher.h" -#endif - G_BEGIN_DECLS #define OSTREE_DELTAPART_VERSION (0) @@ -235,13 +231,6 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self, gboolean _ostree_repo_remote_name_is_file (const char *remote_name); -#ifdef HAVE_LIBSOUP -OstreeFetcher * -_ostree_repo_remote_new_fetcher (OstreeRepo *self, - const char *remote_name, - GError **error); -#endif - OstreeGpgVerifyResult * _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, GBytes *signed_data, @@ -329,22 +318,4 @@ gboolean _ostree_repo_update_mtime (OstreeRepo *self, GError **error); -/* Load the summary from the cache if the provided .sig file is the same as the - cached version. */ -gboolean -_ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, - const char *remote, - GBytes *summary_sig, - GBytes **summary, - GCancellable *cancellable, - GError **error); - -gboolean -_ostree_repo_cache_summary (OstreeRepo *self, - const char *remote, - GBytes *summary, - GBytes *summary_sig, - GCancellable *cancellable, - GError **error); - G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 2f382856..99b730d8 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -22,13 +22,16 @@ #include "config.h" -#include "libglnx.h" #include "ostree.h" +#include "otutil.h" + +#ifdef HAVE_LIBSOUP + +#include "libglnx.h" #include "ostree-core-private.h" #include "ostree-repo-private.h" #include "ostree-repo-static-delta-private.h" #include "ostree-metalink.h" -#include "otutil.h" #include "ot-fs-utils.h" #include @@ -1778,7 +1781,9 @@ ostree_repo_pull_one_dir (OstreeRepo *self, progress, cancellable, error); } -gboolean +/* Load the summary from the cache if the provided .sig file is the same as the + cached version. */ +static gboolean _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, const char *remote, GBytes *summary_sig, @@ -1840,7 +1845,7 @@ _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, return ret; } -gboolean +static gboolean _ostree_repo_cache_summary (OstreeRepo *self, const char *remote, GBytes *summary, @@ -1880,7 +1885,287 @@ _ostree_repo_cache_summary (OstreeRepo *self, } -/* Documented in ostree-repo.c */ +static OstreeFetcher * +_ostree_repo_remote_new_fetcher (OstreeRepo *self, + const char *remote_name, + GError **error) +{ + OstreeFetcher *fetcher = NULL; + OstreeFetcherConfigFlags fetcher_flags = 0; + gboolean tls_permissive = FALSE; + gboolean success = FALSE; + + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + g_return_val_if_fail (remote_name != NULL, NULL); + + if (!ostree_repo_get_remote_boolean_option (self, remote_name, + "tls-permissive", FALSE, + &tls_permissive, error)) + goto out; + + if (tls_permissive) + fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE; + + fetcher = _ostree_fetcher_new (self->tmp_dir_fd, fetcher_flags); + + { + g_autofree char *tls_client_cert_path = NULL; + g_autofree char *tls_client_key_path = NULL; + + if (!ostree_repo_get_remote_option (self, remote_name, + "tls-client-cert-path", NULL, + &tls_client_cert_path, error)) + goto out; + if (!ostree_repo_get_remote_option (self, remote_name, + "tls-client-key-path", NULL, + &tls_client_key_path, error)) + goto out; + + if ((tls_client_cert_path != NULL) != (tls_client_key_path != NULL)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Remote \"%s\" must specify both " + "\"tls-client-cert-path\" and \"tls-client-key-path\"", + remote_name); + goto out; + } + else if (tls_client_cert_path != NULL) + { + g_autoptr(GTlsCertificate) client_cert = NULL; + + g_assert (tls_client_key_path != NULL); + + client_cert = g_tls_certificate_new_from_files (tls_client_cert_path, + tls_client_key_path, + error); + if (client_cert == NULL) + goto out; + + _ostree_fetcher_set_client_cert (fetcher, client_cert); + } + } + + { + g_autofree char *tls_ca_path = NULL; + + if (!ostree_repo_get_remote_option (self, remote_name, + "tls-ca-path", NULL, + &tls_ca_path, error)) + goto out; + + if (tls_ca_path != NULL) + { + g_autoptr(GTlsDatabase) db = NULL; + + db = g_tls_file_database_new (tls_ca_path, error); + if (db == NULL) + goto out; + + _ostree_fetcher_set_tls_database (fetcher, db); + } + } + + { + g_autofree char *http_proxy = NULL; + + if (!ostree_repo_get_remote_option (self, remote_name, + "proxy", NULL, + &http_proxy, error)) + goto out; + + if (http_proxy != NULL) + _ostree_fetcher_set_proxy (fetcher, http_proxy); + } + + success = TRUE; + +out: + if (!success) + g_clear_object (&fetcher); + + return fetcher; +} + +static gboolean +_ostree_preload_metadata_file (OstreeRepo *self, + OstreeFetcher *fetcher, + SoupURI *base_uri, + const char *filename, + gboolean is_metalink, + GBytes **out_bytes, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + + if (is_metalink) + { + glnx_unref_object OstreeMetalink *metalink = NULL; + GError *local_error = NULL; + + metalink = _ostree_metalink_new (fetcher, filename, + OSTREE_MAX_METADATA_SIZE, + base_uri); + + _ostree_metalink_request_sync (metalink, NULL, out_bytes, NULL, + cancellable, &local_error); + + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + { + g_clear_error (&local_error); + *out_bytes = NULL; + } + else if (local_error != NULL) + { + g_propagate_error (error, local_error); + goto out; + } + } + else + { + SoupURI *uri; + const char *base_path; + g_autofree char *path = NULL; + + base_path = soup_uri_get_path (base_uri); + path = g_build_filename (base_path, filename, NULL); + uri = soup_uri_new_with_base (base_uri, path); + + ret = _ostree_fetcher_request_uri_to_membuf (fetcher, uri, + FALSE, TRUE, + out_bytes, + OSTREE_MAX_METADATA_SIZE, + cancellable, error); + soup_uri_free (uri); + + if (!ret) + goto out; + } + + ret = TRUE; +out: + return ret; +} + +static gboolean +repo_remote_fetch_summary (OstreeRepo *self, + const char *name, + const char *metalink_url_string, + GVariant *options, + GBytes **out_summary, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error) +{ + glnx_unref_object OstreeFetcher *fetcher = NULL; + g_autoptr(GMainContext) mainctx = NULL; + gboolean ret = FALSE; + SoupURI *base_uri = NULL; + gboolean from_cache = FALSE; + g_autofree char *url_override = NULL; + + if (options) + (void) g_variant_lookup (options, "override-url", "&s", &url_override); + + mainctx = g_main_context_new (); + g_main_context_push_thread_default (mainctx); + + fetcher = _ostree_repo_remote_new_fetcher (self, name, error); + if (fetcher == NULL) + goto out; + + { + g_autofree char *url_string = NULL; + if (metalink_url_string) + url_string = g_strdup (metalink_url_string); + else if (url_override) + url_string = g_strdup (url_override); + else if (!ostree_repo_remote_get_url (self, name, &url_string, error)) + goto out; + + base_uri = soup_uri_new (url_string); + if (base_uri == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid URL '%s'", url_string); + goto out; + } + } + + if (!_ostree_preload_metadata_file (self, + fetcher, + base_uri, + "summary.sig", + metalink_url_string ? TRUE : FALSE, + out_signatures, + cancellable, + error)) + goto out; + + if (*out_signatures) + { + if (!_ostree_repo_load_cache_summary_if_same_sig (self, + name, + *out_signatures, + out_summary, + cancellable, + error)) + goto out; + } + + if (*out_summary) + from_cache = TRUE; + else + { + if (!_ostree_preload_metadata_file (self, + fetcher, + base_uri, + "summary", + metalink_url_string ? TRUE : FALSE, + out_summary, + cancellable, + error)) + goto out; + } + + if (!from_cache && *out_summary && *out_signatures) + { + g_autoptr(GError) temp_error = NULL; + + if (!_ostree_repo_cache_summary (self, + name, + *out_summary, + *out_signatures, + cancellable, + &temp_error)) + { + if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED)) + g_debug ("No permissions to save summary cache"); + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); + goto out; + } + } + } + + ret = TRUE; + + out: + if (mainctx) + g_main_context_pop_thread_default (mainctx); + if (base_uri != NULL) + soup_uri_free (base_uri); + return ret; +} + +/* ------------------------------------------------------------------------------------------ + * Below is the libsoup-invariant API; these should match + * the stub functions in the #else clause + * ------------------------------------------------------------------------------------------ + */ + +/* Documented below */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, const char *remote_name_or_baseurl, @@ -2593,3 +2878,147 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref); return ret; } + +/* Documented below */ +gboolean +ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, + const char *name, + GVariant *options, + GBytes **out_summary, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *metalink_url_string = NULL; + g_autoptr(GBytes) summary = NULL; + g_autoptr(GBytes) signatures = NULL; + gboolean ret = FALSE; + gboolean gpg_verify_summary; + + g_return_val_if_fail (OSTREE_REPO (self), FALSE); + g_return_val_if_fail (name != NULL, FALSE); + + if (!ostree_repo_get_remote_option (self, name, "metalink", NULL, + &metalink_url_string, error)) + goto out; + + if (!repo_remote_fetch_summary (self, + name, + metalink_url_string, + options, + &summary, + &signatures, + cancellable, + error)) + goto out; + + if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) + goto out; + + if (gpg_verify_summary && signatures == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)"); + goto out; + } + + /* Verify any summary signatures. */ + if (gpg_verify_summary && summary != NULL && signatures != NULL) + { + glnx_unref_object OstreeGpgVerifyResult *result = NULL; + + result = ostree_repo_verify_summary (self, + name, + summary, + signatures, + cancellable, + error); + if (!ostree_gpg_verify_result_require_valid_signature (result, error)) + goto out; + } + + if (out_summary != NULL) + *out_summary = g_steal_pointer (&summary); + + if (out_signatures != NULL) + *out_signatures = g_steal_pointer (&signatures); + + ret = TRUE; + +out: + return ret; +} + +#else /* HAVE_LIBSOUP */ + +/** + * ostree_repo_pull_with_options: + * @self: Repo + * @remote_name: Name of remote + * @options: A GVariant a{sv} with an extensible set of flags. + * @progress: (allow-none): Progress + * @cancellable: Cancellable + * @error: Error + * + * Like ostree_repo_pull(), but supports an extensible set of flags. + * The following are currently defined: + * + * * refs (as): Array of string refs + * * flags (i): An instance of #OstreeRepoPullFlags + * * subdir (s): Pull just this subdirectory + * * override-remote-name (s): If local, add this remote to refspec + * * gpg-verify (b): GPG verify commits + * * gpg-verify-summary (b): GPG verify summary + * * depth (i): How far in the history to traverse; default is 0, -1 means infinite + * * disable-static-deltas (b): Do not use static deltas + * * require-static-deltas (b): Require static deltas + * * override-commit-ids (as): Array of specific commit IDs to fetch for refs + * * dry-run (b): Only print information on what will be downloaded (requires static deltas) + * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options + */ +gboolean +ostree_repo_pull_with_options (OstreeRepo *self, + const char *remote_name, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error) +{ + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree was built without libsoup, and cannot fetch over HTTP"); + return FALSE; +} + +/** + * ostree_repo_remote_fetch_summary_with_options: + * @self: Self + * @name: name of a remote + * @options: (nullable): A GVariant a{sv} with an extensible set of flags + * @out_summary: (nullable): return location for raw summary data, or %NULL + * @out_signatures: (nullable): return location for raw summary signature + * data, or %NULL + * @cancellable: a #GCancellable + * @error: a #GError + * + * Like ostree_repo_remote_fetch_summary(), but supports an extensible set of flags. + * The following are currently defined: + * + * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options + * + * Returns: %TRUE on success, %FALSE on failure + */ +gboolean +ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, + const char *name, + GVariant *options, + GBytes **out_summary, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error) +{ + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree was built without libsoup, and cannot fetch over HTTP"); + return FALSE; +} + +#endif /* HAVE_LIBSOUP */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index fd02833a..5bda7bcc 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -39,10 +39,6 @@ #include "ot-fs-utils.h" #include "ostree-autocleanups.h" -#ifdef HAVE_LIBSOUP -#include "ostree-metalink.h" -#endif - #include #include @@ -509,109 +505,6 @@ ostree_repo_get_remote_boolean_option (OstreeRepo *self, return ret; } -#ifdef HAVE_LIBSOUP -OstreeFetcher * -_ostree_repo_remote_new_fetcher (OstreeRepo *self, - const char *remote_name, - GError **error) -{ - OstreeFetcher *fetcher = NULL; - OstreeFetcherConfigFlags fetcher_flags = 0; - gboolean tls_permissive = FALSE; - gboolean success = FALSE; - - g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); - g_return_val_if_fail (remote_name != NULL, NULL); - - if (!ostree_repo_get_remote_boolean_option (self, remote_name, - "tls-permissive", FALSE, - &tls_permissive, error)) - goto out; - - if (tls_permissive) - fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE; - - fetcher = _ostree_fetcher_new (self->tmp_dir_fd, fetcher_flags); - - { - g_autofree char *tls_client_cert_path = NULL; - g_autofree char *tls_client_key_path = NULL; - - if (!ostree_repo_get_remote_option (self, remote_name, - "tls-client-cert-path", NULL, - &tls_client_cert_path, error)) - goto out; - if (!ostree_repo_get_remote_option (self, remote_name, - "tls-client-key-path", NULL, - &tls_client_key_path, error)) - goto out; - - if ((tls_client_cert_path != NULL) != (tls_client_key_path != NULL)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Remote \"%s\" must specify both " - "\"tls-client-cert-path\" and \"tls-client-key-path\"", - remote_name); - goto out; - } - else if (tls_client_cert_path != NULL) - { - g_autoptr(GTlsCertificate) client_cert = NULL; - - g_assert (tls_client_key_path != NULL); - - client_cert = g_tls_certificate_new_from_files (tls_client_cert_path, - tls_client_key_path, - error); - if (client_cert == NULL) - goto out; - - _ostree_fetcher_set_client_cert (fetcher, client_cert); - } - } - - { - g_autofree char *tls_ca_path = NULL; - - if (!ostree_repo_get_remote_option (self, remote_name, - "tls-ca-path", NULL, - &tls_ca_path, error)) - goto out; - - if (tls_ca_path != NULL) - { - g_autoptr(GTlsDatabase) db = NULL; - - db = g_tls_file_database_new (tls_ca_path, error); - if (db == NULL) - goto out; - - _ostree_fetcher_set_tls_database (fetcher, db); - } - } - - { - g_autofree char *http_proxy = NULL; - - if (!ostree_repo_get_remote_option (self, remote_name, - "proxy", NULL, - &http_proxy, error)) - goto out; - - if (http_proxy != NULL) - _ostree_fetcher_set_proxy (fetcher, http_proxy); - } - - success = TRUE; - -out: - if (!success) - g_clear_object (&fetcher); - - return fetcher; -} -#endif - static void ostree_repo_finalize (GObject *object) { @@ -1842,181 +1735,6 @@ out: return ret; } -#ifdef HAVE_LIBSOUP -static gboolean -_ostree_preload_metadata_file (OstreeRepo *self, - OstreeFetcher *fetcher, - SoupURI *base_uri, - const char *filename, - gboolean is_metalink, - GBytes **out_bytes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - - if (is_metalink) - { - glnx_unref_object OstreeMetalink *metalink = NULL; - GError *local_error = NULL; - - metalink = _ostree_metalink_new (fetcher, filename, - OSTREE_MAX_METADATA_SIZE, - base_uri); - - _ostree_metalink_request_sync (metalink, NULL, out_bytes, NULL, - cancellable, &local_error); - - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&local_error); - *out_bytes = NULL; - } - else if (local_error != NULL) - { - g_propagate_error (error, local_error); - goto out; - } - } - else - { - SoupURI *uri; - const char *base_path; - g_autofree char *path = NULL; - - base_path = soup_uri_get_path (base_uri); - path = g_build_filename (base_path, filename, NULL); - uri = soup_uri_new_with_base (base_uri, path); - - ret = _ostree_fetcher_request_uri_to_membuf (fetcher, uri, - FALSE, TRUE, - out_bytes, - OSTREE_MAX_METADATA_SIZE, - cancellable, error); - soup_uri_free (uri); - - if (!ret) - goto out; - } - - ret = TRUE; -out: - return ret; -} - -static gboolean -repo_remote_fetch_summary (OstreeRepo *self, - const char *name, - const char *metalink_url_string, - GVariant *options, - GBytes **out_summary, - GBytes **out_signatures, - GCancellable *cancellable, - GError **error) -{ - glnx_unref_object OstreeFetcher *fetcher = NULL; - g_autoptr(GMainContext) mainctx = NULL; - gboolean ret = FALSE; - SoupURI *base_uri = NULL; - gboolean from_cache = FALSE; - g_autofree char *url_override = NULL; - - if (options) - (void) g_variant_lookup (options, "override-url", "&s", &url_override); - - mainctx = g_main_context_new (); - g_main_context_push_thread_default (mainctx); - - fetcher = _ostree_repo_remote_new_fetcher (self, name, error); - if (fetcher == NULL) - goto out; - - { - g_autofree char *url_string = NULL; - if (metalink_url_string) - url_string = g_strdup (metalink_url_string); - else if (url_override) - url_string = g_strdup (url_override); - else if (!ostree_repo_remote_get_url (self, name, &url_string, error)) - goto out; - - base_uri = soup_uri_new (url_string); - if (base_uri == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid URL '%s'", url_string); - goto out; - } - } - - if (!_ostree_preload_metadata_file (self, - fetcher, - base_uri, - "summary.sig", - metalink_url_string ? TRUE : FALSE, - out_signatures, - cancellable, - error)) - goto out; - - if (*out_signatures) - { - if (!_ostree_repo_load_cache_summary_if_same_sig (self, - name, - *out_signatures, - out_summary, - cancellable, - error)) - goto out; - } - - if (*out_summary) - from_cache = TRUE; - else - { - if (!_ostree_preload_metadata_file (self, - fetcher, - base_uri, - "summary", - metalink_url_string ? TRUE : FALSE, - out_summary, - cancellable, - error)) - goto out; - } - - if (!from_cache && *out_summary && *out_signatures) - { - g_autoptr(GError) temp_error = NULL; - - if (!_ostree_repo_cache_summary (self, - name, - *out_summary, - *out_signatures, - cancellable, - &temp_error)) - { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED)) - g_debug ("No permissions to save summary cache"); - else - { - g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; - } - } - } - - ret = TRUE; - - out: - if (mainctx) - g_main_context_pop_thread_default (mainctx); - if (base_uri != NULL) - soup_uri_free (base_uri); - return ret; -} -#endif - /** * ostree_repo_remote_fetch_summary: * @self: Self @@ -2057,99 +1775,6 @@ ostree_repo_remote_fetch_summary (OstreeRepo *self, error); } -/** - * ostree_repo_remote_fetch_summary_with_options: - * @self: Self - * @name: name of a remote - * @options: (nullable): A GVariant a{sv} with an extensible set of flags - * @out_summary: (nullable): return location for raw summary data, or %NULL - * @out_signatures: (nullable): return location for raw summary signature - * data, or %NULL - * @cancellable: a #GCancellable - * @error: a #GError - * - * Like ostree_repo_remote_fetch_summary(), but supports an extensible set of flags. - * The following are currently defined: - * - * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options - * - * Returns: %TRUE on success, %FALSE on failure - */ -gboolean -ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, - const char *name, - GVariant *options, - GBytes **out_summary, - GBytes **out_signatures, - GCancellable *cancellable, - GError **error) -{ -#ifdef HAVE_LIBSOUP - g_autofree char *metalink_url_string = NULL; - g_autoptr(GBytes) summary = NULL; - g_autoptr(GBytes) signatures = NULL; - gboolean ret = FALSE; - gboolean gpg_verify_summary; - - g_return_val_if_fail (OSTREE_REPO (self), FALSE); - g_return_val_if_fail (name != NULL, FALSE); - - if (!ostree_repo_get_remote_option (self, name, "metalink", NULL, - &metalink_url_string, error)) - goto out; - - if (!repo_remote_fetch_summary (self, - name, - metalink_url_string, - options, - &summary, - &signatures, - cancellable, - error)) - goto out; - - if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) - goto out; - - if (gpg_verify_summary && signatures == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)"); - goto out; - } - - /* Verify any summary signatures. */ - if (gpg_verify_summary && summary != NULL && signatures != NULL) - { - glnx_unref_object OstreeGpgVerifyResult *result = NULL; - - result = ostree_repo_verify_summary (self, - name, - summary, - signatures, - cancellable, - error); - if (!ostree_gpg_verify_result_require_valid_signature (result, error)) - goto out; - } - - if (out_summary != NULL) - *out_summary = g_steal_pointer (&summary); - - if (out_signatures != NULL) - *out_signatures = g_steal_pointer (&signatures); - - ret = TRUE; - -out: - return ret; -#else - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "This version of ostree was built without libsoup, and cannot fetch over HTTP"); - return FALSE; -#endif -} - static gboolean ostree_repo_mode_to_string (OstreeRepoMode mode, const char **out_mode, @@ -4143,44 +3768,6 @@ ostree_repo_pull_one_dir (OstreeRepo *self, return FALSE; } -/** - * ostree_repo_pull_with_options: - * @self: Repo - * @remote_name: Name of remote - * @options: A GVariant a{sv} with an extensible set of flags. - * @progress: (allow-none): Progress - * @cancellable: Cancellable - * @error: Error - * - * Like ostree_repo_pull(), but supports an extensible set of flags. - * The following are currently defined: - * - * * refs (as): Array of string refs - * * flags (i): An instance of #OstreeRepoPullFlags - * * subdir (s): Pull just this subdirectory - * * override-remote-name (s): If local, add this remote to refspec - * * gpg-verify (b): GPG verify commits - * * gpg-verify-summary (b): GPG verify summary - * * depth (i): How far in the history to traverse; default is 0, -1 means infinite - * * disable-static-deltas (b): Do not use static deltas - * * require-static-deltas (b): Require static deltas - * * override-commit-ids (as): Array of specific commit IDs to fetch for refs - * * dry-run (b): Only print information on what will be downloaded (requires static deltas) - * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options - */ -gboolean -ostree_repo_pull_with_options (OstreeRepo *self, - const char *remote_name, - GVariant *options, - OstreeAsyncProgress *progress, - GCancellable *cancellable, - GError **error) -{ - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "This version of ostree was built without libsoup, and cannot fetch over HTTP"); - return FALSE; -} - #endif /** @@ -4214,7 +3801,6 @@ _formatted_time_remaining_from_seconds (guint64 seconds_remaining) return g_string_free (description, FALSE); } - /** * ostree_repo_pull_default_console_progress_changed: * @progress: Async progress From 693b4ec4bed733421a5bd32738265e291ef92d8f Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Wed, 1 Jun 2016 17:47:08 -0600 Subject: [PATCH 096/128] pull: Move pull and pull_onedir back into ostree-repo.c They call pull_with_options, which will fail anyway. Closes: #327 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 43 -------------------------------- src/libostree/ostree-repo.c | 27 +++++++++++++------- 2 files changed, 18 insertions(+), 52 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 99b730d8..66495c17 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1738,49 +1738,6 @@ validate_variant_is_csum (GVariant *csum, return ret; } -/* documented in ostree-repo.c */ -gboolean -ostree_repo_pull (OstreeRepo *self, - const char *remote_name, - char **refs_to_fetch, - OstreeRepoPullFlags flags, - OstreeAsyncProgress *progress, - GCancellable *cancellable, - GError **error) -{ - return ostree_repo_pull_one_dir (self, remote_name, NULL, refs_to_fetch, flags, progress, cancellable, error); -} - -/* Documented in ostree-repo.c */ -gboolean -ostree_repo_pull_one_dir (OstreeRepo *self, - const char *remote_name, - const char *dir_to_pull, - char **refs_to_fetch, - OstreeRepoPullFlags flags, - OstreeAsyncProgress *progress, - GCancellable *cancellable, - GError **error) -{ - GVariantBuilder builder; - g_autoptr(GVariant) options = NULL; - - g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); - - if (dir_to_pull) - g_variant_builder_add (&builder, "{s@v}", "subdir", - g_variant_new_variant (g_variant_new_string (dir_to_pull))); - g_variant_builder_add (&builder, "{s@v}", "flags", - g_variant_new_variant (g_variant_new_int32 (flags))); - if (refs_to_fetch) - g_variant_builder_add (&builder, "{s@v}", "refs", - g_variant_new_variant (g_variant_new_strv ((const char *const*) refs_to_fetch, -1))); - - options = g_variant_ref_sink (g_variant_builder_end (&builder)); - return ostree_repo_pull_with_options (self, remote_name, options, - progress, cancellable, error); -} - /* Load the summary from the cache if the provided .sig file is the same as the cached version. */ static gboolean diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 5bda7bcc..86d36cd0 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3697,7 +3697,6 @@ ostree_repo_read_commit (OstreeRepo *self, return ret; } -#ifndef HAVE_LIBSOUP /** * ostree_repo_pull: * @self: Repo @@ -3734,9 +3733,7 @@ ostree_repo_pull (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "This version of ostree was built without libsoup, and cannot fetch over HTTP"); - return FALSE; + return ostree_repo_pull_one_dir (self, remote_name, NULL, refs_to_fetch, flags, progress, cancellable, error); } /** @@ -3763,12 +3760,24 @@ ostree_repo_pull_one_dir (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "This version of ostree was built without libsoup, and cannot fetch over HTTP"); - return FALSE; -} + GVariantBuilder builder; + g_autoptr(GVariant) options = NULL; -#endif + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); + + if (dir_to_pull) + g_variant_builder_add (&builder, "{s@v}", "subdir", + g_variant_new_variant (g_variant_new_string (dir_to_pull))); + g_variant_builder_add (&builder, "{s@v}", "flags", + g_variant_new_variant (g_variant_new_int32 (flags))); + if (refs_to_fetch) + g_variant_builder_add (&builder, "{s@v}", "refs", + g_variant_new_variant (g_variant_new_strv ((const char *const*) refs_to_fetch, -1))); + + options = g_variant_ref_sink (g_variant_builder_end (&builder)); + return ostree_repo_pull_with_options (self, remote_name, options, + progress, cancellable, error); +} /** * _formatted_time_remaining_from_seconds From 1b88dc7f906cf2fba75c6278de523a562f0a599a Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Sat, 4 Jun 2016 16:16:05 -0600 Subject: [PATCH 097/128] docs: Get API docs working again This changes around a few things that didn't work for me: * Section names seem to be ostree-* instead of libostree-* * Also XML files are ostree-* (they didn't show up at all) - gtk-doc doesn't seem to parse const _OSTREE_PUBLIC correctly * pull documentation is now on the actual functions rather than stubs * Update gitignore with some more files And there some changes to make gtk-doc give fewer warnings (not finished) Closes: #327 Approved by: cgwalters --- apidoc/.gitignore | 2 + apidoc/ostree-docs.xml | 30 ++++---- apidoc/ostree-sections.txt | 53 ++++++++++---- src/libostree/ostree-async-progress.c | 2 +- src/libostree/ostree-core-private.h | 5 +- src/libostree/ostree-core.c | 2 +- src/libostree/ostree-core.h | 13 ++-- src/libostree/ostree-deployment-private.h | 32 ++++++--- src/libostree/ostree-diff.c | 2 + src/libostree/ostree-diff.h | 6 ++ src/libostree/ostree-gpg-verify-result.c | 2 +- src/libostree/ostree-lzma-compressor.c | 2 +- src/libostree/ostree-lzma-decompressor.c | 8 +++ src/libostree/ostree-mutable-tree.c | 2 +- src/libostree/ostree-mutable-tree.h | 3 +- src/libostree/ostree-repo-pull.c | 88 +++++++++++------------ src/libostree/ostree-repo.c | 8 +-- src/libostree/ostree-repo.h | 30 +++++--- src/libostree/ostree-sepolicy.c | 2 +- src/libostree/ostree-sysroot-private.h | 4 ++ src/libostree/ostree-sysroot-upgrader.c | 2 +- src/libostree/ostree-sysroot.c | 2 +- 22 files changed, 186 insertions(+), 114 deletions(-) diff --git a/apidoc/.gitignore b/apidoc/.gitignore index 77cacd4d..80d2c112 100644 --- a/apidoc/.gitignore +++ b/apidoc/.gitignore @@ -32,6 +32,7 @@ /ostree.args /ostree.hierarchy /ostree.interfaces +/ostree-overrides.txt /ostree.pdf /ostree.prerequisites /ostree.signals @@ -48,4 +49,5 @@ /tmpl/*.bak /tmpl/ostree-unused.sgml /xml +/version.xml _libs diff --git a/apidoc/ostree-docs.xml b/apidoc/ostree-docs.xml index c5ea28e6..8721ffa8 100644 --- a/apidoc/ostree-docs.xml +++ b/apidoc/ostree-docs.xml @@ -13,25 +13,23 @@ API Reference - - - - - - - - - - - - - - - + + + + + + + + + + + + + + API Index - diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 3003da2b..6dca9ecc 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -1,5 +1,5 @@
-libostree-async-progress +ostree-async-progress OstreeAsyncProgress ostree_async_progress_new ostree_async_progress_new_and_connect @@ -22,7 +22,7 @@ ostree_async_progress_get_type
-libostree-bootconfig-parser +ostree-bootconfig-parser OstreeBootconfigParser ostree_bootconfig_parser_new ostree_bootconfig_parser_clone @@ -40,7 +40,7 @@ ostree_bootconfig_parser_get_type
-libostree-chain-input-stream +ostree-chain-input-stream OstreeChainInputStream ostree_chain_input_stream_new @@ -56,7 +56,7 @@ ostree_chain_input_stream_get_type
-libostree-checksum-input-stream +ostree-checksum-input-stream OstreeChecksumInputStream ostree_checksum_input_stream_new @@ -72,7 +72,7 @@ ostree_checksum_input_stream_get_type
-libostree-core +ostree-core OSTREE_MAX_METADATA_SIZE OSTREE_MAX_METADATA_WARN_SIZE OSTREE_MAX_RECURSION @@ -133,7 +133,7 @@ ostree_commit_get_timestamp
-libostree-deployment +ostree-deployment OstreeDeployment ostree_deployment_hash ostree_deployment_equal @@ -162,7 +162,7 @@ ostree_deployment_get_type
-libostree-diff +ostree-diff OstreeDiffFlags OstreeDiffItem ostree_diff_item_ref @@ -174,7 +174,7 @@ ostree_diff_item_get_type
-libostree-gpg-verify-result +ostree-gpg-verify-result OstreeGpgVerifyResult OstreeGpgSignatureAttr ostree_gpg_verify_result_count_all @@ -193,8 +193,33 @@ OSTREE_TYPE_GPG_VERIFY_RESULT ostree_gpg_verify_result_get_type
+ostree-lzma-compressor + +OSTREE_IS_LZMA_COMPRESSOR +OSTREE_IS_LZMA_COMPRESSOR_CLASS +OSTREE_LZMA_COMPRESSOR +OSTREE_LZMA_COMPRESSOR_CLASS +OSTREE_LZMA_COMPRESSOR_GET_CLASS +OSTREE_TYPE_LZMA_COMPRESSOR +OstreeLzmaCompressor +OstreeLzmaCompressorClass + +
-libostree-mutable-tree +ostree-lzma-decompressor + +OSTREE_IS_LZMA_DECOMPRESSOR +OSTREE_IS_LZMA_DECOMPRESSOR_CLASS +OSTREE_LZMA_DECOMPRESSOR +OSTREE_LZMA_DECOMPRESSOR_CLASS +OSTREE_LZMA_DECOMPRESSOR_GET_CLASS +OSTREE_TYPE_LZMA_DECOMPRESSOR +OstreeLzmaDecompressor +OstreeLzmaDecompressorClass +
+ +
+ostree-mutable-tree OstreeMutableTree ostree_mutable_tree_new ostree_mutable_tree_set_metadata_checksum @@ -220,7 +245,7 @@ ostree_mutable_tree_get_type
-libostree-repo +ostree-repo OstreeRepo OstreeRepoMode ostree_repo_mode_from_string @@ -362,7 +387,7 @@ ostree_repo_transaction_stats_get_type
-libostree-repo-file +ostree-repo-file OstreeRepoFile ostree_repo_file_ensure_resolved ostree_repo_file_get_xattrs @@ -388,7 +413,7 @@ ostree_repo_file_get_type
-libostree-sepolicy +ostree-sepolicy OstreeSePolicy ostree_sepolicy_new ostree_sepolicy_get_path @@ -407,7 +432,7 @@ ostree_sepolicy_get_type
-libostree-sysroot +ostree-sysroot OstreeSysroot ostree_sysroot_new ostree_sysroot_new_default @@ -451,7 +476,7 @@ ostree_sysroot_get_type
-libostree-sysroot-upgrader +ostree-sysroot-upgrader OstreeSysrootUpgrader ostree_sysroot_upgrader_new ostree_sysroot_upgrader_new_for_os diff --git a/src/libostree/ostree-async-progress.c b/src/libostree/ostree-async-progress.c index 59656fc9..0851fd86 100644 --- a/src/libostree/ostree-async-progress.c +++ b/src/libostree/ostree-async-progress.c @@ -23,7 +23,7 @@ #include "ostree-async-progress.h" /** - * SECTION:libostree-async-progress + * SECTION:ostree-async-progress * @title: Progress notification system for asynchronous operations * @short_description: Values representing progress * diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index a92aa4c7..91d52f1b 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -83,7 +83,7 @@ _ostree_make_temporary_symlink_at (int tmp_dirfd, GFileInfo * _ostree_header_gfile_info_new (mode_t mode, uid_t uid, gid_t gid); -/* XX + / + checksum-2 + . + extension, but let's just use 256 for a +/* XX/checksum-2.extension, but let's just use 256 for a * bit of overkill. */ #define _OSTREE_LOOSE_PATH_MAX (256) @@ -112,8 +112,7 @@ _ostree_get_relative_static_delta_part_path (const char *from, const char *to, guint i); -static inline char * -_ostree_get_commitpartial_path (const char *checksum) +static inline char * _ostree_get_commitpartial_path (const char *checksum) { return g_strconcat ("state/", checksum, ".commitpartial", NULL); } diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 135d15d4..8e8424fc 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -47,7 +47,7 @@ zlib_file_header_parse (GVariant *metadata, GError **error); /** - * SECTION:libostree-core + * SECTION:ostree-core * @title: Core repository-independent functions * @short_description: Create, validate, and convert core data types * diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 4d788b20..4a0e60e9 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -59,7 +59,7 @@ G_BEGIN_DECLS * @OSTREE_OBJECT_TYPE_DIR_TREE: List of children (trees or files), and metadata * @OSTREE_OBJECT_TYPE_DIR_META: Directory metadata * @OSTREE_OBJECT_TYPE_COMMIT: Toplevel object, refers to tree and dirmeta for root - * @OSTREE_OBJECT_TYPE_COMMIT_TOMBSTONE: Toplevel object, refers to a deleted commit + * @OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT: Toplevel object, refers to a deleted commit * * Enumeration for core object types; %OSTREE_OBJECT_TYPE_FILE is for * content, the other types are metadata. @@ -153,9 +153,9 @@ typedef enum { /** * OstreeRepoMode: - * @OSTREE_REPO_MODE_BARE: Files are stored as themselves; can only be written as root + * @OSTREE_REPO_MODE_BARE: Files are stored as themselves; checkouts are hardlinks; can only be written as root * @OSTREE_REPO_MODE_ARCHIVE_Z2: Files are compressed, should be owned by non-root. Can be served via HTTP - * @OSTREE_REPO_MODE_BARE_USER: Files are stored as themselves, except ownership; can be written by user + * @OSTREE_REPO_MODE_BARE_USER: Files are stored as themselves, except ownership; can be written by user. Hardlinks work only in user checkouts. * * See the documentation of #OstreeRepo for more information about the * possible modes. @@ -166,8 +166,8 @@ typedef enum { OSTREE_REPO_MODE_BARE_USER } OstreeRepoMode; -const _OSTREE_PUBLIC -GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype); +_OSTREE_PUBLIC +const GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype); _OSTREE_PUBLIC gboolean ostree_validate_checksum_string (const char *sha256, @@ -242,7 +242,8 @@ void ostree_object_from_string (const char *str, gchar **out_checksum, OstreeObjectType *out_objtype); -_OSTREE_PUBLIC gboolean +_OSTREE_PUBLIC +gboolean ostree_content_stream_parse (gboolean compressed, GInputStream *input, guint64 input_length, diff --git a/src/libostree/ostree-deployment-private.h b/src/libostree/ostree-deployment-private.h index 856a3987..01fdf1bf 100644 --- a/src/libostree/ostree-deployment-private.h +++ b/src/libostree/ostree-deployment-private.h @@ -24,19 +24,33 @@ G_BEGIN_DECLS + +/** + * OstreeDeployment: + * @parent_instance: + * @index: Global offset + * @osname: + * @csum: OSTree checksum of tree + * @deployserial: How many times this particular csum appears in deployment list + * @bootcsum: Checksum of kernel+initramfs + * @bootserial: An integer assigned to this tree per its ${bootcsum} + * @bootconfig: Bootloader configuration + * @origin: How to construct an upgraded version of this tree + * @unlocked: The unlocked state + */ struct _OstreeDeployment { GObject parent_instance; - int index; /* Global offset */ - char *osname; /* osname */ - char *csum; /* OSTree checksum of tree */ - int deployserial; /* How many times this particular csum appears in deployment list */ - char *bootcsum; /* Checksum of kernel+initramfs */ - int bootserial; /* An integer assigned to this tree per its ${bootcsum} */ - OstreeBootconfigParser *bootconfig; /* Bootloader configuration */ - GKeyFile *origin; /* How to construct an upgraded version of this tree */ - OstreeDeploymentUnlockedState unlocked; /* The unlocked state */ + int index; + char *osname; + char *csum; + int deployserial; + char *bootcsum; + int bootserial; + OstreeBootconfigParser *bootconfig; + GKeyFile *origin; + OstreeDeploymentUnlockedState unlocked; }; void _ostree_deployment_set_bootcsum (OstreeDeployment *self, const char *bootcsum); diff --git a/src/libostree/ostree-diff.c b/src/libostree/ostree-diff.c index 84961028..69a67b76 100644 --- a/src/libostree/ostree-diff.c +++ b/src/libostree/ostree-diff.c @@ -210,6 +210,8 @@ diff_add_dir_recurse (GFile *d, * @modified: (element-type OstreeDiffItem): Modified files * @removed: (element-type Gio.File): Removed files * @added: (element-type Gio.File): Added files + * @cancellable: Cancellable + * @error: Error * * Compute the difference between directory @a and @b as 3 separate * sets of #OstreeDiffItem in @modified, @removed, and @added. diff --git a/src/libostree/ostree-diff.h b/src/libostree/ostree-diff.h index f4db23ef..781bf768 100644 --- a/src/libostree/ostree-diff.h +++ b/src/libostree/ostree-diff.h @@ -27,11 +27,17 @@ G_BEGIN_DECLS +/** + * OstreeDiffFlags: + */ typedef enum { OSTREE_DIFF_FLAGS_NONE = 0, OSTREE_DIFF_FLAGS_IGNORE_XATTRS = (1 << 0) } OstreeDiffFlags; +/** + * OstreeDiffItem: + */ typedef struct _OstreeDiffItem OstreeDiffItem; struct _OstreeDiffItem { diff --git a/src/libostree/ostree-gpg-verify-result.c b/src/libostree/ostree-gpg-verify-result.c index d72856c9..fa4614d1 100644 --- a/src/libostree/ostree-gpg-verify-result.c +++ b/src/libostree/ostree-gpg-verify-result.c @@ -27,7 +27,7 @@ #include "ostree-gpg-verify-result-private.h" /** - * SECTION: libostree-gpg-verify-result + * SECTION: ostree-gpg-verify-result * @title: GPG signature verification results * @short_description: Inspect detached GPG signatures * diff --git a/src/libostree/ostree-lzma-compressor.c b/src/libostree/ostree-lzma-compressor.c index 1ec03c41..8d9dcbc9 100644 --- a/src/libostree/ostree-lzma-compressor.c +++ b/src/libostree/ostree-lzma-compressor.c @@ -32,7 +32,7 @@ enum { /** * SECTION:ostree-lzma-compressor - * @short_description: LZMA compressor + * @title: LZMA compressor * * An implementation of #GConverter that compresses data using * LZMA. diff --git a/src/libostree/ostree-lzma-decompressor.c b/src/libostree/ostree-lzma-decompressor.c index b46e8fb2..6376e389 100644 --- a/src/libostree/ostree-lzma-decompressor.c +++ b/src/libostree/ostree-lzma-decompressor.c @@ -29,6 +29,14 @@ enum { PROP_0, }; +/** + * SECTION:ostree-lzma-decompressor + * @title: LZMA decompressor + * + * An implementation of #GConverter that decompresses data using + * LZMA. + */ + static void _ostree_lzma_decompressor_iface_init (GConverterIface *iface); struct _OstreeLzmaDecompressor diff --git a/src/libostree/ostree-mutable-tree.c b/src/libostree/ostree-mutable-tree.c index d0f21f37..5540cc7c 100644 --- a/src/libostree/ostree-mutable-tree.c +++ b/src/libostree/ostree-mutable-tree.c @@ -27,7 +27,7 @@ #include "ostree-core.h" /** - * SECTION:libostree-mutable-tree + * SECTION:ostree-mutable-tree * @title: In-memory modifiable filesystem tree * @short_description: Modifiable filesystem tree * diff --git a/src/libostree/ostree-mutable-tree.h b/src/libostree/ostree-mutable-tree.h index 30425d8d..1b642c42 100644 --- a/src/libostree/ostree-mutable-tree.h +++ b/src/libostree/ostree-mutable-tree.h @@ -84,7 +84,8 @@ gboolean ostree_mutable_tree_lookup (OstreeMutableTree *self, OstreeMutableTree **out_subdir, GError **error); -_OSTREE_PUBLIC gboolean +_OSTREE_PUBLIC +gboolean ostree_mutable_tree_ensure_parent_dirs (OstreeMutableTree *self, GPtrArray *split_path, const char *metadata_checksum, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 66495c17..cbd225bf 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2122,7 +2122,31 @@ repo_remote_fetch_summary (OstreeRepo *self, * ------------------------------------------------------------------------------------------ */ -/* Documented below */ +/** + * ostree_repo_pull_with_options: + * @self: Repo + * @remote_name: Name of remote + * @options: A GVariant a{sv} with an extensible set of flags. + * @progress: (allow-none): Progress + * @cancellable: Cancellable + * @error: Error + * + * Like ostree_repo_pull(), but supports an extensible set of flags. + * The following are currently defined: + * + * * refs (as): Array of string refs + * * flags (i): An instance of #OstreeRepoPullFlags + * * subdir (s): Pull just this subdirectory + * * override-remote-name (s): If local, add this remote to refspec + * * gpg-verify (b): GPG verify commits + * * gpg-verify-summary (b): GPG verify summary + * * depth (i): How far in the history to traverse; default is 0, -1 means infinite + * * disable-static-deltas (b): Do not use static deltas + * * require-static-deltas (b): Require static deltas + * * override-commit-ids (as): Array of specific commit IDs to fetch for refs + * * dry-run (b): Only print information on what will be downloaded (requires static deltas) + * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options + */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, const char *remote_name_or_baseurl, @@ -2836,7 +2860,24 @@ ostree_repo_pull_with_options (OstreeRepo *self, return ret; } -/* Documented below */ +/** + * ostree_repo_remote_fetch_summary_with_options: + * @self: Self + * @name: name of a remote + * @options: (nullable): A GVariant a{sv} with an extensible set of flags + * @out_summary: (nullable): return location for raw summary data, or %NULL + * @out_signatures: (nullable): return location for raw summary signature + * data, or %NULL + * @cancellable: a #GCancellable + * @error: a #GError + * + * Like ostree_repo_remote_fetch_summary(), but supports an extensible set of flags. + * The following are currently defined: + * + * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options + * + * Returns: %TRUE on success, %FALSE on failure + */ gboolean ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, const char *name, @@ -2908,31 +2949,6 @@ out: #else /* HAVE_LIBSOUP */ -/** - * ostree_repo_pull_with_options: - * @self: Repo - * @remote_name: Name of remote - * @options: A GVariant a{sv} with an extensible set of flags. - * @progress: (allow-none): Progress - * @cancellable: Cancellable - * @error: Error - * - * Like ostree_repo_pull(), but supports an extensible set of flags. - * The following are currently defined: - * - * * refs (as): Array of string refs - * * flags (i): An instance of #OstreeRepoPullFlags - * * subdir (s): Pull just this subdirectory - * * override-remote-name (s): If local, add this remote to refspec - * * gpg-verify (b): GPG verify commits - * * gpg-verify-summary (b): GPG verify summary - * * depth (i): How far in the history to traverse; default is 0, -1 means infinite - * * disable-static-deltas (b): Do not use static deltas - * * require-static-deltas (b): Require static deltas - * * override-commit-ids (as): Array of specific commit IDs to fetch for refs - * * dry-run (b): Only print information on what will be downloaded (requires static deltas) - * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options - */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, const char *remote_name, @@ -2946,24 +2962,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, return FALSE; } -/** - * ostree_repo_remote_fetch_summary_with_options: - * @self: Self - * @name: name of a remote - * @options: (nullable): A GVariant a{sv} with an extensible set of flags - * @out_summary: (nullable): return location for raw summary data, or %NULL - * @out_signatures: (nullable): return location for raw summary signature - * data, or %NULL - * @cancellable: a #GCancellable - * @error: a #GError - * - * Like ostree_repo_remote_fetch_summary(), but supports an extensible set of flags. - * The following are currently defined: - * - * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options - * - * Returns: %TRUE on success, %FALSE on failure - */ gboolean ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, const char *name, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 86d36cd0..0cff64e9 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -43,7 +43,7 @@ #include /** - * SECTION:libostree-repo + * SECTION:ostree-repo * @title: Content-addressed object store * @short_description: A git-like storage system for operating system binaries * @@ -2309,6 +2309,8 @@ ostree_repo_set_disable_fsync (OstreeRepo *self, * @self: An #OstreeRepo * @dfd: directory fd * @path: subpath in @dfd + * @cancellable: a #GCancellable + * @error: a #GError * * Set a custom location for the cache directory used for e.g. * per-remote summary caches. Setting this manually is useful when @@ -4183,9 +4185,7 @@ out: /** * ostree_repo_sign_delta: - * @self: Self - * @from_commit: SHA256 of starting commit to sign, or %NULL - * @to_commit: SHA256 of target commit to sign + * * This function is deprecated, sign the summary file instead. * Add a GPG signature to a static delta. */ diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index b81ccc93..ce280ee6 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -115,6 +115,10 @@ gboolean ostree_repo_remote_delete (OstreeRepo *self, GCancellable *cancellable, GError **error); +/** + * OstreeRepoRemoteChange: + * The remote change operation. + */ typedef enum { OSTREE_REPO_REMOTE_CHANGE_ADD, OSTREE_REPO_REMOTE_CHANGE_ADD_IF_NOT_EXISTS, @@ -223,6 +227,10 @@ gboolean ostree_repo_write_config (OstreeRepo *self, * were written to the repository in this transaction. * @content_bytes_written: The amount of data added to the repository, * in bytes, counting only content objects. + * @padding1: reserved + * @padding2: reserved + * @padding3: reserved + * @padding4: reserved * * A list of statistics for each transaction that may be * interesting for reporting purposes. @@ -699,7 +707,8 @@ typedef enum { OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1 } OstreeRepoCheckoutOverwriteMode; -_OSTREE_PUBLIC gboolean +_OSTREE_PUBLIC +gboolean ostree_repo_checkout_tree (OstreeRepo *self, OstreeRepoCheckoutMode mode, OstreeRepoCheckoutOverwriteMode overwrite_mode, @@ -866,14 +875,16 @@ typedef enum { OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE = (1 << 0) } OstreeRepoCommitTraverseFlags; -_OSTREE_PUBLIC gboolean +_OSTREE_PUBLIC +gboolean ostree_repo_commit_traverse_iter_init_commit (OstreeRepoCommitTraverseIter *iter, OstreeRepo *repo, GVariant *commit, OstreeRepoCommitTraverseFlags flags, GError **error); -_OSTREE_PUBLIC gboolean +_OSTREE_PUBLIC +gboolean ostree_repo_commit_traverse_iter_init_dirtree (OstreeRepoCommitTraverseIter *iter, OstreeRepo *repo, GVariant *dirtree, @@ -903,8 +914,8 @@ void ostree_repo_commit_traverse_iter_get_dir (OstreeRepoCommitTraverseIter *ite char **out_content_checksum, char **out_meta_checksum); -_OSTREE_PUBLIC void -ostree_repo_commit_traverse_iter_clear (OstreeRepoCommitTraverseIter *iter); +_OSTREE_PUBLIC +void ostree_repo_commit_traverse_iter_clear (OstreeRepoCommitTraverseIter *iter); _OSTREE_PUBLIC void ostree_repo_commit_traverse_iter_cleanup (void *p); @@ -923,7 +934,8 @@ typedef enum { OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY } OstreeRepoPruneFlags; -_OSTREE_PUBLIC gboolean +_OSTREE_PUBLIC +gboolean ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit, GCancellable *cancellable, GError **error); @@ -961,7 +973,8 @@ gboolean ostree_repo_pull (OstreeRepo *self, GCancellable *cancellable, GError **error); -_OSTREE_PUBLIC gboolean +_OSTREE_PUBLIC +gboolean ostree_repo_pull_one_dir (OstreeRepo *self, const char *remote_name, const char *dir_to_pull, @@ -1000,7 +1013,8 @@ gboolean ostree_repo_sign_delta (OstreeRepo *self, GCancellable *cancellable, GError **error); -_OSTREE_PUBLIC gboolean +_OSTREE_PUBLIC +gboolean ostree_repo_add_gpg_signature_summary (OstreeRepo *self, const gchar **key_id, const gchar *homedir, diff --git a/src/libostree/ostree-sepolicy.c b/src/libostree/ostree-sepolicy.c index 3b1a391b..8e49428f 100644 --- a/src/libostree/ostree-sepolicy.c +++ b/src/libostree/ostree-sepolicy.c @@ -32,7 +32,7 @@ #include "ostree-bootloader-syslinux.h" /** - * SECTION:libostree-sepolicy + * SECTION:ostree-sepolicy * @title: SELinux policy management * @short_description: Read SELinux policy and manage filesystem labels * diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index d210a36f..1fa8e83c 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -34,6 +34,10 @@ typedef enum { } OstreeSysrootDebugFlags; +/** + * OstreeSysroot: + * Internal struct + */ struct OstreeSysroot { GObject parent; diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index 5c8f9d1d..92b8dc83 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -25,7 +25,7 @@ #include "ostree-sysroot-upgrader.h" /** - * SECTION:libostree-sysroot-upgrader + * SECTION:ostree-sysroot-upgrader * @title: Simple upgrade class * @short_description: Upgrade OSTree systems * diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 8949f8f5..7cc4c9f5 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -37,7 +37,7 @@ find_booted_deployment (OstreeSysroot *self, GError **error); /** - * SECTION:libostree-sysroot + * SECTION:ostree-sysroot * @title: Root partition mount point * @short_description: Manage physical root filesystem * From 04ca15cb0e860a928bd342652d0c4ba0789a987b Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Fri, 3 Jun 2016 14:14:49 -0600 Subject: [PATCH 098/128] related-projects: Update with more information and projects In particular, NixOS has changed somewhat, and Conda is worth looking at. Also it seems reasonable to mention rpm-ostree / Gnome Continuous. Closes: #331 Approved by: cgwalters --- docs/manual/related-projects.md | 110 ++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 26 deletions(-) diff --git a/docs/manual/related-projects.md b/docs/manual/related-projects.md index 896c7655..bb0258e7 100644 --- a/docs/manual/related-projects.md +++ b/docs/manual/related-projects.md @@ -78,6 +78,10 @@ available. All of the above also applies if one replaces "BTRFS" with "LVM snapshots" except for the reflinks. +OSTree supports using "bare-user" repositories, which do not require +root to use. Using a filesystem-level layer without root is more +difficult and would likely require a setuid helper or privileged service. + Finally, see the next portion around ChromiumOS for why a hybrid but integrated package/image system improves on this. @@ -147,37 +151,54 @@ See [this comment](http://blog.verbum.org/2013/08/26/ostree-v2013-6-released/#comment-1169) for a comparison. -## NixOS +## NixOS / Nix -See [NixOS](http://nixos.org/). It was a very influential project for -OSTree. NixOS and OSTree both support the idea of independent "roots" -that are bootable. +See [NixOS](http://nixos.org/). It was a very influential project for OSTree. +NixOS and OSTree both support the idea of independent "roots" that are bootable. -In NixOS, the entire system is based on checksums of package inputs -(build dependencies) - see [Nix store](http://nixos.org/nix/manual/#chap-package-management/). A both -positive and negative of the Nix model is that a change in the build -dependencies (e.g. being built with a newer gcc), requires a cascading -rebuild of everything. +In NixOS, files in a package are accessed by a path depending on the checksums +of package inputs (build dependencies) - see +[Nix store](http://nixos.org/nix/manual/#chap-package-management/). +However, OSTree uses a commit/deploy model - it isn't tied to any particular +directory layout, and you can put whatever data you want inside an OSTree, for +example the standard FHS layout. A both positive and negative of the Nix model +is that a change in the build dependencies (e.g. being built with a newer gcc), +requires a cascading rebuild of everything. It's good because it makes it easy +to do massive system-wide changes such as gcc upgrades, and allows installing +multiple versions of packages at once. However, a security update to e.g. glibc +forces a rebuild of everything from scratch, and so Nix is not practical at +scale. OSTree supports using a build system that just rebuilds individual +components (packages) as they change, without forcing a rebuild of their +dependencies. -In OSTree, the checksums are of object *content* (including extended -attributes). This means that any data that's identical is -transparently, automatically shared on disk. It's possible to ask the -Nix store to deduplicate, (via hard links and immutable bit), but this -is significantly less efficient than the OSTree approach. The Nix use -of the ext immutable bit is racy, since it has to be briefly removed -to make a hard link. +Nix automatically detects runtime package dependencies by scanning content for +hashes. OSTree only supports only system-level images, and doesn't do dependency +management. Nix can store arbitrary files, using nix-store --add, but, more +commonly, paths are added as the result of running a derivation file generated +using the Nix language. OSTree is build-system agnostic; filesystem trees are +committed using a simple C API, and this is the only way to commit files. -At the lowest level, OSTree is just "git for binaries" - it isn't tied -strongly to any particular build system. You can put whatever data you -want inside an OSTree repository, built however you like. So for -example, while one could make a build system that did the "purely -functional" approach of Nix, it also works to have a build system that -just rebuilds individual components (packages) as they change, without -forcing a rebuild of their dependencies. +OSTree automatically shares the storage of identical data using hard links into +a content-addressed store. Nix can deduplicate using hard links as well, using +the auto-optimise-store option, but this is not on by default, and Nix does not +guarantee that all of its files are in the content-addressed store. OSTree +provides a git-like command line interface for browsing the content-addressed +store, while Nix does not have this functionality. -The author of OSTree believes that while Nix has some good ideas, -forcing a rebuild of everything for a security update to e.g. glibc is -not practical at scale. +Nix used to use the immutable bit to prevent modifications to /nix/store, but +now it uses a read-only bind mount. The bind mount can be privately remounted, +allowing per-process privileged write access. OSTree uses the immutable +bit on the root of the deployment, and mounts /usr as read-only. + +NixOS supports switching OS images on-the-fly, by maintaining both booted-system +and current-system roots. It is not clear how well this approach works. OSTree +currently requries a reboot to switch images. + +Finally, NixOS supports installing user-specific packages from trusted +repositories without requiring root, using a trusted daemon. +[Flatpak](https://lwn.net/Articles/687909/), based on OSTree, similarly has a +policykit-based system helper that allows you to authenticate via polkit to +install into the system repository. ## Solaris IPS @@ -205,3 +226,40 @@ See [bmap](https://source.tizen.org/documentation/reference/bmaptool/introduction). A tool for optimized copying of disk images. Intended for offline use, so not directly comparable. + +## Git + +Although OSTree has been called "Git for Binaries", and the two share the idea +of a hashed content store, the implementation details are quite different. +OSTree supports extended attributes and uses SHA256 instead of Git's SHA1. It +"checks out" files via hardlinks, rather than copying, and thus requires the +checkout to be immutable. At the moment, OSTree commits may have at most one +parent, as opposed to Git which allows an arbitrary number. Git uses a +smart-delta protocol for updates, while OSTree uses 1 HTTP request per changed +file, or can generate static deltas. + +## Conda + +[Conda](http://conda.pydata.org/docs/) is an "OS-agnostic, system-level binary +package manager and ecosystem"; although most well-known for its accompanying +Python distribution anaconda, its scope has been expanding quickly. The package +format is very similar to well-known ones such as RPM. However, unlike typical +RPMs, the packages are built to be relocatable. Also, the package manager runs +natively on Windows. Conda's main advantage is its ability to install +collections of packages into "environments" by unpacking them all to the same +directory. Conda reduces duplication across environments using hardlinks, +similar to OSTree's sharing between deployments (although Conda uses package / +file path instead of file hash). Overall, it is quite similar to rpm-ostree in +functionality and scope. + +## rpm-ostree + +This builds on top of ostree to support building RPMs into OSTree images, and +even composing RPMs on-the-fly using an overlay filesystem. It is being +developed by Fedora, Red Hat, and CentOS as part of Project Atomic. + +## GNOME Continuous + +This is a service that incrementally rebuilds and tests GNOME on every commit. +The need to make and distribute snapshots for this system was the original +inspiration for ostree. From 0f7bf7be95864bb165fd52748edba847db2418b3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 11 May 2016 13:32:40 -0400 Subject: [PATCH 099/128] lib: Fix a leak in remote parsing As the docs say, `g_regex_match()` still allocates a match even if it returns `FALSE`. Using `g_autoptr` is just plain better. Closes: #292 Approved by: krnowak --- src/libostree/ostree-repo.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 0cff64e9..f32a4815 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -130,7 +130,7 @@ static OstreeRemote * ost_remote_new_from_keyfile (GKeyFile *keyfile, const gchar *group) { - GMatchInfo *match = NULL; + g_autoptr(GMatchInfo) match = NULL; OstreeRemote *remote; static gsize regex_initialized; @@ -157,8 +157,6 @@ ost_remote_new_from_keyfile (GKeyFile *keyfile, ot_keyfile_copy_group (keyfile, remote->options, group); - g_match_info_unref (match); - return remote; } From c015fe13fb5e887a6aa62a3a2830affb6fe17e50 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 9 Jun 2016 10:49:03 -0400 Subject: [PATCH 100/128] lib: Add OSTREE_SUPPRESS_SYNCFS environment variable Just to work around valgrind not understanding the `syncfs()` syscall in EL7 right now. Closes: #292 Approved by: krnowak --- src/libostree/ostree-repo-commit.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 6166c652..fc6ffd24 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1469,10 +1469,16 @@ ostree_repo_commit_transaction (OstreeRepo *self, goto out; } - if (syncfs (self->tmp_dir_fd) < 0) + /* FIXME: Added since valgrind in el7 doesn't know about + * `syncfs`...we should delete this later. + */ + if (g_getenv ("OSTREE_SUPRESS_SYNCFS") == NULL) { - glnx_set_error_from_errno (error); - goto out; + if (syncfs (self->tmp_dir_fd) < 0) + { + glnx_set_error_from_errno (error); + goto out; + } } if (!rename_pending_loose_objects (self, cancellable, error)) From 70af1d26b1de686373733f02c4c01151910619e9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 11 May 2016 15:54:48 -0400 Subject: [PATCH 101/128] tests: Modernize valgrind infrastructure The recent memleak fixes motivated me to look at the bitrotted code to run invocations of `ostree` in the test suite underneath valgrind. There are a few things here. First, update suppressions file from libhif, since I recently worked on it. When running *uninstalled* as we now support, we need `libtool --mode=execute` in the mix so it expands out to the uninstalled binary and we don't valgrind the intermediate shell. However, it's harder than that because we chdir into a tmpdir, which defeats the libtool logic. AFAICS, the only fix for this is to determine the realbin path before we chdir, and then unfortunately we need to change every use of `ostree` to `${OSTREE}` =( Then this immediately breaks for me on RHEL7 because my ancient copy of `valgrind-3.10.0-16.el7.x86_64` is unaware of syscall 306, i.e. `syncfs`. But let's do this first before I dive into that. Closes: #292 Approved by: krnowak --- Makefile-tests.am | 11 +- src/libostree/ostree-repo-commit.c | 2 +- tests/glib.supp | 535 +++++++++++++++++++++++++++++ tests/libtest.sh | 2 +- tests/ostree-valgrind.supp | 199 ----------- tests/ostree.supp | 1 + 6 files changed, 547 insertions(+), 203 deletions(-) create mode 100644 tests/glib.supp delete mode 100644 tests/ostree-valgrind.supp create mode 100644 tests/ostree.supp diff --git a/Makefile-tests.am b/Makefile-tests.am index 58397d37..50c01273 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -25,9 +25,11 @@ include $(top_srcdir)/buildutil/glib-tap.mk TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ GI_TYPELIB_PATH=$$(cd $(top_builddir) && pwd) \ LD_LIBRARY_PATH=$$(cd $(top_builddir)/.libs && pwd) \ - PATH=$$(cd $(top_builddir) && pwd):$${PATH} \ + PATH=$$(cd $(top_builddir)/tests && pwd):$${PATH} \ $(NULL) +uninstalled_test_data = tests/ostree-symlink-stamp + uninstalled_test_scripts = tests/test-symbols.sh test_scripts = \ @@ -231,7 +233,12 @@ EXTRA_DIST += \ tests/libreaddir-rand.so: Makefile $(AM_V_GEN) ln -fns ../.libs/libreaddir-rand.so tests ALL_LOCAL_RULES += tests/libreaddir-rand.so -CLEANFILES += tests/libreaddir-rand.so +CLEANFILES += tests/libreaddir-rand.so tests/ostree-symlink-stamp tests/ostree + +tests/ostree-symlink-stamp: + @real_bin=`cd $(top_builddir) && libtool --mode=execute echo ostree`; \ + ln -sf "$${real_bin}" tests/ostree; \ + touch $@ # Unfortunately the glib test data APIs don't actually handle # non-recursive Automake, so we change our code to canonically look diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index fc6ffd24..685eadd0 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1472,7 +1472,7 @@ ostree_repo_commit_transaction (OstreeRepo *self, /* FIXME: Added since valgrind in el7 doesn't know about * `syncfs`...we should delete this later. */ - if (g_getenv ("OSTREE_SUPRESS_SYNCFS") == NULL) + if (g_getenv ("OSTREE_SUPPRESS_SYNCFS") == NULL) { if (syncfs (self->tmp_dir_fd) < 0) { diff --git a/tests/glib.supp b/tests/glib.supp new file mode 100644 index 00000000..33449070 --- /dev/null +++ b/tests/glib.supp @@ -0,0 +1,535 @@ +# This GLib suppressions file is known to be used at least by: +# +# - rpm-software-management/libhif +# +# Please use the upstream verison in libhif for changes. +{ + gobject_init_1 + Memcheck:Leak + ... + fun:gobject_init +} +{ + g_type_register_static_1 + Memcheck:Leak + ... + fun:g_type_register_static +} +{ + g_type_register_fundamental + Memcheck:Leak + ... + fun:g_type_register_fundamental +} +{ + g_type_init_with_debug_flags + Memcheck:Leak + ... + fun:g_type_init_with_debug_flags +} +{ + g_type_class_ref_1 + Memcheck:Leak + ... + fun:type_iface_vtable_base_init_Wm + ... + fun:g_type_class_ref +} +{ + g_type_class_ref_2 + Memcheck:Leak + ... + fun:type_class_init_Wm + ... + fun:g_type_class_ref +} +{ + g_type_add_interface_static + Memcheck:Leak + ... + fun:g_type_add_interface_static +} +{ + g_param_spec_internal + Memcheck:Leak + ... + fun:g_type_class_ref + fun:g_type_create_instance + fun:g_param_spec_internal +} +{ + g_param_spec_enum + Memcheck:Leak + ... + fun:g_type_class_ref + fun:g_param_spec_enum +} +{ + g_param_spec_flags + Memcheck:Leak + ... + fun:g_type_class_ref + fun:g_param_spec_flags +} +{ + g_quark_from_static_string + Memcheck:Leak + ... + fun:g_quark_from_static_string +} +{ + g_quark_from_string + Memcheck:Leak + ... + fun:g_quark_from_string +} +{ + g_value_register_transform_func + Memcheck:Leak + ... + fun:g_value_register_transform_func +} +{ + test_run_seed + Memcheck:Leak + ... + fun:g_rand_new_with_seed_array + fun:test_run_seed + ... + fun:g_test_run_suite +} +{ + g_test_init + Memcheck:Leak + ... + fun:g_rand_new_with_seed_array + ... + fun:g_test_init +} +{ + g_intern_static_string + Memcheck:Leak + ... + fun:g_intern_static_string +} +{ + g_main_context_push_thread_default + Memcheck:Leak + ... + fun:g_queue_new + fun:g_main_context_push_thread_default +} +{ + g_main_context_push_thread_default_inlined + Memcheck:Leak + ... + fun:g_slice_alloc0 + fun:g_main_context_push_thread_default +} +{ + g_dbus_error_register_error + Memcheck:Leak + ... + fun:g_dbus_error_register_error +} +{ + g_param_spec_pool_insert + Memcheck:Leak + ... + fun:g_param_spec_pool_insert +} +{ + g_main_context_default + Memcheck:Leak + ... + fun:g_main_context_default +} +{ + g_main_context_check + Memcheck:Leak + ... + fun:g_ptr_array_add + fun:g_main_context_check +} +{ + g_test_run_suite + Memcheck:Leak + ... + fun:g_slist_copy + fun:g_test_run_suite_internal + fun:g_test_run_suite +} +{ + g_dbus_interface_info_cache_build + Memcheck:Leak + ... + fun:g_dbus_interface_info_cache_build +} +{ + g_cancellable_push_current + Memcheck:Leak + ... + fun:thread_memory_from_self + ... + fun:g_cancellable_push_current +} +{ + _g_io_module_get_default + Memcheck:Leak + ... + fun:g_io_module_new + fun:g_io_modules_scan_all_in_directory_with_scope + fun:_g_io_modules_ensure_loaded + fun:_g_io_module_get_default +} +{ + g_io_scheduler_push_job + Memcheck:Leak + ... + fun:init_scheduler + fun:g_once_impl + fun:g_io_scheduler_push_job +} +{ + g_io_scheduler_push_job_2 + Memcheck:Leak + ... + fun:g_system_thread_new + ... + fun:g_io_scheduler_push_job +} +{ + g_bus_get_sync__available_connections + Memcheck:Leak + ... + fun:g_hash_table_new + fun:initable_init + fun:g_initable_init + fun:g_bus_get_sync +} +{ + g_socket_connection_factory_register_type + Memcheck:Leak + ... + fun:g_socket_connection_factory_register_type +} +{ + g_test_add_vtable + Memcheck:Leak + ... + fun:g_test_add_vtable +} +{ + g_mutex_lock + Memcheck:Leak + ... + fun:g_mutex_impl_new + fun:g_mutex_get_impl + fun:g_mutex_lock +} +{ + g_thread_self + Memcheck:Leak + ... + fun:g_thread_self +} +{ + g_rec_mutex_lock + Memcheck:Leak + ... + fun:g_rec_mutex_impl_new + fun:g_rec_mutex_get_impl + fun:g_rec_mutex_lock +} +{ + test_case_run + Memcheck:Leak + ... + fun:g_malloc0 + fun:test_case_run + ... + fun:g_test_run_suite +} +{ + g_get_charset + Memcheck:Leak + ... + fun:g_get_charset +} +{ + g_test_run_suite__timer_new + Memcheck:Leak + ... + fun:g_timer_new + fun:test_case_run + ... + fun:g_test_run_suite +} +{ + g_test_run_suite__timer_new2 + Memcheck:Leak + ... + fun:g_timer_new + fun:test_case_run_suite_internal + ... + fun:g_test_run_suite +} +{ + g_test_run_suite__strconcat + Memcheck:Leak + ... + fun:g_strconcat + fun:test_case_run + ... + fun:g_test_run_suite + fun:g_test_run +} +{ + g_type_interface_add_prerequisite + Memcheck:Leak + ... + fun:g_type_interface_add_prerequisite +} +{ + + Memcheck:Leak + ... + fun:g_slist_copy + fun:g_test_run_suite_internal + ... + fun:g_test_run_suite +} +{ + g_set_prgname + Memcheck:Leak + ... + fun:g_set_prgname +} +{ + g_test_run_suite__strconcat_2 + Memcheck:Leak + ... + fun:g_strconcat + fun:g_test_run_suite_internal +} +{ + g_test_run_suite__strdup + Memcheck:Leak + ... + fun:g_strdup + fun:g_test_run_suite_internal +} +{ + g_private_get + Memcheck:Leak + ... + fun:g_private_get +} +{ + g_private_set + Memcheck:Leak + ... + fun:g_private_set +} +{ + g_static_mutex_get_mutex_impl + Memcheck:Leak + ... + fun:g_static_mutex_get_mutex_impl +} +{ + g_variant_type_info_unref + Memcheck:Leak + ... + fun:g_hash_table_remove + fun:g_variant_type_info_unref +} +{ + g_rw_lock_reader_lock + Memcheck:Leak + ... + fun:g_rw_lock_impl_new + fun:g_rw_lock_get_impl + fun:g_rw_lock_reader_lock +} +{ + g_child_watch_finalize__rt_sigaction + Memcheck:Param + rt_sigaction(act->sa_flags) + fun:__libc_sigaction + ... + fun:g_child_watch_finalize +} +{ + g_dbus_worker_new + Memcheck:Leak + fun:calloc + ... + fun:_g_dbus_worker_new +} +{ + gdbus_shared_thread_func + Memcheck:Leak + match-leak-kinds: definite + ... + fun:g_malloc + ... + fun:gdbus_shared_thread_func +} +{ + g_task_start_task_thread + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:g_malloc + fun:g_slice_alloc + fun:g_slice_alloc0 + ... + fun:g_thread_pool_push + fun:g_task_start_task_thread +} +{ + g_get_language_names + Memcheck:Leak + match-leak-kinds: definite + fun:calloc + fun:g_malloc0 + fun:g_get_language_names +} +{ + g_get_filename_charsets + Memcheck:Leak + match-leak-kinds: definite + ... + fun:g_get_filename_charsets + fun:g_filename_display_name +} +{ + g_main_current_source + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:g_malloc + ... + fun:g_main_current_source + fun:g_task_return + fun:g_task_thread_pool_thread +} +{ + g_once_init_enter + Memcheck:Leak + match-leak-kinds: definite + ... + fun:g_once_init_enter +} +{ + g_child_watch_source_new + Memcheck:Leak + match-leak-kinds: definite + ... + fun:g_thread_new + ... + fun:g_child_watch_source_new +} +{ + continue_writing_in_idle_cb + Memcheck:Leak + match-leak-kinds: definite + ... + fun:g_task_new + ... + fun:continue_writing_in_idle_cb + fun:g_main_context_dispatch +} +{ + g_main_current_source + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + ... + fun:g_main_current_source +} +{ + g_thread_pool_push + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + ... + fun:g_thread_pool_push +} +{ + leak_test_dbus_dispose + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + ... + fun:g_main_loop_run + fun:g_test_dbus_down +} +{ + leak_test_dbus_down + Memcheck:Leak + match-leak-kinds: definite + fun:calloc + fun:g_malloc0 + fun:g_main_loop_new + fun:g_test_dbus_down +} +{ + leak_socket_client_connect + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:g_malloc + fun:g_slice_alloc + fun:g_slice_alloc0 + fun:g_socket_client_connect_async + fun:g_socket_client_connect_to_uri_async +} +{ + leak_signal_handlers_disconnect_matched + Memcheck:Leak + match-leak-kinds: definite + fun:calloc + fun:g_malloc0 + ... + fun:g_slice_alloc + ... + fun:g_signal_handlers_disconnect_matched +} +{ + g_tls_connection_gnutls_init_priorities + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:g_malloc + fun:g_strdup + fun:g_tls_connection_gnutls_init_priorities +} +{ + g_tls_connection_gnutls_heisenbug_likely_same_as_above + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:g_malloc + fun:g_strdup + ... + fun:g_tls_client_connection_new +} +{ + g_unix_signal_add_full + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:g_malloc + ... + fun:g_thread_new + ... + fun:g_unix_signal_add_full +} +{ + glib_worker_1 + Memcheck:Leak + ... + fun:glib_worker_main +} diff --git a/tests/libtest.sh b/tests/libtest.sh index 34976e23..2916bdbb 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -77,7 +77,7 @@ if test -n "${OT_TESTS_DEBUG:-}"; then fi if test -n "${OT_TESTS_VALGRIND:-}"; then - CMD_PREFIX="env G_SLICE=always-malloc valgrind -q --leak-check=full --num-callers=30 --suppressions=${test_srcdir}/ostree-valgrind.supp" + CMD_PREFIX="env G_SLICE=always-malloc OSTREE_SUPPRESS_SYNCFS=1 valgrind -q --leak-check=full --num-callers=30 --suppressions=${test_srcdir}/glib.supp --suppressions=${test_srcdir}/ostree.supp" else CMD_PREFIX="env LD_PRELOAD=${test_builddir}/libreaddir-rand.so" fi diff --git a/tests/ostree-valgrind.supp b/tests/ostree-valgrind.supp deleted file mode 100644 index 00efd87b..00000000 --- a/tests/ostree-valgrind.supp +++ /dev/null @@ -1,199 +0,0 @@ -{ - g_type_init_with_debug_flags calloc - Memcheck:Leak - fun:calloc - ... - fun:g_type_init_with_debug_flags - ... -} - -{ - g_type_add_interface_static malloc - Memcheck:Leak - fun:malloc - ... - fun:g_type_add_interface_static - ... -} - -{ - g_type_add_interface_dynamic malloc - Memcheck:Leak - fun:malloc - ... - fun:g_type_add_interface_dynamic - ... -} - -{ - g_type_class_ref malloc - Memcheck:Leak - fun:malloc - ... - fun:g_type_class_ref - ... -} - -{ - g_type_register_dynamic malloc - Memcheck:Leak - fun:malloc - ... - fun:g_type_register_dynamic - ... -} - -{ - g_type_init_with_debug_flags malloc - Memcheck:Leak - fun:malloc - ... - fun:g_type_init_with_debug_flags - ... -} - -{ - g_type_init_with_debug_flags realloc - Memcheck:Leak - fun:realloc - ... - fun:g_type_init_with_debug_flags - ... -} - -{ - g_test_add_vtable malloc - Memcheck:Leak - fun:malloc - ... - fun:g_test_add_vtable - ... -} - -{ - g_test_init - Memcheck:Leak - fun:malloc - ... - fun:g_test_init - ... -} - -{ - g_type_register_static malloc - Memcheck:Leak - fun:malloc - ... - fun:g_type_register_static - ... -} - -{ - g_type_register_static realloc - Memcheck:Leak - fun:realloc - ... - fun:g_type_register_static - ... -} - -{ - g_type_register_fundamental never freed - Memcheck:Leak - fun:malloc - ... - fun:g_type_register_fundamental - ... -} - -{ - g_type_class_ref never finalized - Memcheck:Leak - fun:calloc - ... - fun:g_type_class_ref - ... -} - -{ - DBusGValue qdata - Memcheck:Leak - fun:realloc - fun:g_realloc - fun:g_type_set_qdata - fun:_dbus_g_value_types_init - ... -} - -{ - gettext conditional jump - Memcheck:Cond - fun:__GI___strcasecmp_l - fun:__gconv_open - fun:_nl_find_msg - fun:__dcigettext - ... -} - -{ - gettext uninitialized value - Memcheck:Value8 - fun:__GI___strcasecmp_l - fun:__gconv_open - fun:_nl_find_msg - fun:__dcigettext - ... -} - -{ - font config invalid reads - Memcheck:Addr4 - ... - fun:FcConfigParseAndLoad - ... -} - -{ - dynamic loader conditional jump - Memcheck:Cond - fun:index - fun:expand_dynamic_string_token - fun:_dl_map_object - fun:map_doit - fun:_dl_catch_error - fun:do_preload - fun:dl_main - ... -} - -{ - g_vfs_get_local - Memcheck:Leak - ... - fun:g_vfs_get_local - ... -} - -{ - _g_io_modules_ensure_loaded - Memcheck:Leak - ... - fun:_g_io_modules_ensure_loaded - ... -} - -{ - _g_io_module_get_default - Memcheck:Leak - ... - fun:_g_io_module_get_default - ... -} - -{ - _dl_allocate_tls - Memcheck:Leak - ... - fun:_dl_allocate_tls - ... -} diff --git a/tests/ostree.supp b/tests/ostree.supp new file mode 100644 index 00000000..b81ea51b --- /dev/null +++ b/tests/ostree.supp @@ -0,0 +1 @@ +# Use this to suppress "possibly lost" for global statics From 8e3d558f2316ba627ca706461cb453df62acdba4 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 10 Jun 2016 12:10:37 +0200 Subject: [PATCH 102/128] libotutil: Fix leak of VariantMapData Closes: #333 Approved by: giuseppe --- src/libotutil/ot-variant-utils.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index 9d4a62d2..315bbeb2 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -157,6 +157,7 @@ variant_map_data_destroy (gpointer data) { VariantMapData *mdata = data; (void) munmap (mdata->addr, mdata->len); + g_free (mdata); } gboolean From bd4e250e9372bd719cfb2831e5bcf7a5d7141253 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 10 Jun 2016 12:12:07 +0200 Subject: [PATCH 103/128] pull: Fix leak of debug temporary strings Closes: #333 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index cbd225bf..ecc26d9d 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -633,6 +633,7 @@ content_fetch_on_write_complete (GObject *object, const char *expected_checksum; g_autofree guchar *csum = NULL; g_autofree char *checksum = NULL; + g_autofree char *checksum_obj = NULL; if (!ostree_repo_write_content_finish ((OstreeRepo*)object, result, &csum, error)) @@ -643,7 +644,8 @@ content_fetch_on_write_complete (GObject *object, ostree_object_name_deserialize (fetch_data->object, &expected_checksum, &objtype); g_assert (objtype == OSTREE_OBJECT_TYPE_FILE); - g_debug ("write of %s complete", ostree_object_to_string (checksum, objtype)); + checksum_obj = ostree_object_to_string (checksum, objtype); + g_debug ("write of %s complete", checksum_obj); if (strcmp (checksum, expected_checksum) != 0) { @@ -679,6 +681,7 @@ content_fetch_on_complete (GObject *object, g_autoptr(GInputStream) object_input = NULL; g_autofree char *temp_path = NULL; const char *checksum; + g_autofree char *checksum_obj = NULL; OstreeObjectType objtype; temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error); @@ -688,7 +691,8 @@ content_fetch_on_complete (GObject *object, ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype); g_assert (objtype == OSTREE_OBJECT_TYPE_FILE); - g_debug ("fetch of %s complete", ostree_object_to_string (checksum, objtype)); + checksum_obj = ostree_object_to_string (checksum, objtype); + g_debug ("fetch of %s complete", checksum_obj); if (pull_data->is_mirror && pull_data->repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2) { @@ -800,13 +804,15 @@ meta_fetch_on_complete (GObject *object, g_autoptr(GVariant) metadata = NULL; g_autofree char *temp_path = NULL; const char *checksum; + g_autofree char *checksum_obj = NULL; OstreeObjectType objtype; GError *local_error = NULL; GError **error = &local_error; glnx_fd_close int fd = -1; ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype); - g_debug ("fetch of %s%s complete", ostree_object_to_string (checksum, objtype), + checksum_obj = ostree_object_to_string (checksum, objtype); + g_debug ("fetch of %s%s complete", checksum_obj, fetch_data->is_detached_meta ? " (detached)" : ""); temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error); From 785f894de16dd59a3ac8c0ddf3a7aa41d550d0de Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 10 Jun 2016 12:12:33 +0200 Subject: [PATCH 104/128] pull: Fix leak of FetchObjectData in some cases Closes: #333 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index ecc26d9d..64ecc75b 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -809,6 +809,7 @@ meta_fetch_on_complete (GObject *object, GError *local_error = NULL; GError **error = &local_error; glnx_fd_close int fd = -1; + gboolean free_fetch_data = FALSE; ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype); checksum_obj = ostree_object_to_string (checksum, objtype); @@ -874,6 +875,8 @@ meta_fetch_on_complete (GObject *object, if (!fetch_data->object_is_stored) enqueue_one_object_request (pull_data, checksum, objtype, FALSE, FALSE); + + free_fetch_data = TRUE; } else { @@ -911,7 +914,7 @@ meta_fetch_on_complete (GObject *object, pull_data->n_outstanding_metadata_fetches--; pull_data->n_fetched_metadata++; check_outstanding_requests_handle_error (pull_data, local_error); - if (local_error) + if (local_error || free_fetch_data) { g_variant_unref (fetch_data->object); g_free (fetch_data); From c9c4ab851e98c5d893766fe99e3885161208c30d Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 10 Jun 2016 12:12:58 +0200 Subject: [PATCH 105/128] pull: Fix leaked uri in request_static_delta_superblock_sync Closes: #333 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 64ecc75b..c28596ff 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1458,6 +1458,7 @@ request_static_delta_superblock_sync (OtPullData *pull_data, if (out_delta_superblock) *out_delta_superblock = g_steal_pointer (&ret_delta_superblock); out: + g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free); return ret; } From f6b70a32e6fc6e31c35a5e2b63d7a9142a7ebffd Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 10 Jun 2016 12:13:23 +0200 Subject: [PATCH 106/128] pull: Fix leak of refs_to_fetch Closes: #333 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index c28596ff..53847340 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2182,7 +2182,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, guint64 end_time; OstreeRepoPullFlags flags = 0; const char *dir_to_pull = NULL; - char **refs_to_fetch = NULL; + g_autofree char **refs_to_fetch = NULL; char **override_commit_ids = NULL; GSource *update_timeout = NULL; gboolean disable_static_deltas = FALSE; From d258b124a2e062a3b1adc7f6a7baef9d276020c4 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 10 Jun 2016 13:28:08 +0200 Subject: [PATCH 107/128] tests: Exit valgrind tests if a leak is detected This fails the test if OT_TESTS_VALGRIND is set and valgrind detects a leak. Closes: #334 Approved by: cgwalters --- tests/libtest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 2916bdbb..3350cb20 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -77,7 +77,7 @@ if test -n "${OT_TESTS_DEBUG:-}"; then fi if test -n "${OT_TESTS_VALGRIND:-}"; then - CMD_PREFIX="env G_SLICE=always-malloc OSTREE_SUPPRESS_SYNCFS=1 valgrind -q --leak-check=full --num-callers=30 --suppressions=${test_srcdir}/glib.supp --suppressions=${test_srcdir}/ostree.supp" + CMD_PREFIX="env G_SLICE=always-malloc OSTREE_SUPPRESS_SYNCFS=1 valgrind -q --error-exitcode=1 --leak-check=full --num-callers=30 --suppressions=${test_srcdir}/glib.supp --suppressions=${test_srcdir}/ostree.supp" else CMD_PREFIX="env LD_PRELOAD=${test_builddir}/libreaddir-rand.so" fi From 0cf0fa815414eebb3e642429085fa75dda05c24d Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sat, 11 Jun 2016 19:07:40 +0100 Subject: [PATCH 108/128] test-sysroot.js: set "strict mode" when sourcing libtest.sh As with the C tests in commit 08580118, this makes sure the test fails as soon as something goes wrong. Signed-off-by: Simon McVittie Closes: #335 Approved by: giuseppe --- tests/test-sysroot.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-sysroot.js b/tests/test-sysroot.js index 9468d2fb..7c31659d 100644 --- a/tests/test-sysroot.js +++ b/tests/test-sysroot.js @@ -37,7 +37,7 @@ function libtestExec(shellCode) { let testdatadir = GLib.getenv("G_TEST_SRCDIR"); let libtestPath = GLib.build_filenamev([testdatadir, 'libtest.sh']) let proc = GSystem.Subprocess.new_simple_argv(['bash', '-c', - '. ' + GLib.shell_quote(libtestPath) + '; ' + shellCode], + 'set -xeuo pipefail; . ' + GLib.shell_quote(libtestPath) + '; ' + shellCode], GSystem.SubprocessStreamDisposition.INHERIT, GSystem.SubprocessStreamDisposition.INHERIT, null); From b9e18b83fd195c980c5dffab4bedf3fa79b19a2e Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 10:25:21 -0400 Subject: [PATCH 109/128] Link libreaddir-rand to libdl It uses dlsym(). There's no point in being extra-portable here because OSTree only targets Linux anyway. Signed-off-by: Simon McVittie Closes: #336 Approved by: cgwalters --- Makefile-tests.am | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index 50c01273..f1e8e676 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -133,7 +133,10 @@ endif test_ltlibraries = libreaddir-rand.la libreaddir_rand_la_SOURCES = tests/readdir-rand.c libreaddir_rand_la_CFLAGS = $(OT_INTERNAL_GIO_UNIX_CFLAGS) -libreaddir_rand_la_LIBADD = $(OT_INTERNAL_GIO_UNIX_LIBS) +libreaddir_rand_la_LIBADD = \ + -ldl \ + $(OT_INTERNAL_GIO_UNIX_LIBS) \ + $(NULL) libreaddir_rand_la_LDFLAGS = -avoid-version if !ENABLE_INSTALLED_TESTS libreaddir_rand_la_LDFLAGS += -rpath $(abs_builddir) From c930eefeeada9a9a820d650fcc3481709d9d070b Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 10:31:29 +0100 Subject: [PATCH 110/128] autogen.sh: rely on VPATH for submodules' Makefile-*.am.inc This avoids hard-coding the autogen-time ${srcdir} into our build system, and might fix "make distcheck". We don't need to specify ${srcdir} explicitly in the various variables, because Automake always uses make's VPATH feature to look in both ${builddir} and ${srcdir} at build time. Signed-off-by: Simon McVittie Closes: #337 Approved by: cgwalters --- autogen.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autogen.sh b/autogen.sh index 0eb65550..581f3dee 100755 --- a/autogen.sh +++ b/autogen.sh @@ -33,8 +33,8 @@ if ! test -f libglnx/README.md || ! test -f bsdiff/README.md; then git submodule update --init fi # Workaround automake bug with subdir-objects and computed paths -sed -e 's,$(libglnx_srcpath),'${srcdir}/libglnx,g < libglnx/Makefile-libglnx.am >libglnx/Makefile-libglnx.am.inc -sed -e 's,$(libbsdiff_srcpath),'${srcdir}/bsdiff,g < bsdiff/Makefile-bsdiff.am >bsdiff/Makefile-bsdiff.am.inc +sed -e 's,$(libglnx_srcpath),libglnx,g' < libglnx/Makefile-libglnx.am >libglnx/Makefile-libglnx.am.inc +sed -e 's,$(libbsdiff_srcpath),bsdiff,g' < bsdiff/Makefile-bsdiff.am >bsdiff/Makefile-bsdiff.am.inc autoreconf --force --install --verbose From 19cfe2d73a2f311f5a52482c858f829a7f91d3fb Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 10:36:44 +0100 Subject: [PATCH 111/128] build: distribute libostree.sym in tarballs Signed-off-by: Simon McVittie Closes: #337 Approved by: cgwalters --- Makefile-libostree.am | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 13013c1f..d6b83528 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -149,6 +149,8 @@ libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$( libostree_1_la_LDFLAGS = -version-number 1:0:0 -Bsymbolic-functions -Wl,--version-script=$(top_srcdir)/src/libostree/libostree.sym libostree_1_la_LIBADD = libotutil.la libbupsplit.la libglnx.la libbsdiff.la libostree-kernel-args.la $(OT_INTERNAL_GIO_UNIX_LIBS) $(OT_INTERNAL_GPGME_LIBS) $(OT_DEP_LZMA_LIBS) $(OT_DEP_ZLIB_LIBS) +EXTRA_DIST += src/libostree/libostree.sym + if USE_LIBARCHIVE libostree_1_la_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_LIBARCHIVE_LIBS) From c337d4dab552ce8ce5241044b7255e44f6ee0521 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 10:39:46 +0100 Subject: [PATCH 112/128] build: distribute test scripts in tarballs Signed-off-by: Simon McVittie Closes: #337 Approved by: cgwalters --- Makefile-tests.am | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index f1e8e676..d8ab0179 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -30,9 +30,9 @@ TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ uninstalled_test_data = tests/ostree-symlink-stamp -uninstalled_test_scripts = tests/test-symbols.sh +dist_uninstalled_test_scripts = tests/test-symbols.sh -test_scripts = \ +dist_test_scripts = \ tests/test-basic.sh \ tests/test-pull-subpath.sh \ tests/test-archivez.sh \ @@ -83,15 +83,15 @@ test_scripts = \ $(NULL) if BUILDOPT_FUSE -test_scripts += tests/test-rofiles-fuse.sh +dist_test_scripts += tests/test-rofiles-fuse.sh endif # This one uses corrupt-repo-ref.js if BUILDOPT_GJS -test_scripts += tests/test-corruption.sh +dist_test_scripts += tests/test-corruption.sh endif -installed_test_data = tests/archive-test.sh \ +dist_installed_test_data = tests/archive-test.sh \ tests/pull-test.sh \ tests/libtest.sh \ tests/admin-test.sh \ @@ -102,7 +102,7 @@ installed_test_data = tests/archive-test.sh \ tests/pre-endian-deltas-repo-little.tar.xz \ $(NULL) -test_extra_scripts = tests/bootloader-entries-crosscheck.py \ +dist_test_extra_scripts = tests/bootloader-entries-crosscheck.py \ tests/ostree-grub-generator # We can't use nobase_ as we need to strip off the tests/, can't @@ -124,7 +124,7 @@ gpgvinsttest_DATA = $(addprefix tests/gpg-verify-data/, \ endif if BUILDOPT_GJS -installed_test_scripts = tests/test-core.js \ +dist_installed_test_scripts = tests/test-core.js \ tests/test-sizes.js \ tests/test-sysroot.js \ $(NULL) From 906575271c025621613a62f0313bfeaabf28a694 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 10:43:34 +0100 Subject: [PATCH 113/128] build: distribute TAP helper scripts in tarballs Signed-off-by: Simon McVittie Closes: #337 Approved by: cgwalters --- Makefile-tests.am | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile-tests.am b/Makefile-tests.am index d8ab0179..d8e300ca 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -19,6 +19,11 @@ include $(top_srcdir)/buildutil/glib-tap.mk +EXTRA_DIST += \ + buildutil/tap-driver.sh \ + buildutil/tap-test \ + $(NULL) + # We should probably consider flipping the default for DEBUG. Also, # include the builddir in $PATH so we find our just-built ostree # binary. From 70a11189559856ec5aabed5ee8c62d0792a19ca8 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 10:45:48 +0100 Subject: [PATCH 114/128] build: distribute more test files Signed-off-by: Simon McVittie Closes: #337 Approved by: cgwalters --- Makefile-tests.am | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index d8e300ca..88618c46 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -114,17 +114,17 @@ dist_test_extra_scripts = tests/bootloader-entries-crosscheck.py \ # use plain installed_ as we do need the gpghome/ prefix. if ENABLE_INSTALLED_TESTS gpginsttestdir = $(installed_testdir)/gpghome -gpginsttest_DATA = tests/gpghome/secring.gpg \ +dist_gpginsttest_DATA = tests/gpghome/secring.gpg \ tests/gpghome/pubring.gpg \ tests/gpghome/trustdb.gpg \ tests/gpghome/key1.asc \ tests/gpghome/key2.asc \ tests/gpghome/key3.asc gpginsttest_trusteddir = $(installed_testdir)/gpghome/trusted -gpginsttest_trusted_DATA = tests/gpghome/trusted/pubring.gpg +dist_gpginsttest_trusted_DATA = tests/gpghome/trusted/pubring.gpg gpgvinsttestdir = $(installed_testdir)/gpg-verify-data -gpgvinsttest_DATA = $(addprefix tests/gpg-verify-data/, \ +dist_gpgvinsttest_DATA = $(addprefix tests/gpg-verify-data/, \ gpg.conf lgpl2 lgpl2.sig pubring.gpg secring.gpg trustdb.gpg) endif @@ -230,6 +230,7 @@ tests_test_gpg_verify_result_LDADD = $(TESTS_LDADD) $(OT_INTERNAL_GPGME_LIBS) EXTRA_DIST += \ tests/libostreetest.h \ + tests/libtest.sh \ tests/gpg-verify-data/README.md \ tests/gpg-verify-data/lgpl2 \ tests/gpg-verify-data/lgpl2.sig \ From 8011d8c62736f17e88487337e8637c3d208dbdcf Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 10:51:11 +0100 Subject: [PATCH 115/128] libtest: make gpghome in tmpdir writeable Otherwise, during "make distcheck" it will be read-only (because it is a copy of the read-only ${srcdir} with permissions preserved), and deletion will fail during cleanup. Signed-off-by: Simon McVittie Closes: #337 Approved by: cgwalters --- tests/libtest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/libtest.sh b/tests/libtest.sh index 3350cb20..16d5c0e0 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -69,6 +69,7 @@ export TEST_GPG_KEYID_3="DF444D67" # this by copying locally. echo "Copying gpghome to ${test_tmpdir}" cp -a "${test_srcdir}/gpghome" ${test_tmpdir} +chmod -R u+w "${test_tmpdir}" export TEST_GPG_KEYHOME=${test_tmpdir}/gpghome export OSTREE_GPG_HOME=${test_tmpdir}/gpghome/trusted From 4801e9b30ba036cc1b18c07f1efa544755fba382 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 09:21:19 -0400 Subject: [PATCH 116/128] libostreetest: include libtest.sh from srcdir, not builddir Signed-off-by: Simon McVittie Closes: #337 Approved by: cgwalters --- tests/libostreetest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/libostreetest.c b/tests/libostreetest.c index aff6c1e8..81f743e9 100644 --- a/tests/libostreetest.c +++ b/tests/libostreetest.c @@ -32,7 +32,7 @@ gboolean ot_test_run_libtest (const char *cmd, GError **error) { gboolean ret = FALSE; - const char *builddir = g_getenv ("G_TEST_BUILDDIR"); + const char *srcdir = g_getenv ("G_TEST_SRCDIR"); int estatus; g_autoptr(GPtrArray) argv = g_ptr_array_new (); g_autoptr(GString) cmdstr = g_string_new (""); @@ -41,7 +41,7 @@ ot_test_run_libtest (const char *cmd, GError **error) g_ptr_array_add (argv, "-c"); g_string_append (cmdstr, "set -xeuo pipefail; . "); - g_string_append (cmdstr, builddir); + g_string_append (cmdstr, srcdir); g_string_append (cmdstr, "/tests/libtest.sh; "); g_string_append (cmdstr, cmd); From 14ff4f94fb8d92b65bbd084b19e3f27f947d6bd2 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sun, 12 Jun 2016 09:32:51 -0400 Subject: [PATCH 117/128] build: re-create tests/ostree-symlink-stamp if Makefile changes The filename of the real ostree executable could be either .libs/ostree or .libs/lt-ostree. Signed-off-by: Simon McVittie Closes: #337 Approved by: cgwalters --- Makefile-tests.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index 88618c46..e5cf9554 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -244,7 +244,7 @@ tests/libreaddir-rand.so: Makefile ALL_LOCAL_RULES += tests/libreaddir-rand.so CLEANFILES += tests/libreaddir-rand.so tests/ostree-symlink-stamp tests/ostree -tests/ostree-symlink-stamp: +tests/ostree-symlink-stamp: Makefile @real_bin=`cd $(top_builddir) && libtool --mode=execute echo ostree`; \ ln -sf "$${real_bin}" tests/ostree; \ touch $@ From 7847bc7394c5de82f7993c7ab1f6ffc1719683c8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 9 Jun 2016 14:30:14 -0400 Subject: [PATCH 118/128] lib: Port some manual close() cleanups to be glnx_fd_close Just noticed this while reading some code, we didn't have many manual `out: close()` bits left, this pushes us over the edge to autocleanup almost everywhere. Closes: #332 Approved by: jlebon --- src/libostree/ostree-linuxfsutil.c | 4 +--- src/libostree/ostree-repo-checkout.c | 4 +--- src/libostree/ostree-repo.c | 6 +---- src/libostree/ostree-sysroot-deploy.c | 32 +++++++-------------------- src/libotutil/ot-gio-utils.c | 12 +++------- 5 files changed, 14 insertions(+), 44 deletions(-) diff --git a/src/libostree/ostree-linuxfsutil.c b/src/libostree/ostree-linuxfsutil.c index 4151fa34..b270a94e 100644 --- a/src/libostree/ostree-linuxfsutil.c +++ b/src/libostree/ostree-linuxfsutil.c @@ -107,7 +107,7 @@ _ostree_linuxfs_alter_immutable_flag (GFile *path, GError **error) { gboolean ret = FALSE; - int fd = -1; + glnx_fd_close int fd = -1; if (g_cancellable_set_error_if_cancelled (cancellable, error)) return FALSE; @@ -129,7 +129,5 @@ _ostree_linuxfs_alter_immutable_flag (GFile *path, ret = TRUE; out: - if (fd != -1) - (void) close (fd); return ret; } diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 227227b6..5f7cf0bd 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -632,7 +632,7 @@ checkout_tree_at (OstreeRepo *self, { gboolean ret = FALSE; gboolean did_exist = FALSE; - int destination_dfd = -1; + glnx_fd_close int destination_dfd = -1; int res; g_autoptr(GVariant) xattrs = NULL; g_autoptr(GFileEnumerator) dir_enum = NULL; @@ -779,8 +779,6 @@ checkout_tree_at (OstreeRepo *self, ret = TRUE; out: - if (destination_dfd != -1) - (void) close (destination_dfd); return ret; } diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index f32a4815..c40abb78 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2558,7 +2558,7 @@ load_metadata_internal (OstreeRepo *self, { gboolean ret = FALSE; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; - int fd = -1; + glnx_fd_close int fd = -1; g_autoptr(GInputStream) ret_stream = NULL; g_autoptr(GVariant) ret_variant = NULL; @@ -2586,8 +2586,6 @@ load_metadata_internal (OstreeRepo *self, mfile = g_mapped_file_new_from_fd (fd, FALSE, error); if (!mfile) goto out; - (void) close (fd); /* Ignore errors, we have it mapped */ - fd = -1; ret_variant = g_variant_new_from_data (ostree_metadata_variant_type (objtype), g_mapped_file_get_contents (mfile), g_mapped_file_get_length (mfile), @@ -2632,8 +2630,6 @@ load_metadata_internal (OstreeRepo *self, ot_transfer_out_value (out_variant, &ret_variant); ot_transfer_out_value (out_stream, &ret_stream); out: - if (fd != -1) - (void) close (fd); return ret; } diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index f0e7ecca..4616bab1 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -172,8 +172,8 @@ copy_dir_recurse (int src_parent_dfd, GError **error) { gboolean ret = FALSE; - int src_dfd = -1; - int dest_dfd = -1; + glnx_fd_close int src_dfd = -1; + glnx_fd_close int dest_dfd = -1; DIR *srcd = NULL; struct dirent *dent; @@ -240,10 +240,6 @@ copy_dir_recurse (int src_parent_dfd, /* Note the srcd owns src_dfd */ src_dfd = -1; } - if (src_dfd != -1) - (void) close (src_dfd); - if (dest_dfd != -1) - (void) close (dest_dfd); return ret; } @@ -257,8 +253,8 @@ ensure_directory_from_template (int orig_etc_fd, GError **error) { gboolean ret = FALSE; - int src_dfd = -1; - int target_dfd = -1; + glnx_fd_close int src_dfd = -1; + glnx_fd_close int target_dfd = -1; g_assert (path != NULL); g_assert (*path != '/' && *path != '\0'); @@ -316,10 +312,6 @@ ensure_directory_from_template (int orig_etc_fd, target_dfd = -1; } out: - if (src_dfd != -1) - (void) close (src_dfd); - if (target_dfd != -1) - (void) close (target_dfd); return ret; } @@ -341,7 +333,7 @@ copy_modified_config_file (int orig_etc_fd, gboolean ret = FALSE; struct stat modified_stbuf; struct stat new_stbuf; - int dest_parent_dfd = -1; + glnx_fd_close int dest_parent_dfd = -1; if (fstatat (modified_etc_fd, path, &modified_stbuf, AT_SYMLINK_NOFOLLOW) < 0) { @@ -431,8 +423,6 @@ copy_modified_config_file (int orig_etc_fd, ret = TRUE; out: - if (dest_parent_dfd != -1) - (void) close (dest_parent_dfd); return ret; } @@ -459,9 +449,9 @@ merge_etc_changes (GFile *orig_etc, g_autoptr(GPtrArray) removed = NULL; g_autoptr(GPtrArray) added = NULL; guint i; - int orig_etc_fd = -1; - int modified_etc_fd = -1; - int new_etc_fd = -1; + glnx_fd_close int orig_etc_fd = -1; + glnx_fd_close int modified_etc_fd = -1; + glnx_fd_close int new_etc_fd = -1; modified = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_diff_item_unref); removed = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref); @@ -538,12 +528,6 @@ merge_etc_changes (GFile *orig_etc, ret = TRUE; out: - if (orig_etc_fd != -1) - (void) close (orig_etc_fd); - if (modified_etc_fd != -1) - (void) close (modified_etc_fd); - if (new_etc_fd != -1) - (void) close (new_etc_fd); return ret; } diff --git a/src/libotutil/ot-gio-utils.c b/src/libotutil/ot-gio-utils.c index a9da4504..eb9b94f5 100644 --- a/src/libotutil/ot-gio-utils.c +++ b/src/libotutil/ot-gio-utils.c @@ -357,7 +357,7 @@ ot_gfile_replace_contents_fsync (GFile *path, GError **error) { gboolean ret = FALSE; - int parent_dfd; + glnx_fd_close int parent_dfd = -1; const char *target_basename = glnx_basename (gs_file_get_path_cached (path)); g_autoptr(GFile) parent = NULL; @@ -374,8 +374,6 @@ ot_gfile_replace_contents_fsync (GFile *path, ret = TRUE; out: - if (parent_dfd != -1) - (void) close (parent_dfd); return ret; } @@ -412,7 +410,7 @@ ot_util_fsync_directory (GFile *dir, GError **error) { gboolean ret = FALSE; - int dfd = -1; + glnx_fd_close int dfd = -1; if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (dir), TRUE, &dfd, error)) @@ -426,8 +424,6 @@ ot_util_fsync_directory (GFile *dir, ret = TRUE; out: - if (dfd != -1) - (void) close (dfd); return ret; } @@ -446,7 +442,7 @@ ot_util_ensure_directory_and_fsync (GFile *dir, GError **error) { gboolean ret = FALSE; - int parentfd = -1; + glnx_fd_close int parentfd = -1; const char *basename = glnx_basename (gs_file_get_path_cached (dir)); g_autoptr(GFile) parent = g_file_get_parent (dir); @@ -495,7 +491,5 @@ ot_util_ensure_directory_and_fsync (GFile *dir, ret = TRUE; out: - if (parentfd != -1) - (void) close (parentfd); return ret; } From f8fcdba0a542507917fe49a7b81b8932c6ce115e Mon Sep 17 00:00:00 2001 From: Yu Qi Zhang Date: Tue, 14 Jun 2016 14:13:06 +0000 Subject: [PATCH 119/128] refs: add "ostree refs --create" and unit tests Added the ability to create a ref (much like a git tag) for an existing commit through "ostree refs EXISTING --create=NEWREF". Previously the only way to create a new ref was by creating a new commit, but refs --create allows multiple refs to point to the same commit. The command will fail if: - None/more than one existing ref is specified - The specified EXISTING tag does not exist, or was not specified - The specified NEWREF already exists, or is the name of a folder Add unit tests in tests-ref.sh to verify above functionality Closes: #340 Approved by: jlebon --- src/ostree/ot-builtin-refs.c | 46 +++++++++++++++++++++++++++++++++++- tests/test-refs.sh | 31 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-refs.c b/src/ostree/ot-builtin-refs.c index af90c841..10647ec6 100644 --- a/src/ostree/ot-builtin-refs.c +++ b/src/ostree/ot-builtin-refs.c @@ -28,10 +28,12 @@ static gboolean opt_delete; static gboolean opt_list; +static char *opt_create; static GOptionEntry options[] = { { "delete", 0, 0, G_OPTION_ARG_NONE, &opt_delete, "Delete refs which match PREFIX, rather than listing them", NULL }, { "list", 0, 0, G_OPTION_ARG_NONE, &opt_list, "Do not remove the prefix from the refs", NULL }, + { "create", 0, 0, G_OPTION_ARG_STRING, &opt_create, "Create a new ref for an existing commit", "NEWREF" }, { NULL } }; @@ -48,10 +50,16 @@ static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellab cancellable, error)) goto out; } + else if (opt_create) + { + if (!ostree_repo_list_refs_ext (repo, NULL, &refs, OSTREE_REPO_LIST_REFS_EXT_NONE, + cancellable, error)) + goto out; + } else if (!ostree_repo_list_refs (repo, refspec_prefix, &refs, cancellable, error)) goto out; - if (!opt_delete) + if (!opt_delete && !opt_create) { g_hash_table_iter_init (&hashiter, refs); while (g_hash_table_iter_next (&hashiter, &hashkey, &hashvalue)) @@ -60,7 +68,30 @@ static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellab g_print ("%s\n", ref); } } + else if (opt_create) + { + g_autofree char *checksum = NULL; + g_autofree char *checksum_existing = NULL; + + if (!ostree_repo_resolve_rev (repo, opt_create, TRUE, &checksum_existing, error)) + goto out; + + if (checksum_existing != NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "--create specified but ref %s already exists", opt_create); + goto out; + } + + if (!ostree_repo_resolve_rev (repo, refspec_prefix, FALSE, &checksum, error)) + goto out; + + if (!ostree_repo_set_ref_immediate (repo, NULL, opt_create, checksum, + cancellable, error)) + goto out; + } else + /* delete */ { g_hash_table_iter_init (&hashiter, refs); while (g_hash_table_iter_next (&hashiter, &hashkey, &hashvalue)) @@ -97,6 +128,12 @@ ostree_builtin_refs (int argc, char **argv, GCancellable *cancellable, GError ** if (argc >= 2) { + if (opt_create && argc > 2) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "You must specify only 1 existing ref when creating a new ref"); + goto out; + } for (i = 1; i < argc; i++) if (!do_ref (repo, argv[i], cancellable, error)) goto out; @@ -110,6 +147,13 @@ ostree_builtin_refs (int argc, char **argv, GCancellable *cancellable, GError ** "At least one PREFIX is required when deleting refs"); goto out; } + else if (opt_create) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "You must specify an existing ref when creating a new ref"); + goto out; + } + ret = do_ref (repo, NULL, cancellable, error); } diff --git a/tests/test-refs.sh b/tests/test-refs.sh index bcebae9e..3d229031 100755 --- a/tests/test-refs.sh +++ b/tests/test-refs.sh @@ -65,4 +65,35 @@ ${CMD_PREFIX} ostree refs --repo=repo | wc -l > refscount.delete3 assert_file_has_content refscount.delete3 "^3$" assert_not_file_has_content reflist '^test-1$' +#Add a few more commits, to test --create +${CMD_PREFIX} ostree --repo=repo commit --branch=ctest -m ctest -s ctest tree +${CMD_PREFIX} ostree --repo=repo commit --branch=foo/ctest -m ctest -s ctest tree + +${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount +assert_file_has_content refscount "^5$" + +if ${CMD_PREFIX} ostree --repo=repo refs --create=ctest-new; then + assert_not_reached "refs --create unexpectedly succeeded without specifying an existing ref!" +fi +if ${CMD_PREFIX} ostree --repo=repo refs ctest --create; then + assert_not_reached "refs --create unexpectedly succeeded without specifying the ref to create!" +fi +if ${CMD_PREFIX} ostree --repo=repo refs does-not-exist --create=ctest-new; then + assert_not_reached "refs --create unexpectedly succeeded for a prefix that doesn't exist!" +fi +if ${CMD_PREFIX} ostree --repo=repo refs ctest --create=foo; then + assert_not_reached "refs --create unexpectedly succeeded for a prefix that is already in use by a folder!" +fi +if ${CMD_PREFIX} ostree --repo=repo refs foo/ctest --create=ctest; then + assert_not_reached "refs --create unexpectedly succeeded in overwriting an existing prefix!" +fi + +#Check to see if any uncleaned tmp files were created after failed --create +${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount.create1 +assert_file_has_content refscount.create1 "^5$" + +${CMD_PREFIX} ostree --repo=repo refs ctest --create ctest-new +${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount.create2 +assert_file_has_content refscount.create2 "^6$" + echo "ok refs" From 85f202d0b294a60495997b05be07f7b400b37cb5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 15 Jun 2016 09:20:11 -0400 Subject: [PATCH 120/128] manual: Link to mender.io Came out of a discussion on the list. Closes: #344 Approved by: jlebon --- docs/manual/related-projects.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/manual/related-projects.md b/docs/manual/related-projects.md index bb0258e7..d37e2cfe 100644 --- a/docs/manual/related-projects.md +++ b/docs/manual/related-projects.md @@ -134,6 +134,11 @@ believes that at the moment, the "CL updater" is not truly atomic in the sense that because it applies updates live, there is a window where the OS root may be inconsistent. +## Mender.io + +[Mender.io](https://mender.io/) is another implementation of the dual +partition approach. + ## OLPC update OSTree is basically a generalization of olpc-update, except using From ab47a8a0306d79fd13d618fd13c26b3134053011 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Jun 2016 10:07:26 -0400 Subject: [PATCH 121/128] Add "archive" as an alias for "archive-z2" I find the "-z2" is really a long ago relic of the past when I changed the format. We no longer have anything to do with the original `archive`, so let's start allowing people to type `--mode=archive` which just looks saner. At some point later I'll update the docs too, but it'll be an annoying transition period as we'll have to say "On older OSTree, use -z2" etc. Closes: #346 Approved by: giuseppe --- src/libostree/ostree-repo.c | 3 ++- tests/basic-test.sh | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index c40abb78..5e3e31df 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1816,7 +1816,8 @@ ostree_repo_mode_from_string (const char *mode, ret_mode = OSTREE_REPO_MODE_BARE; else if (strcmp (mode, "bare-user") == 0) ret_mode = OSTREE_REPO_MODE_BARE_USER; - else if (strcmp (mode, "archive-z2") == 0) + else if (strcmp (mode, "archive-z2") == 0 || + strcmp (mode, "archive") == 0) ret_mode = OSTREE_REPO_MODE_ARCHIVE_Z2; else { diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 0fe372dc..003df893 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -278,7 +278,7 @@ echo "ok prune" cd ${test_tmpdir} rm repo3 -rf -${CMD_PREFIX} ostree --repo=repo3 init --mode=archive-z2 +${CMD_PREFIX} ostree --repo=repo3 init --mode=archive ${CMD_PREFIX} ostree --repo=repo3 pull-local --remote=aremote repo test2 rm repo3/refs/remotes -rf mkdir repo3/refs/remotes @@ -405,7 +405,7 @@ echo "ok commit of fifo was rejected" cd ${test_tmpdir} rm repo2 -rf mkdir repo2 -${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2 +${CMD_PREFIX} ostree --repo=repo2 init --mode=archive ${CMD_PREFIX} ostree --repo=repo2 pull-local repo rm -rf test2-checkout ${CMD_PREFIX} ostree --repo=repo2 checkout -U --disable-cache test2 test2-checkout From 30c34b2f55e9d301fcc1bd719b59112d8f8ed654 Mon Sep 17 00:00:00 2001 From: Yu Qi Zhang Date: Thu, 16 Jun 2016 18:51:10 +0000 Subject: [PATCH 122/128] libglnx porting: delete temp files on failure of file creation We noticed the temp files being left over in ostree when (mistakenly) trying to create refs with names in use by folders. This fix removes temp files created by glnx_file_replace_contents_at on failure. Closes: #348 Approved by: cgwalters --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index afe3c3a8..a6d08657 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit afe3c3a86178c29ceaa3a5e46397ab2fa97202b3 +Subproject commit a6d08657aa868a0d5c7b5dd494e16f65415a148f From 651f4bc3b9e847ec16245a2b5f13c965ea80e5bb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Jun 2016 21:51:13 -0400 Subject: [PATCH 123/128] repo: Avoid a possible divide by zero in progress The previous code was subject to a divide by zero if less than a second had passed. Rework it so we only do the divide if more than a second has passed. Closes: #349 Approved by: Mathnerd314 --- src/libostree/ostree-repo.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 5e3e31df..e86685bf 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3865,20 +3865,23 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress guint64 start_time = ostree_async_progress_get_uint64 (progress, "start-time"); guint64 total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); guint64 current_time = g_get_monotonic_time (); - guint64 bytes_sec = bytes_transferred / ((current_time - start_time) / G_USEC_PER_SEC); - guint64 est_time_remaining = (total_delta_part_size - bytes_transferred) / bytes_sec; g_autofree char *formatted_bytes_transferred = g_format_size_full (bytes_transferred, 0); g_autofree char *formatted_bytes_sec = NULL; g_autofree char *formatted_est_time_remaining = NULL; - if (!bytes_sec) // Ignore first second + /* Ignore the first second, or when we haven't transferred any + * data, since those could cause divide by zero below. + */ + if ((current_time - start_time) < G_USEC_PER_SEC || bytes_transferred == 0) { formatted_bytes_sec = g_strdup ("-"); formatted_est_time_remaining = g_strdup ("- "); } else { + guint64 bytes_sec = bytes_transferred / ((current_time - start_time) / G_USEC_PER_SEC); + guint64 est_time_remaining = (total_delta_part_size - bytes_transferred) / bytes_sec; formatted_bytes_sec = g_format_size (bytes_sec); formatted_est_time_remaining = _formatted_time_remaining_from_seconds (est_time_remaining); } From d03ae448d04efefc5f2944ef9f3fafcf9e7b743d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Jun 2016 11:25:37 -0400 Subject: [PATCH 124/128] manual: Discuss mirroring This should likely be its own section, but it makes enough sense here for now too. Closes: #347 Approved by: yuqi-zhang --- docs/manual/repository-management.md | 32 +++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/manual/repository-management.md b/docs/manual/repository-management.md index b83f6c15..b6da9629 100644 --- a/docs/manual/repository-management.md +++ b/docs/manual/repository-management.md @@ -4,11 +4,12 @@ Once you have a build system going, if you actually want client systems to retrieve the content, you will quickly feel a need for "repository management". -OSTree itself does not currently come with tools to do this. One -reason is that how content is delivered and managed has concerns very -specific to the organization. For example, some operating system -content vendors may want integration with a specific errata -notification system. +The command line tool `ostree` does cover some core functionality, but +doesn't include very high level workflows. One reason is that how +content is delivered and managed has concerns very specific to the +organization. For example, some operating system content vendors may +want integration with a specific errata notification system when +generating commits. In this section, we will describe some high level ideas and methods for managing content in OSTree repositories, mostly independent of any @@ -21,6 +22,27 @@ repositories today is the [Pulp Project](http://www.pulpproject.org/), which has a [Pulp OSTree plugin](https://pulp-ostree.readthedocs.org/en/latest/). +## Mirroring repositories + +It's very common to want to perform a full or partial mirror, in +particular across organizational boundaries (e.g. an upstream OS +provider, and a user that wants offline and faster access to the +content). OSTree supports both full and partial mirroring of the base +`archive-z2` content, although not yet of static deltas. + +To create a mirror, first create an `archive-z2` repository (you don't +need to run this as root), then add the upstream as a remote, then use +`pull --mirror`. + +``` +ostree --repo=repo init --mode=archive-z2 +ostree --repo=repo remote add exampleos https://exampleos.com/ostree/repo +ostree --repo=repo pull --mirror exampleos:exampleos/x86_64/standard +``` + +You can use the `--depth=-1` option to retrieve all history, or a +positive integer like `3` to retrieve just the last 3 commits. + ## Separate development vs release repositories By default, OSTree accumulates server side history. This is actually From 71e768d23030c4eb5c74ea4df9b8c63544be21fb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Jun 2016 09:36:51 -0400 Subject: [PATCH 125/128] build: Fix libreaddir-rand to honor global CFLAGS It's unfortunate that in automake one has to explicitly include the globa `$(AM_CFLAGS)` if one sets `CFLAGS`, and similarly for other variables. I'm trying to use `-fsanitize=address`, and not including it was causing linker failures. We also weren't inheriting the global warnings etc., so I had to fix a decl-after-statement. Closes: #351 Approved by: jlebon --- Makefile-tests.am | 4 ++-- tests/readdir-rand.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index e5cf9554..80903071 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -137,12 +137,12 @@ endif test_ltlibraries = libreaddir-rand.la libreaddir_rand_la_SOURCES = tests/readdir-rand.c -libreaddir_rand_la_CFLAGS = $(OT_INTERNAL_GIO_UNIX_CFLAGS) +libreaddir_rand_la_CFLAGS = $(AM_CFLAGS) $(OT_INTERNAL_GIO_UNIX_CFLAGS) libreaddir_rand_la_LIBADD = \ -ldl \ $(OT_INTERNAL_GIO_UNIX_LIBS) \ $(NULL) -libreaddir_rand_la_LDFLAGS = -avoid-version +libreaddir_rand_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version if !ENABLE_INSTALLED_TESTS libreaddir_rand_la_LDFLAGS += -rpath $(abs_builddir) endif diff --git a/tests/readdir-rand.c b/tests/readdir-rand.c index ee43756f..afef387c 100644 --- a/tests/readdir-rand.c +++ b/tests/readdir-rand.c @@ -109,12 +109,12 @@ readdir (DIR *dirp) { if (g_random_boolean ()) { + struct dirent *copy; if (!de) { de = dir_entries_new (); g_hash_table_insert (direntcache, dirp, de); } - struct dirent *copy; copy = g_memdup (ret, sizeof (struct dirent)); g_ptr_array_add (de->entries, copy); } From d262fc2e0ff80472594bd931971512971ff4af04 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Jun 2016 09:43:51 -0400 Subject: [PATCH 126/128] tests: Support OT_SKIP_READDIR_RAND For some reason I'm really not inclined to debug right now, `libtool` is eating the `-fsanitize=address` argument when linking `libreaddir-rand.so`, which causes failures since we're trying to `LD_PRELOAD`. Closes: #351 Approved by: jlebon --- tests/libtest.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 16d5c0e0..2d064299 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -80,7 +80,14 @@ fi if test -n "${OT_TESTS_VALGRIND:-}"; then CMD_PREFIX="env G_SLICE=always-malloc OSTREE_SUPPRESS_SYNCFS=1 valgrind -q --error-exitcode=1 --leak-check=full --num-callers=30 --suppressions=${test_srcdir}/glib.supp --suppressions=${test_srcdir}/ostree.supp" else - CMD_PREFIX="env LD_PRELOAD=${test_builddir}/libreaddir-rand.so" + # In some cases the LD_PRELOAD may cause obscure problems, + # e.g. right now it breaks for me with -fsanitize=address, so + # let's allow users to skip it. + if test -z "${OT_SKIP_READDIR_RAND:-}"; then + CMD_PREFIX="env LD_PRELOAD=${test_builddir}/libreaddir-rand.so" + else + CMD_PREFIX="" + fi fi assert_streq () { From 535033a4f01f772e696ad2201b34403203b98bea Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Jun 2016 22:16:27 -0400 Subject: [PATCH 127/128] pull: Ensure we always process queue only from main thread I was easily reproducing a hang on pulls with thousands of requests on current git master. The initial symptom seemed to be that there are multiple code paths where we don't invoke `session_thread_process_pending_queue()`. We really need to do that any time we remove something from the outstanding queue, to ensure it gets filled again. A further issue is that we were tying the lifecycle of the pending object to the `GTask`, but the task could be unref'd from the main thread (via a `GSource` on the main thread), and that introduced threadsafety issues, because the hash table and other data suddenly could be concurrently modified. Both of these need to be fixed together. First, we introduce `Arc`, and ensure that both the main and worker threads hold references. Second, we ensure that we re-process the queue *immediately* whenever a task is done, inside the worker thread, rather than doing it incidentally via an unref. This architecture is quite similar to what the outside pull code is doing. Closes: #350 Approved by: jlebon --- src/libostree/ostree-fetcher.c | 57 ++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index d956d959..313df6a8 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -67,7 +67,12 @@ typedef struct { guint64 total_downloaded; } ThreadClosure; +static void +session_thread_process_pending_queue (ThreadClosure *thread_closure); + typedef struct { + volatile int ref_count; + ThreadClosure *thread_closure; SoupURI *uri; @@ -186,10 +191,22 @@ pending_task_compare (gconstpointer a, (priority_a < priority_b) ? -1 : 1; } -static void -pending_uri_free (OstreeFetcherPendingURI *pending) +static OstreeFetcherPendingURI * +pending_uri_ref (OstreeFetcherPendingURI *pending) { - g_hash_table_remove (pending->thread_closure->outstanding, pending); + g_return_val_if_fail (pending != NULL, NULL); + g_return_val_if_fail (pending->ref_count > 0, NULL); + + g_atomic_int_inc (&pending->ref_count); + + return pending; +} + +static void +pending_uri_unref (OstreeFetcherPendingURI *pending) +{ + if (!g_atomic_int_dec_and_test (&pending->ref_count)) + return; g_clear_pointer (&pending->thread_closure, thread_closure_unref); @@ -331,8 +348,7 @@ session_thread_process_pending_queue (ThreadClosure *thread_closure) pending = g_task_get_task_data (task); cancellable = g_task_get_cancellable (task); - /* pending_uri_free() removes this. */ - g_hash_table_add (thread_closure->outstanding, pending); + g_hash_table_add (thread_closure->outstanding, pending_uri_ref (pending)); soup_request_send_async (pending->request, cancellable, @@ -540,7 +556,7 @@ _ostree_fetcher_constructed (GObject *object) self->thread_closure->tmpdir_dfd = -1; self->thread_closure->tmpdir_lock = empty_lockfile; - self->thread_closure->outstanding = g_hash_table_new (NULL, NULL); + self->thread_closure->outstanding = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify)pending_uri_unref); self->thread_closure->output_stream_set = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) NULL, (GDestroyNotify) g_object_unref); @@ -742,6 +758,18 @@ on_stream_read (GObject *object, GAsyncResult *result, gpointer user_data); +static void +remove_pending_rerun_queue (OstreeFetcherPendingURI *pending) +{ + /* Hold a temporary ref to ensure the reference to + * pending->thread_closure is valid. + */ + pending_uri_ref (pending); + g_hash_table_remove (pending->thread_closure->outstanding, pending); + session_thread_process_pending_queue (pending->thread_closure); + pending_uri_unref (pending); +} + static void on_out_splice_complete (GObject *object, GAsyncResult *result, @@ -770,7 +798,10 @@ on_out_splice_complete (GObject *object, out: if (local_error) - g_task_return_error (task, local_error); + { + g_task_return_error (task, local_error); + remove_pending_rerun_queue (pending); + } g_object_unref (task); } @@ -802,6 +833,7 @@ on_stream_read (GObject *object, g_task_return_pointer (task, g_strdup (pending->out_tmpfile), (GDestroyNotify) g_free); + remove_pending_rerun_queue (pending); } else { @@ -837,7 +869,10 @@ on_stream_read (GObject *object, out: if (local_error) - g_task_return_error (task, local_error); + { + g_task_return_error (task, local_error); + remove_pending_rerun_queue (pending); + } g_object_unref (task); } @@ -883,6 +918,7 @@ on_request_sent (GObject *object, g_strdup (pending->out_tmpfile), (GDestroyNotify) g_free); } + remove_pending_rerun_queue (pending); goto out; } else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code)) @@ -947,6 +983,7 @@ on_request_sent (GObject *object, g_task_return_pointer (task, g_object_ref (pending->request_body), (GDestroyNotify) g_object_unref); + remove_pending_rerun_queue (pending); } out: @@ -955,6 +992,7 @@ on_request_sent (GObject *object, if (pending->request_body) (void) g_input_stream_close (pending->request_body, NULL, NULL); g_task_return_error (task, local_error); + remove_pending_rerun_queue (pending); } g_object_unref (task); @@ -979,6 +1017,7 @@ ostree_fetcher_request_uri_internal (OstreeFetcher *self, /* SoupRequest is created in session thread. */ pending = g_new0 (OstreeFetcherPendingURI, 1); + pending->ref_count = 1; pending->thread_closure = thread_closure_ref (self->thread_closure); pending->uri = soup_uri_copy (uri); pending->max_size = max_size; @@ -986,7 +1025,7 @@ ostree_fetcher_request_uri_internal (OstreeFetcher *self, task = g_task_new (self, cancellable, callback, user_data); g_task_set_source_tag (task, source_tag); - g_task_set_task_data (task, pending, (GDestroyNotify) pending_uri_free); + g_task_set_task_data (task, pending, (GDestroyNotify) pending_uri_unref); /* We'll use the GTask priority for our own priority queue. */ g_task_set_priority (task, priority); From 6c7e191147023942feb7454359557bb3551467a4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 20 Jun 2016 10:46:50 -0400 Subject: [PATCH 128/128] Release 2016.6 --- configure.ac | 2 +- src/libostree/libostree.sym | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index d1b732f7..18f9f277 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.63]) -AC_INIT([ostree], [2016.5], [walters@verbum.org]) +AC_INIT([ostree], [2016.6], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 3e17828a..e8cca9d8 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -333,11 +333,6 @@ global: ostree_repo_set_cache_dir; } LIBOSTREE_2016.4; -/* NOTE NOTE NOTE - * Versions above here are released. Only add symbols below this line. - * NOTE NOTE NOTE - */ - LIBOSTREE_2016.6 { global: ostree_gpg_verify_result_require_valid_signature; @@ -345,3 +340,16 @@ global: ostree_repo_gpg_verify_data; ostree_repo_remote_fetch_summary_with_options; } LIBOSTREE_2016.5; + +/* NOTE NOTE NOTE + * Versions above here are released. Only add symbols below this line. + * NOTE NOTE NOTE + */ + +/* Uncomment this when adding a new symbol */ +/* +LIBOSTREE_2016.7 { +global: + ostree_some_new_symbol; +} LIBOSTREE_2016.6; +*/