From 46c3fc5d76653d7e5497c0782ef21a7d422e5f96 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 13 Jan 2016 10:34:32 -0500 Subject: [PATCH 01/54] repo: Note global transaction resume is legacy See docs for details. https://github.com/GNOME/ostree/pull/169 --- src/libostree/ostree-repo-commit.c | 3 ++- src/libostree/ostree-repo-pull.c | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 09190285..5ea49efa 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1144,7 +1144,8 @@ ostree_repo_scan_hardlinks (OstreeRepo *self, * ostree_repo_prepare_transaction: * @self: An #OstreeRepo * @out_transaction_resume: (allow-none) (out): Whether this transaction - * is resuming from a previous one. + * is resuming from a previous one. This is a legacy state, now OSTree + * pulls use per-commit `state/.commitpartial` files. * @cancellable: Cancellable * @error: Error * diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 907ed454..c6c91082 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -48,7 +48,7 @@ typedef struct { GCancellable *cancellable; OstreeAsyncProgress *progress; - gboolean transaction_resuming; + gboolean legacy_transaction_resuming; enum { OSTREE_PULL_PHASE_FETCHING_REFS, OSTREE_PULL_PHASE_FETCHING_OBJECTS @@ -1227,7 +1227,7 @@ scan_one_metadata_object_c (OtPullData *pull_data, } else if (is_stored) { - gboolean do_scan = pull_data->transaction_resuming || is_requested || pull_data->commitpartial_exists; + gboolean do_scan = pull_data->legacy_transaction_resuming || is_requested || pull_data->commitpartial_exists; /* For commits, always refetch detached metadata. */ if (objtype == OSTREE_OBJECT_TYPE_COMMIT) @@ -2182,11 +2182,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->fetcher == NULL) goto out; - if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->transaction_resuming, + if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->legacy_transaction_resuming, cancellable, error)) goto out; - g_debug ("resuming transaction: %s", pull_data->transaction_resuming ? "true" : " false"); + if (pull_data->legacy_transaction_resuming) + g_debug ("resuming legacy transaction"); g_hash_table_iter_init (&hash_iter, commits_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) From dc9239dd7b09ef5e104309b4dbf0e136889da274 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 13 Jan 2016 10:15:21 -0500 Subject: [PATCH 02/54] sysroot: Don't individually fsync dirs in checkout, rely on syncfs Originally, a lot of the `fsync()` calls here were added for the wrong reason - I was chasing a bug that ended up being the extlinux bootloader not parsing 64 bit ext4 filesystems. But since it looked like corruption, I tried adding a lot more `fsync()` calls. All we should have to do is use `syncfs()`. If that doesn't work, it's a kernel bug. I'm making this change because skipping the individual fsyncs can be a major performance win - it's easier for the FS to optimize, we do more in parallel, etc. https://bugzilla.gnome.org/show_bug.cgi?id=757117 --- doc/ostree-sections.txt | 1 + src/libostree/ostree-repo.c | 13 +++++++++++++ src/libostree/ostree-repo.h | 2 ++ src/libostree/ostree-sysroot-deploy.c | 18 ++++++++++++++---- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/doc/ostree-sections.txt b/doc/ostree-sections.txt index d448d6de..4b22e25b 100644 --- a/doc/ostree-sections.txt +++ b/doc/ostree-sections.txt @@ -220,6 +220,7 @@ ostree_repo_new_for_sysroot_path ostree_repo_new_default ostree_repo_open ostree_repo_set_disable_fsync +ostree_repo_get_disable_fsync ostree_repo_is_system ostree_repo_is_writable ostree_repo_create diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 415421af..d5b9aeaf 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2384,6 +2384,19 @@ ostree_repo_set_disable_fsync (OstreeRepo *self, self->disable_fsync = disable_fsync; } +/** + * ostree_repo_get_disable_fsync: + * @self: An #OstreeRepo + * + * For more information see ostree_repo_set_disable_fsync(). + * + * Returns: Whether or not fsync() is enabled for this repo. + */ +gboolean +ostree_repo_get_disable_fsync (OstreeRepo *self) +{ + return self->disable_fsync; +} /* Replace the contents of a file, honoring the repository's fsync * policy. diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 5bc25204..fd2b7f06 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -56,6 +56,8 @@ gboolean ostree_repo_open (OstreeRepo *self, void ostree_repo_set_disable_fsync (OstreeRepo *self, gboolean disable_fsync); +gboolean ostree_repo_get_disable_fsync (OstreeRepo *self); + gboolean ostree_repo_is_system (OstreeRepo *repo); gboolean ostree_repo_is_writable (OstreeRepo *self, diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 041b0b25..1524a866 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -540,10 +540,20 @@ checkout_deployment_tree (OstreeSysroot *sysroot, if (!glnx_shutil_rm_rf_at (osdeploy_dfd, checkout_target_name, cancellable, error)) goto out; - if (!ostree_repo_checkout_tree_at (repo, &checkout_opts, osdeploy_dfd, - checkout_target_name, csum, - cancellable, error)) - goto out; + /* We end up using syncfs for the entire filesystem, so turn off + * OstreeRepo level fsync. + */ + { gboolean fsync_was_disabled = ostree_repo_get_disable_fsync (repo); + gboolean checkout_success; + + ostree_repo_set_disable_fsync (repo, TRUE); + checkout_success = ostree_repo_checkout_tree_at (repo, &checkout_opts, osdeploy_dfd, + checkout_target_name, csum, + cancellable, error); + ostree_repo_set_disable_fsync (repo, fsync_was_disabled); + if (!checkout_success) + goto out; + } if (!glnx_opendirat (osdeploy_dfd, checkout_target_name, TRUE, &ret_fd, error)) goto out; From a13b56f91c898ac5c24ea587183d7e99dea3c4cb Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Fri, 15 Jan 2016 10:24:08 +0100 Subject: [PATCH 03/54] diff: do not traverse parent commits The object count displayed included also the number of parent commits. Signed-off-by: Giuseppe Scrivano --- src/ostree/ot-builtin-diff.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-builtin-diff.c b/src/ostree/ot-builtin-diff.c index 2a27f5d2..b5e0c5a8 100644 --- a/src/ostree/ot-builtin-diff.c +++ b/src/ostree/ot-builtin-diff.c @@ -194,9 +194,9 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError ** if (!ostree_repo_resolve_rev (repo, target, FALSE, &rev_b, error)) goto out; - if (!ostree_repo_traverse_commit (repo, rev_a, -1, &reachable_a, cancellable, error)) + if (!ostree_repo_traverse_commit (repo, rev_a, 0, &reachable_a, cancellable, error)) goto out; - if (!ostree_repo_traverse_commit (repo, rev_b, -1, &reachable_b, cancellable, error)) + if (!ostree_repo_traverse_commit (repo, rev_b, 0, &reachable_b, cancellable, error)) goto out; a_size = g_hash_table_size (reachable_a); From cd0a9d3435d052f4f994673f1682e4c15b897458 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 23 Jan 2016 15:07:55 -0500 Subject: [PATCH 04/54] Add a checkout option to skip fsync This is a better followup to dc9239dd7b09ef5e104309b4dbf0e136889da274 since I wanted to do fsync-less checkouts in rpm-ostree too, and replicating the "turn off fsync temporarily" was in retrospect just a hack. We can simply add a boolean to the checkout options. https://github.com/GNOME/ostree/pull/172 --- src/libostree/ostree-repo-checkout.c | 34 +++++++++++++++------------ src/libostree/ostree-repo.h | 3 ++- src/libostree/ostree-sysroot-deploy.c | 23 +++++++----------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index ad309e75..0d0e7ef4 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -102,9 +102,16 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, return ret; } +static gboolean +fsync_is_enabled (OstreeRepo *self, + OstreeRepoCheckoutOptions *options) +{ + return !(self->disable_fsync || options->disable_fsync); +} + static gboolean write_regular_file_content (OstreeRepo *self, - OstreeRepoCheckoutMode mode, + OstreeRepoCheckoutOptions *options, GOutputStream *output, GFileInfo *file_info, GVariant *xattrs, @@ -113,6 +120,7 @@ write_regular_file_content (OstreeRepo *self, GError **error) { gboolean ret = FALSE; + const OstreeRepoCheckoutMode mode = options->mode; int fd; int res; @@ -154,12 +162,12 @@ write_regular_file_content (OstreeRepo *self, } } - if (!self->disable_fsync) + if (fsync_is_enabled (self, options)) { if (fsync (fd) == -1) { gs_set_error_from_errno (error, errno); - goto out; + goto out; } } @@ -238,7 +246,7 @@ checkout_file_from_input_at (OstreeRepo *self, temp_out = g_unix_output_stream_new (fd, TRUE); fd = -1; /* Transfer ownership */ - if (!write_regular_file_content (self, options->mode, temp_out, file_info, xattrs, input, + if (!write_regular_file_content (self, options, temp_out, file_info, xattrs, input, cancellable, error)) goto out; } @@ -298,7 +306,7 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo, cancellable, error)) goto out; - if (!write_regular_file_content (repo, options->mode, temp_out, file_info, xattrs, input, + if (!write_regular_file_content (repo, options, temp_out, file_info, xattrs, input, cancellable, error)) goto out; } @@ -726,17 +734,13 @@ checkout_tree_at (OstreeRepo *self, } } - /* Finally, fsync to ensure all entries are on disk. Ultimately - * this should be configurable for the case where we're constructing - * buildroots. - */ - if (!self->disable_fsync) + if (fsync_is_enabled (self, options)) { - if (fsync (destination_dfd) == -1) - { - gs_set_error_from_errno (error, errno); - goto out; - } + if (fsync (destination_dfd) == -1) + { + gs_set_error_from_errno (error, errno); + goto out; + } } ret = TRUE; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index fd2b7f06..d4d0f418 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -532,7 +532,8 @@ typedef struct { OstreeRepoCheckoutOverwriteMode overwrite_mode; guint enable_uncompressed_cache : 1; - guint unused : 31; + guint disable_fsync : 1; + guint reserved : 30; const char *subpath; diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 1524a866..3dcf39ff 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -528,6 +528,11 @@ checkout_deployment_tree (OstreeSysroot *sysroot, glnx_fd_close int osdeploy_dfd = -1; int ret_fd; + /* We end up using syncfs for the entire filesystem, so turn off + * OstreeRepo level fsync. + */ + checkout_opts.disable_fsync = TRUE; + osdeploy_path = g_strconcat ("ostree/deploy/", ostree_deployment_get_osname (deployment), "/deploy", NULL); checkout_target_name = g_strdup_printf ("%s.%d", csum, ostree_deployment_get_deployserial (deployment)); @@ -540,20 +545,10 @@ checkout_deployment_tree (OstreeSysroot *sysroot, if (!glnx_shutil_rm_rf_at (osdeploy_dfd, checkout_target_name, cancellable, error)) goto out; - /* We end up using syncfs for the entire filesystem, so turn off - * OstreeRepo level fsync. - */ - { gboolean fsync_was_disabled = ostree_repo_get_disable_fsync (repo); - gboolean checkout_success; - - ostree_repo_set_disable_fsync (repo, TRUE); - checkout_success = ostree_repo_checkout_tree_at (repo, &checkout_opts, osdeploy_dfd, - checkout_target_name, csum, - cancellable, error); - ostree_repo_set_disable_fsync (repo, fsync_was_disabled); - if (!checkout_success) - goto out; - } + if (!ostree_repo_checkout_tree_at (repo, &checkout_opts, osdeploy_dfd, + checkout_target_name, csum, + cancellable, error)) + goto out; if (!glnx_opendirat (osdeploy_dfd, checkout_target_name, TRUE, &ret_fd, error)) goto out; From 91a1f91440a974dfce4c0e1bedc2756d4f632972 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 26 Jan 2016 16:48:37 -0500 Subject: [PATCH 05/54] refs: Add a missing `goto out` for error handling If the `refs/remotes` directory doesn't exist, we'd trip an assertion. --- src/libostree/ostree-repo-refs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index 6d3bd51e..4fc415b2 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -551,6 +551,8 @@ ostree_repo_list_refs (OstreeRepo *self, remote_enumerator = g_file_enumerate_children (self->remote_heads_dir, OSTREE_GIO_FAST_QUERYINFO, 0, cancellable, error); + if (!remote_enumerator) + goto out; while (TRUE) { From 3d2322db7a609fb4653ce6bf6c4b96b0c315a5b7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 27 Jan 2016 10:43:57 -0500 Subject: [PATCH 06/54] grub2: Don't delete grub2.cfg.old file we just copied The original intention here was that we'd keey around a copy of the file so that grub2 could eventually learn how to do atomic updates by checking for a "fully written" marker in the *new* file, and if it didn't exist, falling back to grub2.cfg.old. I haven't yet proposed that upstream, but we might as well stop deleting the file since it's useful as a backup at least. Reported-by: Gatis Paeglis --- src/libostree/ostree-bootloader-grub2.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libostree/ostree-bootloader-grub2.c b/src/libostree/ostree-bootloader-grub2.c index 1f899143..42ed5acc 100644 --- a/src/libostree/ostree-bootloader-grub2.c +++ b/src/libostree/ostree-bootloader-grub2.c @@ -388,8 +388,6 @@ rm -f ${grub_cfg}.new if (!g_file_copy (self->config_path_efi, config_path_efi_old, G_FILE_COPY_OVERWRITE, cancellable, NULL, NULL, error)) goto out; - if (!ot_gfile_ensure_unlinked (config_path_efi_old, cancellable, error)) - goto out; /* NOTE: NON-ATOMIC REPLACEMENT; WE can't do anything else on FAT; * see https://bugzilla.gnome.org/show_bug.cgi?id=724246 From 5ebe43859d284a003e240e0cfe66b5185681e993 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 27 Jan 2016 11:44:10 -0500 Subject: [PATCH 07/54] tests: Use "bash strict mode" I noticed in the static deltas tests, there were some tests that should have been under `-o pipefail` to ensure we properly propagate errors. There were a few places where we were referencing undefined variables. Overall, this is clearly a good idea IMO. --- tests/admin-test.sh | 2 +- tests/archive-test.sh | 2 +- tests/basic-test.sh | 2 +- tests/libtest.sh | 20 +++++++------------ tests/pull-test.sh | 2 +- tests/test-admin-deploy-2.sh | 3 +-- tests/test-admin-deploy-clean.sh | 2 +- .../test-admin-deploy-etcmerge-cornercases.sh | 2 +- tests/test-admin-deploy-grub2.sh | 2 +- tests/test-admin-deploy-karg.sh | 2 +- tests/test-admin-deploy-switch.sh | 2 +- tests/test-admin-deploy-syslinux.sh | 2 +- tests/test-admin-deploy-uboot.sh | 2 +- tests/test-admin-instutil-set-kargs.sh | 2 +- tests/test-admin-locking.sh | 2 +- tests/test-admin-pull-deploy-commit.sh | 2 +- tests/test-admin-upgrade-not-backwards.sh | 2 +- tests/test-admin-upgrade-unconfigured.sh | 2 +- tests/test-archivez.sh | 2 +- tests/test-auto-summary.sh | 2 +- tests/test-basic-user.sh | 2 +- tests/test-basic.sh | 2 +- tests/test-commit-sign.sh | 4 ++-- tests/test-corruption.sh | 2 +- tests/test-delta.sh | 2 +- tests/test-gpg-signed-commit.sh | 4 ++-- tests/test-help.sh | 4 ++-- tests/test-libarchive.sh | 4 ++-- tests/test-local-pull-depth.sh | 2 +- tests/test-local-pull.sh | 2 +- tests/test-oldstyle-partial.sh | 2 +- tests/test-prune.sh | 2 +- tests/test-pull-archive-z.sh | 2 +- tests/test-pull-commit-only.sh | 2 +- tests/test-pull-corruption.sh | 4 ++-- tests/test-pull-depth.sh | 2 +- tests/test-pull-large-metadata.sh | 2 +- tests/test-pull-metalink.sh | 2 +- tests/test-pull-mirror-summary.sh | 2 +- tests/test-pull-resume.sh | 2 +- tests/test-pull-subpath.sh | 2 +- tests/test-pull-summary-sigs.sh | 2 +- tests/test-remote-add.sh | 2 +- tests/test-remote-gpg-import.sh | 2 +- tests/test-repo-checkout-subpath.sh | 2 +- tests/test-reset-nonlinear.sh | 2 +- tests/test-setuid.sh | 2 +- tests/test-xattrs.sh | 2 +- 48 files changed, 59 insertions(+), 66 deletions(-) diff --git a/tests/admin-test.sh b/tests/admin-test.sh index edbf6651..a587d79b 100755 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -16,7 +16,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..10" diff --git a/tests/archive-test.sh b/tests/archive-test.sh index 9d9c0a28..e6f67cf5 100755 --- a/tests/archive-test.sh +++ b/tests/archive-test.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail $OSTREE checkout test2 checkout-test2 echo "ok checkout" diff --git a/tests/basic-test.sh b/tests/basic-test.sh index e8f1e9d2..d39f32cf 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..48" diff --git a/tests/libtest.sh b/tests/libtest.sh index 885c6403..bd806f6d 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -33,11 +33,11 @@ cp -a ${SRCDIR}/gpghome ${test_tmpdir} export TEST_GPG_KEYHOME=${test_tmpdir}/gpghome export OSTREE_GPG_HOME=${test_tmpdir}/gpghome/trusted -if test -n "${OT_TESTS_DEBUG}"; then +if test -n "${OT_TESTS_DEBUG:-}"; then set -x fi -if test -n "$OT_TESTS_VALGRIND"; then +if test -n "${OT_TESTS_VALGRIND:-}"; then CMD_PREFIX="env G_SLICE=always-malloc valgrind -q --leak-check=full --num-callers=30 --suppressions=${SRCDIR}/ostree-valgrind.supp" else CMD_PREFIX="env LD_PRELOAD=${SRCDIR}/libreaddir-rand.so" @@ -139,8 +139,8 @@ setup_test_repository () { setup_fake_remote_repo1() { mode=$1 - commit_opts=$2 - args=$3 + commit_opts=${2:-} + args=${3:-} shift oldpwd=`pwd` mkdir ostree-srv @@ -272,7 +272,7 @@ EOF mkdir ${test_tmpdir}/httpd cd httpd ln -s ${test_tmpdir} ostree - ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args + ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port port=$(cat ${test_tmpdir}/httpd-port) echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address cd ${oldpwd} @@ -280,15 +280,9 @@ EOF os_repository_new_commit () { - boot_checksum_iteration=$1 - content_iteration=$2 + boot_checksum_iteration=${1:-0} + content_iteration=${2:-0} echo "BOOT ITERATION: $boot_checksum_iteration" - if test -z "$boot_checksum_iteration"; then - boot_checksum_iteration=0 - fi - if test -z "$content_iteration"; then - content_iteration=0 - fi cd ${test_tmpdir}/osdata rm boot/* echo "new: a kernel ${boot_checksum_iteration}" > boot/vmlinuz-3.6.0 diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 42979fe9..6f0b651a 100755 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail function repo_init() { cd ${test_tmpdir} diff --git a/tests/test-admin-deploy-2.sh b/tests/test-admin-deploy-2.sh index d6117de0..ef6b5953 100755 --- a/tests/test-admin-deploy-2.sh +++ b/tests/test-admin-deploy-2.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh @@ -49,7 +49,6 @@ os_repository_new_commit "1" bootcsum3=${bootcsum} ${CMD_PREFIX} ostree admin upgrade --os=testos -rev=${newrev} newrev=$(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse testos/buildmaster/x86_64-runtime) assert_not_streq ${rev} ${newrev} assert_not_streq ${bootcsum1} ${bootcsum2} diff --git a/tests/test-admin-deploy-clean.sh b/tests/test-admin-deploy-clean.sh index 19f71e61..58283108 100644 --- a/tests/test-admin-deploy-clean.sh +++ b/tests/test-admin-deploy-clean.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-etcmerge-cornercases.sh b/tests/test-admin-deploy-etcmerge-cornercases.sh index 0541a670..4b0d781b 100644 --- a/tests/test-admin-deploy-etcmerge-cornercases.sh +++ b/tests/test-admin-deploy-etcmerge-cornercases.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-grub2.sh b/tests/test-admin-deploy-grub2.sh index 94c4bc03..8da294d7 100755 --- a/tests/test-admin-deploy-grub2.sh +++ b/tests/test-admin-deploy-grub2.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-karg.sh b/tests/test-admin-deploy-karg.sh index 9decec67..a8c1e594 100644 --- a/tests/test-admin-deploy-karg.sh +++ b/tests/test-admin-deploy-karg.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-switch.sh b/tests/test-admin-deploy-switch.sh index 7e1e173a..4a52000c 100755 --- a/tests/test-admin-deploy-switch.sh +++ b/tests/test-admin-deploy-switch.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-syslinux.sh b/tests/test-admin-deploy-syslinux.sh index 97ee32df..5883f76d 100755 --- a/tests/test-admin-deploy-syslinux.sh +++ b/tests/test-admin-deploy-syslinux.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-uboot.sh b/tests/test-admin-deploy-uboot.sh index 219db14b..c22af6f3 100755 --- a/tests/test-admin-deploy-uboot.sh +++ b/tests/test-admin-deploy-uboot.sh @@ -18,7 +18,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-instutil-set-kargs.sh b/tests/test-admin-instutil-set-kargs.sh index 04f98c01..33b2b74e 100644 --- a/tests/test-admin-instutil-set-kargs.sh +++ b/tests/test-admin-instutil-set-kargs.sh @@ -18,7 +18,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-locking.sh b/tests/test-admin-locking.sh index 6b432202..5f00f571 100644 --- a/tests/test-admin-locking.sh +++ b/tests/test-admin-locking.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-pull-deploy-commit.sh b/tests/test-admin-pull-deploy-commit.sh index 08c1e6bd..e1f7def6 100644 --- a/tests/test-admin-pull-deploy-commit.sh +++ b/tests/test-admin-pull-deploy-commit.sh @@ -19,7 +19,7 @@ # See https://github.com/GNOME/ostree/pull/145 -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-upgrade-not-backwards.sh b/tests/test-admin-upgrade-not-backwards.sh index c42dc242..1b99e25d 100644 --- a/tests/test-admin-upgrade-not-backwards.sh +++ b/tests/test-admin-upgrade-not-backwards.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-upgrade-unconfigured.sh b/tests/test-admin-upgrade-unconfigured.sh index cbc1e753..38df710d 100644 --- a/tests/test-admin-upgrade-unconfigured.sh +++ b/tests/test-admin-upgrade-unconfigured.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-archivez.sh b/tests/test-archivez.sh index d5b7fc35..5db973ff 100755 --- a/tests/test-archivez.sh +++ b/tests/test-archivez.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-auto-summary.sh b/tests/test-auto-summary.sh index 2452e553..6cb52e8c 100755 --- a/tests/test-auto-summary.sh +++ b/tests/test-auto-summary.sh @@ -18,7 +18,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh index c1705254..f53de89a 100755 --- a/tests/test-basic-user.sh +++ b/tests/test-basic-user.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-basic.sh b/tests/test-basic.sh index 3c55756e..ae55aab2 100755 --- a/tests/test-basic.sh +++ b/tests/test-basic.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-commit-sign.sh b/tests/test-commit-sign.sh index 0dfbb768..2db671ee 100755 --- a/tests/test-commit-sign.sh +++ b/tests/test-commit-sign.sh @@ -17,9 +17,9 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail -if ! ${CMD_PREFIX} ostree --version | grep -q -e '\+gpgme'; then +if ! ostree --version | grep -q -e '\+gpgme'; then exit 77 fi diff --git a/tests/test-corruption.sh b/tests/test-corruption.sh index 9ad6aaf2..ef0e94ef 100755 --- a/tests/test-corruption.sh +++ b/tests/test-corruption.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..2" diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 2dddefee..b31d65e9 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-gpg-signed-commit.sh b/tests/test-gpg-signed-commit.sh index ba361c7a..b713da2d 100644 --- a/tests/test-gpg-signed-commit.sh +++ b/tests/test-gpg-signed-commit.sh @@ -18,9 +18,9 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail -if ! ${CMD_PREFIX} ostree --version | grep -q -e '\+gpgme'; then +if ! ostree --version | grep -q -e '\+gpgme'; then exit 77 fi diff --git a/tests/test-help.sh b/tests/test-help.sh index 37d9fa1d..ca555b1b 100755 --- a/tests/test-help.sh +++ b/tests/test-help.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh @@ -45,7 +45,7 @@ test_recursive() { if [ $? = 0 ] ; then echo 1>&2 "missing subcommand but 0 exit status"; exit 1 fi - set -e + set -euo pipefail # error message and usage goes to standard error assert_file_has_content err "[Uu]sage" assert_file_has_content err "$cmd" diff --git a/tests/test-libarchive.sh b/tests/test-libarchive.sh index c875f6b0..92e24083 100755 --- a/tests/test-libarchive.sh +++ b/tests/test-libarchive.sh @@ -17,9 +17,9 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail -if ! ${CMD_PREFIX} ostree --version | grep -q -e '\+libarchive'; then +if ! ostree --version | grep -q -e '\+libarchive'; then exit 77 fi diff --git a/tests/test-local-pull-depth.sh b/tests/test-local-pull-depth.sh index 4f8988dc..e89d0914 100755 --- a/tests/test-local-pull-depth.sh +++ b/tests/test-local-pull-depth.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index df5342de..a9beb083 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-oldstyle-partial.sh b/tests/test-oldstyle-partial.sh index b7da9a86..220b0831 100644 --- a/tests/test-oldstyle-partial.sh +++ b/tests/test-oldstyle-partial.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-prune.sh b/tests/test-prune.sh index 28695b8f..a1322d90 100644 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-archive-z.sh b/tests/test-pull-archive-z.sh index 0d208528..6482f6f9 100755 --- a/tests/test-pull-archive-z.sh +++ b/tests/test-pull-archive-z.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-commit-only.sh b/tests/test-pull-commit-only.sh index 136cc3b9..775b2f71 100755 --- a/tests/test-pull-commit-only.sh +++ b/tests/test-pull-commit-only.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-corruption.sh b/tests/test-pull-corruption.sh index e81e4229..1df31943 100755 --- a/tests/test-pull-corruption.sh +++ b/tests/test-pull-corruption.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh @@ -55,4 +55,4 @@ gjs --help >/dev/null 2>&1 || exit 77 gjs $(dirname $0)/corrupt-repo-ref.js ${repopath} main || true assert_file_has_content corrupted-status.txt 'Changed byte' do_corrupt_pull_test -echo "ok corruption $iteration" +echo "ok corruption" diff --git a/tests/test-pull-depth.sh b/tests/test-pull-depth.sh index 35191cc4..7a52f9cb 100644 --- a/tests/test-pull-depth.sh +++ b/tests/test-pull-depth.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-large-metadata.sh b/tests/test-pull-large-metadata.sh index f86ce4be..c50d7943 100644 --- a/tests/test-pull-large-metadata.sh +++ b/tests/test-pull-large-metadata.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-metalink.sh b/tests/test-pull-metalink.sh index 84ef3739..52d2d503 100755 --- a/tests/test-pull-metalink.sh +++ b/tests/test-pull-metalink.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-mirror-summary.sh b/tests/test-pull-mirror-summary.sh index 958044f9..de55b59b 100755 --- a/tests/test-pull-mirror-summary.sh +++ b/tests/test-pull-mirror-summary.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-resume.sh b/tests/test-pull-resume.sh index 25ee907e..1e7220d0 100755 --- a/tests/test-pull-resume.sh +++ b/tests/test-pull-resume.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-subpath.sh b/tests/test-pull-subpath.sh index f2abbb3b..70348322 100644 --- a/tests/test-pull-subpath.sh +++ b/tests/test-pull-subpath.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-summary-sigs.sh b/tests/test-pull-summary-sigs.sh index 7afca9a3..dbcc67a5 100644 --- a/tests/test-pull-summary-sigs.sh +++ b/tests/test-pull-summary-sigs.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-remote-add.sh b/tests/test-remote-add.sh index 392dda54..2294a06d 100755 --- a/tests/test-remote-add.sh +++ b/tests/test-remote-add.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-remote-gpg-import.sh b/tests/test-remote-gpg-import.sh index 0dc5424e..fc833493 100755 --- a/tests/test-remote-gpg-import.sh +++ b/tests/test-remote-gpg-import.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-repo-checkout-subpath.sh b/tests/test-repo-checkout-subpath.sh index 343b2614..bf792184 100755 --- a/tests/test-repo-checkout-subpath.sh +++ b/tests/test-repo-checkout-subpath.sh @@ -18,7 +18,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-reset-nonlinear.sh b/tests/test-reset-nonlinear.sh index 1d0f8de6..735f1523 100755 --- a/tests/test-reset-nonlinear.sh +++ b/tests/test-reset-nonlinear.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-setuid.sh b/tests/test-setuid.sh index 02aa9f61..5354d1f3 100755 --- a/tests/test-setuid.sh +++ b/tests/test-setuid.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-xattrs.sh b/tests/test-xattrs.sh index b95707e2..6a83a0bc 100755 --- a/tests/test-xattrs.sh +++ b/tests/test-xattrs.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail touch test-xattrs if ! setfattr -n user.testvalue -v somevalue test-xattrs; then From 313b4720e8b76abf8b7840e79a7a8a0662a82edb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 27 Jan 2016 12:51:07 -0500 Subject: [PATCH 08/54] build: Move man pages into man/ This is preparation for having 3 separate doc build systems (whee): - xsltproc for the man pages - gtk-doc for the API docs - mkdocs for a real manual --- Makefile-man.am | 54 +++++++++++++++++++++++ Makefile.am | 1 + configure.ac | 1 + doc/Makefile.am | 36 --------------- {doc => man}/ostree-admin-cleanup.xml | 0 {doc => man}/ostree-admin-config-diff.xml | 0 {doc => man}/ostree-admin-deploy.xml | 0 {doc => man}/ostree-admin-init-fs.xml | 0 {doc => man}/ostree-admin-instutil.xml | 0 {doc => man}/ostree-admin-os-init.xml | 0 {doc => man}/ostree-admin-set-origin.xml | 0 {doc => man}/ostree-admin-status.xml | 0 {doc => man}/ostree-admin-switch.xml | 0 {doc => man}/ostree-admin-undeploy.xml | 0 {doc => man}/ostree-admin-upgrade.xml | 0 {doc => man}/ostree-admin.xml | 0 {doc => man}/ostree-cat.xml | 0 {doc => man}/ostree-checkout.xml | 0 {doc => man}/ostree-checksum.xml | 0 {doc => man}/ostree-commit.xml | 0 {doc => man}/ostree-config.xml | 0 {doc => man}/ostree-diff.xml | 0 {doc => man}/ostree-fsck.xml | 0 {doc => man}/ostree-gpg-sign.xml | 0 {doc => man}/ostree-init.xml | 0 {doc => man}/ostree-log.xml | 0 {doc => man}/ostree-ls.xml | 0 {doc => man}/ostree-prune.xml | 0 {doc => man}/ostree-pull-local.xml | 0 {doc => man}/ostree-pull.xml | 0 {doc => man}/ostree-refs.xml | 0 {doc => man}/ostree-remote.xml | 0 {doc => man}/ostree-reset.xml | 0 {doc => man}/ostree-rev-parse.xml | 0 {doc => man}/ostree-show.xml | 0 {doc => man}/ostree-static-delta.xml | 0 {doc => man}/ostree-summary.xml | 0 {doc => man}/ostree-trivial-httpd.xml | 0 {doc => man}/ostree.repo-config.xml | 0 {doc => man}/ostree.repo.xml | 0 {doc => man}/ostree.xml | 0 41 files changed, 56 insertions(+), 36 deletions(-) create mode 100644 Makefile-man.am rename {doc => man}/ostree-admin-cleanup.xml (100%) rename {doc => man}/ostree-admin-config-diff.xml (100%) rename {doc => man}/ostree-admin-deploy.xml (100%) rename {doc => man}/ostree-admin-init-fs.xml (100%) rename {doc => man}/ostree-admin-instutil.xml (100%) rename {doc => man}/ostree-admin-os-init.xml (100%) rename {doc => man}/ostree-admin-set-origin.xml (100%) rename {doc => man}/ostree-admin-status.xml (100%) rename {doc => man}/ostree-admin-switch.xml (100%) rename {doc => man}/ostree-admin-undeploy.xml (100%) rename {doc => man}/ostree-admin-upgrade.xml (100%) rename {doc => man}/ostree-admin.xml (100%) rename {doc => man}/ostree-cat.xml (100%) rename {doc => man}/ostree-checkout.xml (100%) rename {doc => man}/ostree-checksum.xml (100%) rename {doc => man}/ostree-commit.xml (100%) rename {doc => man}/ostree-config.xml (100%) rename {doc => man}/ostree-diff.xml (100%) rename {doc => man}/ostree-fsck.xml (100%) rename {doc => man}/ostree-gpg-sign.xml (100%) rename {doc => man}/ostree-init.xml (100%) rename {doc => man}/ostree-log.xml (100%) rename {doc => man}/ostree-ls.xml (100%) rename {doc => man}/ostree-prune.xml (100%) rename {doc => man}/ostree-pull-local.xml (100%) rename {doc => man}/ostree-pull.xml (100%) rename {doc => man}/ostree-refs.xml (100%) rename {doc => man}/ostree-remote.xml (100%) rename {doc => man}/ostree-reset.xml (100%) rename {doc => man}/ostree-rev-parse.xml (100%) rename {doc => man}/ostree-show.xml (100%) rename {doc => man}/ostree-static-delta.xml (100%) rename {doc => man}/ostree-summary.xml (100%) rename {doc => man}/ostree-trivial-httpd.xml (100%) rename {doc => man}/ostree.repo-config.xml (100%) rename {doc => man}/ostree.repo.xml (100%) rename {doc => man}/ostree.xml (100%) diff --git a/Makefile-man.am b/Makefile-man.am new file mode 100644 index 00000000..d2f31ca0 --- /dev/null +++ b/Makefile-man.am @@ -0,0 +1,54 @@ +# Makefile for man/ +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 + +man5_files = ostree.repo.5 ostree.repo-config.5 + +man1_MANS = $(addprefix man/,$(man1_files)) +man5_MANS = $(addprefix man/,$(man5_files)) + +EXTRA_DIST += $(man1_MANS) $(man5_MANS) + +if ENABLE_XSLTPROC + +XSLT_STYLESHEET = http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl + +XSLTPROC_FLAGS = \ + --nonet \ + --stringparam man.output.quietly 1 \ + --stringparam funcsynopsis.style ansi \ + --stringparam man.th.extra1.suppress 1 \ + --stringparam man.authors.section.enabled 0 \ + --stringparam man.copyright.section.enabled 0 + +XSLTPROC_MAN = $(XSLTPROC) $(XSLTPROC_FLAGS) + +%.1: %.xml + $(AM_V_GEN) $(XSLTPROC_MAN) --output $@ $(XSLT_STYLESHEET) $< + +%.5: %.xml + $(AM_V_GEN) $(XSLTPROC_MAN) --output $@ $(XSLT_STYLESHEET) $< + +CLEANFILES += \ + $(man1_MANS) \ + $(man5_MANS) \ + $(NULL) + +endif # ENABLE_GTK_DOC diff --git a/Makefile.am b/Makefile.am index cc44166a..172af0b2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -70,6 +70,7 @@ include Makefile-ostree.am include Makefile-switchroot.am include Makefile-tests.am include Makefile-boot.am +include Makefile-man.am release-tag: git tag -m "Release $(VERSION)" v$(VERSION) diff --git a/configure.ac b/configure.ac index b5bc1e56..46b77fbf 100644 --- a/configure.ac +++ b/configure.ac @@ -127,6 +127,7 @@ AM_CONDITIONAL([ENABLE_GTK_DOC], false) ]) AC_PATH_PROG([XSLTPROC], [xsltproc]) +AM_CONDITIONAL([ENABLE_XSLTPROC], test -n "${XSLTPROC}") AC_ARG_WITH(libarchive, AS_HELP_STRING([--without-libarchive], [Do not use libarchive]), diff --git a/doc/Makefile.am b/doc/Makefile.am index 9406f3ef..c40057e0 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -126,43 +126,7 @@ version.xml: # This includes the standard gtk-doc make rules, copied by gtkdocize. include $(top_srcdir)/gtk-doc.make -man1_MANS = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 - -man5_MANS = ostree.repo.5 ostree.repo-config.5 - -if ENABLE_GTK_DOC - -XSLTPROC_FLAGS = \ - --nonet \ - --stringparam man.output.quietly 1 \ - --stringparam funcsynopsis.style ansi \ - --stringparam man.th.extra1.suppress 1 \ - --stringparam man.authors.section.enabled 0 \ - --stringparam man.copyright.section.enabled 0 - -XSLTPROC_MAN = \ - $(XSLTPROC) $(XSLTPROC_FLAGS) http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl - -.xml.1: - $(AM_V_GEN) $(XSLTPROC_MAN) $< -.xml.5: - $(AM_V_GEN) $(XSLTPROC_MAN) $< - -CLEANFILES += \ - $(man1_MANS) \ - $(man5_MANS) \ - $(NULL) - -endif # ENABLE_GTK_DOC - -MAN_IN_FILES = \ - $(man1_MANS:.1=.xml) \ - $(man5_MANS:.5=.xml) \ - $(NULL) - EXTRA_DIST += \ - $(MAN_IN_FILES) \ version.xml \ - ostree.xml \ ostree-sections.txt \ $(NULL) diff --git a/doc/ostree-admin-cleanup.xml b/man/ostree-admin-cleanup.xml similarity index 100% rename from doc/ostree-admin-cleanup.xml rename to man/ostree-admin-cleanup.xml diff --git a/doc/ostree-admin-config-diff.xml b/man/ostree-admin-config-diff.xml similarity index 100% rename from doc/ostree-admin-config-diff.xml rename to man/ostree-admin-config-diff.xml diff --git a/doc/ostree-admin-deploy.xml b/man/ostree-admin-deploy.xml similarity index 100% rename from doc/ostree-admin-deploy.xml rename to man/ostree-admin-deploy.xml diff --git a/doc/ostree-admin-init-fs.xml b/man/ostree-admin-init-fs.xml similarity index 100% rename from doc/ostree-admin-init-fs.xml rename to man/ostree-admin-init-fs.xml diff --git a/doc/ostree-admin-instutil.xml b/man/ostree-admin-instutil.xml similarity index 100% rename from doc/ostree-admin-instutil.xml rename to man/ostree-admin-instutil.xml diff --git a/doc/ostree-admin-os-init.xml b/man/ostree-admin-os-init.xml similarity index 100% rename from doc/ostree-admin-os-init.xml rename to man/ostree-admin-os-init.xml diff --git a/doc/ostree-admin-set-origin.xml b/man/ostree-admin-set-origin.xml similarity index 100% rename from doc/ostree-admin-set-origin.xml rename to man/ostree-admin-set-origin.xml diff --git a/doc/ostree-admin-status.xml b/man/ostree-admin-status.xml similarity index 100% rename from doc/ostree-admin-status.xml rename to man/ostree-admin-status.xml diff --git a/doc/ostree-admin-switch.xml b/man/ostree-admin-switch.xml similarity index 100% rename from doc/ostree-admin-switch.xml rename to man/ostree-admin-switch.xml diff --git a/doc/ostree-admin-undeploy.xml b/man/ostree-admin-undeploy.xml similarity index 100% rename from doc/ostree-admin-undeploy.xml rename to man/ostree-admin-undeploy.xml diff --git a/doc/ostree-admin-upgrade.xml b/man/ostree-admin-upgrade.xml similarity index 100% rename from doc/ostree-admin-upgrade.xml rename to man/ostree-admin-upgrade.xml diff --git a/doc/ostree-admin.xml b/man/ostree-admin.xml similarity index 100% rename from doc/ostree-admin.xml rename to man/ostree-admin.xml diff --git a/doc/ostree-cat.xml b/man/ostree-cat.xml similarity index 100% rename from doc/ostree-cat.xml rename to man/ostree-cat.xml diff --git a/doc/ostree-checkout.xml b/man/ostree-checkout.xml similarity index 100% rename from doc/ostree-checkout.xml rename to man/ostree-checkout.xml diff --git a/doc/ostree-checksum.xml b/man/ostree-checksum.xml similarity index 100% rename from doc/ostree-checksum.xml rename to man/ostree-checksum.xml diff --git a/doc/ostree-commit.xml b/man/ostree-commit.xml similarity index 100% rename from doc/ostree-commit.xml rename to man/ostree-commit.xml diff --git a/doc/ostree-config.xml b/man/ostree-config.xml similarity index 100% rename from doc/ostree-config.xml rename to man/ostree-config.xml diff --git a/doc/ostree-diff.xml b/man/ostree-diff.xml similarity index 100% rename from doc/ostree-diff.xml rename to man/ostree-diff.xml diff --git a/doc/ostree-fsck.xml b/man/ostree-fsck.xml similarity index 100% rename from doc/ostree-fsck.xml rename to man/ostree-fsck.xml diff --git a/doc/ostree-gpg-sign.xml b/man/ostree-gpg-sign.xml similarity index 100% rename from doc/ostree-gpg-sign.xml rename to man/ostree-gpg-sign.xml diff --git a/doc/ostree-init.xml b/man/ostree-init.xml similarity index 100% rename from doc/ostree-init.xml rename to man/ostree-init.xml diff --git a/doc/ostree-log.xml b/man/ostree-log.xml similarity index 100% rename from doc/ostree-log.xml rename to man/ostree-log.xml diff --git a/doc/ostree-ls.xml b/man/ostree-ls.xml similarity index 100% rename from doc/ostree-ls.xml rename to man/ostree-ls.xml diff --git a/doc/ostree-prune.xml b/man/ostree-prune.xml similarity index 100% rename from doc/ostree-prune.xml rename to man/ostree-prune.xml diff --git a/doc/ostree-pull-local.xml b/man/ostree-pull-local.xml similarity index 100% rename from doc/ostree-pull-local.xml rename to man/ostree-pull-local.xml diff --git a/doc/ostree-pull.xml b/man/ostree-pull.xml similarity index 100% rename from doc/ostree-pull.xml rename to man/ostree-pull.xml diff --git a/doc/ostree-refs.xml b/man/ostree-refs.xml similarity index 100% rename from doc/ostree-refs.xml rename to man/ostree-refs.xml diff --git a/doc/ostree-remote.xml b/man/ostree-remote.xml similarity index 100% rename from doc/ostree-remote.xml rename to man/ostree-remote.xml diff --git a/doc/ostree-reset.xml b/man/ostree-reset.xml similarity index 100% rename from doc/ostree-reset.xml rename to man/ostree-reset.xml diff --git a/doc/ostree-rev-parse.xml b/man/ostree-rev-parse.xml similarity index 100% rename from doc/ostree-rev-parse.xml rename to man/ostree-rev-parse.xml diff --git a/doc/ostree-show.xml b/man/ostree-show.xml similarity index 100% rename from doc/ostree-show.xml rename to man/ostree-show.xml diff --git a/doc/ostree-static-delta.xml b/man/ostree-static-delta.xml similarity index 100% rename from doc/ostree-static-delta.xml rename to man/ostree-static-delta.xml diff --git a/doc/ostree-summary.xml b/man/ostree-summary.xml similarity index 100% rename from doc/ostree-summary.xml rename to man/ostree-summary.xml diff --git a/doc/ostree-trivial-httpd.xml b/man/ostree-trivial-httpd.xml similarity index 100% rename from doc/ostree-trivial-httpd.xml rename to man/ostree-trivial-httpd.xml diff --git a/doc/ostree.repo-config.xml b/man/ostree.repo-config.xml similarity index 100% rename from doc/ostree.repo-config.xml rename to man/ostree.repo-config.xml diff --git a/doc/ostree.repo.xml b/man/ostree.repo.xml similarity index 100% rename from doc/ostree.repo.xml rename to man/ostree.repo.xml diff --git a/doc/ostree.xml b/man/ostree.xml similarity index 100% rename from doc/ostree.xml rename to man/ostree.xml From 32c360b5a0188ec0139218d53db7767698c8bef6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 27 Jan 2016 13:29:21 -0500 Subject: [PATCH 09/54] build: Rename doc/ -> apidoc/ This is preparation for introducing a `mkdocs` manual under `doc/` which should be significantly more useful for the world at large than the minimal manual that exists there now. --- Makefile.am | 2 +- {doc => apidoc}/.gitignore | 0 {doc => apidoc}/Makefile.am | 0 {doc => apidoc}/adapting-existing.xml | 0 {doc => apidoc}/atomic-upgrades.xml | 0 {doc => apidoc}/deployment.xml | 0 {doc => apidoc}/ostree-docs.xml | 0 {doc => apidoc}/ostree-sections.txt | 0 {doc => apidoc}/overview.xml | 0 {doc => apidoc}/repo.xml | 0 configure.ac | 5 +++-- 11 files changed, 4 insertions(+), 3 deletions(-) rename {doc => apidoc}/.gitignore (100%) rename {doc => apidoc}/Makefile.am (100%) rename {doc => apidoc}/adapting-existing.xml (100%) rename {doc => apidoc}/atomic-upgrades.xml (100%) rename {doc => apidoc}/deployment.xml (100%) rename {doc => apidoc}/ostree-docs.xml (100%) rename {doc => apidoc}/ostree-sections.txt (100%) rename {doc => apidoc}/overview.xml (100%) rename {doc => apidoc}/repo.xml (100%) diff --git a/Makefile.am b/Makefile.am index 172af0b2..da904da2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,7 +32,7 @@ DISTCHECK_CONFIGURE_FLAGS += --enable-gtk-doc --disable-maintainer-mode SUBDIRS += . if ENABLE_GTK_DOC -SUBDIRS += doc +SUBDIRS += apidoc endif EXTRA_DIST += autogen.sh COPYING README.md diff --git a/doc/.gitignore b/apidoc/.gitignore similarity index 100% rename from doc/.gitignore rename to apidoc/.gitignore diff --git a/doc/Makefile.am b/apidoc/Makefile.am similarity index 100% rename from doc/Makefile.am rename to apidoc/Makefile.am diff --git a/doc/adapting-existing.xml b/apidoc/adapting-existing.xml similarity index 100% rename from doc/adapting-existing.xml rename to apidoc/adapting-existing.xml diff --git a/doc/atomic-upgrades.xml b/apidoc/atomic-upgrades.xml similarity index 100% rename from doc/atomic-upgrades.xml rename to apidoc/atomic-upgrades.xml diff --git a/doc/deployment.xml b/apidoc/deployment.xml similarity index 100% rename from doc/deployment.xml rename to apidoc/deployment.xml diff --git a/doc/ostree-docs.xml b/apidoc/ostree-docs.xml similarity index 100% rename from doc/ostree-docs.xml rename to apidoc/ostree-docs.xml diff --git a/doc/ostree-sections.txt b/apidoc/ostree-sections.txt similarity index 100% rename from doc/ostree-sections.txt rename to apidoc/ostree-sections.txt diff --git a/doc/overview.xml b/apidoc/overview.xml similarity index 100% rename from doc/overview.xml rename to apidoc/overview.xml diff --git a/doc/repo.xml b/apidoc/repo.xml similarity index 100% rename from doc/repo.xml rename to apidoc/repo.xml diff --git a/configure.ac b/configure.ac index 46b77fbf..140b6c21 100644 --- a/configure.ac +++ b/configure.ac @@ -232,7 +232,7 @@ AM_CONDITIONAL(BUILDOPT_STATIC_DELTAS, test x$enable_static_deltas = xyes) AC_CONFIG_FILES([ Makefile -doc/Makefile +apidoc/Makefile src/libostree/ostree-1.pc ]) AC_OUTPUT @@ -248,7 +248,8 @@ echo " SELinux: $with_selinux libarchive (parse tar files directly): $with_libarchive static deltas: $enable_static_deltas - documentation: $enable_gtk_doc + man pages (xsltproc): $XSLTPROC + api docs (gtk-doc): $enable_gtk_doc gjs-based tests: $have_gjs dracut: $with_dracut mkinitcpio: $with_mkinitcpio" From 64ebe2b82ac25f1933f39a5cf8a24a5e0096cf6f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 27 Jan 2016 16:56:16 -0500 Subject: [PATCH 10/54] Rewrite manual in mkdocs I don't much like Docbook (and am considering converting the man pages too), but let's start with the manual. I looked at various documentation generators (there are a lot), and I had a few requirements: - Markdown - Packaged in Fedora - Suitable for upload to a static webserver `mkdocs` seems to fit the bill. --- CONTRIBUTING.md | 122 +------------- README.md | 4 +- apidoc/Makefile.am | 5 - apidoc/adapting-existing.xml | 267 ------------------------------- apidoc/atomic-upgrades.xml | 181 --------------------- apidoc/deployment.xml | 158 ------------------ apidoc/ostree-docs.xml | 8 +- apidoc/overview.xml | 155 ------------------ apidoc/repo.xml | 127 --------------- docs/CONTRIBUTING.md | 121 ++++++++++++++ docs/index.md | 1 + docs/manual/adapting-existing.md | 159 ++++++++++++++++++ docs/manual/atomic-upgrades.md | 116 ++++++++++++++ docs/manual/deployment.md | 90 +++++++++++ docs/manual/introduction.md | 110 +++++++++++++ docs/manual/repo.md | 81 ++++++++++ mkdocs.yml | 10 ++ 17 files changed, 692 insertions(+), 1023 deletions(-) mode change 100644 => 120000 CONTRIBUTING.md delete mode 100644 apidoc/adapting-existing.xml delete mode 100644 apidoc/atomic-upgrades.xml delete mode 100644 apidoc/deployment.xml delete mode 100644 apidoc/overview.xml delete mode 100644 apidoc/repo.xml create mode 100644 docs/CONTRIBUTING.md create mode 120000 docs/index.md create mode 100644 docs/manual/adapting-existing.md create mode 100644 docs/manual/atomic-upgrades.md create mode 100644 docs/manual/deployment.md create mode 100644 docs/manual/introduction.md create mode 100644 docs/manual/repo.md create mode 100644 mkdocs.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 4458256e..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,121 +0,0 @@ -Submitting patches ------------------- - -You can: - - 1. Send mail to ostree-list@gnome.org, with the patch attached - 1. Submit a pull request against https://github.com/GNOME/ostree - 1. Attach them to https://bugzilla.gnome.org/ - -Please look at "git log" and match the commit log style. - -Running the test suite ----------------------- - -Currently, ostree uses https://wiki.gnome.org/GnomeGoals/InstalledTests -To run just ostree's tests: - - ./configure ... --enable-installed-tests - gnome-desktop-testing-runner -p 0 ostree/ - -Also, there is a regular: - - make check - -That runs a different set of tests. - -Coding style ------------- - -Indentation is GNU. Files should start with the appropriate mode lines. - -Use GCC `__attribute__((cleanup))` wherever possible. If interacting -with a third party library, try defining local cleanup macros. - -Use GError and GCancellable where appropriate. - -Prefer returning `gboolean` to signal success/failure, and have output -values as parameters. - -Prefer linear control flow inside functions (aside from standard -loops). In other words, avoid "early exits" or use of `goto` besides -`goto out;`. - -This is an example of an "early exit": - - static gboolean - myfunc (...) - { - gboolean ret = FALSE; - - /* some code */ - - /* some more code */ - - if (condition) - return FALSE; - - /* some more code */ - - ret = TRUE; - out: - return ret; - } - -If you must shortcut, use: - - if (condition) - { - ret = TRUE; - goto out; - } - -A consequence of this restriction is that you are encouraged to avoid -deep nesting of loops or conditionals. Create internal static helper -functions, particularly inside loops. For example, rather than: - - while (condition) - { - /* some code */ - if (condition) - { - for (i = 0; i < somevalue; i++) - { - if (condition) - { - /* deeply nested code */ - } - - /* more nested code */ - } - } - } - -Instead do this: - - static gboolean - helperfunc (..., GError **error) - { - if (condition) - { - /* deeply nested code */ - } - - /* more nested code */ - - return ret; - } - - while (condition) - { - /* some code */ - if (!condition) - continue; - - for (i = 0; i < somevalue; i++) - { - if (!helperfunc (..., i, error)) - goto out; - } - } - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 120000 index 00000000..49d1b98f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +docs/CONTRIBUTING.md \ No newline at end of file diff --git a/README.md b/README.md index ac070da8..493fa925 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ versions support extended validation using However, in order to build from a git clone, you must update the submodules. If you're packaging OSTree and want a tarball, I recommend using a "recursive git archive" script. There are several -available online; [this -code](https://git.gnome.org/browse/ostree/tree/packaging/Makefile.dist-packaging#n11) +available online; +[this code](https://git.gnome.org/browse/ostree/tree/packaging/Makefile.dist-packaging#n11) in OSTree is an example. Once you have a git clone or recursive archive, building is the diff --git a/apidoc/Makefile.am b/apidoc/Makefile.am index c40057e0..59cd8096 100644 --- a/apidoc/Makefile.am +++ b/apidoc/Makefile.am @@ -95,11 +95,6 @@ HTML_IMAGES= # Extra SGML files that are included by $(DOC_MAIN_SGML_FILE). # e.g. content_files=running.sgml building.sgml changes-2.0.sgml content_files= \ - overview.xml \ - repo.xml \ - deployment.xml \ - atomic-upgrades.xml \ - adapting-existing.xml \ $(NULL) # SGML files where gtk-doc abbrevations (#GtkWidget) are expanded diff --git a/apidoc/adapting-existing.xml b/apidoc/adapting-existing.xml deleted file mode 100644 index 5d1e0011..00000000 --- a/apidoc/adapting-existing.xml +++ /dev/null @@ -1,267 +0,0 @@ - - -]> - - Adapting existing mainstream distributions - - System layout - - First, OSTree encourages systems to implement UsrMove. - This is simply to avoid the need for more bind mounts. By - default OSTree's dracut hook creates a read-only bind mount over - /usr; you can of course - generate individual bind-mounts for /bin, all the /lib variants, etc. So it is not - intended to be a hard requirement. - - - - Remember, because by default the system is booted into a - chroot equivalent, there has to be some way - to refer to the actual physical root filesystem. Therefore, - your operating system tree should contain an empty /sysroot directory; at boot time, - OSTree will make this a bind mount to the physical / root - directory. There is precedent for this name in the initramfs - context. You should furthermore make a toplevel symbolic link - /ostree which points to - /sysroot/ostree, so that - the OSTree tool at runtime can consistently find the system data - regardless of whether it's operating on a physical root or - inside a deployment. - - - - Because OSTree only preserves /var across upgrades (each - deployment's chroot directory will be garbage collected - eventually), you will need to choose how to handle other - toplevel writable directories specified by the Filesystem Hierarchy - Standard. Your operating system may of course choose - not to support some of these such as /usr/local, but following is the - recommended set: - - - - /home to /var/home - - - - - /opt to /var/opt - - - - - /srv to /var/srv - - - - - /root to /var/roothome - - - - - /usr/local to /var/local - - - - - /mnt to /var/mnt - - - - - /tmp to /sysroot/tmp - - - - - - - Furthermore, since /var - is empty by default, your operating system will need to - dynamically create the targets of these at - boot. A good way to do this is using - systemd-tmpfiles, if your OS uses systemd. - For example: - - - - - - - - Particularly note here the double indirection of /home. By default, each - deployment will share the global toplevel /home directory on the physical - root filesystem. It is then up to higher levels of management - tools to keep /etc/passwd or equivalent - synchronized between operating systems. - - - Each deployment can easily be reconfigured to have its own home - directory set simply by making /var/home a real directory. - - - - - Booting and initramfs technology - - OSTree comes with optional dracut+systemd integration code that - parses the ostree= kernel command line - argument in the initramfs, and then sets up the read-only bind - mount on /usr, a bind - mount on the deployment's /sysroot to the physical /, and then finally uses - mount(MS_MOVE) to make the deployment root appear to be the - root filesystem before telling systemd to switch root. - - - - If you are not using dracut or systemd, using OSTree should still - be possible, but you will have to write the integration code. Patches - to support other initramfs technologies and init systems, if sufficiently - clean, will likely be accepted upstream. - - - - A further specific note regarding sysvinit: - OSTree used to support recording device files such the - /dev/initctl FIFO, but no longer does. - It's recommended to just patch your initramfs to create this at - boot. - - - - - /usr/lib/passwd - - Unlike traditional package systems, OSTree trees contain - numeric uid and gids. Furthermore, it does - not have a %post type mechanism where - useradd could be invoked. In order to ship - an OS that contains both system users and users dynamically - created on client machines, you will need to choose a solution - for /etc/passwd. The core problem is that - if you add a user to the system for a daemon, the OSTree upgrade - process for /etc will - simply notice that because /etc/passwd - differs from the previous default, it will keep the modified - config file, and your new OS user will not be visible. - - - The solution chosen for the gnome-continuous - operating system is to create - /usr/lib/passwd, and to include a NSS - module nss-altfiles - which instructs glibc to read from it. Then, the build system - places all system users there, freeing up - /etc/passwd to be purely a database of - local users. See also a more recent effort from Systemd - stateless. - - - - - Adapting existing package managers - - The largest endeavor is likely to be redesigning your - distribution's package manager to be on top of OSTree, - particularly if you want to keep compatibility with the "old - way" of installing into the physical /. This section will use examples - from both dpkg and rpm as - the author has familiarity with both; but the abstract concepts - should apply to most traditional package managers. - - - - There are many levels of possible integration; initially, we - will describe the most naive implementation which is the - simplest but also the least efficient. We will assume here that - the admin is booted into an OSTree-enabled system, and wants to - add a set of packages. - - - - Many package managers store their state in /var; but since in the OSTree model - that directory is shared between independent versions, the - package database must first be found in the per-deployment - /usr directory. It - becomes read-only; remember, all upgrades involve constructing a - new filesystem tree, so your package manager will also need to - create a copy of its database. Most likely, if you want to - continue supporting non-OSTree deployments, simply have your - package manager fall back to the legacy /var location if the one in - /usr is not found. - - - - To install a set of new packages (without removing any existing - ones), enumerate the set of packages in the currently booted - deployment, and perform dependency resolution to compute the - complete set of new packages. Download and unpack these new - packages to a temporary directory. - - - - Now, because we are merely installing new packages and not - removing anything, we can make the major optimization of reusing - our existing filesystem tree, and merely - layering the composed filesystem tree of - these new packages on top. A command lke this: ostree - commit -b osname/releasename/description - --tree=ref=osname/releasenamename/description - --tree=dir=/var/tmp/newpackages.13A8D0/ will create a - new commit in the - osname/releasename/description - branch. The OSTree SHA256 checksum of all the files in - /var/tmp/newpackages.13A8D0/ will be computed, but we will not - re-checksum the present existing tree. In this layering model, - earlier directories will take precedence, but files in later - layers will silently override earlier layers. - - - - Then to actually deploy this tree for the next boot: - ostree admin deploy - osname/releasenamename/description - - - - - diff --git a/apidoc/atomic-upgrades.xml b/apidoc/atomic-upgrades.xml deleted file mode 100644 index fd949fbf..00000000 --- a/apidoc/atomic-upgrades.xml +++ /dev/null @@ -1,181 +0,0 @@ - - -]> - - Atomic Upgrades - - You can turn off the power anytime you want... - - OSTree is designed to implement fully atomic and safe upgrades; - more generally, atomic transitions between lists of bootable - deployments. If the system crashes or you pull the power, you - will have either the old system, or the new one. - - - - - Simple upgrades via HTTP - - First, the most basic model OSTree supports is one where it - replicates pre-generated filesystem trees from a server over - HTTP, tracking exactly one ref, which is stored in the .origin file for the deployment. - The command ostree admin upgrade implements - this. - - - - To begin a simple upgrade, OSTree fetches the contents of the - ref from the remote server. Suppose we're tracking a ref named - exampleos/buildmaster/x86_64-runtime. - OSTree fetches the URL - http://example.com/repo/refs/exampleos/buildmaster/x86_64-runtime, - which contains a SHA256 checksum. This determines the tree to - deploy, and /etc will be - merged from currently booted tree. - - - - If we do not have this commit, then, then we perform a pull - process. At present (without static deltas), this involves - quite simply just fetching each individual object that we do not - have, asynchronously. Put in other words, we only download - changed files (zlib-compressed). Each object has its checksum - validated and is stored in /ostree/repo/objects/. - - - - Once the pull is complete, we have all the objects locally - we need to perform a deployment. - - - - - Upgrades via external tools (e.g. package managers) - - - As mentioned in the introduction, OSTree is also designed to - allow a model where filesystem trees are computed on the client. - It is completely agnostic as to how those trees are generated; - they could be computed with traditional packages, packages with - post-deployment scripts on top, or built by developers directly - from revision control locally, etc. - - - - At a practical level, most package managers today - (dpkg and rpm) operate - "live" on the currently booted filesystem. The way they could - work with OSTree is instead to take the list of installed - packages in the currently booted tree, and compute a new - filesystem from that. A later chapter describes in more details - how this could work: . - - - - For the purposes of this section, let's assume that we have a - newly generated filesystem tree stored in the repo (which shares - storage with the existing booted tree). We can then move on to - checking it back out of the repo into a deployment. - - - - - Assembling a new deployment directory - - Given a commit to deploy, OSTree first allocates a directory for - it. This is of the form /boot/loader/entries/ostree-osname-checksum.serial.conf. - The serial is normally 0, but if a - given commit is deployed more than once, it will be incremented. - This is supported because the previous deployment may have - configuration in /etc - that we do not want to use or overwrite. - - - - Now that we have a deployment directory, a 3-way merge is - performed between the (by default) currently booted deployment's - /etc, its default - configuration, and the new deployment (based on its /usr/etc). - - - - - Atomically swapping boot configuration - - At this point, a new deployment directory has been created as a - hardlink farm; the running system is untouched, and the - bootloader configuration is untouched. We want to add this deployment - to the "deployment list". - - - - To support a more general case, OSTree supports atomic - transitioning between arbitrary sets of deployments, with the - restriction that the currently booted deployment must always be - in the new set. In the normal case, we have exactly one - deployment, which is the booted one, and we want to add the new - deployment to the list. A more complex command might allow - creating 100 deployments as part of one atomic transaction, so - that one can set up an automated system to bisect across them. - - - - The bootversion - - OSTree allows swapping between boot configurations by - implementing the "swapped directory pattern" in /boot. This means it is a - symbolic link to one of two directories /ostree/boot.[0|1]. - To swap the contents atomically, if the current version is - 0, we create /ostree/boot.1, populate it with - the new contents, then atomically swap the symbolic link. Finally, - the old contents can be garbage collected at any point. - - - - - The /ostree/boot directory - - However, we want to optimize for the case where we the set of - kernel/initramfs pairs is the same between both the old and - new deployment lists. This happens when doing an upgrade that - does not include the kernel; think of a simple translation - update. OSTree optimizes for this case because on some - systems /boot may be on - a separate medium such as flash storage not optimized for - significant amounts of write traffic. - - - - To implement this, OSTree also maintains the directory - /ostree/boot.bootversion, - which is a set of symbolic links to the deployment - directories. The bootversion here - must match the version of /boot. However, in order to - allow atomic transitions of this - directory, this is also a swapped directory, so just like - /boot, it has a version - of 0 or 1 appended. - - - - Each bootloader entry has a special ostree= - argument which refers to one of these symbolic links. This is - parsed at runtime in the initramfs. - - - - - - - diff --git a/apidoc/deployment.xml b/apidoc/deployment.xml deleted file mode 100644 index 489850ae..00000000 --- a/apidoc/deployment.xml +++ /dev/null @@ -1,158 +0,0 @@ - - -]> - - Deployments - - Overview - - Built on top of the OSTree versioning filesystem core is a layer - that knows how to deploy, parallel install, and manage Unix-like - operating systems (accessible via ostree - admin). The core content of these operating systems - are treated as read-only, but they transparently share storage. - - - - A deployment is physically located at a path of the form - /ostree/deploy/osname/deploy/checksum. - OSTree is designed to boot directly into exactly one deployment - at a time; each deployment is intended to be a target for - chroot() or equivalent. - - - - - - "osname": Group of deployments that share /var - - Each deployment is grouped in exactly one "osname". From - above, you can see that an osname is physically represented in - the /ostree/deploy/osname - directory. For example, OSTree can allow parallel installing - Debian in /ostree/deploy/debian and Red Hat - Enterprise Linux in /ostree/deploy/rhel (subject to - operating system support, present released versions of these - operating systems may not support this). - - - - Each osname has exactly one copy of the traditional Unix - /var, stored physically - in /ostree/deploy/osname/var. - OSTree provides support tools for systemd - to create a Linux bind mount that ensures the booted - deployment sees the shared copy of /var. - - - - OSTree does not touch the contents of /var. Operating system components - such as daemon services are required to create any directories - they require there at runtime (e.g. /var/cache/daemonname), - and to manage upgrading data formats inside those directories. - - - - - Contents of a deployment - - A deployment begins with a specific commit (represented as a - SHA256 hash) in the OSTree repository in /ostree/repo. This commit refers - to a filesystem tree that represents the underlying basis of a - deployment. For short, we will call this the "tree", to - distinguish it from the concept of a deployment. - - - - First, the tree must include a kernel stored as /boot/vmlinuz-checksum. - The checksum should be a SHA256 hash of the kernel contents; - it must be pre-computed before storing the kernel in the - repository. Optionally, the tree can contain an initramfs, - stored as /boot/initramfs-checksum. - If this exists, the checksum must include both the kernel and - initramfs contents. OSTree will use this to determine which - kernels are shared. The rationale for this is to avoid - computing checksums on the client by default. - - - - The deployment should not have a traditional UNIX /etc; instead, it should include - /usr/etc. This is the - "default configuration". When OSTree creates a deployment, it - performs a 3-way merge using the old - default configuration, the active system's /etc, and the new default - configuration. In the final filesystem tree for a deployment - then, /etc is a regular - writable directory. - - - - Besides the exceptions of /var and /etc then, the rest of the - contents of the tree are checked out as hard links into the - repository. It's strongly recommended that operating systems - ship all of their content in /usr, but this is not a hard - requirement. - - - - Finally, a deployment may have a .origin file, stored next to its - directory. This file tells ostree admin - upgrade how to upgrade it. At the moment, OSTree only - supports upgrading a single refspec. However, in the future - OSTree may support a syntax for composing layers of trees, for - example. - - - - - - The system /boot - - While OSTree parallel installs deployments cleanly inside the - /ostree directory, - ultimately it has to control the system's /boot directory. The way this - works is via the boot - loader specification, which is a standard for - bootloader-independent drop-in configuration files. - - - When a tree is deployed, it will have a configuration file - generated of the form /boot/loader/entries/ostree-osname-checksum.serial.conf. - This configuration file will include a special - ostree= kernel argument that allows the - initramfs to find (and chroot() into) the - specified deployment. - - - - At present, not all bootloaders implement the BootLoaderSpec, - so OSTree contains code for some of these to regenerate native - config files (such as - /boot/syslinux/syslinux.conf based on the - entries. - - - - diff --git a/apidoc/ostree-docs.xml b/apidoc/ostree-docs.xml index d2e9f7b8..c5ea28e6 100644 --- a/apidoc/ostree-docs.xml +++ b/apidoc/ostree-docs.xml @@ -7,16 +7,10 @@ ]> - OSTree Manual + OSTree API references for OSTree &version; - - - - - - API Reference diff --git a/apidoc/overview.xml b/apidoc/overview.xml deleted file mode 100644 index 94bf9c78..00000000 --- a/apidoc/overview.xml +++ /dev/null @@ -1,155 +0,0 @@ - - -]> - - OSTree Overview - - Introduction - - OSTree an upgrade system for Linux-based operating systems that - performs atomic upgrades of complete filesystem trees. It is - not a package system; rather, it is intended to complement them. - A primary model is composing packages on a server, and then - replicating them to clients. - - - - The underlying architecture might be summarized as "git for - operating system binaries". It operates in userspace, and will - work on top of any Linux filesystem. At its core is a git-like - content-addressed object store, and layered on top of that is - bootloader configuration, management of - /etc, and other functions to perform an - upgrade beyond just replicating files. - - - - You can use OSTree standalone in the pure replication model, - but another approach is to add a package manager on top, - thus creating a hybrid tree/package system. - - - - - - Comparison with "package managers" - - Because OSTree is designed for deploying core operating - systems, a comparison with traditional "package managers" such - as dpkg and rpm is illustrative. Packages are traditionally - composed of partial filesystem trees with metadata and scripts - attached, and these are dynamically assembled on the client - machine, after a process of dependency resolution. - - - - In contrast, OSTree only supports recording and deploying - complete (bootable) filesystem trees. It - has no built-in knowledge of how a given filesystem tree was - generated or the origin of individual files, or dependencies, - descriptions of individual components. Put another way, OSTree - only handles delivery and deployment; you will likely still want - to include inside each tree metadata about the individual - components that went into the tree. For example, a system - administrator may want to know what version of OpenSSL was - included in your tree, so you should support the equivalent of - rpm -q or dpkg -L. - - - - The OSTree core emphasizes replicating read-only OS trees via - HTTP, and where the OS includes (if desired) an entirely - separate mechanism to install applications, stored in /var if they're system global, or - /home for per-user - application installation. An example application mechanism is - Docker. - - - - However, it is entirely possible to use OSTree underneath a - package system, where the contents of /usr are computed on the client. - For example, when installing a package, rather than changing the - currently running filesystem, the package manager could assemble - a new filesystem tree that layers the new packages on top of a - base tree, record it in the local OSTree repository, and then - set it up for the next boot. To support this model, OSTree - provides an (introspectable) C shared library. - - - - - Comparison with block/image replication - - OSTree shares some similarity with "dumb" replication and - stateless deployments, such as the model common in "cloud" - deployments where nodes are booted from an (effectively) - readonly disk, and user data is kept on a different volumes. - The advantage of "dumb" replication, shared by both OSTree and - the cloud model, is that it's reliable - and predictable. - - - But unlike many default image-based deployments, OSTree supports - exactly two persistent writable directories that are preserved - across upgrades: /etc and - /var. - - - Because OSTree operates at the Unix filesystem layer, it works - on top of any filesystem or block storage layout; it's possible - to replicate a given filesystem tree from an OSTree repository - into plain ext4, BTRFS, XFS, or in general any Unix-compatible - filesystem that supports hard links. Note: OSTree will - transparently take advantage of some BTRFS features if deployed - on it. - - - - - Atomic transitions between parallel-installable read-only filesystem trees - - Another deeply fundamental difference between both package - managers and image-based replication is that OSTree is - designed to parallel-install multiple - versions of multiple - independent operating systems. OSTree - relies on a new toplevel ostree directory; it can in fact - parallel install inside an existing OS or distribution - occupying the physical / root. - - - On each client machine, there is an OSTree repository stored - in /ostree/repo, and a - set of "deployments" stored in /ostree/deploy/OSNAME/CHECKSUM. - Each deployment is primarily composed of a set of hardlinks - into the repository. This means each version is deduplicated; - an upgrade process only costs disk space proportional to the - new files, plus some constant overhead. - - - The model OSTree emphasizes is that the OS read-only content - is kept in the classic Unix /usr; it comes with code to - create a Linux read-only bind mount to prevent inadvertent - corruption. There is exactly one /var writable directory shared - between each deployment for a given OS. The OSTree core code - does not touch content in this directory; it is up to the code - in each operating system for how to manage and upgrade state. - - - Finally, each deployment has its own writable copy of the - configuration store /etc. On upgrade, OSTree will - perform a basic 3-way diff, and apply any local changes to the - new copy, while leaving the old untouched. - - - diff --git a/apidoc/repo.xml b/apidoc/repo.xml deleted file mode 100644 index 5379d3a0..00000000 --- a/apidoc/repo.xml +++ /dev/null @@ -1,127 +0,0 @@ - - -]> - - Anatomy of an OSTree repository - - Core object types and data model - - OSTree is deeply inspired by git; the core layer is a userspace - content-addressed versioning filesystem. It is worth taking - some time to familiarize yourself with Git - Internals, as this section will assume some knowledge of - how git works. - - - - Its object types are similar to git; it has commit objects and - content objects. Git has "tree" objects, whereas OSTree splits - them into "dirtree" and "dirmeta" objects. But unlike git, - OSTree's checksums are SHA256. And most crucially, its content - objects include uid, gid, and extended attributes (but still no - timestamps). - - - - Commit objects - - A commit object contains metadata such as a timestamp, a log - message, and most importantly, a reference to a - dirtree/dirmeta pair of checksums which describe the root - directory of the filesystem. - - - Also like git, each commit in OSTree can have a parent. It is - designed to store a history of your binary builds, just like git - stores a history of source control. However, OSTree also makes - it easy to delete data, under the assumption that you can - regenerate it from source code. - - - - - Dirtree objects - - A dirtree contains a sorted array of (filename, checksum) - pairs for content objects, and a second sorted array of - (filename, dirtree checksum, dirmeta checksum), which are - subdirectories. - - - - - Dirmeta objects - - In git, tree objects contain the metadata such as permissions - for their children. But OSTree splits this into a separate - object to avoid duplicating extended attribute listings. - - - - - Content objects - - Unlike the first three object types which are metadata, - designed to be mmap()ed, the content object - has a separate internal header and payload sections. The - header contains uid, gid, mode, and symbolic link target (for - symlinks), as well as extended attributes. After the header, - for regular files, the content follows. - - - - - - Repository types and locations - - - Also unlike git, an OSTree repository can be in one of two - separate modes: bare and - archive-z2. A bare repository is one where - content files are just stored as regular files; it's designed to - be the source of a "hardlink farm", where each operating system - checkout is merely links into it. If you want to store files - owned by e.g. root in this mode, you must run OSTree as root. - In contrast, the archive-z2 mode is designed - for serving via plain HTTP. Like tar files, it can be - read/written by non-root users. - - - - On an OSTree-deployed system, the "system repository" is - /ostree/repo. It can be - read by any uid, but only written by root. Unless the - --repo argument is given to the - ostree command, it will operate on the system - repository. - - - - - Refs - - Like git, OSTree uses "refs" to which are text files that point - to particular commits (i.e. filesystem trees). For example, the - gnome-ostree operating system creates trees named like - gnome-ostree/buildmaster/x86_64-runtime and - gnome-ostree/buildmaster/x86_64-devel-debug. - These two refs point to two different generated filesystem - trees. In this example, the "runtime" tree contains just enough - to run a basic GNOME system, and "devel-debug" contains all of - the developer tools. - - - - The ostree supports a simple syntax using the - carat ^ to refer to the parent of a given - commit. For example, - gnome-ostree/buildmaster/x86_64-runtime^ - refers to the previous build, and - gnome-ostree/buildmaster/x86_64-runtime^^ - refers to the one before that. - - - diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 00000000..4458256e --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,121 @@ +Submitting patches +------------------ + +You can: + + 1. Send mail to ostree-list@gnome.org, with the patch attached + 1. Submit a pull request against https://github.com/GNOME/ostree + 1. Attach them to https://bugzilla.gnome.org/ + +Please look at "git log" and match the commit log style. + +Running the test suite +---------------------- + +Currently, ostree uses https://wiki.gnome.org/GnomeGoals/InstalledTests +To run just ostree's tests: + + ./configure ... --enable-installed-tests + gnome-desktop-testing-runner -p 0 ostree/ + +Also, there is a regular: + + make check + +That runs a different set of tests. + +Coding style +------------ + +Indentation is GNU. Files should start with the appropriate mode lines. + +Use GCC `__attribute__((cleanup))` wherever possible. If interacting +with a third party library, try defining local cleanup macros. + +Use GError and GCancellable where appropriate. + +Prefer returning `gboolean` to signal success/failure, and have output +values as parameters. + +Prefer linear control flow inside functions (aside from standard +loops). In other words, avoid "early exits" or use of `goto` besides +`goto out;`. + +This is an example of an "early exit": + + static gboolean + myfunc (...) + { + gboolean ret = FALSE; + + /* some code */ + + /* some more code */ + + if (condition) + return FALSE; + + /* some more code */ + + ret = TRUE; + out: + return ret; + } + +If you must shortcut, use: + + if (condition) + { + ret = TRUE; + goto out; + } + +A consequence of this restriction is that you are encouraged to avoid +deep nesting of loops or conditionals. Create internal static helper +functions, particularly inside loops. For example, rather than: + + while (condition) + { + /* some code */ + if (condition) + { + for (i = 0; i < somevalue; i++) + { + if (condition) + { + /* deeply nested code */ + } + + /* more nested code */ + } + } + } + +Instead do this: + + static gboolean + helperfunc (..., GError **error) + { + if (condition) + { + /* deeply nested code */ + } + + /* more nested code */ + + return ret; + } + + while (condition) + { + /* some code */ + if (!condition) + continue; + + for (i = 0; i < somevalue; i++) + { + if (!helperfunc (..., i, error)) + goto out; + } + } + diff --git a/docs/index.md b/docs/index.md new file mode 120000 index 00000000..32d46ee8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/docs/manual/adapting-existing.md b/docs/manual/adapting-existing.md new file mode 100644 index 00000000..858c82d3 --- /dev/null +++ b/docs/manual/adapting-existing.md @@ -0,0 +1,159 @@ +# Adapting existing mainstream distributions + +## System layout + +First, OSTree encourages systems to implement +[UsrMove](http://www.freedesktop.org/wiki/Software/systemd/TheCaseForTheUsrMerge/) +This is simply to avoid the need for more bind mounts. By default +OSTree's dracut hook creates a read-only bind mount over `/usr`; you +can of course generate individual bind-mounts for `/bin`, all the +`/lib` variants, etc. So it is not intended to be a hard requirement. + +Remember, because by default the system is booted into a `chroot` +equivalent, there has to be some way to refer to the actual physical +root filesystem. Therefore, your operating system tree should contain +an empty `/sysroot` directory; at boot time, OSTree will make this a +bind mount to the physical / root directory. There is precedent for +this name in the initramfs context. You should furthermore make a +toplevel symbolic link `/ostree` which points to `/sysroot/ostree`, so +that the OSTree tool at runtime can consistently find the system data +regardless of whether it's operating on a physical root or inside a +deployment. + +Because OSTree only preserves `/var` across upgrades (each +deployment's chroot directory will be garbage collected +eventually), you will need to choose how to handle other +toplevel writable directories specified by the [Filesystem Hierarchy Standard](http://www.pathname.com/fhs/") +Your operating system may of course choose +not to support some of these such as `/usr/local`, but following is the +recommended set: + + - `/home` → `/var/home` + - `/opt` → `/var/opt` + - `/srv` → `/var/srv` + - `/root` → `/var/roothome` + - `/usr/local` → `/var/local` + - `/mnt` → `/var/mnt` + - `/tmp` → `/sysroot/tmp` + +Furthermore, since `/var` is empty by default, your operating system +will need to dynamically create the targets of +these at boot. A good way to do this is using `systemd-tmpfiles`, if +your OS uses systemd. For example: + +``` +d /var/log/journal 0755 root root - +L /var/home - - - - ../sysroot/home +d /var/opt 0755 root root - +d /var/srv 0755 root root - +d /var/roothome 0700 root root - +d /var/usrlocal 0755 root root - +d /var/usrlocal/bin 0755 root root - +d /var/usrlocal/etc 0755 root root - +d /var/usrlocal/games 0755 root root - +d /var/usrlocal/include 0755 root root - +d /var/usrlocal/lib 0755 root root - +d /var/usrlocal/man 0755 root root - +d /var/usrlocal/sbin 0755 root root - +d /var/usrlocal/share 0755 root root - +d /var/usrlocal/src 0755 root root - +d /var/mnt 0755 root root - +d /run/media 0755 root root - +``` + +Particularly note here the double indirection of `/home`. By default, +each deployment will share the global toplevel `/home` directory on +the physical root filesystem. It is then up to higher levels of +management tools to keep /etc/passwd or +equivalent synchronized between operating systems. Each deployment +can easily be reconfigured to have its own home directory set simply +by making `/var/home` a real directory. + +## Booting and initramfs technology + +OSTree comes with optional dracut+systemd integration code that parses +the `ostree=` kernel command line argument in the initramfs, and then +sets up the read-only bind mount on `/usr`, a bind mount on the +deployment's `/sysroot` to the physical `/`, and then finally uses +`mount(MS_MOVE)` to make the deployment root appear to be the root +filesystem before telling systemd to switch root. + +If you are not using dracut or systemd, using OSTree should still be +possible, but you will have to write the integration code. Patches to +support other initramfs technologies and init systems, if sufficiently +clean, will likely be accepted upstream. + +A further specific note regarding `sysvinit`: OSTree used to support +recording device files such the `/dev/initctl` FIFO, but no longer +does. It's recommended to just patch your initramfs to create this at +boot. + +## /usr/lib/passwd + +Unlike traditional package systems, OSTree trees contain *numeric* uid +and gids. Furthermore, it does not have a `%post` type mechanism +where `useradd` could be invoked. In order to ship an OS that +contains both system users and users dynamically created on client +machines, you will need to choose a solution for `/etc/passwd`. The +core problem is that if you add a user to the system for a daemon, the +OSTree upgrade process for `/etc` will simply notice that because +`/etc/passwd` differs from the previous default, it will keep the +modified config file, and your new OS user will not be visible. The +solution chosen for the [Gnome Continuous](https://live.gnome.org/Projects/GnomeContinuous) operating +system is to create `/usr/lib/passwd`, and to include a NSS module +[nss-altfiles](https://github.com/aperezdc/nss-altfiles) which +instructs glibc to read from it. Then, the build system places all +system users there, freeing up `/etc/passwd` to be purely a database +of local users. See also a more recent effort from [Systemd Stateless](http://0pointer.de/blog/projects/stateless.html) + +## Adapting existing package managers + +The largest endeavor is likely to be redesigning your distribution's +package manager to be on top of OSTree, particularly if you want to +keep compatibility with the "old way" of installing into the physical +`/`. This section will use examples from both `dpkg` and `rpm` as the +author has familiarity with both; but the abstract concepts should +apply to most traditional package managers. + +There are many levels of possible integration; initially, we will +describe the most naive implementation which is the simplest but also +the least efficient. We will assume here that the admin is booted +into an OSTree-enabled system, and wants to add a set of packages. + +Many package managers store their state in `/var`; but since in the +OSTree model that directory is shared between independent versions, +the package database must first be found in the per-deployment `/usr` +directory. It becomes read-only; remember, all upgrades involve +constructing a new filesystem tree, so your package manager will also +need to create a copy of its database. Most likely, if you want to +continue supporting non-OSTree deployments, simply have your package +manager fall back to the legacy `/var` location if the one in `/usr` +is not found. + +To install a set of new packages (without removing any existing ones), +enumerate the set of packages in the currently booted deployment, and +perform dependency resolution to compute the complete set of new +packages. Download and unpack these new packages to a temporary +directory. + +Now, because we are merely installing new packages and not +removing anything, we can make the major optimization of reusing +our existing filesystem tree, and merely +*layering* the composed filesystem tree of +these new packages on top. A command like this: + +``` +ostree commit -b osname/releasename/description +--tree=ref=$osname/releasename/description +--tree=dir=/var/tmp/newpackages.13A8D0/ +``` + +will create a new commit in the `$osname/releasename/description` +branch. The OSTree SHA256 checksum of all the files in +`/var/tmp/newpackages.13A8D0/` will be computed, but we will not +re-checksum the present existing tree. In this layering model, +earlier directories will take precedence, but files in later layers +will silently override earlier layers. + +Then to actually deploy this tree for the next boot: +`ostree admin deploy osname/releasename/description` diff --git a/docs/manual/atomic-upgrades.md b/docs/manual/atomic-upgrades.md new file mode 100644 index 00000000..9ce2c8ae --- /dev/null +++ b/docs/manual/atomic-upgrades.md @@ -0,0 +1,116 @@ +# Atomic Upgrades + +## You can turn off the power anytime you want... + +OSTree is designed to implement fully atomic and safe upgrades; +more generally, atomic transitions between lists of bootable +deployments. If the system crashes or you pull the power, you +will have either the old system, or the new one. + +## Simple upgrades via HTTP + +First, the most basic model OSTree supports is one where it replicates +pre-generated filesystem trees from a server over HTTP, tracking +exactly one ref, which is stored in the `.origin` file for the +deployment. The command `ostree admin upgrade` +implements this. + +o begin a simple upgrade, OSTree fetches the contents of the ref from +the remote server. Suppose we're tracking a ref named +`exampleos/buildmaster/x86_64-runtime`. OSTree fetches the URL +`http://$example.com/repo/refs/exampleos/buildmaster/x86_64-runtime`, +which contains a SHA256 checksum. This determines the tree to deploy, +and `/etc` will be merged from currently booted tree. + +If we do not have this commit, then, then we perform a pull process. +At present (without static deltas), this involves quite simply just +fetching each individual object that we do not have, asynchronously. +Put in other words, we only download changed files (zlib-compressed). +Each object has its checksum validated and is stored in `/ostree/repo/objects/`. + +Once the pull is complete, we have all the objects locally +we need to perform a deployment. + +## Upgrades via external tools (e.g. package managers) + +As mentioned in the introduction, OSTree is also designed to allow a +model where filesystem trees are computed on the client. It is +completely agnostic as to how those trees are generated; hey could be +computed with traditional packages, packages with post-deployment +scripts on top, or built by developers directly from revision control +locally, etc. + +At a practical level, most package managers today (`dpkg` and `rpm`) +operate "live" on the currently booted filesystem. The way they could +work with OSTree is instead to take the list of installed packages in +the currently booted tree, and compute a new filesystem from that. A +later chapter describes in more details how this could work: +[adapting-existing.md](Adapting Existing Systems). + +For the purposes of this section, let's assume that we have a +newly generated filesystem tree stored in the repo (which shares +storage with the existing booted tree). We can then move on to +checking it back out of the repo into a deployment. + +## Assembling a new deployment directory + +Given a commit to deploy, OSTree first allocates a directory for +it. This is of the form `/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. +he $serial is normally 0, but if a +given commit is deployed more than once, it will be incremented. +his is supported because the previous deployment may have +configuration in `/etc` +hat we do not want to use or overwrite. + +Now that we have a deployment directory, a 3-way merge is +performed between the (by default) currently booted deployment's +`/etc`, its default +configuration, and the new deployment (based on its `/usr/etc`). + +## Atomically swapping boot configuration + +At this point, a new deployment directory has been created as a +hardlink farm; the running system is untouched, and the bootloader +configuration is untouched. We want to add this deployment o the +"deployment list". + +To support a more general case, OSTree supports atomic ransitioning +between arbitrary sets of deployments, with the restriction that the +currently booted deployment must always be in the new set. In the +normal case, we have exactly one deployment, which is the booted one, +and we want to add the new deployment to the list. A more complex +command might allow creating 100 deployments as part of one atomic +transaction, so that one can set up an automated system to bisect +across them. + +## The bootversion + +OSTree allows swapping between boot configurations by implementing the +"swapped directory pattern" in `/boot`. This means it is a symbolic +link to one of two directories `/ostree/boot.[0|1]`. To swap the +contents atomically, if the current version is `0`, we create +`/ostree/boot.1`, populate it with the new contents, then atomically +swap the symbolic link. Finally, the old contents can be garbage +collected at any point. + +## The /ostree/boot directory + +However, we want to optimize for the case where we the set of +kernel/initramfs pairs is the same between both the old and new +deployment lists. This happens when doing an upgrade that does not +include the kernel; think of a simple translation update. OSTree +optimizes for this case because on some systems `/boot` may be on a +separate medium such as flash storage not optimized for significant +amounts of write traffic. + +To implement this, OSTree also maintains the directory +`/ostree/boot.bootversion`, which is a set +of symbolic links to the deployment directories. The +bootversion here must match the version of +`/boot`. However, in order to allow atomic transitions of +this directory, this is also a swapped directory, +so just like `/boot`, it has a version of `0` or `1` appended. + +Each bootloader entry has a special `ostree=` argument which refers to +one of these symbolic links. This is parsed at runtime in the +initramfs. diff --git a/docs/manual/deployment.md b/docs/manual/deployment.md new file mode 100644 index 00000000..53b0b662 --- /dev/null +++ b/docs/manual/deployment.md @@ -0,0 +1,90 @@ +# Deployments + +## Overview + +Built on top of the OSTree versioning filesystem core is a layer +that knows how to deploy, parallel install, and manage Unix-like +operating systems (accessible via `ostree admin`). The core content of these operating systems +are treated as read-only, but they transparently share storage. + +A deployment is physically located at a path of the form +`/ostree/deploy/$osname/deploy/$checksum`. +OSTree is designed to boot directly into exactly one deployment +at a time; each deployment is intended to be a target for +`chroot()` or equivalent. + +### "osname": Group of deployments that share /var + +Each deployment is grouped in exactly one "osname". From above, you +can see that an osname is physically represented in the +`/ostree/deploy/$osname` directory. For example, OSTree can allow +parallel installing Debian in `/ostree/deploy/debian` and Red Hat +Enterprise Linux in `/ostree/deploy/rhel` (subject to operating system +support, present released versions of these operating systems may not +support this). + +Each osname has exactly one copy of the traditional Unix `/var`, +stored physically in `/ostree/deploy/$osname/var`. OSTree provides +support tools for `systemd` to create a Linux bind mount that ensures +the booted deployment sees the shared copy of `/var`. + +OSTree does not touch the contents of `/var`. Operating system +components such as daemon services are required to create any +directories they require there at runtime +(e.g. `/var/cache/$daemonname`), and to manage upgrading data formats +inside those directories. + +### Contents of a deployment + +A deployment begins with a specific commit (represented as a +SHA256 hash) in the OSTree repository in `/ostree/repo`. This commit refers +to a filesystem tree that represents the underlying basis of a +deployment. For short, we will call this the "tree", to +distinguish it from the concept of a deployment. + +First, the tree must include a kernel stored as +`/boot/vmlinuz-$checksum`. The checksum should be a SHA256 hash of +the kernel contents; it must be pre-computed before storing the kernel +in the repository. Optionally, the tree can contain an initramfs, +stored as `/boot/initramfs-$checksum`. If this exists, the checksum +must include both the kernel and initramfs contents. OSTree will use +this to determine which kernels are shared. The rationale for this is +to avoid computing checksums on the client by default. + +The deployment should not have a traditional UNIX `/etc`; instead, it +should include `/usr/etc`. This is the "default configuration". When +OSTree creates a deployment, it performs a 3-way merge using the +old default configuration, the active system's +`/etc`, and the new default configuration. In the final filesystem +tree for a deployment then, `/etc` is a regular writable directory. + +Besides the exceptions of `/var` and `/etc` then, the rest of the +contents of the tree are checked out as hard links into the +repository. It's strongly recommended that operating systems ship all +of their content in `/usr`, but this is not a hard requirement. + +Finally, a deployment may have a `.origin` file, stored next to its +directory. This file tells `ostree admin upgrade` how to upgrade it. +At the moment, OSTree only supports upgrading a single refspec. +However, in the future OSTree may support a syntax for composing +layers of trees, for example. + +### The system /boot + +While OSTree parallel installs deployments cleanly inside the +`/ostree` directory, ultimately it has to control the system's `/boot` +directory. The way this works is via the +[Boot Loader Specification](http://www.freedesktop.org/wiki/Specifications/BootLoaderSpec), +which is a standard for bootloader-independent drop-in configuration +files. + +When a tree is deployed, it will have a configuration file generated +of the form +`/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. This +configuration file will include a special `ostree=` kernel argument +that allows the initramfs to find (and `chroot()` into) the specified +deployment. + +At present, not all bootloaders implement the BootLoaderSpec, so +OSTree contains code for some of these to regenerate native config +files (such as `/boot/syslinux/syslinux.conf` based on the entries. diff --git a/docs/manual/introduction.md b/docs/manual/introduction.md new file mode 100644 index 00000000..b0d47390 --- /dev/null +++ b/docs/manual/introduction.md @@ -0,0 +1,110 @@ +# OSTree Overview + +## Introduction + +OSTree an upgrade system for Linux-based operating systems that +performs atomic upgrades of complete filesystem trees. It is +not a package system; rather, it is intended to complement them. +A primary model is composing packages on a server, and then +replicating them to clients. + +The underlying architecture might be summarized as "git for +operating system binaries". It operates in userspace, and will +work on top of any Linux filesystem. At its core is a git-like +content-addressed object store, and layered on top of that is +bootloader configuration, management of +`/etc`, and other functions to perform an +upgrade beyond just replicating files. + +You can use OSTree standalone in the pure replication model, +but another approach is to add a package manager on top, +thus creating a hybrid tree/package system. + +## Comparison with "package managers" + +Because OSTree is designed for deploying core operating +systems, a comparison with traditional "package managers" such +as dpkg and rpm is illustrative. Packages are traditionally +composed of partial filesystem trees with metadata and scripts +attached, and these are dynamically assembled on the client +machine, after a process of dependency resolution. + +In contrast, OSTree only supports recording and deploying +*complete* (bootable) filesystem trees. It +has no built-in knowledge of how a given filesystem tree was +generated or the origin of individual files, or dependencies, +descriptions of individual components. Put another way, OSTree +only handles delivery and deployment; you will likely still want +to include inside each tree metadata about the individual +components that went into the tree. For example, a system +administrator may want to know what version of OpenSSL was +included in your tree, so you should support the equivalent of +`rpm -q` or `dpkg -L`. + +The OSTree core emphasizes replicating read-only OS trees via +HTTP, and where the OS includes (if desired) an entirely +separate mechanism to install applications, stored in `/var` if they're system global, or +`/home` for per-user +application installation. An example application mechanism is +http://docker.io/ + +However, it is entirely possible to use OSTree underneath a +package system, where the contents of `/usr` are computed on the client. +For example, when installing a package, rather than changing the +currently running filesystem, the package manager could assemble +a new filesystem tree that layers the new packages on top of a +base tree, record it in the local OSTree repository, and then +set it up for the next boot. To support this model, OSTree +provides an (introspectable) C shared library. + +## Comparison with block/image replication + +OSTree shares some similarity with "dumb" replication and +stateless deployments, such as the model common in "cloud" +deployments where nodes are booted from an (effectively) +readonly disk, and user data is kept on a different volumes. +The advantage of "dumb" replication, shared by both OSTree and +the cloud model, is that it's *reliable* +and *predictable*. + +But unlike many default image-based deployments, OSTree supports +exactly two persistent writable directories that are preserved across +upgrades: `/etc` and `/var`. + +Because OSTree operates at the Unix filesystem layer, it works +on top of any filesystem or block storage layout; it's possible +to replicate a given filesystem tree from an OSTree repository +into plain ext4, BTRFS, XFS, or in general any Unix-compatible +filesystem that supports hard links. Note: OSTree will +transparently take advantage of some BTRFS features if deployed +on it. + +## Atomic transitions between parallel-installable read-only filesystem trees + +Another deeply fundamental difference between both package +managers and image-based replication is that OSTree is +designed to parallel-install *multiple versions* of multiple +*independent* operating systems. OSTree +relies on a new toplevel `ostree` directory; it can in fact +parallel install inside an existing OS or distribution +occupying the physical `/` root. + +On each client machine, there is an OSTree repository stored +in `/ostree/repo`, and a set of "deployments" stored in `/ostree/deploy/$OSNAME/$CHECKSUM`. +Each deployment is primarily composed of a set of hardlinks +into the repository. This means each version is deduplicated; +an upgrade process only costs disk space proportional to the +new files, plus some constant overhead. + +The model OSTree emphasizes is that the OS read-only content +is kept in the classic Unix `/usr`; it comes with code to +create a Linux read-only bind mount to prevent inadvertent +corruption. There is exactly one `/var` writable directory shared +between each deployment for a given OS. The OSTree core code +does not touch content in this directory; it is up to the code +in each operating system for how to manage and upgrade state. + +Finally, each deployment has its own writable copy of the +configuration store `/etc`. On upgrade, OSTree will +perform a basic 3-way diff, and apply any local changes to the +new copy, while leaving the old untouched. diff --git a/docs/manual/repo.md b/docs/manual/repo.md new file mode 100644 index 00000000..a3e64bd3 --- /dev/null +++ b/docs/manual/repo.md @@ -0,0 +1,81 @@ +# Anatomy of an OSTree repository + +## Core object types and data model + +OSTree is deeply inspired by git; the core layer is a userspace +content-addressed versioning filesystem. It is worth taking some time +to familiarize yourself with +[Git Internals](http://git-scm.com/book/en/Git-Internals), as this +section will assume some knowledge of how git works. + +Its object types are similar to git; it has commit objects and content +objects. Git has "tree" objects, whereas OSTree splits them into +"dirtree" and "dirmeta" objects. But unlike git, OSTree's checksums +are SHA256. And most crucially, its content objects include uid, gid, +and extended attributes (but still no timestamps). + +### Commit objects + +A commit object contains metadata such as a timestamp, a log +message, and most importantly, a reference to a +dirtree/dirmeta pair of checksums which describe the root +directory of the filesystem. +Also like git, each commit in OSTree can have a parent. It is +designed to store a history of your binary builds, just like git +stores a history of source control. However, OSTree also makes +it easy to delete data, under the assumption that you can +regenerate it from source code. + +### Dirtree objects + +A dirtree contains a sorted array of (filename, checksum) +pairs for content objects, and a second sorted array of +(filename, dirtree checksum, dirmeta checksum), which are +subdirectories. + +### Dirmeta objects + +In git, tree objects contain the metadata such as permissions +for their children. But OSTree splits this into a separate +object to avoid duplicating extended attribute listings. + +### Content objects + +Unlike the first three object types which are metadata, designed to be +`mmap()`ed, the content object has a separate internal header and +payload sections. The header contains uid, gid, mode, and symbolic +link target (for symlinks), as well as extended attributes. After the +header, for regular files, the content follows. + +# Repository types and locations + +Also unlike git, an OSTree repository can be in one of two separate +modes: `bare` and `archive-z2`. A bare repository is one where +content files are just stored as regular files; it's designed to be +the source of a "hardlink farm", where each operating system checkout +is merely links into it. If you want to store files owned by +e.g. root in this mode, you must run OSTree as root. In contrast, the +`archive-z2` mode is designed for serving via plain HTTP. Like tar +files, it can be read/written by non-root users. + +On an OSTree-deployed system, the "system repository" is +`/ostree/repo`. It can be read by any uid, but only written by root. +Unless the `--repo` argument is given to the ostree +command, it will operate on the system repository. + +## Refs + +Like git, OSTree uses "refs" to which are text files that point to +particular commits (i.e. filesystem trees). For example, the +gnome-ostree operating system creates trees named like +`exampleos/buildmaster/x86_64-runtime` and +`exampleos/buildmaster/x86_64-devel-debug`. These two refs point to +two different generated filesystem trees. In this example, the +"runtime" tree contains just enough to run a basic system, and +"devel-debug" contains all of the developer tools and debuginfo. + +The `ostree` supports a simple syntax using the carat `^` to refer to +the parent of a given commit. For example, +`exampleos/buildmaster/x86_64-runtime^` refers to the previous build, +and `exampleos/buildmaster/x86_64-runtime^^` refers to the one before +that. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..b11dbc87 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,10 @@ +site_name: My Docs +pages: + - Home: 'index.md' + - Contributing: 'CONTRIBUTING.md' + - Manual: + - Introduction: 'manual/introduction.md' + - Repository: 'manual/repo.md' + - Deployments: 'manual/deployment.md' + - Atomic Upgrades: 'manual/atomic-upgrades.md' + - Adapting Existing Systems: 'manual/adapting-existing.md' From 5bab946b809aaf030770a57389d74766e8981030 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 28 Jan 2016 09:30:16 -0500 Subject: [PATCH 11/54] apidoc: Remove unnecessary srcdir != builddir workaround It seems to be fine with `gtk-doc-1.19-3.el7.noarch`, so let's drop this workaround, as it causes `make` warnings. --- apidoc/Makefile.am | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apidoc/Makefile.am b/apidoc/Makefile.am index 59cd8096..dcb009ef 100644 --- a/apidoc/Makefile.am +++ b/apidoc/Makefile.am @@ -111,10 +111,6 @@ expand_content_files= \ # e.g. GTKDOC_LIBS=$(top_builddir)/gtk/$(gtktargetlib) GTKDOC_LIBS= -# Hacks around gtk-doc brokenness for out of tree builds -ostree-sections.txt: $(srcdir)/ostree-sections.txt - cp $< $@ - version.xml: echo -n $(VERSION) > "$@" From ca57ec4aa56e0e3ecfb4c5f4f34b84633a22fb66 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 26 Jan 2016 21:34:31 -0500 Subject: [PATCH 12/54] repo: Port -refs.c to openat() I'd like to incrementally convert all of `ostree-repo*.c` to fd-relative usage, so that we can sanely introduce `ostree_repo_new_at()` which doesn't involve GFile. This one is medium risk, but passes the test suite. --- libglnx | 2 +- src/libostree/ostree-repo-private.h | 2 - src/libostree/ostree-repo-refs.c | 251 +++++++++++++++++----------- src/libostree/ostree-repo.c | 4 - tests/admin-test.sh | 2 + 5 files changed, 158 insertions(+), 103 deletions(-) diff --git a/libglnx b/libglnx index 03138641..76952275 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 03138641298fd6799f46b16423871f959332bacf +Subproject commit 769522753c25537e520adc322fa62e5390272add diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index b6ea3177..463b3dd7 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -50,8 +50,6 @@ struct OstreeRepo { int repo_dir_fd; GFile *tmp_dir; int tmp_dir_fd; - GFile *local_heads_dir; - GFile *remote_heads_dir; GFile *objects_dir; GFile *state_dir; int objects_dir_fd; diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index 4fc415b2..68e34f20 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -25,19 +25,19 @@ static gboolean add_ref_to_set (const char *remote, - GFile *base, - GFile *child, + int base_fd, + const char *path, GHashTable *refs, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; char *contents; - char *relpath; gsize len; GString *refname; - if (!g_file_load_contents (child, cancellable, &contents, &len, NULL, error)) + contents = glnx_file_get_contents_utf8_at (base_fd, path, &len, cancellable, error); + if (!contents) goto out; g_strchomp (contents); @@ -48,9 +48,7 @@ add_ref_to_set (const char *remote, g_string_append (refname, remote); g_string_append_c (refname, ':'); } - relpath = g_file_get_relative_path (base, child); - g_string_append (refname, relpath); - g_free (relpath); + g_string_append (refname, path); g_hash_table_insert (refs, g_string_free (refname, FALSE), contents); @@ -116,44 +114,69 @@ write_checksum_file_at (OstreeRepo *self, return ret; } +static gboolean +openat_ignore_enoent (int dfd, + const char *path, + int *out_fd, + GError **error) +{ + gboolean ret = FALSE; + int target_fd = -1; + + target_fd = openat (dfd, path, O_CLOEXEC | O_RDONLY); + if (target_fd < 0) + { + if (errno != ENOENT) + { + glnx_set_error_from_errno (error); + goto out; + } + } + + ret = TRUE; + *out_fd = target_fd; + out: + return ret; +} + static gboolean find_ref_in_remotes (OstreeRepo *self, const char *rev, - GFile **out_file, + int *out_fd, GError **error) { gboolean ret = FALSE; - g_autoptr(GFileEnumerator) dir_enum = NULL; - g_autoptr(GFile) ret_file = NULL; + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + glnx_fd_close int ret_fd = -1; - dir_enum = g_file_enumerate_children (self->remote_heads_dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, error); - if (!dir_enum) + if (!glnx_dirfd_iterator_init_at (self->repo_dir_fd, "refs/remotes", TRUE, &dfd_iter, error)) goto out; while (TRUE) { - GFileInfo *file_info; - GFile *child; - if (!gs_file_enumerator_iterate (dir_enum, &file_info, &child, - NULL, error)) + struct dirent *dent = NULL; + glnx_fd_close int remote_dfd = -1; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, NULL, error)) goto out; - if (file_info == NULL) + if (dent == NULL) break; - if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_DIRECTORY) + + if (dent->d_type != DT_DIR) continue; - g_clear_object (&ret_file); - ret_file = g_file_resolve_relative_path (child, rev); - if (!g_file_query_exists (ret_file, NULL)) - g_clear_object (&ret_file); - else + if (!glnx_opendirat (dfd_iter.fd, dent->d_name, TRUE, &remote_dfd, error)) + goto out; + + if (!openat_ignore_enoent (remote_dfd, rev, &ret_fd, error)) + goto out; + + if (ret_fd != -1) break; } ret = TRUE; - ot_transfer_out_value (out_file, &ret_file); + *out_fd = ret_fd; ret_fd = -1; out: return ret; } @@ -210,9 +233,8 @@ resolve_refspec (OstreeRepo *self, { gboolean ret = FALSE; __attribute__((unused)) GCancellable *cancellable = NULL; - GError *temp_error = NULL; g_autofree char *ret_rev = NULL; - g_autoptr(GFile) child = NULL; + glnx_fd_close int target_fd = -1; g_return_val_if_fail (ref != NULL, FALSE); @@ -223,40 +245,42 @@ resolve_refspec (OstreeRepo *self, } else if (remote != NULL) { - child = ot_gfile_resolve_path_printf (self->remote_heads_dir, "%s/%s", - remote, ref); - if (!g_file_query_exists (child, NULL)) - g_clear_object (&child); + const char *remote_ref = glnx_strjoina ("refs/remotes/", remote, "/", ref); + + if (!openat_ignore_enoent (self->repo_dir_fd, remote_ref, &target_fd, error)) + goto out; } else { - child = g_file_resolve_relative_path (self->local_heads_dir, ref); + const char *local_ref = glnx_strjoina ("refs/heads/", ref); - if (!g_file_query_exists (child, NULL)) + if (!openat_ignore_enoent (self->repo_dir_fd, local_ref, &target_fd, error)) + goto out; + + if (target_fd == -1) { - g_clear_object (&child); + local_ref = glnx_strjoina ("refs/remotes/", ref); - child = g_file_resolve_relative_path (self->remote_heads_dir, ref); + if (!openat_ignore_enoent (self->repo_dir_fd, local_ref, &target_fd, error)) + goto out; - if (!g_file_query_exists (child, NULL)) + if (target_fd == -1) { - g_clear_object (&child); - - if (!find_ref_in_remotes (self, ref, &child, error)) + if (!find_ref_in_remotes (self, ref, &target_fd, error)) goto out; } } } - if (child) + if (target_fd != -1) { - if ((ret_rev = gs_file_load_contents_utf8 (child, NULL, &temp_error)) == NULL) + ret_rev = glnx_fd_readall_utf8 (target_fd, NULL, NULL, error); + if (!ret_rev) { - g_propagate_error (error, temp_error); - g_prefix_error (error, "Couldn't open ref '%s': ", gs_file_get_path_cached (child)); + g_prefix_error (error, "Couldn't open ref '%s': ", ref); goto out; } - + g_strchomp (ret_rev); if (!ostree_validate_checksum_string (ret_rev, error)) goto out; @@ -433,43 +457,50 @@ ostree_repo_resolve_rev (OstreeRepo *self, static gboolean enumerate_refs_recurse (OstreeRepo *repo, const char *remote, - GFile *base, - GFile *dir, + int base_dfd, + GString *base_path, + int child_dfd, + const char *path, GHashTable *refs, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - g_autoptr(GFileEnumerator) enumerator = NULL; + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); - if (!enumerator) + if (!glnx_dirfd_iterator_init_at (child_dfd, path, FALSE, &dfd_iter, error)) goto out; while (TRUE) { - GFileInfo *file_info = NULL; - GFile *child = NULL; + guint len = base_path->len; + struct dirent *dent = NULL; - if (!gs_file_enumerator_iterate (enumerator, &file_info, &child, - NULL, error)) + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) goto out; - if (file_info == NULL) + if (dent == NULL) break; - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) + g_string_append (base_path, dent->d_name); + + if (dent->d_type == DT_DIR) { - if (!enumerate_refs_recurse (repo, remote, base, child, refs, cancellable, error)) + g_string_append_c (base_path, '/'); + + if (!enumerate_refs_recurse (repo, remote, base_dfd, base_path, + dfd_iter.fd, dent->d_name, + refs, cancellable, error)) goto out; + } - else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) + else if (dent->d_type == DT_REG) { - if (!add_ref_to_set (remote, base, child, refs, + if (!add_ref_to_set (remote, base_dfd, base_path->str, refs, cancellable, error)) goto out; } + + g_string_truncate (base_path, len); } ret = TRUE; @@ -505,35 +536,55 @@ ostree_repo_list_refs (OstreeRepo *self, if (refspec_prefix) { - g_autoptr(GFile) dir = NULL; - g_autoptr(GFile) child = NULL; - g_autoptr(GFileInfo) info = NULL; + struct stat stbuf; + const char *prefix_path; + const char *path; if (!ostree_parse_refspec (refspec_prefix, &remote, &ref_prefix, error)) goto out; if (remote) - dir = g_file_get_child (self->remote_heads_dir, remote); - else - dir = g_object_ref (self->local_heads_dir); - - child = g_file_resolve_relative_path (dir, ref_prefix); - if (!ot_gfile_query_info_allow_noent (child, OSTREE_GIO_FAST_QUERYINFO, 0, - &info, cancellable, error)) - goto out; - - if (info) { - if (g_file_info_get_file_type (info) == G_FILE_TYPE_DIRECTORY) + prefix_path = glnx_strjoina ("refs/remotes/", remote, "/"); + path = glnx_strjoina (prefix_path, ref_prefix); + } + else + { + prefix_path = "refs/heads/"; + path = glnx_strjoina (prefix_path, ref_prefix); + } + + if (fstatat (self->repo_dir_fd, path, &stbuf, 0) < 0) + { + if (errno != ENOENT) { - if (!enumerate_refs_recurse (self, remote, child, child, - ret_all_refs, - cancellable, error)) + glnx_set_error_from_errno (error); + goto out; + } + } + else + { + if (S_ISDIR (stbuf.st_mode)) + { + glnx_fd_close int base_fd = -1; + g_autoptr(GString) base_path = g_string_new (""); + + if (!glnx_opendirat (self->repo_dir_fd, path, TRUE, &base_fd, error)) + goto out; + + if (!enumerate_refs_recurse (self, remote, base_fd, base_path, + base_fd, ".", + ret_all_refs, cancellable, error)) goto out; } else { - if (!add_ref_to_set (remote, dir, child, ret_all_refs, + glnx_fd_close int prefix_dfd = -1; + + if (!glnx_opendirat (self->repo_dir_fd, prefix_path, TRUE, &prefix_dfd, error)) + goto out; + + if (!add_ref_to_set (remote, prefix_dfd, ref_prefix, ret_all_refs, cancellable, error)) goto out; } @@ -541,33 +592,41 @@ ostree_repo_list_refs (OstreeRepo *self, } else { - g_autoptr(GFileEnumerator) remote_enumerator = NULL; - - if (!enumerate_refs_recurse (self, NULL, self->local_heads_dir, self->local_heads_dir, - ret_all_refs, - cancellable, error)) + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + g_autoptr(GString) base_path = g_string_new (""); + glnx_fd_close int refs_heads_dfd = -1; + + if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &refs_heads_dfd, error)) goto out; - remote_enumerator = g_file_enumerate_children (self->remote_heads_dir, OSTREE_GIO_FAST_QUERYINFO, - 0, - cancellable, error); - if (!remote_enumerator) + if (!enumerate_refs_recurse (self, NULL, refs_heads_dfd, base_path, + refs_heads_dfd, ".", + ret_all_refs, cancellable, error)) + goto out; + + g_string_truncate (base_path, 0); + + if (!glnx_dirfd_iterator_init_at (self->repo_dir_fd, "refs/remotes", TRUE, &dfd_iter, error)) goto out; while (TRUE) { - GFileInfo *info; - GFile *child; - const char *name; + struct dirent *dent; + glnx_fd_close int remote_dfd = -1; - if (!gs_file_enumerator_iterate (remote_enumerator, &info, &child, - cancellable, error)) + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) goto out; - if (!info) + if (!dent) break; - name = g_file_info_get_name (info); - if (!enumerate_refs_recurse (self, name, child, child, + if (dent->d_type != DT_DIR) + continue; + + if (!glnx_opendirat (dfd_iter.fd, dent->d_name, TRUE, &remote_dfd, error)) + goto out; + + if (!enumerate_refs_recurse (self, dent->d_name, remote_dfd, base_path, + remote_dfd, ".", ret_all_refs, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index d5b9aeaf..21362084 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -518,8 +518,6 @@ ostree_repo_finalize (GObject *object) g_clear_object (&self->tmp_dir); if (self->tmp_dir_fd) (void) close (self->tmp_dir_fd); - g_clear_object (&self->local_heads_dir); - g_clear_object (&self->remote_heads_dir); g_clear_object (&self->objects_dir); if (self->objects_dir_fd != -1) (void) close (self->objects_dir_fd); @@ -605,8 +603,6 @@ ostree_repo_constructed (GObject *object) g_assert (self->repodir != NULL); self->tmp_dir = g_file_resolve_relative_path (self->repodir, "tmp"); - self->local_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/heads"); - self->remote_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/remotes"); self->objects_dir = g_file_get_child (self->repodir, "objects"); self->deltas_dir = g_file_get_child (self->repodir, "deltas"); diff --git a/tests/admin-test.sh b/tests/admin-test.sh index a587d79b..c4644d3a 100755 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -155,6 +155,8 @@ assert_not_streq ${rev} ${newrev} assert_file_has_content sysroot/ostree/deploy/testos/deploy/${newrev}.0/etc/os-release 'NAME=TestOS' ${CMD_PREFIX} ostree admin status validate_bootloader +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo refs testos:testos > reftest.txt +assert_file_has_content reftest.txt testos:buildmaster/x86_64-runtime echo "ok upgrade" From c74ea8b075168168820528d0423e46428de1c84d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 28 Jan 2016 15:22:10 -0500 Subject: [PATCH 13/54] build: Add --disable-man Gnome Continuous doesn't have docbook, so copy what we do for glib. --- Makefile-man.am | 4 ++-- configure.ac | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Makefile-man.am b/Makefile-man.am index d2f31ca0..299e3177 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -26,7 +26,7 @@ man5_MANS = $(addprefix man/,$(man5_files)) EXTRA_DIST += $(man1_MANS) $(man5_MANS) -if ENABLE_XSLTPROC +if ENABLE_MAN XSLT_STYLESHEET = http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl @@ -51,4 +51,4 @@ CLEANFILES += \ $(man5_MANS) \ $(NULL) -endif # ENABLE_GTK_DOC +endif diff --git a/configure.ac b/configure.ac index 140b6c21..c36134b9 100644 --- a/configure.ac +++ b/configure.ac @@ -126,8 +126,22 @@ enable_gtk_doc=no AM_CONDITIONAL([ENABLE_GTK_DOC], false) ]) -AC_PATH_PROG([XSLTPROC], [xsltproc]) -AM_CONDITIONAL([ENABLE_XSLTPROC], test -n "${XSLTPROC}") +AC_ARG_ENABLE(man, + [AS_HELP_STRING([--enable-man], + [generate man pages [default=auto]])],, + enable_man=maybe) + +AS_IF([test "$enable_man" != no], [ + AC_PATH_PROG([XSLTPROC], [xsltproc]) + AS_IF([test -z "$XSLTPROC"], [ + AS_IF([test "$enable_man" = yes], [ + AC_MSG_ERROR([xsltproc is required for --enable-man]) + ]) + enable_man=no + ]) + enable_man=yes +]) +AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) AC_ARG_WITH(libarchive, AS_HELP_STRING([--without-libarchive], [Do not use libarchive]), @@ -248,7 +262,7 @@ echo " SELinux: $with_selinux libarchive (parse tar files directly): $with_libarchive static deltas: $enable_static_deltas - man pages (xsltproc): $XSLTPROC + man pages (xsltproc): $enable_man api docs (gtk-doc): $enable_gtk_doc gjs-based tests: $have_gjs dracut: $with_dracut From fa9e547e09199d81018ca51dc784bc7253388c2d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 28 Jan 2016 14:53:38 -0500 Subject: [PATCH 14/54] lib: Add a #define OSTREE_SHA256_DIGEST_LEN 32 And use it internally. This way it's a bit less magical. --- src/libostree/ostree-core.c | 24 +++++++++---------- src/libostree/ostree-core.h | 2 ++ src/libostree/ostree-repo-commit.c | 4 ++-- src/libostree/ostree-repo-pull.c | 6 ++--- .../ostree-repo-static-delta-compilation.c | 8 +++---- src/libostree/ostree-repo-static-delta-core.c | 2 +- 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 5f24e7c9..11c61f95 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1069,7 +1069,7 @@ int ostree_cmp_checksum_bytes (const guchar *a, const guchar *b) { - return memcmp (a, b, 32); + return memcmp (a, b, OSTREE_SHA256_DIGEST_LEN); } /** @@ -1151,7 +1151,7 @@ ostree_checksum_inplace_to_bytes (const char *checksum, guint i; guint j; - for (i = 0, j = 0; i < 32; i += 1, j += 2) + for (i = 0, j = 0; i < OSTREE_SHA256_DIGEST_LEN; i += 1, j += 2) { gint big, little; @@ -1177,7 +1177,7 @@ ostree_checksum_inplace_to_bytes (const char *checksum, guchar * ostree_checksum_to_bytes (const char *checksum) { - guchar *ret = g_malloc (32); + guchar *ret = g_malloc (OSTREE_SHA256_DIGEST_LEN); ostree_checksum_inplace_to_bytes (checksum, ret); return ret; } @@ -1191,9 +1191,9 @@ ostree_checksum_to_bytes (const char *checksum) GVariant * ostree_checksum_to_bytes_v (const char *checksum) { - guchar result[32]; + guchar result[OSTREE_SHA256_DIGEST_LEN]; ostree_checksum_inplace_to_bytes (checksum, result); - return ot_gvariant_new_bytearray ((guchar*)result, 32); + return ot_gvariant_new_bytearray ((guchar*)result, OSTREE_SHA256_DIGEST_LEN); } /** @@ -1210,7 +1210,7 @@ ostree_checksum_inplace_from_bytes (const guchar *csum, static const gchar hexchars[] = "0123456789abcdef"; guint i, j; - for (i = 0, j = 0; i < 32; i++, j += 2) + for (i = 0, j = 0; i < OSTREE_SHA256_DIGEST_LEN; i++, j += 2) { guchar byte = csum[i]; buf[j] = hexchars[byte >> 4]; @@ -1242,7 +1242,7 @@ ostree_checksum_b64_inplace_from_bytes (const guchar *csum, * a lot easier to reuse GLib's base64 encoder and postprocess it * to replace the '/' with '_'. */ - outlen = g_base64_encode_step (csum, 32, FALSE, tmpbuf, &state, &save); + outlen = g_base64_encode_step (csum, OSTREE_SHA256_DIGEST_LEN, FALSE, tmpbuf, &state, &save); outlen += g_base64_encode_close (FALSE, tmpbuf+outlen, &state, &save); g_assert (outlen == 44); @@ -1299,7 +1299,7 @@ ostree_checksum_bytes_peek (GVariant *bytes) gsize n_elts; const guchar *ret; ret = g_variant_get_fixed_array (bytes, &n_elts, 1); - if (G_UNLIKELY (n_elts != 32)) + if (G_UNLIKELY (n_elts != OSTREE_SHA256_DIGEST_LEN)) return NULL; return ret; } @@ -1434,20 +1434,20 @@ _ostree_get_relative_static_delta_path (const char *from, const char *to, const char *target) { - guint8 csum_to[32]; + guint8 csum_to[OSTREE_SHA256_DIGEST_LEN]; char to_b64[44]; - guint8 csum_to_copy[32]; + guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN]; GString *ret = g_string_new ("deltas/"); ostree_checksum_inplace_to_bytes (to, csum_to); ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64); ostree_checksum_b64_inplace_to_bytes (to_b64, csum_to_copy); - g_assert (memcmp (csum_to, csum_to_copy, 32) == 0); + g_assert (memcmp (csum_to, csum_to_copy, OSTREE_SHA256_DIGEST_LEN) == 0); if (from != NULL) { - guint8 csum_from[32]; + guint8 csum_from[OSTREE_SHA256_DIGEST_LEN]; char from_b64[44]; ostree_checksum_inplace_to_bytes (from, csum_from); diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 637f8172..c5b42a75 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -50,6 +50,8 @@ G_BEGIN_DECLS */ #define OSTREE_MAX_RECURSION (256) +#define OSTREE_SHA256_DIGEST_LEN (32) + /** * OstreeObjectType: * @OSTREE_OBJECT_TYPE_FILE: Content; regular file, symbolic link diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 5ea49efa..a7c16192 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -412,12 +412,12 @@ add_size_index_to_metadata (OstreeRepo *self, for (i = 0; i < sorted_keys->len; i++) { - guint8 csum[32]; + guint8 csum[OSTREE_SHA256_DIGEST_LEN]; const char *e_checksum = sorted_keys->pdata[i]; GString *buffer = g_string_new (NULL); ostree_checksum_inplace_to_bytes (e_checksum, csum); - g_string_append_len (buffer, (char*)csum, 32); + g_string_append_len (buffer, (char*)csum, sizeof (csum)); e_size = g_hash_table_lookup (self->object_sizes, e_checksum); _ostree_write_varuint64 (buffer, e_size->archived); diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index c6c91082..a0a1fc9c 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -123,7 +123,7 @@ typedef struct { } FetchStaticDeltaData; typedef struct { - guchar csum[32]; + guchar csum[OSTREE_SHA256_DIGEST_LEN]; OstreeObjectType objtype; guint recursion_depth; } ScanObjectQueueData; @@ -1150,7 +1150,7 @@ queue_scan_one_metadata_object (OtPullData *pull_data, OstreeObjectType objtype, guint recursion_depth) { - guchar buf[32]; + guchar buf[OSTREE_SHA256_DIGEST_LEN]; ostree_checksum_inplace_to_bytes (csum, buf); queue_scan_one_metadata_object_c (pull_data, buf, objtype, recursion_depth); } @@ -2067,7 +2067,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, { const char *delta; GVariant *csum_v = NULL; - guchar *csum_data = g_malloc (32); + guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN); g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i); g_variant_get_child (ref, 0, "&s", &delta); diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 530b3a75..2071bb68 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -228,7 +228,7 @@ objtype_checksum_array_new (GPtrArray *objects) GVariant *serialized_key = objects->pdata[i]; OstreeObjectType objtype; const char *checksum; - guint8 csum[32]; + guint8 csum[OSTREE_SHA256_DIGEST_LEN]; guint8 objtype_v; ostree_object_name_deserialize (serialized_key, &checksum, &objtype); @@ -678,7 +678,7 @@ process_one_rollsum (OstreeRepo *repo, { gsize mode_offset, xattr_offset, from_csum_offset; gboolean reading_payload = TRUE; - guchar source_csum[32]; + guchar source_csum[OSTREE_SHA256_DIGEST_LEN]; guint i; write_content_mode_xattrs (repo, current_part, content_finfo, content_xattrs, @@ -799,7 +799,7 @@ process_one_bsdiff (OstreeRepo *repo, g_ptr_array_add (current_part->objects, ostree_object_name_serialize (to_checksum, OSTREE_OBJECT_TYPE_FILE)); { gsize mode_offset, xattr_offset; - guchar source_csum[32]; + guchar source_csum[OSTREE_SHA256_DIGEST_LEN]; write_content_mode_xattrs (repo, current_part, content_finfo, content_xattrs, &mode_offset, &xattr_offset); @@ -1411,7 +1411,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, cancellable, error)) goto out; - checksum_bytes = g_bytes_new (part_checksum, 32); + checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN); objtype_checksum_array = objtype_checksum_array_new (part_builder->objects); delta_part_header = g_variant_new ("(u@aytt@ay)", OSTREE_DELTAPART_VERSION, diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index e3490fd6..b2ffddc5 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -131,7 +131,7 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, g_autofree char *buf = g_strconcat (name1, name2, NULL); GString *out = g_string_new (""); char checksum[65]; - guchar csum[32]; + guchar csum[OSTREE_SHA256_DIGEST_LEN]; const char *dash = strchr (buf, '-'); ostree_checksum_b64_inplace_to_bytes (buf, csum); From 944b3abab1cff15f676800bd52a990bbb2914515 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 28 Jan 2016 15:34:38 -0500 Subject: [PATCH 15/54] build: Hoist man conditional higher So we actually build with `--disable-man`. --- Makefile-man.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile-man.am b/Makefile-man.am index 299e3177..f70a577c 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -17,6 +17,8 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. +if ENABLE_MAN + man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 man5_files = ostree.repo.5 ostree.repo-config.5 @@ -26,8 +28,6 @@ man5_MANS = $(addprefix man/,$(man5_files)) EXTRA_DIST += $(man1_MANS) $(man5_MANS) -if ENABLE_MAN - XSLT_STYLESHEET = http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl XSLTPROC_FLAGS = \ From 98d5f6e3db376ca0a7ab5f505474a0105bee19a7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 27 Jan 2016 22:02:36 -0500 Subject: [PATCH 16/54] static-delta: Add `show` subcommand Right now though, almost all of the details of deltas are private, so we can't do the "honest thing" and have the command line just use the shared library. Eventually some of this should appear in the API, but for now add command line which is useful for debugging. --- src/libostree/ostree-cmdprivate.c | 4 +- src/libostree/ostree-cmdprivate.h | 1 + src/libostree/ostree-repo-static-delta-core.c | 93 +++++++++++++++++++ .../ostree-repo-static-delta-private.h | 11 +++ src/libotutil/ot-variant-utils.c | 22 +++++ src/libotutil/ot-variant-utils.h | 7 ++ src/ostree/ot-builtin-static-delta.c | 35 +++++++ 7 files changed, 172 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-cmdprivate.c b/src/libostree/ostree-cmdprivate.c index 99834937..74c32a3c 100644 --- a/src/libostree/ostree-cmdprivate.c +++ b/src/libostree/ostree-cmdprivate.c @@ -23,6 +23,7 @@ #include "ostree-cmdprivate.h" #include "ostree-repo-private.h" #include "ostree-core-private.h" +#include "ostree-repo-static-delta-private.h" #include "ostree-sysroot.h" #include "ostree-bootloader-grub2.h" @@ -44,7 +45,8 @@ const OstreeCmdPrivateVTable * ostree_cmd__private__ (void) { static OstreeCmdPrivateVTable table = { - impl_ostree_generate_grub2_config + impl_ostree_generate_grub2_config, + _ostree_repo_static_delta_dump }; return &table; diff --git a/src/libostree/ostree-cmdprivate.h b/src/libostree/ostree-cmdprivate.h index 317a7592..7746406b 100644 --- a/src/libostree/ostree-cmdprivate.h +++ b/src/libostree/ostree-cmdprivate.h @@ -26,6 +26,7 @@ G_BEGIN_DECLS typedef struct { gboolean (* ostree_generate_grub2_config) (OstreeSysroot *sysroot, int bootversion, int target_fd, GCancellable *cancellable, GError **error); + gboolean (* ostree_static_delta_dump) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); } OstreeCmdPrivateVTable; const OstreeCmdPrivateVTable * diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index b2ffddc5..bc1b460c 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -22,6 +22,7 @@ #include "ostree-core-private.h" #include "ostree-repo-private.h" +#include "ostree-cmdprivate.h" #include "ostree-repo-static-delta-private.h" #include "otutil.h" @@ -404,3 +405,95 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, out: return ret; } + +gboolean +_ostree_repo_static_delta_dump (OstreeRepo *self, + const char *delta_id, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + g_autofree char *from = NULL; + g_autofree char *to = NULL; + g_autofree char *superblock_path = NULL; + glnx_fd_close int superblock_fd = -1; + g_autoptr(GVariant) delta_superblock = NULL; + guint64 total_size = 0, total_usize = 0; + guint64 total_fallback_size = 0, total_fallback_usize = 0; + guint i; + + _ostree_parse_delta_name (delta_id, &from, &to); + superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); + + if (!ot_util_variant_map_at (self->repo_dir_fd, superblock_path, + (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, + TRUE, &delta_superblock, error)) + goto out; + + g_print ("Delta: %s\n", delta_id); + { guint64 ts; + g_variant_get_child (delta_superblock, 1, "t", &ts); + g_print ("Timestamp: %" G_GUINT64_FORMAT "\n", GUINT64_FROM_BE (ts)); + } + { g_autoptr(GVariant) recurse = NULL; + g_variant_get_child (delta_superblock, 5, "@ay", &recurse); + g_print ("Number of parents: %u\n", (guint)(g_variant_get_size (recurse) / (OSTREE_SHA256_DIGEST_LEN * 2))); + } + { g_autoptr(GVariant) fallback = NULL; + guint n_fallback; + + g_variant_get_child (delta_superblock, 7, "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT, &fallback); + n_fallback = g_variant_n_children (fallback); + + g_print ("Number of fallback entries: %u\n", n_fallback); + + for (i = 0; i < n_fallback; i++) + { + guint64 size, usize; + g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize); + total_fallback_size += size; + total_fallback_usize += usize; + } + { g_autofree char *sizestr = g_format_size (total_fallback_size); + g_autofree char *usizestr = g_format_size (total_fallback_usize); + g_print ("Total Fallback Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_size, sizestr); + g_print ("Total Fallback Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_usize, usizestr); + } + } + { g_autoptr(GVariant) meta_entries = NULL; + guint n_parts; + + g_variant_get_child (delta_superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries); + n_parts = g_variant_n_children (meta_entries); + g_print ("Number of parts: %u\n", n_parts); + + for (i = 0; i < n_parts; i++) + { + guint32 version; + guint64 size, usize; + g_autoptr(GVariant) objects = NULL; + g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects); + total_size += size; + total_usize += usize; + g_print ("Part%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n", + i, (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN), size, usize); + } + } + + { g_autofree char *sizestr = g_format_size (total_size); + g_autofree char *usizestr = g_format_size (total_usize); + g_print ("Total Part Size: %" G_GUINT64_FORMAT " (%s)\n", total_size, sizestr); + g_print ("Total Part Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_usize, usizestr); + } + { guint64 overall_size = total_size + total_fallback_size; + guint64 overall_usize = total_usize + total_fallback_usize; + g_autofree char *sizestr = g_format_size (overall_size); + g_autofree char *usizestr = g_format_size (overall_usize); + g_print ("Total Size: %" G_GUINT64_FORMAT " (%s)\n", overall_size, sizestr); + g_print ("Total Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", overall_usize, usizestr); + } + + ret = TRUE; + out: + return ret; +} diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index de32ec65..b0e82ade 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -103,6 +103,11 @@ G_BEGIN_DECLS */ #define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}tayay" OSTREE_COMMIT_GVARIANT_STRING "aya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")" +gboolean _ostree_static_delta_dump (OstreeRepo *repo, + const char *delta_id, + GCancellable *cancellable, + GError **error); + gboolean _ostree_static_delta_part_validate (OstreeRepo *repo, GInputStream *in, guint part_offset, @@ -177,4 +182,10 @@ _ostree_delta_compute_similar_objects (OstreeRepo *repo, GCancellable *cancellable, GError **error); +gboolean +_ostree_repo_static_delta_dump (OstreeRepo *repo, + const char *delta_id, + GCancellable *cancellable, + GError **error); + G_END_DECLS diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index 1210a785..ed650268 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -154,6 +154,28 @@ ot_util_variant_map (GFile *src, return ret; } +gboolean +ot_util_variant_map_at (int dfd, + const char *path, + const GVariantType *type, + gboolean trusted, + GVariant **out_variant, + GError **error) +{ + glnx_fd_close int fd = -1; + g_autoptr(GVariant) ret_variant = NULL; + + fd = openat (dfd, path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + { + glnx_set_error_from_errno (error); + g_prefix_error (error, "Opening %s: ", path); + return FALSE; + } + + return ot_util_variant_map_fd (fd, 0, type, trusted, out_variant, error); +} + typedef struct { gpointer addr; gsize len; diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h index 34b6eb5d..1a7abe0e 100644 --- a/src/libotutil/ot-variant-utils.h +++ b/src/libotutil/ot-variant-utils.h @@ -48,6 +48,13 @@ gboolean ot_util_variant_map (GFile *src, GVariant **out_variant, GError **error); +gboolean ot_util_variant_map_at (int dfd, + const char *path, + const GVariantType *type, + gboolean trusted, + GVariant **out_variant, + GError **error); + gboolean ot_util_variant_map_fd (int fd, goffset offset, const GVariantType *type, diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index 903e5fd6..c3c99ba0 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -23,6 +23,7 @@ #include "ot-main.h" #include "ot-builtins.h" #include "ostree.h" +#include "ostree-cmdprivate.h" #include "ot-main.h" #include "otutil.h" @@ -38,6 +39,7 @@ static gboolean opt_disable_bsdiff; #define BUILTINPROTO(name) static gboolean ot_static_delta_builtin_ ## name (int argc, char **argv, GCancellable *cancellable, GError **error) BUILTINPROTO(list); +BUILTINPROTO(show); BUILTINPROTO(generate); BUILTINPROTO(apply_offline); @@ -45,6 +47,7 @@ BUILTINPROTO(apply_offline); static OstreeCommand static_delta_subcommands[] = { { "list", ot_static_delta_builtin_list }, + { "show", ot_static_delta_builtin_show }, { "generate", ot_static_delta_builtin_generate }, { "apply-offline", ot_static_delta_builtin_apply_offline }, { NULL, NULL } @@ -129,6 +132,38 @@ ot_static_delta_builtin_list (int argc, char **argv, GCancellable *cancellable, return ret; } +static gboolean +ot_static_delta_builtin_show (int argc, char **argv, GCancellable *cancellable, GError **error) +{ + gboolean ret = FALSE; + GOptionContext *context; + glnx_unref_object OstreeRepo *repo = NULL; + const char *delta_id = NULL; + + context = g_option_context_new ("SHOW - Dump information on a delta"); + + if (!ostree_option_context_parse (context, list_options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) + goto out; + + if (argc < 3) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "DELTA must be specified"); + goto out; + } + + delta_id = argv[2]; + + if (!ostree_cmd__private__ ()->ostree_static_delta_dump (repo, delta_id, cancellable, error)) + goto out; + + ret = TRUE; + out: + if (context) + g_option_context_free (context); + return ret; +} + static gboolean ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellable, GError **error) { From ba272008a70cd6d5d26644620f31bb2fef50cee8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 3 Feb 2016 10:22:01 -0500 Subject: [PATCH 17/54] packaging: Sync spec file with Fedora --- packaging/ostree.spec.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/ostree.spec.in b/packaging/ostree.spec.in index 8fb219d6..8f9cacfd 100644 --- a/packaging/ostree.spec.in +++ b/packaging/ostree.spec.in @@ -20,10 +20,12 @@ BuildRequires: pkgconfig(e2p) # Extras BuildRequires: pkgconfig(libarchive) BuildRequires: pkgconfig(libselinux) +BuildRequires: libcap-devel BuildRequires: gpgme-devel BuildRequires: pkgconfig(systemd) BuildRequires: /usr/bin/g-ir-scanner BuildRequires: dracut +BuildRequires: bison # Runtime requirements Requires: dracut From b0163d618274004990a0a24c5f1d2564861e4861 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Fri, 29 Jan 2016 11:02:17 +0100 Subject: [PATCH 18/54] build: 'make clean' removes parse-datetime.c and fix make dist while at it. Signed-off-by: Giuseppe Scrivano --- Makefile-libostree.am | 5 ++++- Makefile-ostree.am | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 238cb0b4..23fa9494 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -182,7 +182,10 @@ pkgconfig_DATA += src/libostree/ostree-1.pc gpgreadme_DATA = src/libostree/README-gpg gpgreadmedir = $(pkgdatadir)/trusted.gpg.d -EXTRA_DIST += src/libostree/README-gpg src/libostree/bupsplit.h +EXTRA_DIST += src/libostree/README-gpg src/libostree/bupsplit.h \ + src/libostree/ostree-enumtypes.h.template \ + src/libostree/ostree-enumtypes.c.template \ + src/libostree/ostree-deployment-private.h install-mkdir-remotes-d-hook: mkdir -p $(DESTDIR)$(sysconfdir)/ostree/remotes.d diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 384479da..76fb26f3 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -89,6 +89,8 @@ ostree_SOURCES += \ src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile $(AM_V_GEN) $(YACC) $< -o $@ +EXTRA_DIST += src/ostree/parse-datetime.y +CLEANFILES += src/ostree/parse-datetime.c ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(srcdir)/src/ostree \ $(NULL) From 371856b6e4447a9136fe1ab6f80b2bdcacf77315 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 8 Feb 2016 14:24:24 +0100 Subject: [PATCH 19/54] docs-md: Delete (obsoleted by docs/) This was just a prototype start at markdown docs that I never picked back up. Now that we have the Markdown/mkdocs done, delete this. --- docs-md/atomic-upgrades.md | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 docs-md/atomic-upgrades.md diff --git a/docs-md/atomic-upgrades.md b/docs-md/atomic-upgrades.md deleted file mode 100644 index 76cd0250..00000000 --- a/docs-md/atomic-upgrades.md +++ /dev/null @@ -1,12 +0,0 @@ -Atomic upgrades and rollback ----------------------------- - -Traditional package managers operate "live" on the running system. -This means it's possible for interrupted updates to result in a -half-updated system. This model also makes it significantly harder to -support rollbacks when updates fail. - -In contrast, OSTree always creates a *new* root whenever it's -performing an update. This new root shares storage via hardlinks with -the current system. Upon success, the bootloader configuration will -be updated. From 0c15c9d6dd056b7890c37c9ac47a410fb17fd19f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 8 Feb 2016 14:35:09 +0100 Subject: [PATCH 20/54] README.md: Update to link to Read The Docs, describe a bit better --- README.md | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 493fa925..10f7c32f 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,48 @@ OSTree ====== +New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ ) + +----- + OSTree is a tool that combines a "git-like" model for committing and downloading bootable filesystem trees, along with a layer for deploying them and managing the bootloader configuration. -Traditional package managers (dpkg/rpm) build filesystem trees on the -client side. In contrast, the primary focus of OSTree is on -replicating trees composed on a server. +OSTree is like git in that it checksums individual files and has a +content-addressed-object store. It's unlike git in that it "checks +out" the files via hardlinks, and they should thus be immutable. +Therefore, another way to think of OSTree is that it's just a more +polished version of +[Linux VServer hardlinks](http://linux-vserver.org/index.php?title=util-vserver:Vhashify&oldid=2285). **Features:** - - Atomic upgrades and rollback - - GPG signatures and "pinned TLS" support + - Atomic upgrades and rollback for the system + - Replicating content incrementally over HTTP via GPG signatures and "pinned TLS" support - Support for parallel installing more than just 2 bootable roots - - Binary history on the server side + - Binary history on the server side (and client) - Introspectable shared library API for build and deployment systems +This last point is important - you should think of the OSTree command +line as effectively a "demo" for the shared library. The intent is that +package managers, system upgrade tools, container build tools and the like +use OSTree as a "deduplicating hardlink store". + Projects using OSTree --------------------- [rpm-ostree](https://github.com/projectatomic/rpm-ostree) is a tool that uses OSTree as a shared library, and supports committing RPMs -into an OSTree repository, and deploying them on the client. +into an OSTree repository, and deploying them on the client. This is +appropriate for "fixed purpose" systems. There is in progress work +for more sophisticated hybrid models, deeply integrating the RPM +packaging with OSTree. -[Project Atomic](http://www.projectatomic.io/) uses rpm-ostree -to provide a minimal host for Docker formatted Linux containers. +[Project Atomic](http://www.projectatomic.io/) uses rpm-ostree to +provide a minimal host for Docker formatted Linux containers. +Replicating a base immutable OS, then using Docker for applications +meshes together two different tools with different tradeoffs. [xdg-app](https://github.com/alexlarsson/xdg-app) uses OSTree for desktop application containers. @@ -63,12 +80,11 @@ make install DESTDIR=/path/to/dest More documentation ------------------ +New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ ) + Some more information is available on the old wiki page: https://wiki.gnome.org/Projects/OSTree -The intent is for that wiki page content to be migrated into Markdown -in this git repository. - Contributing ------------ From 8702ec7b3eb9988109ba9ae3a0e801edd9e81885 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 27 Jan 2016 11:06:30 -0500 Subject: [PATCH 21/54] build: Remove --disable-static-deltas option I'm confident now we can declare them stable. --- configure.ac | 11 +---------- src/libostree/ostree-repo-pull.c | 2 -- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/configure.ac b/configure.ac index c36134b9..03cc746a 100644 --- a/configure.ac +++ b/configure.ac @@ -235,15 +235,6 @@ AS_IF([test "x$found_introspection" = xyes], [ ], [have_gjs=no]) AM_CONDITIONAL(BUILDOPT_GJS, test x$have_gjs = xyes) -AC_ARG_ENABLE(static_deltas, - AS_HELP_STRING([--enable-static-deltas], - [Enable static delta code (default: yes)]),, - [enable_static_deltas=yes]) -AS_IF([test x$enable_static_deltas = xyes], [ - AC_DEFINE([BUILDOPT_STATIC_DELTAS], 1, [Define if static deltas are enabled]) -]) -AM_CONDITIONAL(BUILDOPT_STATIC_DELTAS, test x$enable_static_deltas = xyes) - AC_CONFIG_FILES([ Makefile apidoc/Makefile @@ -261,7 +252,7 @@ echo " libsoup TLS client certs: $have_libsoup_client_certs SELinux: $with_selinux libarchive (parse tar files directly): $with_libarchive - static deltas: $enable_static_deltas + static deltas: yes (always enabled now) man pages (xsltproc): $enable_man api docs (gtk-doc): $enable_gtk_doc gjs-based tests: $have_gjs diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index a0a1fc9c..6e6f0cd2 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2208,14 +2208,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, &from_revision, error)) goto out; -#ifdef BUILDOPT_STATIC_DELTAS if (!disable_static_deltas && (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0)) { if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision, &delta_superblock, cancellable, error)) goto out; } -#endif if (!delta_superblock) { From 56fc249d08d784ec481b71f2b13e2dbe509b0d9a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 6 Feb 2016 13:56:19 +0100 Subject: [PATCH 22/54] lib: Create an internal static delta parsing/opening function We had code to deal with opening/checksumming/decompressing static deltas in a few places. I'd like to teach `ostree static-delta show` how to display more information, and this will allow it to just use `_ostree_static_delta_part_open()` too. --- src/libostree/ostree-repo-pull.c | 123 ++++----- src/libostree/ostree-repo-static-delta-core.c | 244 +++++++++++++++--- .../ostree-repo-static-delta-private.h | 33 +-- .../ostree-repo-static-delta-processing.c | 141 +--------- 4 files changed, 284 insertions(+), 257 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 6e6f0cd2..750c68f5 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -935,8 +935,7 @@ static_deltapart_fetch_on_complete (GObject *object, g_autoptr(GVariant) metadata = NULL; g_autofree char *temp_path = NULL; g_autoptr(GInputStream) in = NULL; - g_autofree char *actual_checksum = NULL; - g_autofree guint8 *csum = NULL; + g_autoptr(GVariant) part = NULL; GError *local_error = NULL; GError **error = &local_error; gs_fd_close int fd = -1; @@ -950,54 +949,33 @@ static_deltapart_fetch_on_complete (GObject *object, fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC); if (fd == -1) { - gs_set_error_from_errno (error, errno); + glnx_set_error_from_errno (error); goto out; } + + /* From here on, if we fail to apply the delta, we'll re-fetch it */ + if (unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0) < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + in = g_unix_input_stream_new (fd, FALSE); - /* TODO - consider making async */ - if (!ot_gio_checksum_stream (in, &csum, pull_data->cancellable, error)) + /* TODO - make async */ + if (!_ostree_static_delta_part_open (in, NULL, 0, fetch_data->expected_checksum, + &part, pull_data->cancellable, error)) goto out; - actual_checksum = ostree_checksum_from_bytes (csum); - - if (strcmp (actual_checksum, fetch_data->expected_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted static delta part; checksum expected='%s' actual='%s'", - fetch_data->expected_checksum, actual_checksum); - goto out; - } - - /* Might as well close the fd here */ - (void) g_input_stream_close (in, NULL, NULL); - - { - GMappedFile *mfile = NULL; - g_autoptr(GBytes) delta_data = NULL; - - mfile = g_mapped_file_new_from_fd (fd, FALSE, error); - if (!mfile) - goto out; - delta_data = g_mapped_file_get_bytes (mfile); - g_mapped_file_unref (mfile); - - /* Unlink now while we're holding an open fd, so that on success - * or error, the file will be gone. This is particularly - * important if say we hit e.g. ENOSPC. - */ - (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0); - - _ostree_static_delta_part_execute_async (pull_data->repo, - fetch_data->objects, - delta_data, - /* Trust checksums if summary was gpg signed */ - pull_data->gpg_verify_summary && pull_data->summary_data_sig, - pull_data->cancellable, - on_static_delta_written, - fetch_data); - pull_data->n_outstanding_deltapart_write_requests++; - } + _ostree_static_delta_part_execute_async (pull_data->repo, + fetch_data->objects, + part, + /* Trust checksums if summary was gpg signed */ + pull_data->gpg_verify_summary && pull_data->summary_data_sig, + pull_data->cancellable, + on_static_delta_written, + fetch_data); + pull_data->n_outstanding_deltapart_write_requests++; out: g_assert (pull_data->n_outstanding_deltapart_fetches > 0); @@ -1604,10 +1582,10 @@ process_one_static_delta (OtPullData *pull_data, FetchStaticDeltaData *fetch_data; g_autoptr(GVariant) csum_v = NULL; g_autoptr(GVariant) objects = NULL; - g_autoptr(GVariant) part_data = NULL; - g_autoptr(GBytes) delta_data = NULL; + g_autoptr(GBytes) inline_part_bytes = NULL; guint64 size, usize; guint32 version; + const gboolean trusted = pull_data->gpg_verify_summary && pull_data->summary_data_sig; header = g_variant_get_child_value (headers, i); g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects); @@ -1623,31 +1601,6 @@ process_one_static_delta (OtPullData *pull_data, if (!csum) goto out; - deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i); - - part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)")); - if (part_data) - { - g_autofree char *actual_checksum = NULL; - g_autofree char *expected_checksum = ostree_checksum_from_bytes_v (csum_v); - - delta_data = g_variant_get_data_as_bytes (part_data); - - /* For inline parts we are relying on per-commit GPG, so this isn't strictly necessary for security. - * See https://github.com/GNOME/ostree/pull/139 - */ - actual_checksum = g_compute_checksum_for_bytes (G_CHECKSUM_SHA256, delta_data); - if (strcmp (actual_checksum, expected_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted static delta part; checksum expected='%s' actual='%s'", - expected_checksum, actual_checksum); - goto out; - } - } - - pull_data->total_deltapart_size += size; - if (!_ostree_repo_static_delta_part_have_all_objects (pull_data->repo, objects, &have_all, @@ -1663,18 +1616,38 @@ process_one_static_delta (OtPullData *pull_data, continue; } + deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i); + + { g_autoptr(GVariant) part_datav = + g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)")); + + if (part_datav) + inline_part_bytes = g_variant_get_data_as_bytes (part_datav); + } + + pull_data->total_deltapart_size += size; + fetch_data = g_new0 (FetchStaticDeltaData, 1); fetch_data->pull_data = pull_data; fetch_data->objects = g_variant_ref (objects); fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v); - if (delta_data != NULL) + if (inline_part_bytes != NULL) { + g_autoptr(GInputStream) memin = g_memory_input_stream_new_from_bytes (inline_part_bytes); + g_autoptr(GVariant) inline_delta_part = NULL; + + /* For inline parts we are relying on per-commit GPG, so don't bother checksumming. */ + if (!_ostree_static_delta_part_open (memin, inline_part_bytes, + OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM, + NULL, &inline_delta_part, + cancellable, error)) + goto out; + _ostree_static_delta_part_execute_async (pull_data->repo, fetch_data->objects, - delta_data, - /* Trust checksums if summary was gpg signed */ - pull_data->gpg_verify_summary && pull_data->summary_data_sig, + inline_delta_part, + trusted, pull_data->cancellable, on_static_delta_written, fetch_data); diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index bc1b460c..e9d5c15e 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -20,9 +20,14 @@ #include "config.h" +#include +#include +#include #include "ostree-core-private.h" #include "ostree-repo-private.h" +#include "ostree-lzma-decompressor.h" #include "ostree-cmdprivate.h" +#include "ostree-checksum-input-stream.h" #include "ostree-repo-static-delta-private.h" #include "otutil.h" @@ -226,31 +231,45 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, { gboolean ret = FALSE; guint i, n; - g_autoptr(GFile) meta_file = NULL; - g_autoptr(GFile) dir = NULL; + const char *dir_or_file_path = NULL; + glnx_fd_close int meta_fd = -1; + glnx_fd_close int dfd = -1; g_autoptr(GVariant) meta = NULL; g_autoptr(GVariant) headers = NULL; g_autoptr(GVariant) metadata = NULL; g_autoptr(GVariant) fallback = NULL; g_autofree char *to_checksum = NULL; g_autofree char *from_checksum = NULL; - GFileType file_type; + dir_or_file_path = gs_file_get_path_cached (dir_or_file); - file_type = g_file_query_file_type (dir_or_file, 0, cancellable); - if (file_type == G_FILE_TYPE_DIRECTORY) + /* First, try opening it as a directory */ + dfd = glnx_opendirat_with_errno (AT_FDCWD, dir_or_file_path, TRUE); + if (dfd < 0) { - dir = g_object_ref (dir_or_file); - meta_file = g_file_get_child (dir, "superblock"); - } - else - { - meta_file = g_object_ref (dir_or_file); - dir = g_file_get_parent (meta_file); + if (errno != ENOTDIR) + { + glnx_set_error_from_errno (error); + goto out; + } + else + { + g_autofree char *dir = dirname (g_strdup (dir_or_file_path)); + + if (!glnx_opendirat (AT_FDCWD, dir, TRUE, &dfd, error)) + goto out; + } } - if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), - FALSE, &meta, error)) + meta_fd = openat (dfd, "superblock", O_RDONLY | O_CLOEXEC); + if (meta_fd < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + if (!ot_util_variant_map_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), + FALSE, &meta, error)) goto out; /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */ @@ -330,14 +349,18 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, guint64 size; guint64 usize; const guchar *csum; + char checksum[65]; gboolean have_all; + g_autoptr(GInputStream) part_in = NULL; g_autoptr(GBytes) delta_data = NULL; - g_autoptr(GVariant) part_data = NULL; + g_autoptr(GVariant) inline_part_data = NULL; g_autoptr(GVariant) header = NULL; g_autoptr(GVariant) csum_v = NULL; g_autoptr(GVariant) objects = NULL; - g_autoptr(GBytes) bytes = NULL; + g_autoptr(GVariant) part = NULL; g_autofree char *deltapart_path = NULL; + OstreeStaticDeltaOpenFlags delta_open_flags = + skip_validation ? OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM : 0; header = g_variant_get_child_value (headers, i); g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects); @@ -362,41 +385,56 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, csum = ostree_checksum_bytes_peek_validate (csum_v, error); if (!csum) goto out; + ostree_checksum_inplace_from_bytes (csum, checksum); deltapart_path = _ostree_get_relative_static_delta_part_path (from_checksum, to_checksum, i); - part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)")); - if (part_data) + inline_part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)")); + if (inline_part_data) { - bytes = g_variant_get_data_as_bytes (part_data); + g_autoptr(GBytes) inline_part_bytes = g_variant_get_data_as_bytes (inline_part_data); + part_in = g_memory_input_stream_new_from_bytes (inline_part_bytes); + + /* For inline parts, we don't checksum, because it's + * included with the metadata, so we're not trying to + * protect against MITM or such. Non-security related + * checksums should be done at the underlying storage layer. + */ + delta_open_flags |= OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM; + + if (!_ostree_static_delta_part_open (part_in, inline_part_bytes, + delta_open_flags, + NULL, + &part, + cancellable, error)) + goto out; } else { - g_autoptr(GFile) part_path = ot_gfile_resolve_path_printf (dir, "%u", i); - GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error); - if (!mfile) - goto out; + g_autofree char *relpath = g_strdup_printf ("%u", i); /* TODO avoid malloc here */ + glnx_fd_close int part_fd = openat (dfd, relpath, O_RDONLY | O_CLOEXEC); + if (part_fd < 0) + { + glnx_set_error_from_errno (error); + g_prefix_error (error, "Opening deltapart '%s': ", deltapart_path); + goto out; + } - bytes = g_mapped_file_get_bytes (mfile); - g_mapped_file_unref (mfile); - } + part_in = g_unix_input_stream_new (part_fd, FALSE); - if (!skip_validation) - { - g_autoptr(GInputStream) in = g_memory_input_stream_new_from_bytes (bytes); - - g_autofree char *expected_checksum = ostree_checksum_from_bytes (csum); - if (!_ostree_static_delta_part_validate (self, in, i, - expected_checksum, - cancellable, error)) + if (!_ostree_static_delta_part_open (part_in, NULL, + delta_open_flags, + checksum, + &part, + cancellable, error)) goto out; } - if (!_ostree_static_delta_part_execute (self, objects, bytes, skip_validation, + if (!_ostree_static_delta_part_execute (self, objects, part, skip_validation, cancellable, error)) { - g_prefix_error (error, "executing delta part %i: ", i); + g_prefix_error (error, "Executing delta part %i: ", i); goto out; } } @@ -406,6 +444,140 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, return ret; } +gboolean +_ostree_static_delta_part_open (GInputStream *part_in, + GBytes *inline_part_bytes, + OstreeStaticDeltaOpenFlags flags, + const char *expected_checksum, + GVariant **out_part, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + const gboolean trusted = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED) > 0; + const gboolean skip_checksum = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM) > 0; + gsize bytes_read; + guint8 comptype; + g_autoptr(GChecksum) checksum = NULL; + g_autoptr(GInputStream) checksum_in = NULL; + g_autoptr(GVariant) ret_part = NULL; + GInputStream *source_in; + + /* We either take a fd or a GBytes reference */ + g_return_val_if_fail (G_IS_FILE_DESCRIPTOR_BASED (part_in) || inline_part_bytes != NULL, FALSE); + g_return_val_if_fail (skip_checksum || expected_checksum != NULL, FALSE); + + if (!skip_checksum) + { + checksum = g_checksum_new (G_CHECKSUM_SHA256); + checksum_in = (GInputStream*)ostree_checksum_input_stream_new (part_in, checksum); + source_in = checksum_in; + } + else + { + source_in = part_in; + } + + { guint8 buf[1]; + /* First byte is compression type */ + if (!g_input_stream_read_all (source_in, buf, sizeof(buf), &bytes_read, + cancellable, error)) + { + g_prefix_error (error, "Reading initial compression flag byte: "); + goto out; + } + comptype = buf[0]; + } + + switch (comptype) + { + case 0: + if (!inline_part_bytes) + { + int part_fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)part_in); + + /* No compression, no checksums - a fast path */ + if (!ot_util_variant_map_fd (part_fd, 1, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), + trusted, &ret_part, error)) + goto out; + } + else + { + g_autoptr(GBytes) content_bytes = g_bytes_new_from_bytes (inline_part_bytes, 1, + g_bytes_get_size (inline_part_bytes) - 1); + ret_part = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), + content_bytes, trusted); + } + + if (!skip_checksum) + g_checksum_update (checksum, g_variant_get_data (ret_part), + g_variant_get_size (ret_part)); + + break; + case 'x': + { + g_autofree char *tmppath = g_strdup ("/var/tmp/ostree-delta-XXXXXX"); + g_autoptr(GConverter) decomp = (GConverter*) _ostree_lzma_decompressor_new (); + g_autoptr(GInputStream) convin = g_converter_input_stream_new (source_in, decomp); + g_autoptr(GOutputStream) unpacked_out = NULL; + glnx_fd_close int unpacked_fd = -1; + gssize n_bytes_written; + + unpacked_fd = g_mkstemp_full (tmppath, O_RDWR | O_CLOEXEC, 0640); + if (unpacked_fd < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + /* Now make it autocleanup on process exit - in the future, we + * should consider caching unpacked deltas as well. + */ + if (unlink (tmppath) < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + unpacked_out = g_unix_output_stream_new (unpacked_fd, FALSE); + + n_bytes_written = g_output_stream_splice (unpacked_out, convin, + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | + G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, + cancellable, error); + if (n_bytes_written < 0) + goto out; + + if (!ot_util_variant_map_fd (unpacked_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), + trusted, &ret_part, error)) + goto out; + } + break; + default: + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid compression type '%u'", comptype); + goto out; + } + + if (checksum) + { + const char *actual_checksum = g_checksum_get_string (checksum); + g_assert (expected_checksum != NULL); + if (strcmp (actual_checksum, expected_checksum) != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Checksum mismatch in static delta part; expected=%s actual=%s", + expected_checksum, actual_checksum); + goto out; + } + } + + ret = TRUE; + *out_part = g_steal_pointer (&ret_part); + out: + return ret; +} + gboolean _ostree_repo_static_delta_dump (OstreeRepo *self, const char *delta_id, diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index b0e82ade..55777fcd 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -103,35 +103,36 @@ G_BEGIN_DECLS */ #define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}tayay" OSTREE_COMMIT_GVARIANT_STRING "aya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")" +typedef enum { + OSTREE_STATIC_DELTA_OPEN_FLAGS_NONE = 0, + OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM = (1 << 0), + OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED = (1 << 1) +} OstreeStaticDeltaOpenFlags; + +gboolean +_ostree_static_delta_part_open (GInputStream *part_in, + GBytes *inline_part_bytes, + OstreeStaticDeltaOpenFlags flags, + const char *expected_checksum, + GVariant **out_part, + GCancellable *cancellable, + GError **error); + gboolean _ostree_static_delta_dump (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); -gboolean _ostree_static_delta_part_validate (OstreeRepo *repo, - GInputStream *in, - guint part_offset, - const char *expected_checksum, - GCancellable *cancellable, - GError **error); - gboolean _ostree_static_delta_part_execute (OstreeRepo *repo, GVariant *header, - GBytes *partdata, + GVariant *part_payload, gboolean trusted, GCancellable *cancellable, GError **error); -gboolean _ostree_static_delta_part_execute_raw (OstreeRepo *repo, - GVariant *header, - GVariant *part, - gboolean trusted, - GCancellable *cancellable, - GError **error); - void _ostree_static_delta_part_execute_async (OstreeRepo *repo, GVariant *header, - GBytes *partdata, + GVariant *part_payload, gboolean trusted, GCancellable *cancellable, GAsyncReadyCallback callback, diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index 10e533c6..74423e7c 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -150,42 +150,12 @@ open_output_target (StaticDeltaExecutionState *state, } gboolean -_ostree_static_delta_part_validate (OstreeRepo *repo, - GInputStream *in, - guint part_offset, - const char *expected_checksum, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - g_autofree guchar *actual_checksum_bytes = NULL; - g_autofree char *actual_checksum = NULL; - - if (!ot_gio_checksum_stream (in, &actual_checksum_bytes, - cancellable, error)) - goto out; - - actual_checksum = ostree_checksum_from_bytes (actual_checksum_bytes); - if (strcmp (actual_checksum, expected_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Checksum mismatch in static delta part %u; expected=%s actual=%s", - part_offset, expected_checksum, actual_checksum); - goto out; - } - - ret = TRUE; - out: - return ret; -} - -gboolean -_ostree_static_delta_part_execute_raw (OstreeRepo *repo, - GVariant *objects, - GVariant *part, - gboolean trusted, - GCancellable *cancellable, - GError **error) +_ostree_static_delta_part_execute (OstreeRepo *repo, + GVariant *objects, + GVariant *part, + gboolean trusted, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; guint8 *checksums_data; @@ -280,99 +250,10 @@ _ostree_static_delta_part_execute_raw (OstreeRepo *repo, return ret; } -static gboolean -decompress_all (GConverter *converter, - GBytes *data, - GBytes **out_uncompressed, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - g_autoptr(GMemoryInputStream) memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes (data); - g_autoptr(GMemoryOutputStream) memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free); - g_autoptr(GInputStream) convin = g_converter_input_stream_new ((GInputStream*)memin, converter); - - { - gssize n_bytes_written = g_output_stream_splice ((GOutputStream*)memout, convin, - G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | - G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, - cancellable, error); - if (n_bytes_written < 0) - goto out; - } - - ret = TRUE; - *out_uncompressed = g_memory_output_stream_steal_as_bytes (memout); - out: - return ret; -} - -gboolean -_ostree_static_delta_part_execute (OstreeRepo *repo, - GVariant *header, - GBytes *part_bytes, - gboolean trusted, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gsize partlen; - const guint8*partdata; - g_autoptr(GBytes) part_payload_bytes = NULL; - g_autoptr(GBytes) payload_data = NULL; - g_autoptr(GVariant) payload = NULL; - guint8 comptype; - - partdata = g_bytes_get_data (part_bytes, &partlen); - - if (partlen < 1) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted 0 length delta part"); - goto out; - } - - /* First byte is compression type */ - comptype = partdata[0]; - /* Then the rest may be compressed or uncompressed */ - part_payload_bytes = g_bytes_new_from_bytes (part_bytes, 1, partlen - 1); - switch (comptype) - { - case 0: - /* No compression */ - payload_data = g_bytes_ref (part_payload_bytes); - break; - case 'x': - { - g_autoptr(GConverter) decomp = - (GConverter*) _ostree_lzma_decompressor_new (); - - if (!decompress_all (decomp, part_payload_bytes, &payload_data, - cancellable, error)) - goto out; - } - break; - default: - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid compression type '%u'", comptype); - goto out; - } - - payload = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), - payload_data, FALSE); - if (!_ostree_static_delta_part_execute_raw (repo, header, payload, trusted, - cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; -} - typedef struct { OstreeRepo *repo; GVariant *header; - GBytes *partdata; + GVariant *part; GCancellable *cancellable; GSimpleAsyncResult *result; gboolean trusted; @@ -385,7 +266,7 @@ static_delta_part_execute_async_data_free (gpointer user_data) g_clear_object (&data->repo); g_variant_unref (data->header); - g_bytes_unref (data->partdata); + g_variant_unref (data->part); g_clear_object (&data->cancellable); g_free (data); } @@ -401,7 +282,7 @@ static_delta_part_execute_thread (GSimpleAsyncResult *res, data = g_simple_async_result_get_op_res_gpointer (res); if (!_ostree_static_delta_part_execute (data->repo, data->header, - data->partdata, + data->part, data->trusted, cancellable, &error)) g_simple_async_result_take_error (res, error); @@ -410,7 +291,7 @@ static_delta_part_execute_thread (GSimpleAsyncResult *res, void _ostree_static_delta_part_execute_async (OstreeRepo *repo, GVariant *header, - GBytes *partdata, + GVariant *part, gboolean trusted, GCancellable *cancellable, GAsyncReadyCallback callback, @@ -421,7 +302,7 @@ _ostree_static_delta_part_execute_async (OstreeRepo *repo, asyncdata = g_new0 (StaticDeltaPartExecuteAsyncData, 1); asyncdata->repo = g_object_ref (repo); asyncdata->header = g_variant_ref (header); - asyncdata->partdata = g_bytes_ref (partdata); + asyncdata->part = g_variant_ref (part); asyncdata->trusted = trusted; asyncdata->cancellable = cancellable ? g_object_ref (cancellable) : NULL; From 0481389afd1c2f51c9c137fc27deec54bfb51fc5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 6 Feb 2016 17:04:53 +0100 Subject: [PATCH 23/54] lib: Expand `ostree static-delta show` to show part stats Now we display stats on the individual parts, such as the blob size and the number of each type of opcode. Most interesting to me is things like how many bsdiff opcodes there are vs new objects, etc. --- src/libostree/ostree-repo-static-delta-core.c | 96 +++++++++++++++++-- .../ostree-repo-static-delta-private.h | 27 ++++-- .../ostree-repo-static-delta-processing.c | 84 +++++++++++++++- 3 files changed, 186 insertions(+), 21 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index e9d5c15e..6369f34d 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -432,6 +432,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, } if (!_ostree_static_delta_part_execute (self, objects, part, skip_validation, + FALSE, NULL, cancellable, error)) { g_prefix_error (error, "Executing delta part %i: ", i); @@ -578,6 +579,89 @@ _ostree_static_delta_part_open (GInputStream *part_in, return ret; } +/* + * Displaying static delta parts + */ + +static gboolean +show_one_part (OstreeRepo *self, + const char *from, + const char *to, + GVariant *meta_entries, + guint i, + guint64 *total_size_ref, + guint64 *total_usize_ref, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint32 version; + guint64 size, usize; + g_autoptr(GVariant) objects = NULL; + g_autoptr(GInputStream) part_in = NULL; + g_autoptr(GVariant) part = NULL; + g_autofree char *part_path = _ostree_get_relative_static_delta_part_path (from, to, i); + gint part_fd = -1; + + g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects); + *total_size_ref += size; + *total_usize_ref += usize; + g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n", + i, (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN), size, usize); + + part_fd = openat (self->repo_dir_fd, part_path, O_RDONLY | O_CLOEXEC); + if (part_fd < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + part_in = g_unix_input_stream_new (part_fd, FALSE); + + if (!_ostree_static_delta_part_open (part_in, NULL, + OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM, + NULL, + &part, + cancellable, error)) + goto out; + + { g_autoptr(GVariant) modes = NULL; + g_autoptr(GVariant) xattrs = NULL; + g_autoptr(GVariant) blob = NULL; + g_autoptr(GVariant) ops = NULL; + OstreeDeltaExecuteStats stats = { { 0, }, }; + + g_variant_get (part, "(@a(uuu)@aa(ayay)@ay@ay)", + &modes, &xattrs, &blob, &ops); + + g_print ("PartPayload%u: nmodes=%" G_GUINT64_FORMAT + " nxattrs=%" G_GUINT64_FORMAT + " blobsize=%" G_GUINT64_FORMAT + " opsize=%" G_GUINT64_FORMAT + "\n", + i, + g_variant_n_children (modes), + g_variant_n_children (xattrs), + g_variant_n_children (blob), + g_variant_n_children (ops)); + + if (!_ostree_static_delta_part_execute (self, objects, + part, TRUE, TRUE, + &stats, cancellable, error)) + goto out; + + { const guint *n_ops = stats.n_ops_executed; + g_print ("PartPayloadOps%u: openspliceclose=%u open=%u write=%u setread=%u " + "unsetread=%u close=%u bspatch=%u\n", + i, n_ops[0], n_ops[1], n_ops[2], n_ops[3], n_ops[4], n_ops[5], n_ops[6]); + } + } + + ret = TRUE; + out: + return ret; +} + gboolean _ostree_repo_static_delta_dump (OstreeRepo *self, const char *delta_id, @@ -641,14 +725,10 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, for (i = 0; i < n_parts; i++) { - guint32 version; - guint64 size, usize; - g_autoptr(GVariant) objects = NULL; - g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects); - total_size += size; - total_usize += usize; - g_print ("Part%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n", - i, (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN), size, usize); + if (!show_one_part (self, from, to, meta_entries, i, + &total_size, &total_usize, + cancellable, error)) + goto out; } } diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index 55777fcd..2da000d6 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -109,6 +109,17 @@ typedef enum { OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED = (1 << 1) } OstreeStaticDeltaOpenFlags; +typedef enum { + OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE = 'S', + OSTREE_STATIC_DELTA_OP_OPEN = 'o', + OSTREE_STATIC_DELTA_OP_WRITE = 'w', + OSTREE_STATIC_DELTA_OP_SET_READ_SOURCE = 'r', + OSTREE_STATIC_DELTA_OP_UNSET_READ_SOURCE = 'R', + OSTREE_STATIC_DELTA_OP_CLOSE = 'c', + OSTREE_STATIC_DELTA_OP_BSPATCH = 'B' +} OstreeStaticDeltaOpCode; +#define OSTREE_STATIC_DELTA_N_OPS 7 + gboolean _ostree_static_delta_part_open (GInputStream *part_in, GBytes *inline_part_bytes, @@ -123,10 +134,16 @@ gboolean _ostree_static_delta_dump (OstreeRepo *repo, GCancellable *cancellable, GError **error); +typedef struct { + guint n_ops_executed[OSTREE_STATIC_DELTA_N_OPS]; +} OstreeDeltaExecuteStats; + gboolean _ostree_static_delta_part_execute (OstreeRepo *repo, GVariant *header, GVariant *part_payload, gboolean trusted, + gboolean stats_only, + OstreeDeltaExecuteStats *stats, GCancellable *cancellable, GError **error); @@ -142,16 +159,6 @@ gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo, GAsyncResult *result, GError **error); -typedef enum { - OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE = 'S', - OSTREE_STATIC_DELTA_OP_OPEN = 'o', - OSTREE_STATIC_DELTA_OP_WRITE = 'w', - OSTREE_STATIC_DELTA_OP_SET_READ_SOURCE = 'r', - OSTREE_STATIC_DELTA_OP_UNSET_READ_SOURCE = 'R', - OSTREE_STATIC_DELTA_OP_CLOSE = 'c', - OSTREE_STATIC_DELTA_OP_BSPATCH = 'B' -} OstreeStaticDeltaOpCode; - gboolean _ostree_static_delta_parse_checksum_array (GVariant *array, guint8 **out_checksums_array, diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index 74423e7c..b44314b6 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -40,6 +40,7 @@ G_STATIC_ASSERT (sizeof (guint) >= sizeof (guint32)); typedef struct { gboolean trusted; + gboolean stats_only; OstreeRepo *repo; guint checksum_index; const guint8 *checksums; @@ -149,11 +150,37 @@ open_output_target (StaticDeltaExecutionState *state, return ret; } +static guint +delta_opcode_index (OstreeStaticDeltaOpCode op) +{ + switch (op) + { + case OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE: + return 0; + case OSTREE_STATIC_DELTA_OP_OPEN: + return 1; + case OSTREE_STATIC_DELTA_OP_WRITE: + return 2; + case OSTREE_STATIC_DELTA_OP_SET_READ_SOURCE: + return 3; + case OSTREE_STATIC_DELTA_OP_UNSET_READ_SOURCE: + return 4; + case OSTREE_STATIC_DELTA_OP_CLOSE: + return 5; + case OSTREE_STATIC_DELTA_OP_BSPATCH: + return 6; + default: + g_assert_not_reached (); + } +} + gboolean _ostree_static_delta_part_execute (OstreeRepo *repo, GVariant *objects, GVariant *part, gboolean trusted, + gboolean stats_only, + OstreeDeltaExecuteStats *stats, GCancellable *cancellable, GError **error) { @@ -171,6 +198,7 @@ _ostree_static_delta_part_execute (OstreeRepo *repo, state->repo = repo; state->async_error = error; state->trusted = trusted; + state->stats_only = stats_only; if (!_ostree_static_delta_parse_checksum_array (objects, &checksums_data, @@ -240,6 +268,8 @@ _ostree_static_delta_part_execute (OstreeRepo *repo, } n_executed++; + if (stats) + stats->n_ops_executed[delta_opcode_index(opcode)]++; } if (state->caught_error) @@ -284,6 +314,7 @@ static_delta_part_execute_thread (GSimpleAsyncResult *res, data->header, data->part, data->trusted, + FALSE, NULL, cancellable, &error)) g_simple_async_result_take_error (res, error); } @@ -419,6 +450,12 @@ dispatch_bspatch (OstreeRepo *repo, if (!read_varuint64 (state, &length, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + if (!state->have_obj) { input_mfile = g_mapped_file_new_from_fd (state->read_source_fd, FALSE, error); @@ -477,6 +514,12 @@ dispatch_open_splice_and_close (OstreeRepo *repo, goto out; if (!validate_ofs (state, offset, length, error)) goto out; + + if (state->stats_only) + { + ret = TRUE; + goto out; + } metadata = g_variant_new_from_data (ostree_metadata_variant_type (state->output_objtype), state->payload_data + offset, length, TRUE, NULL, NULL); @@ -520,6 +563,12 @@ dispatch_open_splice_and_close (OstreeRepo *repo, if (!validate_ofs (state, content_offset, state->content_size, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + /* Fast path for regular files to bare repositories */ if (S_ISREG (state->mode) && (repo->mode == OSTREE_REPO_MODE_BARE || @@ -611,6 +660,8 @@ dispatch_open_splice_and_close (OstreeRepo *repo, ret = TRUE; out: + if (state->stats_only) + (void) dispatch_close (repo, state, cancellable, NULL); if (!ret) g_prefix_error (error, "opcode open-splice-and-close: "); return ret; @@ -626,8 +677,11 @@ dispatch_open (OstreeRepo *repo, g_assert (state->output_target == NULL); /* FIXME - lift this restriction */ - g_assert (repo->mode == OSTREE_REPO_MODE_BARE || - repo->mode == OSTREE_REPO_MODE_BARE_USER); + if (!state->stats_only) + { + g_assert (repo->mode == OSTREE_REPO_MODE_BARE || + repo->mode == OSTREE_REPO_MODE_BARE_USER); + } if (!open_output_target (state, cancellable, error)) goto out; @@ -638,6 +692,12 @@ dispatch_open (OstreeRepo *repo, if (!read_varuint64 (state, &state->content_size, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + if (state->trusted) { if (!_ostree_repo_open_trusted_content_bare (repo, state->checksum, @@ -682,6 +742,12 @@ dispatch_write (OstreeRepo *repo, if (!read_varuint64 (state, &content_offset, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + if (!state->have_obj) { if (state->read_source_fd != -1) @@ -762,6 +828,12 @@ dispatch_set_read_source (OstreeRepo *repo, if (!validate_ofs (state, source_offset, 32, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + g_free (state->read_source_object); state->read_source_object = ostree_checksum_from_bytes (state->payload_data + source_offset); @@ -784,6 +856,12 @@ dispatch_unset_read_source (OstreeRepo *repo, { gboolean ret = FALSE; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + if (state->read_source_fd) { (void) close (state->read_source_fd); @@ -793,7 +871,7 @@ dispatch_unset_read_source (OstreeRepo *repo, g_clear_pointer (&state->read_source_object, g_free); ret = TRUE; - /* out: */ + out: if (!ret) g_prefix_error (error, "opcode unset-read-source: "); return ret; From 5adafd767406820cce260c567a1b936610e8d67a Mon Sep 17 00:00:00 2001 From: Matthew Barnes Date: Tue, 9 Feb 2016 00:58:17 +0000 Subject: [PATCH 24/54] fetcher: Fix hung GTlsInteraction The GTlsInteraction instance must be created in the session thread so it uses the correct GMainContext. --- src/libostree/ostree-fetcher.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index 665286c1..b9223217 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -277,7 +277,12 @@ static void session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure, gpointer data) { - GTlsInteraction *interaction = data; + GTlsCertificate *cert = data; + glnx_unref_object OstreeTlsCertInteraction *interaction = NULL; + + /* The GTlsInteraction instance must be created in the + * session thread so it uses the correct GMainContext. */ + interaction = _ostree_tls_cert_interaction_new (cert); g_object_set (thread_closure->session, SOUP_SESSION_TLS_INTERACTION, @@ -645,7 +650,7 @@ _ostree_fetcher_set_client_cert (OstreeFetcher *self, #ifdef HAVE_LIBSOUP_CLIENT_CERTS session_thread_idle_add (self->thread_closure, session_thread_set_tls_interaction_cb, - _ostree_tls_cert_interaction_new (cert), + g_object_ref (cert), (GDestroyNotify) g_object_unref); #else g_warning ("This version of OSTree is compiled without client side certificate support"); From e9ccdd2d007801ef25cc7283188942d791889c27 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 10 Feb 2016 12:42:54 +0100 Subject: [PATCH 25/54] Import rofiles-fuse While it's not strictly tied to OSTree, let's move https://github.com/cgwalters/rofiles-fuse in here because: - It's *very* useful in concert with OSTree - It's tiny - We can reuse OSTree's test, documentation, etc. infrastructure One thing to consider also is that at some point we could experiment with writing a FUSE filesystem for OSTree. This could internalize a better equivalent of `--link-checkout-speedup`, but on the other hand, the cost of walking filesystem trees for these types of operations is really quite small. But if we did decide to do more FUSE things in OSTree, this is a step towards that too. --- Makefile-man.am | 4 + Makefile-tests.am | 5 + Makefile.am | 1 + configure.ac | 13 + man/rofiles-fuse.xml | 104 ++++++ src/rofiles-fuse/Makefile-inc.am | 23 ++ src/rofiles-fuse/README.md | 48 +++ src/rofiles-fuse/main.c | 598 +++++++++++++++++++++++++++++++ tests/test-rofiles-fuse.sh | 74 ++++ 9 files changed, 870 insertions(+) create mode 100644 man/rofiles-fuse.xml create mode 100644 src/rofiles-fuse/Makefile-inc.am create mode 100644 src/rofiles-fuse/README.md create mode 100644 src/rofiles-fuse/main.c create mode 100755 tests/test-rofiles-fuse.sh diff --git a/Makefile-man.am b/Makefile-man.am index f70a577c..a6090bf4 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -21,6 +21,10 @@ if ENABLE_MAN man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 +if BUILDOPT_FUSE +man1_files += rofiles-fuse.1 +endif + man5_files = ostree.repo.5 ostree.repo-config.5 man1_MANS = $(addprefix man/,$(man1_files)) diff --git a/Makefile-tests.am b/Makefile-tests.am index 9f3feef6..b0466840 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -61,6 +61,11 @@ testfiles = test-basic \ test-auto-summary \ test-prune \ $(NULL) + +if BUILDOPT_FUSE +testfiles += test-rofiles-fuse +endif + insttest_SCRIPTS = $(addprefix tests/,$(testfiles:=.sh)) # This one uses corrupt-repo-ref.js diff --git a/Makefile.am b/Makefile.am index da904da2..e1c942d7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -68,6 +68,7 @@ include Makefile-otutil.am include Makefile-libostree.am include Makefile-ostree.am include Makefile-switchroot.am +include src/rofiles-fuse/Makefile-inc.am include Makefile-tests.am include Makefile-boot.am include Makefile-man.am diff --git a/configure.ac b/configure.ac index 03cc746a..6a34c0d4 100644 --- a/configure.ac +++ b/configure.ac @@ -117,6 +117,8 @@ AS_IF([ test x$have_gpgme = xno ], [ OSTREE_FEATURES="$OSTREE_FEATURES +gpgme" LIBARCHIVE_DEPENDENCY="libarchive >= 2.8.0" +# What's in RHEL7.2. +FUSE_DEPENDENCY="fuse >= 2.9.2" # check for gtk-doc m4_ifdef([GTK_DOC_CHECK], [ @@ -194,6 +196,16 @@ AS_IF([ test x$with_selinux != xno ], [ if test x$with_selinux != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +selinux"; fi AM_CONDITIONAL(USE_SELINUX, test $with_selinux != no) +# Enabled by default because I think people should use it. +AC_ARG_ENABLE(rofiles-fuse, + [AS_HELP_STRING([--enable-rofiles-fuse], + [generate rofiles-fuse helper [default=yes]])],, + enable_rofiles_fuse=yes) +AS_IF([ test $enable_rofiles_fuse != xno ], [ + PKG_CHECK_MODULES(BUILDOPT_FUSE, $FUSE_DEPENDENCY) +], [enable_rofiles_fuse=no]) +AM_CONDITIONAL(BUILDOPT_FUSE, test x$enable_rofiles_fuse = xyes) + AC_ARG_WITH(dracut, AS_HELP_STRING([--with-dracut], [Install dracut module (default: no)]),, @@ -248,6 +260,7 @@ echo " introspection: $found_introspection + rofiles-fuse: $enable_rofiles_fuse libsoup (retrieve remote HTTP repositories): $with_soup libsoup TLS client certs: $have_libsoup_client_certs SELinux: $with_selinux diff --git a/man/rofiles-fuse.xml b/man/rofiles-fuse.xml new file mode 100644 index 00000000..3d282c00 --- /dev/null +++ b/man/rofiles-fuse.xml @@ -0,0 +1,104 @@ + + + + + + + + + rofiles-fuse + rofiles-fuse + + + + Developer + Colin + Walters + walters@verbum.org + + + + + + rofiles-fuse + 1 + + + + rofiles-fuse + Use FUSE to create a view where directories are writable, files are immutable + + + + + rofiles-fuse SRCDIR MNTPOINT + + + + + Description + + + Creating a checkout from an OSTree repository by default + uses hard links, which means an in-place mutation to any + file corrupts the repository and all checkouts. This can be + problematic if one wishes to run arbitrary programs against + such a checkout. For example, RPM %post + scripts or equivalent. + + + + In the case where one wants to create a tree commit derived + from other content, using rofiles-fuse in + concert with ostree commit + --link-checkout-speedup (or the underlying API) + can ensure that only new files are checksummed. + + + + + + Example: Update an OSTree commit + +# Initialize a checkout and mount +$ ostree --repo=repo checkout somebranch branch-checkout +$ mkdir mnt +$ rofiles-fuse branch-checkout mnt + +# Now, arbitrary changes to mnt/ are reflected in branch-checkout +$ echo somenewcontent > mnt/anewfile +$ mkdir mnt/anewdir +$ rm mnt/someoriginalcontent -rf + +# Commit and cleanup +$ fusermount -u mnt +$ ostree --repo=repo commit --link-checkout-speedup -b somebranch -s 'Commit new content' --tree=dir=branch-checkout +$ rm mnt branch-checkout -rf + + + + + See Also + + ostree1 + + + diff --git a/src/rofiles-fuse/Makefile-inc.am b/src/rofiles-fuse/Makefile-inc.am new file mode 100644 index 00000000..5510a2bd --- /dev/null +++ b/src/rofiles-fuse/Makefile-inc.am @@ -0,0 +1,23 @@ +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +bin_PROGRAMS += rofiles-fuse + +rofiles_fuse_SOURCES = src/rofiles-fuse/main.c + +rofiles_fuse_CFLAGS = $(AM_CFLAGS) -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 $(BUILDOPT_FUSE_CFLAGS) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -I$(srcdir)/libglnx $(NULL) +rofiles_fuse_LDADD = libglnx.la $(BUILDOPT_FUSE_LIBS) $(OT_INTERNAL_GIO_UNIX_LIBS) diff --git a/src/rofiles-fuse/README.md b/src/rofiles-fuse/README.md new file mode 100644 index 00000000..1f18afcc --- /dev/null +++ b/src/rofiles-fuse/README.md @@ -0,0 +1,48 @@ +rofiles-fuse +============ + +Create a mountpoint that represents an underlying directory hierarchy, +but where non-directory inodes cannot have content or xattrs changed. +Files can still be unlinked, and new ones created. + +This filesystem is designed for OSTree and other systems that create +"hardlink farms", i.e. filesystem trees deduplicated via hardlinks. + +Normally with hard links, if you change one, you change them all. + +There are two approaches to dealing with that: + - Copy on write: implemented by BTRFS, overlayfs, and http://linux-vserver.org/util-vserver:Vhashify + - Make them read-only: what this FUSE mount does + +Usage +===== + +Let's say that you have immutable data in `/srv/backups/20150410`, and +you want to update it with a new version, storing the result in +`/srv/backups/20150411`. Further assume that all software operating +on the directory does the "create tempfile and `rename()`" dance +rather than in-place edits. + + $ mkdir -p /srv/backups/mnt # Ensure we have a mount point + $ cp -al /srv/backups/20150410 /srv/backups/20150411 + $ rofiles-fuse /srv/backups/20150411 /srv/backups/mnt + +Now we have a "rofiles" mount at `/srv/backups/mnt`. If we try this: + + $ echo new doc content > /srv/backups/mnt/document + bash: /srv/backups/mnt/document: Read-only file system + +It failed because the `>` redirection operator will try to truncate +the existing file. If instead we create `document.tmp` and then +rename it atomically over the old one, it will work: + + $ echo new doc content > /srv/backups/mnt/document.tmp + $ mv /srv/backups/mnt/document.tmp /srv/backups/mnt/document + +Let's unmount: + + $ fusermount -u /srv/backups/mnt + +Now we have two directories `/srv/backups/20150410` +`/srv/backups/20150411` which share all file storage except for the +new document. diff --git a/src/rofiles-fuse/main.c b/src/rofiles-fuse/main.c new file mode 100644 index 00000000..004ad3d2 --- /dev/null +++ b/src/rofiles-fuse/main.c @@ -0,0 +1,598 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2015,2016 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#define FUSE_USE_VERSION 26 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "libglnx.h" + +// Global to store our read-write path +static int basefd = -1; +static GHashTable *created_devino_hash = NULL; + +static inline const char * +ENSURE_RELPATH (const char *path) +{ + return path + strspn (path, "/"); +} + +typedef struct { + dev_t dev; + ino_t ino; +} DevIno; + +static guint +devino_hash (gconstpointer a) +{ + DevIno *a_i = (gpointer)a; + return (guint) (a_i->dev + a_i->ino); +} + +static int +devino_equal (gconstpointer a, + gconstpointer b) +{ + DevIno *a_i = (gpointer)a; + DevIno *b_i = (gpointer)b; + return a_i->dev == b_i->dev + && a_i->ino == b_i->ino; +} + +static gboolean +devino_set_contains (dev_t dev, ino_t ino) +{ + DevIno devino = { dev, ino }; + return g_hash_table_contains (created_devino_hash, &devino); +} + +static gboolean +devino_set_insert (dev_t dev, ino_t ino) +{ + DevIno *devino = g_new (DevIno, 1); + devino->dev = dev; + devino->ino = ino; + return g_hash_table_add (created_devino_hash, devino); +} + +static gboolean +devino_set_remove (dev_t dev, ino_t ino) +{ + DevIno devino = { dev, ino }; + return g_hash_table_remove (created_devino_hash, &devino); +} + +static int +callback_getattr (const char *path, struct stat *st_data) +{ + path = ENSURE_RELPATH (path); + if (!*path) + { + if (fstat (basefd, st_data) == -1) + return -errno; + } + else + { + if (fstatat (basefd, path, st_data, AT_SYMLINK_NOFOLLOW) == -1) + return -errno; + } + return 0; +} + +static int +callback_readlink (const char *path, char *buf, size_t size) +{ + int r; + + path = ENSURE_RELPATH (path); + + /* Note FUSE wants the string to be always nul-terminated, even if + * truncated. + */ + r = readlinkat (basefd, path, buf, size - 1); + if (r == -1) + return -errno; + buf[r] = '\0'; + return 0; +} + +static int +callback_readdir (const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi) +{ + DIR *dp; + struct dirent *de; + int dfd; + + path = ENSURE_RELPATH (path); + + if (!*path) + { + dfd = fcntl (basefd, F_DUPFD_CLOEXEC, 3); + lseek (dfd, 0, SEEK_SET); + } + else + { + dfd = openat (basefd, path, O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | O_NOCTTY); + if (dfd == -1) + return -errno; + } + + /* Transfers ownership of fd */ + dp = fdopendir (dfd); + if (dp == NULL) + return -errno; + + while ((de = readdir (dp)) != NULL) + { + struct stat st; + memset (&st, 0, sizeof (st)); + st.st_ino = de->d_ino; + st.st_mode = de->d_type << 12; + if (filler (buf, de->d_name, &st, 0)) + break; + } + + (void) closedir (dp); + return 0; +} + +static int +callback_mknod (const char *path, mode_t mode, dev_t rdev) +{ + return -EROFS; +} + +static int +callback_mkdir (const char *path, mode_t mode) +{ + path = ENSURE_RELPATH (path); + if (mkdirat (basefd, path, mode) == -1) + return -errno; + return 0; +} + +static int +callback_unlink (const char *path) +{ + struct stat stbuf; + path = ENSURE_RELPATH (path); + + if (fstatat (basefd, path, &stbuf, AT_SYMLINK_NOFOLLOW) == 0) + { + if (!S_ISDIR (stbuf.st_mode)) + devino_set_remove (stbuf.st_dev, stbuf.st_ino); + } + + if (unlinkat (basefd, path, 0) == -1) + return -errno; + return 0; +} + +static int +callback_rmdir (const char *path) +{ + path = ENSURE_RELPATH (path); + if (unlinkat (basefd, path, AT_REMOVEDIR) == -1) + return -errno; + return 0; +} + +static int +callback_symlink (const char *from, const char *to) +{ + struct stat stbuf; + + to = ENSURE_RELPATH (to); + + if (symlinkat (from, basefd, to) == -1) + return -errno; + + if (fstatat (basefd, to, &stbuf, AT_SYMLINK_NOFOLLOW) == -1) + { + fprintf (stderr, "Failed to find newly created symlink '%s': %s\n", + to, g_strerror (errno)); + exit (1); + } + return 0; +} + +static int +callback_rename (const char *from, const char *to) +{ + from = ENSURE_RELPATH (from); + to = ENSURE_RELPATH (to); + if (renameat (basefd, from, basefd, to) == -1) + return -errno; + return 0; +} + +static int +callback_link (const char *from, const char *to) +{ + from = ENSURE_RELPATH (from); + to = ENSURE_RELPATH (to); + if (linkat (basefd, from, basefd, to, 0) == -1) + return -errno; + return 0; +} + +static int +can_write (const char *path) +{ + struct stat stbuf; + if (fstatat (basefd, path, &stbuf, 0) == -1) + { + if (errno == ENOENT) + return 0; + else + return -errno; + } + if (devino_set_contains (stbuf.st_dev, stbuf.st_ino)) + return -EROFS; + return 0; +} + +#define VERIFY_WRITE(path) do { \ + int r = can_write (path); \ + if (r != 0) \ + return r; \ + } while (0) + +static int +callback_chmod (const char *path, mode_t mode) +{ + path = ENSURE_RELPATH (path); + VERIFY_WRITE(path); + if (fchmodat (basefd, path, mode, 0) != 0) + return -errno; + return 0; +} + +static int +callback_chown (const char *path, uid_t uid, gid_t gid) +{ + path = ENSURE_RELPATH (path); + VERIFY_WRITE(path); + if (fchownat (basefd, path, uid, gid, 0) != 0) + return -errno; + return 0; +} + +static int +callback_truncate (const char *path, off_t size) +{ + glnx_fd_close int fd = -1; + + path = ENSURE_RELPATH (path); + VERIFY_WRITE(path); + + fd = openat (basefd, path, O_RDWR | O_CREAT); + if (fd == -1) + return -errno; + + if (ftruncate (fd, size) == -1) + return -errno; + + return 0; +} + +static int +callback_utime (const char *path, struct utimbuf *buf) +{ + struct timespec ts[2]; + + path = ENSURE_RELPATH (path); + + ts[0].tv_sec = buf->actime; + ts[0].tv_nsec = UTIME_OMIT; + ts[1].tv_sec = buf->modtime; + ts[1].tv_nsec = UTIME_OMIT; + + if (utimensat (basefd, path, ts, AT_SYMLINK_NOFOLLOW) == -1) + return -errno; + + return 0; +} + +static int +do_open (const char *path, mode_t mode, struct fuse_file_info *finfo) +{ + const int flags = finfo->flags & O_ACCMODE; + int fd; + struct stat stbuf; + + /* Support read only opens */ + G_STATIC_ASSERT (O_RDONLY == 0); + + path = ENSURE_RELPATH (path); + + if (flags == 0) + fd = openat (basefd, path, flags); + else + { + const int forced_excl_flags = flags | O_CREAT | O_EXCL; + /* Do an exclusive open, don't allow writable fds for existing + files */ + fd = openat (basefd, path, forced_excl_flags, mode); + /* If they didn't specify O_EXCL, give them EROFS if the file + * exists. + */ + if (fd == -1 && (flags & O_EXCL) == 0) + { + if (errno == EEXIST) + errno = EROFS; + } + else if (fd != -1) + { + if (fstat (fd, &stbuf) == -1) + return -errno; + devino_set_insert (stbuf.st_dev, stbuf.st_ino); + } + } + + if (fd == -1) + return -errno; + + finfo->fh = fd; + + return 0; +} + +static int +callback_open (const char *path, struct fuse_file_info *finfo) +{ + return do_open (path, 0, finfo); +} + +static int +callback_create(const char *path, mode_t mode, struct fuse_file_info *finfo) +{ + return do_open (path, mode, finfo); +} + +static int +callback_read (const char *path, char *buf, size_t size, off_t offset, + struct fuse_file_info *finfo) +{ + int r; + r = pread (finfo->fh, buf, size, offset); + if (r == -1) + return -errno; + return r; +} + +static int +callback_write (const char *path, const char *buf, size_t size, off_t offset, + struct fuse_file_info *finfo) +{ + int r; + r = pwrite (finfo->fh, buf, size, offset); + if (r == -1) + return -errno; + return r; +} + +static int +callback_statfs (const char *path, struct statvfs *st_buf) +{ + if (fstatvfs (basefd, st_buf) == -1) + return -errno; + return 0; +} + +static int +callback_release (const char *path, struct fuse_file_info *finfo) +{ + (void) close (finfo->fh); + return 0; +} + +static int +callback_fsync (const char *path, int crap, struct fuse_file_info *finfo) +{ + if (fsync (finfo->fh) == -1) + return -errno; + return 0; +} + +static int +callback_access (const char *path, int mode) +{ + path = ENSURE_RELPATH (path); + + /* Apparently at least GNU coreutils rm calls `faccessat(W_OK)` + * before trying to do an unlink. So...we'll just lie about + * writable access here. + */ + if (faccessat (basefd, path, mode, 0) == -1) + return -errno; + return 0; +} + +static int +callback_setxattr (const char *path, const char *name, const char *value, + size_t size, int flags) +{ + return -ENOTSUP; +} + +static int +callback_getxattr (const char *path, const char *name, char *value, + size_t size) +{ + return -ENOTSUP; +} + +/* + * List the supported extended attributes. + */ +static int +callback_listxattr (const char *path, char *list, size_t size) +{ + return -ENOTSUP; + +} + +/* + * Remove an extended attribute. + */ +static int +callback_removexattr (const char *path, const char *name) +{ + return -ENOTSUP; + +} + +struct fuse_operations callback_oper = { + .getattr = callback_getattr, + .readlink = callback_readlink, + .readdir = callback_readdir, + .mknod = callback_mknod, + .mkdir = callback_mkdir, + .symlink = callback_symlink, + .unlink = callback_unlink, + .rmdir = callback_rmdir, + .rename = callback_rename, + .link = callback_link, + .chmod = callback_chmod, + .chown = callback_chown, + .truncate = callback_truncate, + .utime = callback_utime, + .create = callback_create, + .open = callback_open, + .read = callback_read, + .write = callback_write, + .statfs = callback_statfs, + .release = callback_release, + .fsync = callback_fsync, + .access = callback_access, + + /* Extended attributes support for userland interaction */ + .setxattr = callback_setxattr, + .getxattr = callback_getxattr, + .listxattr = callback_listxattr, + .removexattr = callback_removexattr +}; + +enum +{ + KEY_HELP, + KEY_VERSION, +}; + +static void +usage (const char *progname) +{ + fprintf (stdout, + "usage: %s basepath mountpoint [options]\n" + "\n" + " Makes basepath visible at mountpoint such that files are read-only, directories are writable\n" + "\n" + "general options:\n" + " -o opt,[opt...] mount options\n" + " -h --help print help\n" + "\n", progname); +} + +static int +rofs_parse_opt (void *data, const char *arg, int key, + struct fuse_args *outargs) +{ + (void) data; + + switch (key) + { + case FUSE_OPT_KEY_NONOPT: + if (basefd == -1) + { + basefd = openat (AT_FDCWD, arg, O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | O_NOCTTY); + if (basefd == -1) + { + perror ("openat"); + exit (1); + } + return 0; + } + else + { + return 1; + } + case FUSE_OPT_KEY_OPT: + return 1; + case KEY_HELP: + usage (outargs->argv[0]); + exit (0); + default: + fprintf (stderr, "see `%s -h' for usage\n", outargs->argv[0]); + exit (1); + } + return 1; +} + +static struct fuse_opt rofs_opts[] = { + FUSE_OPT_KEY ("-h", KEY_HELP), + FUSE_OPT_KEY ("--help", KEY_HELP), + FUSE_OPT_KEY ("-V", KEY_VERSION), + FUSE_OPT_KEY ("--version", KEY_VERSION), + FUSE_OPT_END +}; + +int +main (int argc, char *argv[]) +{ + struct fuse_args args = FUSE_ARGS_INIT (argc, argv); + int res; + + res = fuse_opt_parse (&args, &basefd, rofs_opts, rofs_parse_opt); + if (res != 0) + { + fprintf (stderr, "Invalid arguments\n"); + fprintf (stderr, "see `%s -h' for usage\n", argv[0]); + exit (1); + } + if (basefd == -1) + { + fprintf (stderr, "Missing basepath\n"); + fprintf (stderr, "see `%s -h' for usage\n", argv[0]); + exit (1); + } + + created_devino_hash = g_hash_table_new_full (devino_hash, devino_equal, g_free, NULL); + + fuse_main (args.argc, args.argv, &callback_oper, NULL); + + return 0; +} diff --git a/tests/test-rofiles-fuse.sh b/tests/test-rofiles-fuse.sh new file mode 100755 index 00000000..24ee2648 --- /dev/null +++ b/tests/test-rofiles-fuse.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +echo "1..5" + +. $(dirname $0)/libtest.sh +setup_test_repository "bare-user" + +mkdir mnt + +$OSTREE checkout test2 checkout-test2 + +rofiles-fuse checkout-test2 mnt +cleanup_fuse() { + fusermount -u ${test_tmpdir}/mnt || true +} +trap cleanup_fuse EXIT +assert_file_has_content mnt/firstfile first +echo "ok mount" + +if cp /dev/null mnt/firstfile 2>err.txt; then + assert_not_reached "inplace mutation" +fi +assert_file_has_content err.txt "Read-only file system" +assert_file_has_content mnt/firstfile first +assert_file_has_content checkout-test2/firstfile first + +echo "ok failed inplace mutation" + +echo anewfile-for-fuse > mnt/anewfile-for-fuse +assert_file_has_content mnt/anewfile-for-fuse anewfile-for-fuse +assert_file_has_content checkout-test2/anewfile-for-fuse anewfile-for-fuse + +mkdir mnt/newfusedir +for i in $(seq 5); do + echo ${i}-morenewfuse-${i} > mnt/newfusedir/test-morenewfuse.${i} +done +assert_file_has_content checkout-test2/newfusedir/test-morenewfuse.3 3-morenewfuse-3 + +echo "ok new content" + +rm mnt/baz/cow +assert_not_has_file checkout-test2/baz/cow +rm mnt/baz/another -rf +assert_not_has_dir checkout-test2/baz/another + +echo "ok deletion" + +ostree --repo=repo commit -b test2 -s fromfuse --link-checkout-speedup --tree=dir=checkout-test2 + +echo "ok commit" + + + + + From 355f8438ef4fe2720d25e8241aa76797b7029522 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 28 Jan 2016 14:41:27 -0500 Subject: [PATCH 26/54] Add an `export` builtin, and API to write to libarchive At the moment I'm looking at using rpm-ostree to manage RPM inputs which can then be converted into Docker images. It's most convenient if we can stream directly out of libostree rather than doing a checkout + tar combination. There are also backup/debugging etc. reasons to implement `export` as well. --- Makefile-man.am | 2 +- Makefile-ostree.am | 1 + Makefile-tests.am | 1 + src/libostree/ostree-repo-libarchive.c | 249 +++++++++++++++++++++++++ src/libostree/ostree-repo.h | 24 +++ src/ostree/main.c | 1 + src/ostree/ot-builtin-diff.c | 9 +- src/ostree/ot-builtin-export.c | 148 +++++++++++++++ src/ostree/ot-builtins.h | 1 + tests/test-export.sh | 50 +++++ 10 files changed, 484 insertions(+), 2 deletions(-) create mode 100644 src/ostree/ot-builtin-export.c create mode 100755 tests/test-export.sh diff --git a/Makefile-man.am b/Makefile-man.am index a6090bf4..615bf0f0 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -19,7 +19,7 @@ if ENABLE_MAN -man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 +man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 if BUILDOPT_FUSE man1_files += rofiles-fuse.1 diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 76fb26f3..ab4485c6 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -28,6 +28,7 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-checksum.c \ src/ostree/ot-builtin-commit.c \ src/ostree/ot-builtin-diff.c \ + src/ostree/ot-builtin-export.c \ src/ostree/ot-builtin-fsck.c \ src/ostree/ot-builtin-gpg-sign.c \ src/ostree/ot-builtin-init.c \ diff --git a/Makefile-tests.am b/Makefile-tests.am index b0466840..292699ef 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -27,6 +27,7 @@ testfiles = test-basic \ test-remote-add \ test-remote-gpg-import \ test-commit-sign \ + test-export \ test-help \ test-libarchive \ test-pull-archive-z \ diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index e86e4026..3b1b0b83 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -24,6 +24,7 @@ #include "ostree-core-private.h" #include "ostree-repo-private.h" +#include "ostree-repo-file.h" #include "ostree-mutable-tree.h" #ifdef HAVE_LIBARCHIVE @@ -356,3 +357,251 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, return FALSE; #endif } + +#ifdef HAVE_LIBARCHIVE + +static gboolean +file_to_archive_entry_common (GFile *root, + OstreeRepoArchiveOptions *opts, + GFile *path, + GFileInfo *file_info, + struct archive_entry *entry, + GError **error) +{ + gboolean ret = FALSE; + g_autofree char *pathstr = g_file_get_relative_path (root, path); + g_autoptr(GVariant) xattrs = NULL; + time_t ts = (time_t) opts->timestamp_secs; + + if (pathstr && !pathstr[0]) + { + g_free (pathstr); + pathstr = g_strdup ("."); + } + + archive_entry_update_pathname_utf8 (entry, pathstr); + archive_entry_set_ctime (entry, ts, 0); + archive_entry_set_mtime (entry, ts, 0); + archive_entry_set_atime (entry, ts, 0); + archive_entry_set_uid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::uid")); + archive_entry_set_gid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::gid")); + archive_entry_set_mode (entry, g_file_info_get_attribute_uint32 (file_info, "unix::mode")); + + if (!ostree_repo_file_get_xattrs ((OstreeRepoFile*)path, &xattrs, NULL, error)) + goto out; + + if (!opts->disable_xattrs) + { + int i, n; + + n = g_variant_n_children (xattrs); + for (i = 0; i < n; i++) + { + const guint8* name; + g_autoptr(GVariant) value = NULL; + const guint8* value_data; + gsize value_len; + + g_variant_get_child (xattrs, i, "(^&ay@ay)", &name, &value); + value_data = g_variant_get_fixed_array (value, &value_len, 1); + + archive_entry_xattr_add_entry (entry, (char*)name, + (char*) value_data, value_len); + } + } + + ret = TRUE; + out: + return ret; +} + +static gboolean +write_header_free_entry (struct archive *a, + struct archive_entry **entryp, + GError **error) +{ + struct archive_entry *entry = *entryp; + gboolean ret = FALSE; + + if (archive_write_header (a, entry) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + ret = TRUE; + out: + archive_entry_free (entry); + *entryp = NULL; + return ret; +} + +static gboolean +write_directory_to_libarchive_recurse (OstreeRepo *self, + OstreeRepoArchiveOptions *opts, + GFile *root, + GFile *dir, + struct archive *a, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + g_autoptr(GFileInfo) dir_info = NULL; + g_autoptr(GFileEnumerator) dir_enum = NULL; + struct archive_entry *entry = NULL; + + dir_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!dir_info) + goto out; + + entry = archive_entry_new2 (a); + if (!file_to_archive_entry_common (root, opts, dir, dir_info, entry, error)) + goto out; + if (!write_header_free_entry (a, &entry, error)) + goto out; + + dir_enum = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!dir_enum) + goto out; + + while (TRUE) + { + GFileInfo *file_info; + GFile *path; + + if (!gs_file_enumerator_iterate (dir_enum, &file_info, &path, + cancellable, error)) + goto out; + if (file_info == NULL) + break; + + /* First, handle directories recursively */ + if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) + { + if (!write_directory_to_libarchive_recurse (self, opts, root, path, a, + cancellable, error)) + goto out; + + /* Go to the next entry */ + continue; + } + + /* Past here, should be a regular file or a symlink */ + + entry = archive_entry_new2 (a); + if (!file_to_archive_entry_common (root, opts, path, file_info, entry, error)) + goto out; + + switch (g_file_info_get_file_type (file_info)) + { + case G_FILE_TYPE_SYMBOLIC_LINK: + { + archive_entry_set_symlink (entry, g_file_info_get_symlink_target (file_info)); + if (!write_header_free_entry (a, &entry, error)) + goto out; + } + break; + case G_FILE_TYPE_REGULAR: + { + guint8 buf[8192]; + g_autoptr(GInputStream) file_in = NULL; + g_autoptr(GFileInfo) file_info = NULL; + const char *checksum; + + checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)path); + + if (!ostree_repo_load_file (self, checksum, &file_in, &file_info, NULL, + cancellable, error)) + goto out; + + archive_entry_set_size (entry, g_file_info_get_size (file_info)); + + if (archive_write_header (a, entry) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + while (TRUE) + { + gssize bytes_read = g_input_stream_read (file_in, buf, sizeof (buf), + cancellable, error); + if (bytes_read < 0) + goto out; + if (bytes_read == 0) + break; + + { ssize_t r = archive_write_data (a, buf, bytes_read); + if (r != bytes_read) + { + propagate_libarchive_error (error, a); + g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", bytes_read, r); + goto out; + } + } + } + + if (archive_write_finish_entry (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + archive_entry_free (entry); + entry = NULL; + } + break; + default: + g_assert_not_reached (); + } + } + + ret = TRUE; + out: + if (entry) + archive_entry_free (entry); + return ret; +} +#endif + +/** + * ostree_repo_write_tree_to_archive: + * @self: An #OstreeRepo + * @opts: Options controlling conversion + * @root: An #OstreeRepoFile for the base directory + * @archive: A `struct archive`, but specified as void to avoid a dependency on the libarchive headers + * @cancellable: Cancellable + * @error: Error + * + * Import an archive file @archive into the repository, and write its + * file structure to @mtree. + */ +gboolean +ostree_repo_write_tree_to_archive (OstreeRepo *self, + OstreeRepoArchiveOptions *opts, + OstreeRepoFile *root, + void *archive, + GCancellable *cancellable, + GError **error) +{ +#ifdef HAVE_LIBARCHIVE + gboolean ret = FALSE; + struct archive *a = archive; + + if (!write_directory_to_libarchive_recurse (self, opts, (GFile*)root, (GFile*)root, + a, cancellable, error)) + goto out; + + ret = TRUE; + out: + return ret; +#else + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree is not compiled with libarchive support"); + return FALSE; +#endif +} diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index d4d0f418..64e8a028 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -441,6 +441,23 @@ gboolean ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error); +/** + * OstreeRepoWriteArchiveOptions: + * + * An extensible options structure controlling archive creation. Ensure that + * you have entirely zeroed the structure, then set just the desired + * options. This is used by ostree_repo_write_tree_to_archive(). + */ +typedef struct { + guint disable_xattrs : 1; + guint reserved : 31; + + guint64 timestamp_secs; + + guint unused_uint[8]; + gpointer unused_ptrs[8]; +} OstreeRepoArchiveOptions; + gboolean ostree_repo_write_archive_to_mtree (OstreeRepo *self, GFile *archive, OstreeMutableTree *mtree, @@ -449,6 +466,13 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo * GCancellable *cancellable, GError **error); +gboolean ostree_repo_write_tree_to_archive (OstreeRepo *self, + OstreeRepoArchiveOptions *opts, + OstreeRepoFile *root, + void *archive, /* Really struct archive * */ + GCancellable *cancellable, + GError **error); + gboolean ostree_repo_write_mtree (OstreeRepo *self, OstreeMutableTree *mtree, GFile **out_file, diff --git a/src/ostree/main.c b/src/ostree/main.c index 99d7d916..eff3082d 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -40,6 +40,7 @@ static OstreeCommand commands[] = { { "commit", ostree_builtin_commit }, { "config", ostree_builtin_config }, { "diff", ostree_builtin_diff }, + { "export", ostree_builtin_export }, { "fsck", ostree_builtin_fsck }, { "gpg-sign", ostree_builtin_gpg_sign }, { "init", ostree_builtin_init }, diff --git a/src/ostree/ot-builtin-diff.c b/src/ostree/ot-builtin-diff.c index b5e0c5a8..a23ed83f 100644 --- a/src/ostree/ot-builtin-diff.c +++ b/src/ostree/ot-builtin-diff.c @@ -29,10 +29,12 @@ static gboolean opt_stats; static gboolean opt_fs_diff; +static gboolean opt_no_xattrs; static GOptionEntry options[] = { { "stats", 0, 0, G_OPTION_ARG_NONE, &opt_stats, "Print various statistics", NULL }, { "fs-diff", 0, 0, G_OPTION_ARG_NONE, &opt_fs_diff, "Print filesystem diff", NULL }, + { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, { NULL } }; @@ -162,6 +164,11 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_fs_diff) { + OstreeDiffFlags diff_flags = OSTREE_DIFF_FLAGS_NONE; + + if (opt_no_xattrs) + diff_flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS; + if (!parse_file_or_commit (repo, src, &srcf, cancellable, error)) goto out; if (!parse_file_or_commit (repo, target, &targetf, cancellable, error)) @@ -171,7 +178,7 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError ** removed = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); added = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE, srcf, targetf, modified, removed, added, cancellable, error)) + if (!ostree_diff_dirs (diff_flags, srcf, targetf, modified, removed, added, cancellable, error)) goto out; ostree_diff_print (srcf, targetf, modified, removed, added); diff --git a/src/ostree/ot-builtin-export.c b/src/ostree/ot-builtin-export.c new file mode 100644 index 00000000..2d350b8c --- /dev/null +++ b/src/ostree/ot-builtin-export.c @@ -0,0 +1,148 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2016 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "ot-main.h" +#include "ot-builtins.h" +#include "ostree.h" +#include "ostree-repo-file.h" +#include "otutil.h" + +#ifdef HAVE_LIBARCHIVE +#include +#include +#endif + +static char *opt_output_path; +static gboolean opt_no_xattrs; + +static GOptionEntry options[] = { + { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, + { "output", 'o', 0, G_OPTION_ARG_STRING, &opt_output_path, "Output to PATH ", "PATH" }, + { NULL } +}; + +#ifdef HAVE_LIBARCHIVE + +static void +propagate_libarchive_error (GError **error, + struct archive *a) +{ + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "%s", archive_error_string (a)); +} + +#endif + +gboolean +ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError **error) +{ + GOptionContext *context; + glnx_unref_object OstreeRepo *repo = NULL; + gboolean ret = FALSE; + const char *rev; + g_autoptr(GFile) root = NULL; + g_autofree char *commit = NULL; + g_autoptr(GVariant) commit_data = NULL; + struct archive *a; + OstreeRepoArchiveOptions opts = { 0, }; + + context = g_option_context_new ("COMMIT - Stream COMMIT to stdout in tar format"); + + if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) + goto out; + +#ifdef HAVE_LIBARCHIVE + + if (argc <= 1) + { + ot_util_usage_error (context, "A COMMIT argument is required", error); + goto out; + } + rev = argv[1]; + + a = archive_write_new (); + /* Yes, this is hardcoded for now. There is + * archive_write_set_format_filter_by_ext() but it's fairly magic. + * Many programs have support now for GNU tar, so should be a good + * default. I also don't want to lock us into everything libarchive + * supports. + */ + if (archive_write_set_format_gnutar (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + if (archive_write_add_filter_none (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + if (opt_output_path) + { + if (archive_write_open_filename (a, opt_output_path) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + } + else + { + if (archive_write_open_FILE (a, stdout) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + } + + if (opt_no_xattrs) + opts.disable_xattrs = TRUE; + + if (!ostree_repo_read_commit (repo, rev, &root, &commit, cancellable, error)) + goto out; + + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_data, error)) + goto out; + + opts.timestamp_secs = ostree_commit_get_timestamp (commit_data); + + if (!ostree_repo_write_tree_to_archive (repo, &opts, (OstreeRepoFile*)root, a, + cancellable, error)) + goto out; + + if (archive_write_close (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + +#else + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree is not compiled with libarchive support"); + goto out; +#endif + + ret = TRUE; + out: + if (context) + g_option_context_free (context); + return ret; +} diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index 95262ec4..1c862925 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -35,6 +35,7 @@ BUILTINPROTO(checkout); BUILTINPROTO(checksum); BUILTINPROTO(commit); BUILTINPROTO(diff); +BUILTINPROTO(export); BUILTINPROTO(gpg_sign); BUILTINPROTO(init); BUILTINPROTO(log); diff --git a/tests/test-export.sh b/tests/test-export.sh new file mode 100755 index 00000000..18f2f7ae --- /dev/null +++ b/tests/test-export.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_test_repository "archive-z2" + +echo '1..2' + +$OSTREE checkout test2 test2-co +$OSTREE commit --no-xattrs -b test2-noxattrs -s "test2 without xattrs" --tree=dir=test2-co +rm test2-co -rf + +cd ${test_tmpdir} +${OSTREE} 'export' test2-noxattrs -o test2.tar +mkdir t +(cd t && tar xf ../test2.tar) +ostree --repo=repo diff --no-xattrs test2-noxattrs ./t > diff.txt +assert_file_empty diff.txt +rm test2.tar diff.txt t -rf + +echo 'ok export gnutar diff (no xattrs)' + +cd ${test_tmpdir} +${OSTREE} 'export' test2 -o test2.tar +${OSTREE} commit -b test2-from-tar -s 'Import from tar' --tree=tar=test2.tar +ostree --repo=repo diff test2 test2-from-tar +assert_file_empty diff.txt +rm test2.tar diff.txt t -rf + +echo 'ok export import' + From 42c60effbe5037d674ce4b84979ccaf91edfad48 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 11 Feb 2016 13:28:03 -0500 Subject: [PATCH 27/54] pull: Support specifying exact commit to pull via branch@commit I don't know why we didn't do this a long time ago. This extends the pull API to allow grabbing a specific commit, and will set the branch to it. There's some support for this in the deploy engine, but there are a lot of reasons to support it for raw pulls (such as subset mirroring cases). In fact I'm thinking we should also have the override-version logic here too. NOTE: One thing I debated here is inventing a new syntax on the command line. Git doesn't seem to have this functionality (probably because it'd be rarely used). The '@' character at least doesn't conflict with anything. Anyways, I wanted this for some other test cases. Without this, writing tests that go between different commits is more awkward as one must generate the content in one repo, then pull downstream, then generate more content, then pull again. But now I can just keep track of commit IDs and do exactly what I want without synchronizing the tests. --- man/ostree-pull.xml | 27 +++++++++++++-- src/libostree/ostree-repo-pull.c | 57 +++++++++++++++++++++----------- src/libostree/ostree-repo.c | 1 + src/ostree/ot-builtin-pull.c | 38 +++++++++++++++++++-- tests/pull-test.sh | 15 +++++++++ 5 files changed, 114 insertions(+), 24 deletions(-) diff --git a/man/ostree-pull.xml b/man/ostree-pull.xml index d915bcd2..c419307e 100644 --- a/man/ostree-pull.xml +++ b/man/ostree-pull.xml @@ -111,14 +111,37 @@ Boston, MA 02111-1307, USA. Description - Downloads all content corresponding to the provided branch or commit from the given remote. + This command can retrieve just a specific commit, or go all + the way to performing a full mirror of the remote + repository. If no BRANCH is specified, + all branches are retrieved. + + + A special syntax in the @ character allows + specifying a specific commit to retrieve from a branch. This + + Example - $ ostree pull remote_name + $ ostree --repo=repo pull --depth=-1 --mirror remote_name + + Perform a complete mirror of the remote. (This is + likely most useful if your repository is also + archive-z2 mode) + + $ ostree --repo=repo pull remote_name exampleos/x86_64/standard + + Fetch the most recent commit to exampleos/x86_64/standard. + + $ ostree --repo=repo pull remote_name exampleos/x86_64/standard@98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4 + + Download the specific commit starting with + 98ea6e as if it was the latest commit for + exampleos/x86_64/standard. diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 750c68f5..88b468b0 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1762,6 +1762,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, OstreeRepoPullFlags flags = 0; const char *dir_to_pull = NULL; char **refs_to_fetch = NULL; + char **override_commit_ids = NULL; GSource *update_timeout = NULL; gboolean disable_static_deltas = FALSE; @@ -1776,9 +1777,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "override-remote-name", "s", &pull_data->remote_name); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas); + (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); + if (refs_to_fetch && override_commit_ids) + g_return_val_if_fail (g_strv_length (refs_to_fetch) == g_strv_length (override_commit_ids), FALSE); if (dir_to_pull) g_return_val_if_fail (dir_to_pull[0] == '/', FALSE); @@ -2069,8 +2073,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, } else if (refs_to_fetch != NULL) { - char **strviter; - for (strviter = refs_to_fetch; *strviter; strviter++) + char **strviter = refs_to_fetch; + char **commitid_strviter = override_commit_ids ? override_commit_ids : NULL; + + while (*strviter) { const char *branch = *strviter; @@ -2081,8 +2087,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, } else { - g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL); + char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; + g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), commitid); } + + strviter++; + if (commitid_strviter) + commitid_strviter++; } } else @@ -2109,28 +2120,36 @@ ostree_repo_pull_with_options (OstreeRepo *self, while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *branch = key; + const char *override_commitid = value; char *contents = NULL; - if (pull_data->summary) + /* Support specifying "" for an override commitid */ + if (override_commitid && *override_commitid) { - gsize commit_size = 0; - guint64 *malloced_size; - - if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error)) - goto out; - - malloced_size = g_new0 (guint64, 1); - *malloced_size = commit_size; - g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size); + g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), g_strdup (override_commitid)); } - else + else { - if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) - goto out; + if (pull_data->summary) + { + gsize commit_size = 0; + guint64 *malloced_size; + + if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error)) + goto out; + + malloced_size = g_new0 (guint64, 1); + *malloced_size = commit_size; + g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size); + } + else + { + if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) + goto out; + } + /* Transfer ownership of contents */ + g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents); } - - /* Transfer ownership of contents */ - g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents); } /* Create the state directory here - it's new with the commitpartial code, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 21362084..1e9673d1 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3784,6 +3784,7 @@ ostree_repo_pull_one_dir (OstreeRepo *self, * * flags (i): An instance of #OstreeRepoPullFlags * * refs: (as): Array of string refs * * depth: (i): How far in the history to traverse; default is 0, -1 means infinite + * * override-commit-ids: (as): Array of specific commit IDs to fetch for refs */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 6aae8205..63dbc768 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -34,7 +34,7 @@ static gboolean opt_disable_static_deltas; static char* opt_subpath; static int opt_depth = 0; - static GOptionEntry options[] = { +static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "disable-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_disable_static_deltas, "Do not use static deltas", NULL }, @@ -70,6 +70,7 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** OstreeRepoPullFlags pullflags = 0; GSConsole *console = NULL; g_autoptr(GPtrArray) refs_to_fetch = NULL; + g_autoptr(GPtrArray) override_commit_ids = NULL; glnx_unref_object OstreeAsyncProgress *progress = NULL; gulong signal_handler_id = 0; @@ -102,9 +103,36 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (argc > 2) { int i; - refs_to_fetch = g_ptr_array_new (); + refs_to_fetch = g_ptr_array_new_with_free_func (g_free); + for (i = 2; i < argc; i++) - g_ptr_array_add (refs_to_fetch, argv[i]); + { + const char *at = strrchr (argv[i], '@'); + + if (at) + { + guint j; + const char *override_commit_id = at + 1; + + if (!ostree_validate_checksum_string (override_commit_id, error)) + goto out; + + if (!override_commit_ids) + override_commit_ids = g_ptr_array_new_with_free_func (g_free); + + /* Backfill */ + for (j = 2; j < i; i++) + g_ptr_array_add (override_commit_ids, g_strdup ("")); + + g_ptr_array_add (override_commit_ids, g_strdup (override_commit_id)); + g_ptr_array_add (refs_to_fetch, g_strndup (argv[i], at - argv[i])); + } + else + { + g_ptr_array_add (refs_to_fetch, g_strdup (argv[i])); + } + } + g_ptr_array_add (refs_to_fetch, NULL); } } @@ -147,6 +175,10 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_variant_builder_add (&builder, "{s@v}", "disable-static-deltas", g_variant_new_variant (g_variant_new_boolean (opt_disable_static_deltas))); + if (override_commit_ids) + g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", + g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); + if (!ostree_repo_pull_with_options (repo, remote, g_variant_builder_end (&builder), progress, cancellable, error)) goto out; diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 6f0b651a..2233b905 100755 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -72,6 +72,21 @@ $OSTREE show --print-detached-metadata-key=SIGNATURE main > main-meta assert_file_has_content main-meta "HANCOCK" echo "ok pull detached metadata" +cd ${test_tmpdir} +mkdir parentpullrepo +${CMD_PREFIX} ostree --repo=parentpullrepo init --mode=archive-z2 +${CMD_PREFIX} ostree --repo=parentpullrepo remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo +parent_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^) +rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) +${CMD_PREFIX} ostree --repo=parentpullrepo pull origin main@${parent_rev} +${CMD_PREFIX} ostree --repo=parentpullrepo rev-parse origin:main > main.txt +assert_file_has_content main.txt ${parent_rev} +${CMD_PREFIX} ostree --repo=parentpullrepo fsck +${CMD_PREFIX} ostree --repo=parentpullrepo pull origin main +${CMD_PREFIX} ostree --repo=parentpullrepo rev-parse origin:main > main.txt +assert_file_has_content main.txt ${rev} +echo "ok pull specific commit" + cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main From f0b6b358cd012420c7011df1e1aa8a150b50e412 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 12 Feb 2016 12:41:15 -0500 Subject: [PATCH 28/54] manual-tests: New static-delta-generate-crosscheck.sh I wanted to test static deltas vs a repo with "real" content in it from Fedora Atomic. --- .../static-delta-generate-crosscheck.sh | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 manual-tests/static-delta-generate-crosscheck.sh diff --git a/manual-tests/static-delta-generate-crosscheck.sh b/manual-tests/static-delta-generate-crosscheck.sh new file mode 100755 index 00000000..4dbaab98 --- /dev/null +++ b/manual-tests/static-delta-generate-crosscheck.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Generate multiple variants of static deltas in a repo, then use them +# to upgrade in a test repository, verifying that fsck works. +# +# This test is manual so it's easy to run against arbitrary content. + +set -euo pipefail + +repo=$(pwd)/${1} +shift +branch=${1} +shift + +from=$(ostree --repo=${repo} rev-parse "${branch}"^) +to=$(ostree --repo=${repo} rev-parse ${branch}) + +tmpdir=$(mktemp -d /var/tmp/ostree-delta-check.XXXXXX) +cd ${tmpdir} +touch ${tmpdir}/.tmp +echo "Using tmpdir ${tmpdir}" + +cleanup_tmpdir() { + if test -f ${tmpdir}/.tmp; then + rm -rf ${tmpdir} + fi +} + +if test -z "${PRESERVE_TMP:-}"; then + trap cleanup_tmpdir EXIT +fi + +fatal() { + echo "$@" + exit 1 +} + +assert_streq() { + if ! test $1 = $2; then + fatal "assertion failed: $1 = $2" + fi +} + +validate_delta_options() { + mkdir testrepo + ostree --repo=testrepo init --mode=bare-user + ostree --repo=testrepo remote add --set=gpg-verify=false local file://${repo} + ostree --repo=${repo} static-delta generate $@ ${from} ${to} + ostree --repo=testrepo pull local ${branch}@${from} + assert_streq $(ostree --repo=testrepo rev-parse ${branch}) ${from} + ostree --repo=testrepo pull local ${branch} + assert_streq $(ostree --repo=testrepo rev-parse ${branch}) ${to} + ostree --repo=testrepo fsck + rm testrepo -rf +} + +set -x + +validate_delta_options +validate_delta_options --inline +validate_delta_options --disable-bsdiff + From c18193bd19f06802a77dc4e1550f7f2db5560468 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 15 Feb 2016 06:39:49 -0500 Subject: [PATCH 29/54] man/ostree-export.xml: Add to git Forgot to `git add`. --- man/ostree-export.xml | 70 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 man/ostree-export.xml diff --git a/man/ostree-export.xml b/man/ostree-export.xml new file mode 100644 index 00000000..39ad8e69 --- /dev/null +++ b/man/ostree-export.xml @@ -0,0 +1,70 @@ + + + + + + + + + ostree export + OSTree + + + + Developer + Colin + Walters + walters@verbum.org + + + + + + ostree export + 1 + + + + ostree-export + Generate a tar archive from an OSTree commit + + + + + ostree export OPTIONS BRANCH + + + + + Description + + + This command generates a GNU tar formatted archive from an + OSTree commit. This is useful for cases like backups, + converting OSTree commits into Docker images, and the like. + + + + + Example + $ ostree export exampleos/x86_64/standard | gzip > exampleos-standard.tar.gz + + From baaf7450da8a3870e8a42f0cdd4e0ea0ed5018d6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 14 Feb 2016 11:57:59 -0500 Subject: [PATCH 30/54] Support Docker-style whiteouts This is to enable importing Docker layers as ostree commits, then checking them out in a union. The prototype work for this is in: https://github.com/cgwalters/dlayer-ostree Though it will likely ultimately end up in: https://github.com/projectatomic/atomic --- src/libostree/ostree-repo-checkout.c | 47 ++++++++++++++++++++++++---- src/libostree/ostree-repo.h | 3 +- src/ostree/ot-builtin-checkout.c | 7 +++-- tests/basic-test.sh | 36 +++++++++++++++++++++ 4 files changed, 84 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 0d0e7ef4..0879f44b 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -32,6 +32,8 @@ #include "ostree-core-private.h" #include "ostree-repo-private.h" +#define WHITEOUT_PREFIX ".wh." + static gboolean checkout_object_for_uncompressed_cache (OstreeRepo *self, const char *loose_path, @@ -396,20 +398,46 @@ checkout_one_file_at (OstreeRepo *repo, const char *checksum; gboolean is_symlink; gboolean can_cache; - gboolean did_hardlink = FALSE; + gboolean need_copy = TRUE; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; g_autoptr(GInputStream) input = NULL; g_autoptr(GVariant) xattrs = NULL; + gboolean is_whiteout; is_symlink = g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK; checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)source); - /* Try to do a hardlink first, if it's a regular file. This also - * traverses all parent repos. + is_whiteout = !is_symlink && options->process_whiteouts && + g_str_has_prefix (destination_name, WHITEOUT_PREFIX); + + /* First, see if it's a Docker whiteout, + * https://github.com/docker/docker/blob/1a714e76a2cb9008cd19609059e9988ff1660b78/pkg/archive/whiteouts.go */ - if (!is_symlink) + if (is_whiteout) { + const char *name = destination_name + (sizeof (WHITEOUT_PREFIX) - 1); + + if (!name[0]) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid empty whiteout '%s'", name); + goto out; + } + + g_assert (name[0] != '/'); /* Sanity */ + + if (!glnx_shutil_rm_rf_at (destination_dfd, name, cancellable, error)) + goto out; + + need_copy = FALSE; + } + else if (!is_symlink) + { + gboolean did_hardlink = FALSE; + /* Try to do a hardlink first, if it's a regular file. This also + * traverses all parent repos. + */ OstreeRepo *current_repo = repo; while (current_repo) @@ -462,6 +490,8 @@ checkout_one_file_at (OstreeRepo *repo, } current_repo = current_repo->parent_repo; } + + need_copy = !did_hardlink; } can_cache = (options->enable_uncompressed_cache @@ -471,11 +501,14 @@ checkout_one_file_at (OstreeRepo *repo, * it now, stick it in the cache, and then hardlink to that. */ if (can_cache + && !is_whiteout && !is_symlink - && !did_hardlink + && need_copy && repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) { + gboolean did_hardlink; + if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL, cancellable, error)) goto out; @@ -526,10 +559,12 @@ checkout_one_file_at (OstreeRepo *repo, g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name); goto out; } + + need_copy = !did_hardlink; } /* Fall back to copy if we couldn't hardlink */ - if (!did_hardlink) + if (need_copy) { if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs, cancellable, error)) diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 64e8a028..98794b9c 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -557,7 +557,8 @@ typedef struct { guint enable_uncompressed_cache : 1; guint disable_fsync : 1; - guint reserved : 30; + guint process_whiteouts : 1; + guint reserved : 29; const char *subpath; diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index 9929a37b..810a8f72 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -36,6 +36,7 @@ static gboolean opt_allow_noent; static gboolean opt_disable_cache; static char *opt_subpath; static gboolean opt_union; +static gboolean opt_whiteouts; static gboolean opt_from_stdin; static char *opt_from_file; static gboolean opt_disable_fsync; @@ -61,6 +62,7 @@ static GOptionEntry options[] = { { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Do not update or use the internal repository uncompressed object cache", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, { "union", 0, 0, G_OPTION_ARG_NONE, &opt_union, "Keep existing directories, overwrite existing files", NULL }, + { "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL }, { "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL }, { "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL }, { "from-file", 0, 0, G_OPTION_ARG_STRING, &opt_from_file, "Process many checkouts from input file", "FILE" }, @@ -83,7 +85,7 @@ process_one_checkout (OstreeRepo *repo, * `ostree_repo_checkout_tree_at` until such time as we have a more * convenient infrastructure for testing C APIs with data. */ - if (opt_disable_cache) + if (opt_disable_cache || opt_whiteouts) { OstreeRepoCheckoutOptions options = { 0, }; @@ -91,10 +93,11 @@ process_one_checkout (OstreeRepo *repo, options.mode = OSTREE_REPO_CHECKOUT_MODE_USER; if (opt_union) options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; + if (opt_whiteouts) + options.process_whiteouts = TRUE; if (subpath) options.subpath = subpath; - if (!ostree_repo_checkout_tree_at (repo, &options, AT_FDCWD, destination, resolved_commit, diff --git a/tests/basic-test.sh b/tests/basic-test.sh index d39f32cf..c0487d64 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -387,6 +387,42 @@ assert_file_has_content test2-checkout/baz/cow moo assert_has_dir repo2/uncompressed-objects-cache echo "ok disable cache checkout" +# Whiteouts +cd ${test_tmpdir} +mkdir -p overlay/baz/ +touch overlay/baz/.wh.cow +touch overlay/.wh.deeper +touch overlay/anewfile +mkdir overlay/anewdir/ +touch overlay/anewdir/blah +$OSTREE --repo=repo commit -b overlay -s 'overlay' --tree=dir=overlay +rm overlay -rf + +for branch in test2 overlay; do + $OSTREE --repo=repo checkout --union --whiteouts ${branch} overlay-co +done +for f in .wh.deeper baz/cow baz/.wh.cow; do + assert_not_has_file overlay-co/${f} +done +assert_not_has_dir overlay-co/deeper +assert_has_file overlay-co/anewdir/blah +assert_has_file overlay-co/anewfile + +echo "ok whiteouts enabled" + +# Now double check whiteouts are not processed without --whiteouts +rm overlay-co -rf +for branch in test2 overlay; do + $OSTREE --repo=repo checkout --union ${branch} overlay-co +done +for f in .wh.deeper baz/cow baz/.wh.cow; do + assert_has_file overlay-co/${f} +done +assert_not_has_dir overlay-co/deeper +assert_has_file overlay-co/anewdir/blah +assert_has_file overlay-co/anewfile +echo "ok whiteouts disabled" + cd ${test_tmpdir} rm -rf test2-checkout mkdir -p test2-checkout From 2c2fd658ee141979c0310baf6d0b2807503323d5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 17 Feb 2016 10:37:08 -0500 Subject: [PATCH 31/54] packaging: Add a fuse subpackage For rofiles-fuse. Eventually what we should really do is split out the shared library from the binaries? A minimal system shouldn't need rofiles-fuse, it's mainly for doing layered packages and that sort of thing. --- packaging/ostree.spec.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packaging/ostree.spec.in b/packaging/ostree.spec.in index 8f9cacfd..2d6a2660 100644 --- a/packaging/ostree.spec.in +++ b/packaging/ostree.spec.in @@ -19,6 +19,7 @@ BuildRequires: pkgconfig(libgsystem) BuildRequires: pkgconfig(e2p) # Extras BuildRequires: pkgconfig(libarchive) +BuildRequires: pkgconfig(fuse) BuildRequires: pkgconfig(libselinux) BuildRequires: libcap-devel BuildRequires: gpgme-devel @@ -55,6 +56,14 @@ Requires: grub2 %description grub2 GRUB2 integration for OSTree +%package fuse +Summary: FUSE utilities for OSTree +Group: Development/Libraries +Requires: fuse + +%description fuse +%{summary} + %prep %setup -q -n ostree-%{version} @@ -108,3 +117,6 @@ rm -rf $RPM_BUILD_ROOT %files grub2 %{_sysconfdir}/grub.d/*ostree %{_libexecdir}/ostree/grub2* + +%files fuse +%{_bindir}/rofiles-fuse From d3eee15f8421326450b05c2dcac1aea121a92073 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 17 Feb 2016 11:01:21 -0500 Subject: [PATCH 32/54] rofiles-fuse: Fix truncate call to not use O_CREAT I was getting a compilation error with the GCC hardening flags which look for a missing mode with `O_CREAT`. The right fix here is to drop `O_CREAT`, as truncate() should throw `ENOENT` if the file doesn't exist. --- src/rofiles-fuse/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rofiles-fuse/main.c b/src/rofiles-fuse/main.c index 004ad3d2..4337f00f 100644 --- a/src/rofiles-fuse/main.c +++ b/src/rofiles-fuse/main.c @@ -299,7 +299,7 @@ callback_truncate (const char *path, off_t size) path = ENSURE_RELPATH (path); VERIFY_WRITE(path); - fd = openat (basefd, path, O_RDWR | O_CREAT); + fd = openat (basefd, path, O_WRONLY); if (fd == -1) return -errno; From 707c14aeebd738c668572f0362cfcfbd94ac4f49 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 18 Feb 2016 18:21:32 -0500 Subject: [PATCH 33/54] manual: Note that the bare-user mode exists --- docs/manual/repo.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/manual/repo.md b/docs/manual/repo.md index a3e64bd3..3b7a737f 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -49,14 +49,23 @@ header, for regular files, the content follows. # Repository types and locations -Also unlike git, an OSTree repository can be in one of two separate -modes: `bare` and `archive-z2`. A bare repository is one where -content files are just stored as regular files; it's designed to be -the source of a "hardlink farm", where each operating system checkout -is merely links into it. If you want to store files owned by -e.g. root in this mode, you must run OSTree as root. In contrast, the -`archive-z2` mode is designed for serving via plain HTTP. Like tar -files, it can be read/written by non-root users. +Also unlike git, an OSTree repository can be in one of three separate +modes: `bare`, `bare-user`, and `archive-z2`. A bare repository is +one where content files are just stored as regular files; it's +designed to be the source of a "hardlink farm", where each operating +system checkout is merely links into it. If you want to store files +owned by e.g. root in this mode, you must run OSTree as root. + +The `bare-user` is a later addition that is like `bare` in that files +are unpacked, but it can (and should generally) be created as +non-root. In this mode, extended metadata such as owner uid, gid, and +extended attributes are stored but not actually applied. +The `bare-user` mode is useful for build systems that run as non-root +but want to generate root-owned content, as well as non-root container +systems. + +In contrast, the `archive-z2` mode is designed for serving via plain +HTTP. Like tar files, it can be read/written by non-root users. On an OSTree-deployed system, the "system repository" is `/ostree/repo`. It can be read by any uid, but only written by root. From b08b1abccdb6f8614750b31f2e3dc572e7b7bc8a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 18 Feb 2016 11:25:57 -0500 Subject: [PATCH 34/54] Rename libarchive write API to "export", matching command line I was going to add new API for importing, and it was really confusing that what I think of now as import and export both had "write" in the name. It's just clearer to talk about the direction. At the same time, include `Export` in the options structure. This isn't an ABI break as the API isn't in a release. --- src/libostree/ostree-repo-libarchive.c | 18 +++++------ src/libostree/ostree-repo.h | 45 +++++++++++++------------- src/ostree/ot-builtin-export.c | 6 ++-- 3 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 3b1b0b83..19fae757 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -362,7 +362,7 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, static gboolean file_to_archive_entry_common (GFile *root, - OstreeRepoArchiveOptions *opts, + OstreeRepoExportArchiveOptions *opts, GFile *path, GFileInfo *file_info, struct archive_entry *entry, @@ -438,7 +438,7 @@ write_header_free_entry (struct archive *a, static gboolean write_directory_to_libarchive_recurse (OstreeRepo *self, - OstreeRepoArchiveOptions *opts, + OstreeRepoExportArchiveOptions *opts, GFile *root, GFile *dir, struct archive *a, @@ -569,7 +569,7 @@ write_directory_to_libarchive_recurse (OstreeRepo *self, #endif /** - * ostree_repo_write_tree_to_archive: + * ostree_repo_export_tree_to_archive: * @self: An #OstreeRepo * @opts: Options controlling conversion * @root: An #OstreeRepoFile for the base directory @@ -581,12 +581,12 @@ write_directory_to_libarchive_recurse (OstreeRepo *self, * file structure to @mtree. */ gboolean -ostree_repo_write_tree_to_archive (OstreeRepo *self, - OstreeRepoArchiveOptions *opts, - OstreeRepoFile *root, - void *archive, - GCancellable *cancellable, - GError **error) +ostree_repo_export_tree_to_archive (OstreeRepo *self, + OstreeRepoExportArchiveOptions *opts, + OstreeRepoFile *root, + void *archive, + GCancellable *cancellable, + GError **error) { #ifdef HAVE_LIBARCHIVE gboolean ret = FALSE; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 98794b9c..3b1040ab 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -441,22 +441,6 @@ gboolean ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error); -/** - * OstreeRepoWriteArchiveOptions: - * - * An extensible options structure controlling archive creation. Ensure that - * you have entirely zeroed the structure, then set just the desired - * options. This is used by ostree_repo_write_tree_to_archive(). - */ -typedef struct { - guint disable_xattrs : 1; - guint reserved : 31; - - guint64 timestamp_secs; - - guint unused_uint[8]; - gpointer unused_ptrs[8]; -} OstreeRepoArchiveOptions; gboolean ostree_repo_write_archive_to_mtree (OstreeRepo *self, GFile *archive, @@ -466,12 +450,29 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo * GCancellable *cancellable, GError **error); -gboolean ostree_repo_write_tree_to_archive (OstreeRepo *self, - OstreeRepoArchiveOptions *opts, - OstreeRepoFile *root, - void *archive, /* Really struct archive * */ - GCancellable *cancellable, - GError **error); +/** + * OstreeRepoExportArchiveOptions: + * + * An extensible options structure controlling archive creation. Ensure that + * you have entirely zeroed the structure, then set just the desired + * options. This is used by ostree_repo_export_tree_to_archive(). + */ +typedef struct { + guint disable_xattrs : 1; + guint reserved : 31; + + guint64 timestamp_secs; + + guint unused_uint[8]; + gpointer unused_ptrs[8]; +} OstreeRepoExportArchiveOptions; + +gboolean ostree_repo_export_tree_to_archive (OstreeRepo *self, + OstreeRepoExportArchiveOptions *opts, + OstreeRepoFile *root, + void *archive, /* Really struct archive * */ + GCancellable *cancellable, + GError **error); gboolean ostree_repo_write_mtree (OstreeRepo *self, OstreeMutableTree *mtree, diff --git a/src/ostree/ot-builtin-export.c b/src/ostree/ot-builtin-export.c index 2d350b8c..cccb50e7 100644 --- a/src/ostree/ot-builtin-export.c +++ b/src/ostree/ot-builtin-export.c @@ -63,7 +63,7 @@ ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError g_autofree char *commit = NULL; g_autoptr(GVariant) commit_data = NULL; struct archive *a; - OstreeRepoArchiveOptions opts = { 0, }; + OstreeRepoExportArchiveOptions opts = { 0, }; context = g_option_context_new ("COMMIT - Stream COMMIT to stdout in tar format"); @@ -124,8 +124,8 @@ ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError opts.timestamp_secs = ostree_commit_get_timestamp (commit_data); - if (!ostree_repo_write_tree_to_archive (repo, &opts, (OstreeRepoFile*)root, a, - cancellable, error)) + if (!ostree_repo_export_tree_to_archive (repo, &opts, (OstreeRepoFile*)root, a, + cancellable, error)) goto out; if (archive_write_close (a) != ARCHIVE_OK) From 3a555114bc22f599d72d95d71c2579fd2345cf6f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 18 Feb 2016 16:49:30 -0500 Subject: [PATCH 35/54] repo: Add ostree_repo_import_archive_to_mtree This is a more flexible version of the previous ostree_repo_write_archive_to_mtree() which took a file reference. This has an extensible options structure, and in particular now supports `ignore_unsupported_content`. I plan to use this for importing Docker images which contain device nodes. (There's no reason for container images to have those, so we'll just ignore them). Also here, just like the export variant, the caller is responsible for setting up libarchive. --- Makefile-tests.am | 8 + src/libostree/ostree-repo-libarchive.c | 143 +++++++++++++----- src/libostree/ostree-repo.h | 23 +++ tests/test-libarchive-import.c | 200 +++++++++++++++++++++++++ 4 files changed, 334 insertions(+), 40 deletions(-) create mode 100644 tests/test-libarchive-import.c diff --git a/Makefile-tests.am b/Makefile-tests.am index 292699ef..475519d7 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -140,6 +140,10 @@ TESTS = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test- tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \ tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum +if USE_LIBARCHIVE +TESTS += tests/test-libarchive-import +endif + check_PROGRAMS = $(TESTS) TESTS_ENVIRONMENT = \ G_TEST_SRCDIR=$(abs_srcdir)/tests \ @@ -172,6 +176,10 @@ tests_test_checksum_SOURCES = src/libostree/ostree-core.c tests/test-checksum.c tests_test_checksum_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) tests_test_checksum_LDADD = $(TESTS_LDADD) +tests_test_libarchive_import_SOURCES = tests/test-libarchive-import.c +tests_test_libarchive_import_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) +tests_test_libarchive_import_LDADD = $(TESTS_LDADD) + tests_test_keyfile_utils_CFLAGS = $(TESTS_CFLAGS) tests_test_keyfile_utils_LDADD = $(TESTS_LDADD) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 19fae757..64410cb2 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -78,6 +78,7 @@ file_info_from_archive_entry_and_modifier (OstreeRepo *repo, static gboolean import_libarchive_entry_file (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, struct archive *a, struct archive_entry *entry, GFileInfo *file_info, @@ -93,8 +94,27 @@ import_libarchive_entry_file (OstreeRepo *self, if (g_cancellable_set_error_if_cancelled (cancellable, error)) return FALSE; - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) - archive_stream = _ostree_libarchive_input_stream_new (a); + switch (g_file_info_get_file_type (file_info)) + { + case G_FILE_TYPE_REGULAR: + archive_stream = _ostree_libarchive_input_stream_new (a); + break; + case G_FILE_TYPE_SYMBOLIC_LINK: + break; + default: + if (opts->ignore_unsupported_content) + { + ret = TRUE; + goto out; + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unable to import non-regular/non-symlink file '%s'", + archive_entry_pathname (entry)); + goto out; + } + } if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL, &file_object_input, &length, cancellable, error)) @@ -111,6 +131,7 @@ import_libarchive_entry_file (OstreeRepo *self, static gboolean write_libarchive_entry_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, OstreeMutableTree *root, struct archive *a, struct archive_entry *entry, @@ -249,16 +270,19 @@ write_libarchive_entry_to_mtree (OstreeRepo *self, goto out; } - if (!import_libarchive_entry_file (self, a, entry, file_info, &tmp_csum, + if (!import_libarchive_entry_file (self, opts, a, entry, file_info, &tmp_csum, cancellable, error)) goto out; - - g_free (tmp_checksum); - tmp_checksum = ostree_checksum_from_bytes (tmp_csum); - if (!ostree_mutable_tree_replace_file (parent, basename, - tmp_checksum, - error)) - goto out; + + if (tmp_csum) + { + g_free (tmp_checksum); + tmp_checksum = ostree_checksum_from_bytes (tmp_csum); + if (!ostree_mutable_tree_replace_file (parent, basename, + tmp_checksum, + error)) + goto out; + } } } @@ -267,6 +291,71 @@ write_libarchive_entry_to_mtree (OstreeRepo *self, return ret; } #endif + +/** + * ostree_repo_import_archive_to_mtree: + * @self: An #OstreeRepo + * @opts: Options structure, ensure this is zeroed, then set specific variables + * @archive: Really this is "struct archive*" + * @mtree: The #OstreeMutableTree to write to + * @modifier: (allow-none): Optional commit modifier + * @cancellable: Cancellable + * @error: Error + * + * Import an archive file @archive into the repository, and write its + * file structure to @mtree. + */ +gboolean +ostree_repo_import_archive_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, + void *archive, + OstreeMutableTree *mtree, + OstreeRepoCommitModifier *modifier, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + struct archive *a = archive; + struct archive_entry *entry; + g_autofree guchar *tmp_csum = NULL; + int r; + + while (TRUE) + { + r = archive_read_next_header (a, &entry); + if (r == ARCHIVE_EOF) + break; + else if (r != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + /* TODO - refactor this to only create the metadata on demand + * (i.e. if there is a missing parent dir) + */ + if (opts->autocreate_parents && !tmp_csum) + { + g_autoptr(GFileInfo) tmp_dir_info = g_file_info_new (); + + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry)); + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry)); + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); + + if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error)) + goto out; + } + + if (!write_libarchive_entry_to_mtree (self, opts, mtree, a, + entry, modifier, tmp_csum, + cancellable, error)) + goto out; + } + + ret = TRUE; + out: + return ret; +} /** * ostree_repo_write_archive_to_mtree: @@ -293,10 +382,8 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, #ifdef HAVE_LIBARCHIVE gboolean ret = FALSE; struct archive *a = NULL; - struct archive_entry *entry; - int r; g_autoptr(GFileInfo) tmp_dir_info = NULL; - g_autofree guchar *tmp_csum = NULL; + OstreeRepoImportArchiveOptions opts = { 0, }; a = archive_read_new (); #ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL @@ -311,35 +398,11 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, goto out; } - while (TRUE) - { - r = archive_read_next_header (a, &entry); - if (r == ARCHIVE_EOF) - break; - else if (r != ARCHIVE_OK) - { - propagate_libarchive_error (error, a); - goto out; - } + opts.autocreate_parents = !!autocreate_parents; - if (autocreate_parents && !tmp_csum) - { - tmp_dir_info = g_file_info_new (); - - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry)); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry)); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); - - if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error)) - goto out; - } + if (!ostree_repo_import_archive_to_mtree (self, &opts, a, mtree, modifier, cancellable, error)) + goto out; - if (!write_libarchive_entry_to_mtree (self, mtree, a, - entry, modifier, - autocreate_parents ? tmp_csum : NULL, - cancellable, error)) - goto out; - } if (archive_read_close (a) != ARCHIVE_OK) { propagate_libarchive_error (error, a); diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 3b1040ab..4629ea55 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -450,6 +450,29 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo * GCancellable *cancellable, GError **error); +/** + * OstreeRepoImportArchiveOptions: + * + * An extensible options structure controlling archive import. Ensure that + * you have entirely zeroed the structure, then set just the desired + * options. This is used by ostree_repo_import_archive_to_mtree(). + */ +typedef struct { + guint ignore_unsupported_content : 1; + guint autocreate_parents : 1; + guint reserved : 30; + + guint unused_uint[8]; + gpointer unused_ptrs[8]; +} OstreeRepoImportArchiveOptions; + +gboolean ostree_repo_import_archive_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, + void *archive, /* Really struct archive * */ + OstreeMutableTree *mtree, + OstreeRepoCommitModifier *modifier, + GCancellable *cancellable, + GError **error); /** * OstreeRepoExportArchiveOptions: * diff --git a/tests/test-libarchive-import.c b/tests/test-libarchive-import.c new file mode 100644 index 00000000..1dd0c68c --- /dev/null +++ b/tests/test-libarchive-import.c @@ -0,0 +1,200 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" +#include "libglnx.h" +#include +#include +#include +#include + +#include +#include +#include + +typedef struct { + OstreeRepo *repo; + int fd; + char *tmpd; +} TestData; + +static void +test_data_init (TestData *td) +{ + GError *error = NULL; + struct archive *a = archive_write_new (); + struct archive_entry *ae; + + td->tmpd = g_mkdtemp (g_strdup ("/var/tmp/test-libarchive-import-XXXXXX")); + g_assert_cmpint (0, ==, chdir (td->tmpd)); + + td->fd = openat (AT_FDCWD, "foo.tar.gz", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0644); + (void) unlink ("foo.tar.gz"); + + g_assert_no_error (error); + g_assert (td->fd >= 0); + + g_assert_cmpint (0, ==, archive_write_set_format_pax (a)); + g_assert_cmpint (0, ==, archive_write_add_filter_gzip (a)); + g_assert_cmpint (0, ==, archive_write_open_fd (a, td->fd)); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/"); + archive_entry_set_mode (ae, S_IFDIR | 0755); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/file"); + archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_size (ae, 4); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + g_assert_cmpint (4, ==, archive_write_data (a, "foo\n", 4)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/devnull"); + archive_entry_set_mode (ae, S_IFCHR | 0777); + archive_entry_set_devmajor (ae, 1); + archive_entry_set_devminor (ae, 3); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/anotherfile"); + archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_size (ae, 4); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + g_assert_cmpint (4, ==, archive_write_data (a, "bar\n", 4)); + archive_entry_free (ae); + + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_close (a)); + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_free (a)); + + { g_autoptr(GFile) repopath = g_file_new_for_path ("repo"); + td->repo = ostree_repo_new (repopath); + + g_assert_cmpint (0, ==, mkdir ("repo", 0755)); + + ostree_repo_create (td->repo, OSTREE_REPO_MODE_BARE_USER, NULL, &error); + g_assert_no_error (error); + } +} + +static gboolean +spawn_cmdline (const char *cmd, GError **error) +{ + int estatus; + if (!g_spawn_command_line_sync (cmd, NULL, NULL, &estatus, error)) + return FALSE; + if (!g_spawn_check_exit_status (estatus, error)) + return FALSE; + return TRUE; +} + +static void +test_libarchive_error_device_file (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + + g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192)); + + (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); + g_assert (error != NULL); +} + +static void +test_libarchive_ignore_device_file (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + GCancellable *cancellable = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + glnx_unref_object GFile *root = NULL; + g_autofree char *commit_checksum = NULL; + + g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192)); + + opts.ignore_unsupported_content = TRUE; + + if (!ostree_repo_prepare_transaction (td->repo, NULL, cancellable, &error)) + goto out; + + if (!ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error)) + goto out; + + if (!ostree_repo_write_mtree (td->repo, mtree, &root, cancellable, &error)) + goto out; + + if (!ostree_repo_write_commit (td->repo, NULL, "", "", NULL, + OSTREE_REPO_FILE (root), + &commit_checksum, cancellable, &error)) + goto out; + + ostree_repo_transaction_set_ref (td->repo, NULL, "foo", commit_checksum); + + if (!ostree_repo_commit_transaction (td->repo, NULL, cancellable, &error)) + goto out; + + if (!spawn_cmdline ("ostree --repo=repo ls foo file", &error)) + goto out; + + if (!spawn_cmdline ("ostree --repo=repo ls foo anotherfile", &error)) + goto out; + + if (spawn_cmdline ("ostree --repo=repo ls foo devnull", &error)) + g_assert_not_reached (); + g_assert (error != NULL); + g_clear_error (&error); + + out: + g_assert_no_error (error); +} + +int main (int argc, char **argv) +{ + TestData td = {NULL,}; + int r; + + test_data_init (&td); + + g_test_init (&argc, &argv, NULL); + + g_test_add_data_func ("/libarchive/error-device-file", &td, test_libarchive_error_device_file); + g_test_add_data_func ("/libarchive/ignore-device-file", &td, test_libarchive_ignore_device_file); + + r = g_test_run(); + + if (td.tmpd) + (void) glnx_shutil_rm_rf_at (AT_FDCWD, td.tmpd, NULL, NULL); + return r; +} From eba7df0da93fb53c6438646ce11036595a37d76f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 19 Feb 2016 11:58:08 -0500 Subject: [PATCH 36/54] ostree-sysroot: add debug option to help testing If ostree is run in a test setup where it operates as root in a tmp directory, it might cause issues to flag the deployments as immutable. The test harness might simply be doing an `rm -rf` (effectively the case for gnome-desktop-testing-runner), which will then fail. We add a new debug option to the ostree_sysroot object using GLib's GDebugKey functionality to allow our tests to communicate to ostree that we don't want immutable deployments. --- src/libostree/ostree-sysroot-deploy.c | 9 ++++++--- src/libostree/ostree-sysroot-private.h | 8 ++++++++ src/libostree/ostree-sysroot.c | 7 +++++++ tests/libtest.sh | 4 ++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 3dcf39ff..aa034951 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2035,9 +2035,12 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, cancellable, error)) goto out; - if (!ostree_sysroot_deployment_set_mutable (self, new_deployment, FALSE, - cancellable, error)) - goto out; + if (!(self->debug_flags & OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS)) + { + if (!ostree_sysroot_deployment_set_mutable (self, new_deployment, FALSE, + cancellable, error)) + goto out; + } { ostree_cleanup_sepolicy_fscreatecon gpointer dummy = NULL; diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 0c38e269..e0dc24fd 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -27,6 +27,13 @@ G_BEGIN_DECLS +typedef enum { + + /* Don't flag deployments as immutable. */ + OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS = 1 << 0 + +} OstreeSysrootDebugFlags; + struct OstreeSysroot { GObject parent; @@ -46,6 +53,7 @@ struct OstreeSysroot { /* Only access through ostree_sysroot_get_repo() */ OstreeRepo *repo; + OstreeSysrootDebugFlags debug_flags; }; #define OSTREE_SYSROOT_LOCKFILE "ostree/lock" diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 7a4686d8..5ad2713a 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -153,6 +153,13 @@ ostree_sysroot_class_init (OstreeSysrootClass *klass) static void ostree_sysroot_init (OstreeSysroot *self) { + const GDebugKey keys[] = { + { "mutable-deployments", OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS }, + }; + + self->debug_flags = g_parse_debug_string (g_getenv("OSTREE_SYSROOT_DEBUG"), + keys, G_N_ELEMENTS (keys)); + self->sysroot_fd = -1; self->lock = (GLnxLockFile)GLNX_LOCK_FILE_INIT; } diff --git a/tests/libtest.sh b/tests/libtest.sh index bd806f6d..8cc4345f 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -22,6 +22,10 @@ test_tmpdir=$(pwd) export G_DEBUG=fatal-warnings +# Don't flag deployments as immutable so that test harnesses can +# easily clean up. +export OSTREE_SYSROOT_DEBUG=mutable-deployments + export TEST_GPG_KEYID_1="472CDAFA" export TEST_GPG_KEYID_2="CA950D41" export TEST_GPG_KEYID_3="DF444D67" From f2c5ecb996c9169e1e9dadbbc55825b41993a9b2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 19 Feb 2016 08:43:35 -0500 Subject: [PATCH 37/54] pull: Add require-static-deltas pull option For a production release repository, most OS vendors would want to just always use static deltas. Add the ability for the pulls to require it. (I think I'll also add a summary key for this actually in addition, so the repo manager can force it too) --- src/libostree/ostree-repo-pull.c | 18 ++++++++++++++++++ src/ostree/ot-builtin-pull.c | 5 +++++ tests/pull-test.sh | 20 +++++++++++++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 88b468b0..a7db45f9 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1765,6 +1765,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, char **override_commit_ids = NULL; GSource *update_timeout = NULL; gboolean disable_static_deltas = FALSE; + gboolean require_static_deltas = FALSE; if (options) { @@ -1777,6 +1778,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "override-remote-name", "s", &pull_data->remote_name); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas); + (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas); (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); } @@ -1787,6 +1789,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (dir_to_pull) g_return_val_if_fail (dir_to_pull[0] == '/', FALSE); + g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE); + pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0; pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0; @@ -1969,6 +1973,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; } + if (!bytes_summary && require_static_deltas) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Fetch configured to require static deltas, but no summary found"); + goto out; + } + if (bytes_summary) { uri = suburi_new (pull_data->base_uri, "summary.sig", NULL); @@ -2209,6 +2220,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!delta_superblock) { + if (require_static_deltas) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Static deltas required, but none found for %s to %s", + from_revision, to_revision); + goto out; + } g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision); queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, 0); } diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 63dbc768..7fa673cd 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -31,6 +31,7 @@ static gboolean opt_disable_fsync; static gboolean opt_mirror; static gboolean opt_commit_only; static gboolean opt_disable_static_deltas; +static gboolean opt_require_static_deltas; static char* opt_subpath; static int opt_depth = 0; @@ -38,6 +39,7 @@ static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "disable-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_disable_static_deltas, "Do not use static deltas", NULL }, + { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, @@ -175,6 +177,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_variant_builder_add (&builder, "{s@v}", "disable-static-deltas", g_variant_new_variant (g_variant_new_boolean (opt_disable_static_deltas))); + g_variant_builder_add (&builder, "{s@v}", "require-static-deltas", + g_variant_new_variant (g_variant_new_boolean (opt_require_static_deltas))); + if (override_commit_ids) g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 2233b905..04272d5f 100755 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -106,12 +106,18 @@ cd .. rm main-files -rf # Generate delta that we'll use ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main +prev_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^) +ostree --repo=ostree-srv/gnomerepo summary -u cd ${test_tmpdir} -${CMD_PREFIX} ostree --repo=repo pull origin main +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} +${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main ${CMD_PREFIX} ostree --repo=repo fsck cd ${test_tmpdir} +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas origin main ${CMD_PREFIX} ostree --repo=repo fsck @@ -124,6 +130,16 @@ assert_not_has_file baz/saucer echo "ok static delta" +cd ${test_tmpdir} +rm ostree-srv/gnomerepo/deltas -rf +ostree --repo=ostree-srv/gnomerepo summary -u +repo_init +if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main 2>err.txt; then + assert_not_reached "--require-static-deltas unexpectedly succeeded" +fi +assert_file_has_content err.txt "deltas required, but none found" +${CMD_PREFIX} ostree --repo=repo fsck + cd ${test_tmpdir} rm main-files -rf ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files @@ -134,6 +150,7 @@ cd .. rm main-files -rf # Generate new delta that we'll use ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --inline main +ostree --repo=ostree-srv/gnomerepo summary -u cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=repo pull origin main @@ -157,6 +174,7 @@ ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main - cd .. rm main-files -rf ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main +ostree --repo=ostree-srv/gnomerepo summary -u cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=repo pull origin main From 4beb5f4eaf619f1c47762bb872099e24f3efe2f0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 19 Feb 2016 12:28:07 -0500 Subject: [PATCH 38/54] pull: Add a --dry-run option for static deltas One of the design goals with deltas was not just wire efficiency, but also having all the data up front about how much data would be transferred before starting. Let's expose that better by adding a `dry-run` option to the pull API. This requires static deltas to be useful. Basically we simply call the progress callback once with the data from the superblock. --- src/libostree/ostree-repo-pull.c | 40 +++++++++++++++++-- src/ostree/ot-builtin-pull.c | 66 ++++++++++++++++++++++++++++++-- tests/pull-test.sh | 14 +++++++ 3 files changed, 113 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index a7db45f9..efa424f6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -48,6 +48,8 @@ typedef struct { GCancellable *cancellable; OstreeAsyncProgress *progress; + gboolean dry_run; + gboolean dry_run_emitted_progress; gboolean legacy_transaction_resuming; enum { OSTREE_PULL_PHASE_FETCHING_REFS, @@ -78,6 +80,7 @@ typedef struct { guint n_outstanding_deltapart_write_requests; guint n_total_deltaparts; guint64 total_deltapart_size; + guint64 total_deltapart_usize; gint n_requested_metadata; gint n_requested_content; guint n_fetched_deltaparts; @@ -227,6 +230,8 @@ update_progress (gpointer user_data) pull_data->n_total_deltaparts); ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size", pull_data->total_deltapart_size); + ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize", + pull_data->total_deltapart_usize); ostree_async_progress_set_uint (pull_data->progress, "total-delta-superblocks", pull_data->static_delta_superblocks->len); @@ -243,6 +248,9 @@ update_progress (gpointer user_data) else ostree_async_progress_set_status (pull_data->progress, NULL); + if (pull_data->dry_run) + pull_data->dry_run_emitted_progress = TRUE; + return TRUE; } @@ -262,6 +270,9 @@ pull_termination_condition (OtPullData *pull_data) if (pull_data->caught_error) return TRUE; + if (pull_data->dry_run) + return pull_data->dry_run_emitted_progress; + switch (pull_data->phase) { case OSTREE_PULL_PHASE_FETCHING_REFS: @@ -1451,11 +1462,18 @@ process_one_static_delta_fallback (OtPullData *pull_data, if (!ostree_validate_structureof_csum_v (csum_v, error)) goto out; + pull_data->total_deltapart_size += compressed_size; + pull_data->total_deltapart_usize += uncompressed_size; + + if (pull_data->dry_run) + { + ret = TRUE; + goto out; + } + objtype = (OstreeObjectType)objtype_y; checksum = ostree_checksum_from_bytes_v (csum_v); - pull_data->total_deltapart_size += compressed_size; - if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored, cancellable, error)) @@ -1524,6 +1542,7 @@ process_one_static_delta (OtPullData *pull_data, } /* Write the to-commit object */ + if (!pull_data->dry_run) { g_autoptr(GVariant) to_csum_v = NULL; g_autofree char *to_checksum = NULL; @@ -1626,7 +1645,11 @@ process_one_static_delta (OtPullData *pull_data, } pull_data->total_deltapart_size += size; + pull_data->total_deltapart_usize += usize; + if (pull_data->dry_run) + continue; + fetch_data = g_new0 (FetchStaticDeltaData, 1); fetch_data->pull_data = pull_data; fetch_data->objects = g_variant_ref (objects); @@ -1780,6 +1803,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas); (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas); (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); + (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); @@ -1790,6 +1814,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_return_val_if_fail (dir_to_pull[0] == '/', FALSE); g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE); + /* We only do dry runs with static deltas, because we don't really have any + * in-advance information for bare fetches. + */ + g_return_val_if_fail (!pull_data->dry_run || require_static_deltas, FALSE); pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0; pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0; @@ -2243,7 +2271,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->progress) { - update_timeout = g_timeout_source_new_seconds (1); + update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1); g_source_set_priority (update_timeout, G_PRIORITY_HIGH); g_source_set_callback (update_timeout, update_progress, pull_data, NULL); g_source_attach (update_timeout, pull_data->main_context); @@ -2256,6 +2284,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->caught_error) goto out; + + if (pull_data->dry_run) + { + ret = TRUE; + goto out; + } g_assert_cmpint (pull_data->n_outstanding_metadata_fetches, ==, 0); g_assert_cmpint (pull_data->n_outstanding_metadata_write_requests, ==, 0); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 7fa673cd..7c91890f 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -30,6 +30,7 @@ static gboolean opt_disable_fsync; static gboolean opt_mirror; static gboolean opt_commit_only; +static gboolean opt_dry_run; static gboolean opt_disable_static_deltas; static gboolean opt_require_static_deltas; static char* opt_subpath; @@ -42,6 +43,7 @@ static GOptionEntry options[] = { { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL }, + { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { NULL } }; @@ -62,6 +64,39 @@ gpg_verify_result_cb (OstreeRepo *repo, gs_console_begin_status_line (console, "", NULL, NULL); } +static gboolean printed_console_progress; + +static void +dry_run_console_progress_changed (OstreeAsyncProgress *progress, + gpointer user_data) +{ + guint fetched_delta_parts, total_delta_parts; + guint64 total_delta_part_size, total_delta_part_usize; + GString *buf; + + g_assert (!printed_console_progress); + printed_console_progress = TRUE; + + fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts"); + total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts"); + total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); + total_delta_part_usize = ostree_async_progress_get_uint64 (progress, "total-delta-part-usize"); + + buf = g_string_new (""); + + { g_autofree char *formatted_size = + g_format_size (total_delta_part_size); + g_autofree char *formatted_usize = + g_format_size (total_delta_part_usize); + + g_string_append_printf (buf, "Delta update: %u/%u parts, %s to transfer, %s uncompressed", + fetched_delta_parts, total_delta_parts, + formatted_size, formatted_usize); + } + g_print ("%s\n", buf->str); + g_string_free (buf, TRUE); +} + gboolean ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **error) { @@ -99,6 +134,13 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_commit_only) pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY; + if (opt_dry_run && !opt_require_static_deltas) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "--dry-run requires --require-static-deltas"); + goto out; + } + if (strchr (argv[1], ':') == NULL) { remote = g_strdup (argv[1]); @@ -149,11 +191,21 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_ptr_array_add (refs_to_fetch, NULL); } - console = gs_console_get (); - if (console) + if (!opt_dry_run) { - gs_console_begin_status_line (console, "", NULL, NULL); - progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console); + console = gs_console_get (); + if (console) + { + gs_console_begin_status_line (console, "", NULL, NULL); + progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console); + signal_handler_id = g_signal_connect (repo, "gpg-verify-result", + G_CALLBACK (gpg_verify_result_cb), + console); + } + } + else + { + progress = ostree_async_progress_new_and_connect (dry_run_console_progress_changed, console); signal_handler_id = g_signal_connect (repo, "gpg-verify-result", G_CALLBACK (gpg_verify_result_cb), console); @@ -180,6 +232,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_variant_builder_add (&builder, "{s@v}", "require-static-deltas", g_variant_new_variant (g_variant_new_boolean (opt_require_static_deltas))); + g_variant_builder_add (&builder, "{s@v}", "dry-run", + g_variant_new_variant (g_variant_new_boolean (opt_dry_run))); + if (override_commit_ids) g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); @@ -192,6 +247,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (progress) ostree_async_progress_finish (progress); + if (opt_dry_run) + g_assert (printed_console_progress); + ret = TRUE; out: if (signal_handler_id > 0) diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 04272d5f..f73b6818 100755 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -107,12 +107,24 @@ rm main-files -rf # Generate delta that we'll use ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main prev_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^) +new_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) ostree --repo=ostree-srv/gnomerepo summary -u +cd ${test_tmpdir} +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} +${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt +assert_file_has_content out.txt 'Delta update: 0/1 parts' +rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) +assert_streq "${prev_rev}" "${rev}" +${CMD_PREFIX} ostree --repo=repo fsck + cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main +rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) +assert_streq "${new_rev}" "${rev}" ${CMD_PREFIX} ostree --repo=repo fsck cd ${test_tmpdir} @@ -140,6 +152,8 @@ fi assert_file_has_content err.txt "deltas required, but none found" ${CMD_PREFIX} ostree --repo=repo fsck +echo "ok delta required but don't exist" + cd ${test_tmpdir} rm main-files -rf ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files From 6821ca1029c36440a74c17c14c29ea57f2dcfa17 Mon Sep 17 00:00:00 2001 From: Joaquim Rocha Date: Mon, 22 Feb 2016 15:02:47 +0100 Subject: [PATCH 39/54] build: Link ostree with libarchive libarchive (when available) is being used in ot-builtin-export.c so it is necessary to link ostree with it. https://bugzilla.gnome.org/show_bug.cgi?id=762457 --- Makefile-ostree.am | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index ab4485c6..ff7e372b 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -108,3 +108,8 @@ ostree_SOURCES += \ ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS) endif + +if USE_LIBARCHIVE +ostree_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) +ostree_LDADD += $(OT_DEP_LIBARCHIVE_LIBS) +endif From 11b3050fd73562d8d0caed948a5b421a68d0d793 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 22 Feb 2016 14:06:20 -0500 Subject: [PATCH 40/54] docs: Add a new formats section, move static deltas in there The `src/libostree/README-deltas.md` was rather hidden - let's move this into the manual. --- docs/manual/formats.md | 181 +++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + src/libostree/README-deltas.md | 158 ---------------------------- 3 files changed, 182 insertions(+), 158 deletions(-) create mode 100644 docs/manual/formats.md delete mode 100644 src/libostree/README-deltas.md diff --git a/docs/manual/formats.md b/docs/manual/formats.md new file mode 100644 index 00000000..bf7fd0ae --- /dev/null +++ b/docs/manual/formats.md @@ -0,0 +1,181 @@ +# OSTree data formats + +## On the topic of "smart servers" + +One really crucial difference between OSTree and git is that git has a +"smart server". Even when fetching over `https://`, it isn't just a +static webserver, but one that e.g. dynamically computes and +compresses pack files for each client. + +In contrast, the author of OSTree feels that for operating system +updates, many deployments will want to use simple static webservers, +the same target most package systems were designed to use. The +primary advantages are security and compute efficiency. Services like +Amazon S3 and CDNs are a canonical target, as well as a stock static +nginx server. + +## The archive-z2 format + +In the [repo](repo) section, the concept of objects was introduced, +where file/content objects are checksummed and managed individually. +(Unlike a package system, which operates on compressed aggregates). + +The archive-z2 format simply gzip-compresses each content object. +Metadata objects are stored uncompressed. This means that it's easy +to serve via static HTTP. + +When you commit new content, you will see new `.filez` files appearing +in `objects/`. + +## archive-z2 efficiency + +The advantages of `archive-z2`: + + - It's easy to understand and implement + - Can be served directly over plain HTTP by a static webserver + - Clients can download/unpack updates incrementally + - Space efficient on the server + +The biggest disadvantage of this format is that for a client to +perform an update, one HTTP request per changed file is required. In +some scenarios, this actually isn't bad at all, particularly with +techniques to reduce HTTP overhead, such as +[HTTP/2](https://en.wikipedia.org/wiki/HTTP/2). + +In order to make this format work well, you should design your content +such that large data that changes infrequently (e.g. graphic images) +are stored separately from small frequently changing data (application +code). + +Other disadvantages of `archive-z2`: + + - It's quite bad when clients are performing an initial pull (without HTTP/2), + - One doesn't know the total size (compressed or uncompressed) of content + before downloading everything + +## Aside: the bare and bare-user formats + +The most common operation is to pull from an `archive-z2` repository +into a `bare` or `bare-user` formatted repository. These latter two +are not compressed on disk. In other words, pulling to them is +similar to unpacking (but not installing) an RPM/deb package. + +The `bare-user` format is a bit special in that the uid/gid and xattrs +from the content are ignored. This is primarily useful if you want to +have the same OSTree-managed content that can be run on a host system +or an unprivileged container. + +## Static deltas + +OSTree itself was originally focused on a continous delivery model, where +client systems are expected to update regularly. However, many OS vendors +would like to supply content that's updated e.g. once a month or less often. + +For this model, we can do a lot better to support batched updates than +a basic `archive-z2` repo. However, we still want to preserve the +model of "static webserver only". Given this, OSTree has gained the +concept of a "static delta". + +These deltas are targeted to be a delta between two specific commit +objects, including "bsdiff" and "rsync-style" deltas within a content +object. Static deltas also support `from NULL`, where the client can +more efficiently download a commit object from scratch. + +Effectively, we're spending server-side storage (and one-time compute +cost), and gaining efficiency in client network bandwith. + +## Static delta repository layout + +Since static deltas may not exist, the client first needs to attempt +to locate one. Suppose a client wants to retrieve commit `${new}` +while currently running `${current}`. + +The first thing to understand is that in order to save space, these +two commits are "modified base64" - the `/` character is replaced with +`_`. + +Like the commit objects, a "prefix directory" is used to make +management easier for filesystem tools + +A delta is named `$(mbase64 $from)-$(mbase64 $to)`, for example +`GpTyZaVut2jXFPWnO4LJiKEdRTvOw_mFUCtIKW1NIX0-L8f+VVDkEBKNc1Ncd+mDUrSVR4EyybQGCkuKtkDnTwk`, +which in sha256 format is +`1a94f265a56eb768d714f5a73b82c988a11d453bcec3f985502b48296d4d217d-2fc7fe5550e410128d73535c77e98352b495478132c9b4060a4b8ab640e74f09`. + +Finally, the actual content can be found in +`deltas/$fromprefix/$fromsuffix-$to`. + +## Static delta internal structure + +A delta is itself a directory. Inside, there is a file called +`superblock` which contains metadata. The rest of the files will be +integers bearing packs of content. + +The file format of static deltas should be currently considered an +OSTree implementation detail. Obviously, nothing stops one from +writing code which is compatible with OSTree today. However, we would +like the flexibility to expand and change things, and having multiple +codebases makes that more problematic. Please contact the authors +with any requests. + +That said, one critical thing to understand about the design is that +delta payloads are a bit more like "restricted programs" than they are +raw data. There's a "compilation" phase which generates output that +the client executes. + +This "updates as code" model allows for multiple content generation +strategies. The design of this was inspired by that of Chromium: +[http://dev.chromium.org/chromium-os/chromiumos-design-docs/filesystem-autoupdate](ChromiumOS +autoupdate). + +### The delta superblock + +The superblock contains: + + - arbitrary metadata + - delta generation timestamp + - the new commit object + - An array of recursive deltas to apply + - An array of per-part metadata, including total object sizes (compressed and uncompressed), + - An array of fallback objects + +Let's define a delta part, then return to discuss details: + +## A delta part + +A delta part is a combination of a raw blob of data, plus a very +restricted bytecode that operates on it. Say for example two files +happen to share a common section. It's possible for the delta +compilation to include that section once in the delta data blob, then +generate instructions to write out that blob twice when generating +both objects. + +Realistically though, it's very common for most of a delta to just be +"stream of new objects" - if one considers it, it doesn't make sense +to have too much duplication inside operating system content at this +level. + +So then, what's more interesting is that OSTree static deltas support +a per-file delta algorithm called +[bsdiff](https://github.com/mendsley/bsdiff) that most notably works +well on executable code. + +The current delta compiler scans for files with maching basenamesin +each commit that have a similar size, and attempts a bsdiff between +them. (It would make sense later to have a build system provide a +hint for this - for example, files within a same package). + +A generated bsdiff is included in the payload blob, and applying it is +an instruction. + +## Fallback objects + +It's possible for there to be large-ish files which might be resistant +to bsdiff. A good example is that it's common for operating systems +to use an "initramfs", which is itself a compressed filesystem. This +"internal compression" defeats bsdiff analysis. + +For these types of objects, the delta superblock contains an array of +"fallback objects". These objects aren't included in the delta +parts - the client simply fetches them from the underlying `.filez` +object. diff --git a/mkdocs.yml b/mkdocs.yml index b11dbc87..89211c79 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,3 +8,4 @@ pages: - Deployments: 'manual/deployment.md' - Atomic Upgrades: 'manual/atomic-upgrades.md' - Adapting Existing Systems: 'manual/adapting-existing.md' + - Formats: 'manual/formats.md' diff --git a/src/libostree/README-deltas.md b/src/libostree/README-deltas.md deleted file mode 100644 index 9e09e45a..00000000 --- a/src/libostree/README-deltas.md +++ /dev/null @@ -1,158 +0,0 @@ -OSTree Static Object Deltas -=========================== - -Currently, OSTree's "archive-z2" mode stores both metadata and content -objects as individual files in the filesystem. Content objects are -zlib-compressed. - -The advantage of this is model are: - -0) It's easy to understand and implement -1) Can be served directly over plain HTTP by a static webserver -2) Space efficient on the server - -However, it can be inefficient both for large updates and small ones: - -0) For large tree changes (such as going from -runtime to - -devel-debug, or major version upgrades), this can mean thousands - and thousands of HTTP requests. The overhead for that is very - large (until SPDY/HTTP2.0), and will be catastrophically bad if the - webserver is not configured with KeepAlive. -1) Small changes (typo in gnome-shell .js file) still require around - 5 metadata HTTP requests, plus a redownload of the whole file. - -Why not smart servers? -====================== - -Smart servers (custom daemons, or just CGI scripts) as git has are not -under consideration for this proposal. OSTree is designed for the -same use case as GNU/Linux distribution package systems are, where -content is served by a network of volunteer mirrors that will -generally not run custom code. - -In particular, Amazon S3 style dumb content servers is a very -important use case, as is being able to apply updates from static -media like DVD-ROM. - -Finding Static Deltas -===================== - -Since static deltas may not exist, the client first needs to attempt -to locate one. Suppose a client wants to retrieve commit ${new} while -currently running ${current}. The first thing to fetch is the delta -metadata, called "meta". It can be found at -${repo}/deltas/${current}-${new}/meta. - -FIXME: GPG signatures (.metameta?) Or include commit object in meta? -But we would then be forced to verify the commit only after processing -the entirety of the delta, which is dangerous. I think we need to -require signing deltas. - -Delta Bytecode Format -===================== - -A delta-part has the following form: - -byte compression-type (0 = none, 'g' = gzip') -REPEAT[(varint size, delta-part-content)] - -delta-part-content: - byte[] payload - ARRAY[operation] - -The rationale for having delta-part is that it allows easy incremental -resumption of downloads. The client can look at the delta descriptor -and skip downloading delta-parts for which it already has the -contained objects. This is better than simply resuming a gigantic -file because if the client decides to fetch a slightly newer version, -it's very probable that some of the downloading we've already done is -still useful. - -For the actual delta payload, it comes as a stream of pair of -(payload, operation) so that it can be processed while being -decompressed. - -Finally, the delta-part-content is effectively a high level bytecode -for a stack-oriented machine. It iterates on the array of objects in -order. The following operations are available: - -FETCH - Fall back to fetching the current object individually. Move - to the next object. - -WRITE(array[(varint offset, varint length)]) - Write from current input target (default payload) to output. - -GUNZIP(array[(varint offset, varint length)]) - gunzip from current input target (default payload) to output. - -CLOSE - Close the current output target, and proceed to the next; if the - output object was a temporary, the output resets to the current - object. - -# Change the input source to an object -READOBJECT(csum object) - Set object as current input target - -# Change the input source to payload -READPAYLOAD - Set payload as current input target - -Compiling Deltas -================ - -After reading the above, you may be wondering how we actually *make* -these deltas. I envison a strategy similar to that employed by -Chromium autoupdate: -http://www.chromium.org/chromium-os/chromiumos-design-docs/autoupdate-details - -Something like this would be a useful initial algorithm: -1) Compute the set of added objects NEW -2) For each object in NEW: - - Look for a the set of "superficially similar" objects in the - previous tree, using heuristics based first on filename (including - prefix), then on size. Call this set CANDIDATES. - For each entry in CANDIDATES: - - Try doing a bup/librsync style rolling checksum, and compute the - list of changed blocks. - - Try gzip-compressing it -3) Choose the lowest cost method for each NEW object, and partition - the program for each method into deltapart-sized chunks. - -However, there are many other possibilities, that could be used in a -hybrid mode with the above. For example, we could try to find similar -objects, and gzip them together. This would be a *very* useful -strategy for things like the 9000 Boost headers which have massive -amounts of redundant data. - -Notice too that the delta format supports falling back to retrieving -individual objects. For cases like the initramfs which is compressed -inside the tree with gzip, we're not going to find an efficient way to -sync it, so the delta compiler should just fall back to fetching it -individually. - -Which Deltas To Create? -======================= - -Going back to the start, there are two cases to optimize for: - -1) Incremental upgrades between builds -2) Major version upgrades - -A command line operation would look something like this: - -$ ostree --repo=/path/to/repo gendelta --ref-prefix=gnome-ostree/buildmaster/ --strategy=latest --depth=5 - -This would tell ostree to generate deltas from each of the last 4 -commits to each ref (e.g. gnome-ostree/buildmaster/x86_64-runtime) to -the latest commit. It might also be possible of course to have ---strategy=incremental where we generate a delta between each commit. -I suspect that'd be something to do if one has a *lot* of disk space -to spend, and there's a reason for clients to be fetching individual -refs. - -$ ostree --repo=/path/to/repo gendelta --from=gnome-ostree/3.10/x86_64-runtime --to=gnome-ostree/buildmaster/x86_64-runtime - -This is an obvious one - generate a delta from the last stable release -to the current development head. From 30c5fb1a5116e517a9d2e4df9cdc1fbd0249c965 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 22 Feb 2016 11:18:30 -0500 Subject: [PATCH 41/54] libarchive: Make autocreate_parents imply autocreating root dir Some Docker layers are just metadata in the `layer.json`. If one is mapping Docker layers to OSTree commits, one needs to create a dummy root directory, because OSTree doesn't support metadata-only commits. Let's just push that logic down here because it's easier than special casing it in higher levels. --- src/libostree/ostree-repo-libarchive.c | 52 ++++++++++++++++++++---- tests/test-libarchive-import.c | 56 ++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 64410cb2..82c8dd51 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -292,6 +292,23 @@ write_libarchive_entry_to_mtree (OstreeRepo *self, } #endif +static gboolean +create_empty_dir_with_uidgid (OstreeRepo *self, + guint32 uid, + guint32 gid, + guint8 **out_csum, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFileInfo) tmp_dir_info = g_file_info_new (); + + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", uid); + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", gid); + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); + + return _ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, out_csum, cancellable, error); +} + /** * ostree_repo_import_archive_to_mtree: * @self: An #OstreeRepo @@ -320,6 +337,7 @@ ostree_repo_import_archive_to_mtree (OstreeRepo *self, g_autofree guchar *tmp_csum = NULL; int r; + while (TRUE) { r = archive_read_next_header (a, &entry); @@ -336,13 +354,13 @@ ostree_repo_import_archive_to_mtree (OstreeRepo *self, */ if (opts->autocreate_parents && !tmp_csum) { - g_autoptr(GFileInfo) tmp_dir_info = g_file_info_new (); - - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry)); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry)); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); - - if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error)) + /* Here, we auto-pick the first uid/gid we find in the + * archive. Realistically this is probably always going to + * be root, but eh, at least we try to match. + */ + if (!create_empty_dir_with_uidgid (self, archive_entry_uid (entry), + archive_entry_gid (entry), + &tmp_csum, cancellable, error)) goto out; } @@ -352,6 +370,26 @@ ostree_repo_import_archive_to_mtree (OstreeRepo *self, goto out; } + /* If we didn't import anything at all, and autocreation of parents + * is enabled, automatically create a root directory. This is + * useful primarily when importing Docker image layers, which can + * just be metadata. + */ + if (!ostree_mutable_tree_get_metadata_checksum (mtree) && opts->autocreate_parents) + { + char tmp_checksum[65]; + + if (!tmp_csum) + { + /* We didn't have any archive entries to match, so pick uid 0, gid 0. */ + if (!create_empty_dir_with_uidgid (self, 0, 0, &tmp_csum, cancellable, error)) + goto out; + } + + ostree_checksum_inplace_from_bytes (tmp_csum, tmp_checksum); + ostree_mutable_tree_set_metadata_checksum (mtree, tmp_checksum); + } + ret = TRUE; out: return ret; diff --git a/tests/test-libarchive-import.c b/tests/test-libarchive-import.c index 1dd0c68c..928b1491 100644 --- a/tests/test-libarchive-import.c +++ b/tests/test-libarchive-import.c @@ -32,6 +32,7 @@ typedef struct { OstreeRepo *repo; int fd; + int fd_empty; char *tmpd; } TestData; @@ -88,6 +89,19 @@ test_data_init (TestData *td) g_assert_cmpint (ARCHIVE_OK, ==, archive_write_close (a)); g_assert_cmpint (ARCHIVE_OK, ==, archive_write_free (a)); + td->fd_empty = openat (AT_FDCWD, "empty.tar.gz", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0644); + g_assert (td->fd_empty >= 0); + (void) unlink ("empty.tar.gz"); + + a = archive_write_new (); + g_assert (a); + + g_assert_cmpint (0, ==, archive_write_set_format_pax (a)); + g_assert_cmpint (0, ==, archive_write_add_filter_gzip (a)); + g_assert_cmpint (0, ==, archive_write_open_fd (a, td->fd_empty)); + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_close (a)); + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_free (a)); + { g_autoptr(GFile) repopath = g_file_new_for_path ("repo"); td->repo = ostree_repo_new (repopath); @@ -109,6 +123,46 @@ spawn_cmdline (const char *cmd, GError **error) return TRUE; } +static void +test_libarchive_noautocreate_empty (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + + g_assert_cmpint (0, ==, lseek (td->fd_empty, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd_empty, 8192)); + + (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); + g_assert_no_error (error); + g_assert (ostree_mutable_tree_get_metadata_checksum (mtree) == NULL); +} + +static void +test_libarchive_autocreate_empty (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + + opts.autocreate_parents = 1; + + g_assert_cmpint (0, ==, lseek (td->fd_empty, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd_empty, 8192)); + + (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); + g_assert_no_error (error); + g_assert (ostree_mutable_tree_get_metadata_checksum (mtree) != NULL); +} + static void test_libarchive_error_device_file (gconstpointer data) { @@ -189,6 +243,8 @@ int main (int argc, char **argv) g_test_init (&argc, &argv, NULL); + g_test_add_data_func ("/libarchive/noautocreate-empty", &td, test_libarchive_noautocreate_empty); + g_test_add_data_func ("/libarchive/autocreate-empty", &td, test_libarchive_autocreate_empty); g_test_add_data_func ("/libarchive/error-device-file", &td, test_libarchive_error_device_file); g_test_add_data_func ("/libarchive/ignore-device-file", &td, test_libarchive_ignore_device_file); From 9f9d713a5645f6153dbcc03fc891ba8f0fcd566d Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 22 Feb 2016 15:04:38 -0500 Subject: [PATCH 42/54] build: Use threadsafe GPGME https://bugzilla.gnome.org/show_bug.cgi?id=761863 --- Makefile.am | 4 ++-- configure.ac | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile.am b/Makefile.am index e1c942d7..1625419b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -43,8 +43,8 @@ OT_INTERNAL_SOUP_CFLAGS = $(OT_DEP_SOUP_CFLAGS) OT_INTERNAL_SOUP_LIBS = $(OT_DEP_SOUP_LIBS) # This canonicalizes the PKG_CHECK_MODULES or AM_PATH_GPGME results -OT_INTERNAL_GPGME_CFLAGS = $(OT_DEP_GPGME_CFLAGS) $(GPGME_CFLAGS) -OT_INTERNAL_GPGME_LIBS = $(OT_DEP_GPGME_LIBS) $(GPGME_LIBS) +OT_INTERNAL_GPGME_CFLAGS = $(OT_DEP_GPGME_CFLAGS) $(GPGME_PTHREAD_CFLAGS) +OT_INTERNAL_GPGME_LIBS = $(OT_DEP_GPGME_LIBS) $(GPGME_PTHREAD_LIBS) if BUILDOPT_INTROSPECTION include $(INTROSPECTION_MAKEFILE) diff --git a/configure.ac b/configure.ac index 6a34c0d4..f65b93f5 100644 --- a/configure.ac +++ b/configure.ac @@ -106,13 +106,13 @@ AM_CONDITIONAL(BUILDOPT_INTROSPECTION, test "x$found_introspection" = xyes) LIBGPGME_DEPENDENCY="1.1.8" -PKG_CHECK_MODULES(OT_DEP_GPGME, gpgme >= $LIBGPGME_DEPENDENCY, have_gpgme=yes, [ - m4_ifdef([AM_PATH_GPGME], [ - AM_PATH_GPGME($LIBGPGME_DEPENDENCY, have_gpgme=yes, have_gpgme=no) +PKG_CHECK_MODULES(OT_DEP_GPGME, gpgme-pthread >= $LIBGPGME_DEPENDENCY, have_gpgme=yes, [ + m4_ifdef([AM_PATH_GPGME_PTHREAD], [ + AM_PATH_GPGME_PTHREAD($LIBGPGME_DEPENDENCY, have_gpgme=yes, have_gpgme=no) ],[ have_gpgme=no ]) ]) AS_IF([ test x$have_gpgme = xno ], [ - AC_MSG_ERROR([Need GPGME version $LIBGPGME_DEPENDENCY or later]) + AC_MSG_ERROR([Need GPGME_PTHREAD version $LIBGPGME_DEPENDENCY or later]) ]) OSTREE_FEATURES="$OSTREE_FEATURES +gpgme" From f461c02bb55bf2853a3b81ed5c8618040ab54e98 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 22 Feb 2016 16:50:28 -0500 Subject: [PATCH 43/54] gpg: Use gpg_strerror_r for threadsafety These APIs are rather painful...this is why GError exists. --- src/libotutil/ot-gpg-utils.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libotutil/ot-gpg-utils.c b/src/libotutil/ot-gpg-utils.c index aa5b4819..9414088d 100644 --- a/src/libotutil/ot-gpg-utils.c +++ b/src/libotutil/ot-gpg-utils.c @@ -31,6 +31,7 @@ ot_gpgme_error_to_gio_error (gpgme_error_t gpg_error, GError **error) { GIOErrorEnum errcode; + char errbuf[1024]; /* XXX This list is incomplete. Add cases as needed. */ @@ -42,9 +43,11 @@ ot_gpgme_error_to_gio_error (gpgme_error_t gpg_error, /* special case - abort on out-of-memory */ case GPG_ERR_ENOMEM: + (void) gpg_strerror_r (gpg_error, errbuf, sizeof (errbuf)); + errbuf[sizeof(errbuf)-1] = '\0'; g_error ("%s: %s", gpgme_strsource (gpg_error), - gpgme_strerror (gpg_error)); + errbuf); case GPG_ERR_INV_VALUE: errcode = G_IO_ERROR_INVALID_ARGUMENT; @@ -55,9 +58,11 @@ ot_gpgme_error_to_gio_error (gpgme_error_t gpg_error, break; } + (void) gpg_strerror_r (gpg_error, errbuf, sizeof (errbuf)); + errbuf[sizeof(errbuf)-1] = '\0'; g_set_error (error, G_IO_ERROR, errcode, "%s: %s", gpgme_strsource (gpg_error), - gpgme_strerror (gpg_error)); + errbuf); } gboolean From f4d494a0a4bc8682e71b1936536e4e04c38a5ac2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 22 Feb 2016 18:08:01 -0500 Subject: [PATCH 44/54] Release 2016.2 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f65b93f5..2229b145 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.63]) -AC_INIT([ostree], [2016.1], [walters@verbum.org]) +AC_INIT([ostree], [2016.2], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR([build-aux]) From 88ae6f1b9e64926c47e5c1dae36027b8c46816f7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 22 Feb 2016 20:52:33 -0500 Subject: [PATCH 45/54] libarchive: Fix a 32 bit format warning This is actually an error by default with our default CFLAGS. --- src/libostree/ostree-repo-libarchive.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 82c8dd51..b21e222c 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -640,7 +640,7 @@ write_directory_to_libarchive_recurse (OstreeRepo *self, if (r != bytes_read) { propagate_libarchive_error (error, a); - g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", bytes_read, r); + g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", (guint64)bytes_read, r); goto out; } } From 61b4f268622e64610e14883367cbead18c716bc4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 22 Feb 2016 21:11:10 -0500 Subject: [PATCH 46/54] lib: Two more compiler warning fixes --- src/libostree/ostree-repo-libarchive.c | 2 +- src/libostree/ostree-repo.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index b21e222c..1c051592 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -640,7 +640,7 @@ write_directory_to_libarchive_recurse (OstreeRepo *self, if (r != bytes_read) { propagate_libarchive_error (error, a); - g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", (guint64)bytes_read, r); + g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", (guint64)bytes_read, (guint64)r); goto out; } } diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1e9673d1..3b08d445 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4604,8 +4604,8 @@ ostree_repo_regenerate_summary (OstreeRepo *self, gs_free char *from = NULL; gs_free char *to = NULL; gs_free guchar *csum = NULL; - gs_free char *superblock; - gs_fd_close int superblock_file_fd; + gs_free char *superblock = NULL; + gs_fd_close int superblock_file_fd = -1; g_autoptr(GInputStream) in_stream = NULL; _ostree_parse_delta_name (delta_names->pdata[i], &from, &to); From ed7266b10ad75252b7d8026a4329d913a8ada8c1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 22 Feb 2016 22:07:02 -0500 Subject: [PATCH 47/54] deltas: Fix some more 32 bit warnings --- src/libostree/ostree-repo-static-delta-core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 6369f34d..eb6de3bb 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -640,10 +640,10 @@ show_one_part (OstreeRepo *self, " opsize=%" G_GUINT64_FORMAT "\n", i, - g_variant_n_children (modes), - g_variant_n_children (xattrs), - g_variant_n_children (blob), - g_variant_n_children (ops)); + (guint64)g_variant_n_children (modes), + (guint64)g_variant_n_children (xattrs), + (guint64)g_variant_n_children (blob), + (guint64)g_variant_n_children (ops)); if (!_ostree_static_delta_part_execute (self, objects, part, TRUE, TRUE, From 1833142b14255f5ddf0fd2b991a4b5aa5a6ecc92 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 24 Feb 2016 15:17:46 +0100 Subject: [PATCH 48/54] deltas: Fix regression in ostree_repo_static_delta_execute_offline xdg-app passed this a filename directly, and in this case it should be used as is. This regressed to always look for "superblock" in the same directory as the passed in filename. https://bugzilla.gnome.org/show_bug.cgi?id=762617 --- src/libostree/ostree-repo-static-delta-core.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index eb6de3bb..ea5842f7 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -240,6 +240,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, g_autoptr(GVariant) fallback = NULL; g_autofree char *to_checksum = NULL; g_autofree char *from_checksum = NULL; + g_autofree char *basename = NULL; dir_or_file_path = gs_file_get_path_cached (dir_or_file); @@ -255,13 +256,16 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, else { g_autofree char *dir = dirname (g_strdup (dir_or_file_path)); + basename = g_path_get_basename (dir_or_file_path); if (!glnx_opendirat (AT_FDCWD, dir, TRUE, &dfd, error)) goto out; } } + else + basename = g_strdup ("superblock"); - meta_fd = openat (dfd, "superblock", O_RDONLY | O_CLOEXEC); + meta_fd = openat (dfd, basename, O_RDONLY | O_CLOEXEC); if (meta_fd < 0) { glnx_set_error_from_errno (error); From 6c285d2e4ac76e99df848e2dea3cb0633126728a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 24 Feb 2016 12:14:38 +0100 Subject: [PATCH 49/54] rofiles-fuse: Handle operations on the root ENSURE_RELPATH breaks when path is "/". In that case we need to return "." instead of "". https://bugzilla.gnome.org/show_bug.cgi?id=762596 --- src/rofiles-fuse/main.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rofiles-fuse/main.c b/src/rofiles-fuse/main.c index 4337f00f..3c910f1f 100644 --- a/src/rofiles-fuse/main.c +++ b/src/rofiles-fuse/main.c @@ -46,7 +46,10 @@ static GHashTable *created_devino_hash = NULL; static inline const char * ENSURE_RELPATH (const char *path) { - return path + strspn (path, "/"); + path = path + strspn (path, "/"); + if (*path == 0) + return "."; + return path; } typedef struct { From 277220aaa6e73002d5186ba1166c11be07481727 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 23 Feb 2016 11:00:18 -0500 Subject: [PATCH 50/54] deltas: Include an endianness marker We screwed up and had delta integers use host endianness. Start digging out by at least annotating the endianness. https://bugzilla.gnome.org/show_bug.cgi?id=762515 --- .../ostree-repo-static-delta-compilation.c | 19 +++++++++++++ src/libostree/ostree-repo-static-delta-core.c | 28 +++++++++++++++++++ tests/test-delta.sh | 5 ++++ 3 files changed, 52 insertions(+) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 2071bb68..e579e910 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -1306,6 +1306,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self, cancellable, error)) goto out; + /* NOTE: Add user-supplied metadata first. This is used by at least + * xdg-app as a way to provide MIME content sniffing, since the + * metadata appears first in the file. + */ g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}")); if (metadata != NULL) { @@ -1320,6 +1324,21 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } } + { guint8 endianness_char; + switch (G_BYTE_ORDER) + { + case G_LITTLE_ENDIAN: + endianness_char = 'l'; + break; + case G_BIG_ENDIAN: + endianness_char = 'B'; + break; + default: + g_assert_not_reached (); + } + g_variant_builder_add (&metadata_builder, "{sv}", "ostree.endianness", g_variant_new_byte (endianness_char)); + } + if (opt_filename) { g_autoptr(GFile) f = g_file_new_for_path (opt_filename); diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index ea5842f7..6b1dc571 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -678,6 +678,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, g_autofree char *superblock_path = NULL; glnx_fd_close int superblock_fd = -1; g_autoptr(GVariant) delta_superblock = NULL; + g_autoptr(GVariant) delta_meta = NULL; + g_autoptr(GVariantDict) delta_metadict = NULL; guint64 total_size = 0, total_usize = 0; guint64 total_fallback_size = 0, total_fallback_usize = 0; guint i; @@ -690,7 +692,33 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, TRUE, &delta_superblock, error)) goto out; + delta_meta = g_variant_get_child_value (delta_superblock, 0); + delta_metadict = g_variant_dict_new (delta_meta); + g_print ("Delta: %s\n", delta_id); + { guint8 endianness_char; + const char *endianness_description; + + if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char)) + { + switch (endianness_char) + { + case 'l': + endianness_description = "little"; + break; + case 'B': + endianness_description = "big"; + break; + default: + endianness_description = "invalid"; + break; + } + } + else + endianness_description = "unknown"; + + g_print ("Endianness: %s\n", endianness_description); + } { guint64 ts; g_variant_get_child (delta_superblock, 1, "t", &ts); g_print ("Timestamp: %" G_GUINT64_FORMAT "\n", GUINT64_FROM_BE (ts)); diff --git a/tests/test-delta.sh b/tests/test-delta.sh index b31d65e9..84ce8a7c 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -114,6 +114,11 @@ fi echo 'ok generate' +${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show.txt +assert_file_has_content show.txt 'Endianness: \(little\|big\)' + +echo 'ok show' + mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2 ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev} ${CMD_PREFIX} ostree --repo=repo2 fsck From 04d77da0056c89f2f18ae9e47ef121c960f2307b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 24 Feb 2016 14:29:56 -0500 Subject: [PATCH 51/54] deltas: Use endianness marker when parsing Extend the `static-delta show` and `pull` commands to use the endianness information (if available). --- src/libostree/ostree-repo-pull.c | 13 ++- .../ostree-repo-static-delta-compilation.c | 20 +++- src/libostree/ostree-repo-static-delta-core.c | 92 ++++++++++++++----- .../ostree-repo-static-delta-private.h | 46 ++++++++-- src/ostree/ot-builtin-static-delta.c | 38 ++++++++ tests/pull-test.sh | 19 +++- tests/test-delta.sh | 9 ++ 7 files changed, 201 insertions(+), 36 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index efa424f6..0be1b380 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1443,6 +1443,7 @@ request_static_delta_superblock_sync (OtPullData *pull_data, static gboolean process_one_static_delta_fallback (OtPullData *pull_data, + gboolean delta_byteswap, GVariant *fallback_object, GCancellable *cancellable, GError **error) @@ -1462,6 +1463,9 @@ process_one_static_delta_fallback (OtPullData *pull_data, if (!ostree_validate_structureof_csum_v (csum_v, error)) goto out; + compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size); + uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size); + pull_data->total_deltapart_size += compressed_size; pull_data->total_deltapart_usize += uncompressed_size; @@ -1518,11 +1522,14 @@ process_one_static_delta (OtPullData *pull_data, GError **error) { gboolean ret = FALSE; + gboolean delta_byteswap; g_autoptr(GVariant) metadata = NULL; g_autoptr(GVariant) headers = NULL; g_autoptr(GVariant) fallback_objects = NULL; guint i, n; + delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock); + /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */ metadata = g_variant_get_child_value (delta_superblock, 0); headers = g_variant_get_child_value (delta_superblock, 6); @@ -1535,7 +1542,7 @@ process_one_static_delta (OtPullData *pull_data, g_autoptr(GVariant) fallback_object = g_variant_get_child_value (fallback_objects, i); - if (!process_one_static_delta_fallback (pull_data, + if (!process_one_static_delta_fallback (pull_data, delta_byteswap, fallback_object, cancellable, error)) goto out; @@ -1609,6 +1616,10 @@ process_one_static_delta (OtPullData *pull_data, header = g_variant_get_child_value (headers, i); g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects); + version = maybe_swap_endian_u32 (delta_byteswap, version); + size = maybe_swap_endian_u64 (delta_byteswap, size); + usize = maybe_swap_endian_u64 (delta_byteswap, usize); + if (version > OSTREE_DELTAPART_VERSION) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index e579e910..2d02e6a6 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -59,6 +59,7 @@ typedef struct { guint n_rollsum; guint n_bsdiff; guint n_fallback; + gboolean swap_endian; } OstreeStaticDeltaBuilder; typedef enum { @@ -1191,7 +1192,8 @@ get_fallback_headers (OstreeRepo *self, g_variant_new ("(y@aytt)", objtype, ostree_checksum_to_bytes_v (checksum), - compressed_size, uncompressed_size)); + maybe_swap_endian_u64 (builder->swap_endian, compressed_size), + maybe_swap_endian_u64 (builder->swap_endian, uncompressed_size))); } ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder)); @@ -1228,6 +1230,7 @@ get_fallback_headers (OstreeRepo *self, * - bsdiff-enabled: b: Enable bsdiff compression. Default TRUE. * - inline-parts: b: Put part data in header, to get a single file delta. Default FALSE. * - verbose: b: Print diagnostic messages. Default FALSE. + * - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or G_LITTLE_ENDIAN) * - filename: ay: Save delta superblock to this filename, and parts in the same directory. Default saves to repository. */ gboolean @@ -1262,6 +1265,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_autoptr(GVariant) fallback_headers = NULL; g_autoptr(GVariant) detached = NULL; gboolean inline_parts; + guint endianness = G_BYTE_ORDER; g_autoptr(GFile) tmp_dir = NULL; builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref); builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); @@ -1277,6 +1281,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self, max_chunk_size = 32; builder.max_chunk_size_bytes = ((guint64)max_chunk_size) * 1000 * 1000; + (void) g_variant_lookup (params, "endianness", "u", &endianness); + g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE); + + builder.swap_endian = endianness != G_BYTE_ORDER; + { gboolean use_bsdiff; if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff)) use_bsdiff = TRUE; @@ -1325,7 +1334,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } { guint8 endianness_char; - switch (G_BYTE_ORDER) + + switch (endianness) { case G_LITTLE_ENDIAN: endianness_char = 'l'; @@ -1433,10 +1443,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self, checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN); objtype_checksum_array = objtype_checksum_array_new (part_builder->objects); delta_part_header = g_variant_new ("(u@aytt@ay)", - OSTREE_DELTAPART_VERSION, + maybe_swap_endian_u32 (builder.swap_endian, OSTREE_DELTAPART_VERSION), ot_gvariant_new_ay_bytes (checksum_bytes), - (guint64) g_variant_get_size (delta_part), - part_builder->uncompressed_size, + maybe_swap_endian_u64 (builder.swap_endian, (guint64) g_variant_get_size (delta_part)), + maybe_swap_endian_u64 (builder.swap_endian, part_builder->uncompressed_size), ot_gvariant_new_ay_bytes (objtype_checksum_array)); g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header)); diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 6b1dc571..97cdb8c5 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -589,6 +589,7 @@ _ostree_static_delta_part_open (GInputStream *part_in, static gboolean show_one_part (OstreeRepo *self, + gboolean swap_endian, const char *from, const char *to, GVariant *meta_entries, @@ -608,6 +609,8 @@ show_one_part (OstreeRepo *self, gint part_fd = -1; g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects); + size = maybe_swap_endian_u64 (swap_endian, size); + usize = maybe_swap_endian_u64 (swap_endian, usize); *total_size_ref += size; *total_usize_ref += usize; g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n", @@ -666,6 +669,45 @@ show_one_part (OstreeRepo *self, return ret; } +OstreeDeltaEndianness +_ostree_delta_get_endianness (GVariant *superblock) +{ + guint8 endianness_char; + g_autoptr(GVariant) delta_meta = NULL; + g_autoptr(GVariantDict) delta_metadict = NULL; + + delta_meta = g_variant_get_child_value (superblock, 0); + delta_metadict = g_variant_dict_new (delta_meta); + + if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char)) + { + switch (endianness_char) + { + case 'l': + return OSTREE_DELTA_ENDIAN_LITTLE; + case 'B': + return OSTREE_DELTA_ENDIAN_BIG; + default: + return OSTREE_DELTA_ENDIAN_INVALID; + } + } + return OSTREE_DELTA_ENDIAN_UNKNOWN; +} + +gboolean +_ostree_delta_needs_byteswap (GVariant *superblock) +{ + switch (_ostree_delta_get_endianness (superblock)) + { + case OSTREE_DELTA_ENDIAN_BIG: + return G_BYTE_ORDER == G_LITTLE_ENDIAN; + case OSTREE_DELTA_ENDIAN_LITTLE: + return G_BYTE_ORDER == G_BIG_ENDIAN; + default: + return FALSE; + } +} + gboolean _ostree_repo_static_delta_dump (OstreeRepo *self, const char *delta_id, @@ -678,11 +720,11 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, g_autofree char *superblock_path = NULL; glnx_fd_close int superblock_fd = -1; g_autoptr(GVariant) delta_superblock = NULL; - g_autoptr(GVariant) delta_meta = NULL; - g_autoptr(GVariantDict) delta_metadict = NULL; guint64 total_size = 0, total_usize = 0; guint64 total_fallback_size = 0, total_fallback_usize = 0; guint i; + OstreeDeltaEndianness endianness; + gboolean swap_endian = FALSE; _ostree_parse_delta_name (delta_id, &from, &to); superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); @@ -692,30 +734,34 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, TRUE, &delta_superblock, error)) goto out; - delta_meta = g_variant_get_child_value (delta_superblock, 0); - delta_metadict = g_variant_dict_new (delta_meta); + g_print ("%s\n", g_variant_print (delta_superblock, 1)); g_print ("Delta: %s\n", delta_id); - { guint8 endianness_char; - const char *endianness_description; + { const char *endianness_description; - if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char)) + endianness = _ostree_delta_get_endianness (delta_superblock); + + switch (endianness) { - switch (endianness_char) - { - case 'l': - endianness_description = "little"; - break; - case 'B': - endianness_description = "big"; - break; - default: - endianness_description = "invalid"; - break; - } + case OSTREE_DELTA_ENDIAN_BIG: + endianness_description = "big"; + if (G_BYTE_ORDER == G_LITTLE_ENDIAN) + swap_endian = TRUE; + break; + case OSTREE_DELTA_ENDIAN_LITTLE: + endianness_description = "little"; + if (G_BYTE_ORDER == G_BIG_ENDIAN) + swap_endian = TRUE; + break; + case OSTREE_DELTA_ENDIAN_UNKNOWN: + endianness_description = "unknown"; + break; + case OSTREE_DELTA_ENDIAN_INVALID: + endianness_description = "invalid"; + break; + default: + g_assert_not_reached (); } - else - endianness_description = "unknown"; g_print ("Endianness: %s\n", endianness_description); } @@ -739,6 +785,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, { guint64 size, usize; g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize); + size = maybe_swap_endian_u64 (swap_endian, size); + usize = maybe_swap_endian_u64 (swap_endian, usize); total_fallback_size += size; total_fallback_usize += usize; } @@ -757,7 +805,7 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, for (i = 0; i < n_parts; i++) { - if (!show_one_part (self, from, to, meta_entries, i, + if (!show_one_part (self, swap_endian, from, to, meta_entries, i, &total_size, &total_usize, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index 2da000d6..d9e5c456 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -46,10 +46,10 @@ G_BEGIN_DECLS /** * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT: * - * u: version + * u: version (non-canonical endian) * ay checksum - * guint64 size: Total size of delta (sum of parts) - * guint64 usize: Uncompressed size of resulting objects on disk + * guint64 size: Total size of delta (sum of parts) (non-canonical endian) + * guint64 usize: Uncompressed size of resulting objects on disk (non-canonical endian) * ARRAY[(guint8 objtype, csum object)] * * The checksum is of the delta payload, and each entry in the array @@ -64,8 +64,8 @@ G_BEGIN_DECLS * * y: objtype * ay: checksum - * t: compressed size - * t: uncompressed size + * t: compressed size (non-canonical endian) + * t: uncompressed size (non-canonical endian) * * Object to fetch invididually; includes compressed/uncompressed size. */ @@ -79,7 +79,7 @@ G_BEGIN_DECLS * * delta-descriptor: * metadata: a{sv} - * t: timestamp + * t: timestamp (big endian) * from: ay checksum * to: ay checksum * commit: new commit object @@ -196,4 +196,38 @@ _ostree_repo_static_delta_dump (OstreeRepo *repo, GCancellable *cancellable, GError **error); +/* Used for static deltas which due to a historical mistake are + * inconsistent endian. + * + * https://bugzilla.gnome.org/show_bug.cgi?id=762515 + */ +static inline guint32 +maybe_swap_endian_u32 (gboolean swap, + guint32 v) +{ + if (!swap) + return v; + return GUINT32_SWAP_LE_BE (v); +} + +static inline guint64 +maybe_swap_endian_u64 (gboolean swap, + guint64 v) +{ + if (!swap) + return v; + return GUINT64_SWAP_LE_BE (v); +} + +typedef enum { + OSTREE_DELTA_ENDIAN_BIG, + OSTREE_DELTA_ENDIAN_LITTLE, + OSTREE_DELTA_ENDIAN_UNKNOWN, + OSTREE_DELTA_ENDIAN_INVALID +} OstreeDeltaEndianness; + +OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock); + +gboolean _ostree_delta_needs_byteswap (GVariant *superblock); + G_END_DECLS diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index c3c99ba0..36fb63fa 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -32,7 +32,9 @@ static char *opt_to_rev; static char *opt_min_fallback_size; static char *opt_max_bsdiff_size; static char *opt_max_chunk_size; +static char *opt_endianness; static gboolean opt_empty; +static gboolean opt_swap_endianness; static gboolean opt_inline; static gboolean opt_disable_bsdiff; @@ -60,6 +62,8 @@ static GOptionEntry generate_options[] = { { "inline", 0, 0, G_OPTION_ARG_NONE, &opt_inline, "Inline delta parts into main delta", NULL }, { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" }, { "disable-bsdiff", 0, 0, G_OPTION_ARG_NONE, &opt_disable_bsdiff, "Disable use of bsdiff", NULL }, + { "set-endianness", 0, 0, G_OPTION_ARG_STRING, &opt_endianness, "Choose metadata endianness ('l' or 'B')", "ENDIAN" }, + { "swap-endianness", 0, 0, G_OPTION_ARG_NONE, &opt_swap_endianness, "Swap metadata endianness from host order", NULL }, { "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in megabytes for individual HTTP request", NULL}, { "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL}, { "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL}, @@ -194,6 +198,7 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab g_autofree char *to_resolved = NULL; g_autofree char *from_parent_str = NULL; g_autoptr(GVariantBuilder) parambuilder = NULL; + int endianness; g_assert (opt_to_rev); @@ -224,6 +229,37 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab } if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error)) goto out; + + if (opt_endianness) + { + if (strcmp (opt_endianness, "l") == 0) + endianness = G_LITTLE_ENDIAN; + else if (strcmp (opt_endianness, "B") == 0) + endianness = G_BIG_ENDIAN; + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid endianness '%s'", opt_endianness); + goto out; + } + } + else + endianness = G_BYTE_ORDER; + + if (opt_swap_endianness) + { + switch (endianness) + { + case G_LITTLE_ENDIAN: + endianness = G_BIG_ENDIAN; + break; + case G_BIG_ENDIAN: + endianness = G_LITTLE_ENDIAN; + break; + default: + g_assert_not_reached (); + } + } parambuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); if (opt_min_fallback_size) @@ -243,6 +279,8 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab "inline-parts", g_variant_new_boolean (TRUE)); g_variant_builder_add (parambuilder, "{sv}", "verbose", g_variant_new_boolean (TRUE)); + if (opt_endianness || opt_swap_endianness) + g_variant_builder_add (parambuilder, "{sv}", "endianness", g_variant_new_uint32 (endianness)); g_print ("Generating static delta:\n"); g_print (" From: %s\n", from_resolved ? from_resolved : "empty"); diff --git a/tests/pull-test.sh b/tests/pull-test.sh index f73b6818..9c8b41fa 100755 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -113,8 +113,8 @@ ostree --repo=ostree-srv/gnomerepo summary -u cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} -${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt -assert_file_has_content out.txt 'Delta update: 0/1 parts' +${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >dry-run-pull.txt +assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts' rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) assert_streq "${prev_rev}" "${rev}" ${CMD_PREFIX} ostree --repo=repo fsck @@ -142,6 +142,21 @@ assert_not_has_file baz/saucer echo "ok static delta" +cd ${test_tmpdir} +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --swap-endianness main +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u + +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} +${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas --dry-run origin main >byteswapped-dry-run-pull.txt +${CMD_PREFIX} ostree --repo=repo fsck + +if ! diff -u dry-run-pull.txt byteswapped-dry-run-pull.txt; then + assert_not_reached "byteswapped delta differs in size" +fi + +echo "ok pull byteswapped delta" + cd ${test_tmpdir} rm ostree-srv/gnomerepo/deltas -rf ostree --repo=ostree-srv/gnomerepo summary -u diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 84ce8a7c..4679ab8e 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -119,6 +119,15 @@ assert_file_has_content show.txt 'Endianness: \(little\|big\)' echo 'ok show' +${CMD_PREFIX} ostree --repo=repo static-delta generate --swap-endianness --from=${origrev} --to=${newrev} +${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show-swapped.txt +totalsize_orig=$(grep 'Total Size:' show.txt) +totalsize_swapped=$(grep 'Total Size:' show-swapped.txt) +assert_not_streq "${totalsize_orig}" "" +assert_streq "${totalsize_orig}" "${totalsize_swapped}" + +echo 'ok generate + show endian swapped' + mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2 ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev} ${CMD_PREFIX} ostree --repo=repo2 fsck From 7fdf07271031cd4a88af9280b41ece5dc7c1d580 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 24 Feb 2016 17:04:04 -0500 Subject: [PATCH 52/54] deltas: Heuristically detect endianness for older deltas If the average object size is greater than 4GiB, let's assume we're dealing with opposite endianness. I'm fairly confident no one is going to be shipping peta- or exa- byte size ostree deltas, period. Past the gigabyte scale you really want bittorrent or something. --- Makefile-tests.am | 2 + src/libostree/ostree-repo-static-delta-core.c | 74 +++++++++++++++--- .../ostree-repo-static-delta-private.h | 3 +- tests/pre-endian-deltas-repo-big.tar.xz | Bin 0 -> 5760 bytes tests/pre-endian-deltas-repo-little.tar.xz | Bin 0 -> 5752 bytes tests/test-delta.sh | 21 +++++ 6 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 tests/pre-endian-deltas-repo-big.tar.xz create mode 100644 tests/pre-endian-deltas-repo-little.tar.xz diff --git a/Makefile-tests.am b/Makefile-tests.am index 475519d7..6ec58357 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -85,6 +85,8 @@ insttest_DATA = tests/archive-test.sh \ tests/test-basic-user.sh \ tests/test-local-pull.sh \ tests/corrupt-repo-ref.js \ + tests/pre-endian-deltas-repo-big.tar.xz \ + tests/pre-endian-deltas-repo-little.tar.xz \ $(NULL) insttest_SCRIPTS += \ diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 97cdb8c5..0669f691 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -670,15 +670,22 @@ show_one_part (OstreeRepo *self, } OstreeDeltaEndianness -_ostree_delta_get_endianness (GVariant *superblock) +_ostree_delta_get_endianness (GVariant *superblock, + gboolean *out_was_heuristic) { guint8 endianness_char; g_autoptr(GVariant) delta_meta = NULL; g_autoptr(GVariantDict) delta_metadict = NULL; + guint64 total_size = 0; + guint64 total_usize = 0; + guint total_objects = 0; delta_meta = g_variant_get_child_value (superblock, 0); delta_metadict = g_variant_dict_new (delta_meta); + if (out_was_heuristic) + *out_was_heuristic = FALSE; + if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char)) { switch (endianness_char) @@ -691,13 +698,58 @@ _ostree_delta_get_endianness (GVariant *superblock) return OSTREE_DELTA_ENDIAN_INVALID; } } - return OSTREE_DELTA_ENDIAN_UNKNOWN; + + if (out_was_heuristic) + *out_was_heuristic = TRUE; + + { g_autoptr(GVariant) meta_entries = NULL; + guint n_parts; + guint i; + + g_variant_get_child (superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries); + n_parts = g_variant_n_children (meta_entries); + + for (i = 0; i < n_parts; i++) + { + g_autoptr(GVariant) objects = NULL; + guint64 size, usize; + guint n_objects; + + g_variant_get_child (meta_entries, i, "(u@aytt@ay)", NULL, NULL, &size, &usize, &objects); + n_objects = (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN); + + total_objects += n_objects; + total_size += size; + total_usize += usize; + } + + /* If the average object size is greater than 4GiB, let's assume + * we're dealing with opposite endianness. I'm fairly confident + * no one is going to be shipping peta- or exa- byte size ostree + * deltas, period. Past the gigabyte scale you really want + * bittorrent or something. + */ + if ((total_size / total_objects) > G_MAXUINT32) + { + switch (G_BYTE_ORDER) + { + case G_BIG_ENDIAN: + return OSTREE_DELTA_ENDIAN_LITTLE; + case G_LITTLE_ENDIAN: + return OSTREE_DELTA_ENDIAN_BIG; + default: + g_assert_not_reached (); + } + } + + return G_BYTE_ORDER; + } } gboolean _ostree_delta_needs_byteswap (GVariant *superblock) { - switch (_ostree_delta_get_endianness (superblock)) + switch (_ostree_delta_get_endianness (superblock, NULL)) { case OSTREE_DELTA_ENDIAN_BIG: return G_BYTE_ORDER == G_LITTLE_ENDIAN; @@ -738,24 +790,28 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, g_print ("Delta: %s\n", delta_id); { const char *endianness_description; + gboolean was_heuristic; - endianness = _ostree_delta_get_endianness (delta_superblock); + endianness = _ostree_delta_get_endianness (delta_superblock, &was_heuristic); switch (endianness) { case OSTREE_DELTA_ENDIAN_BIG: - endianness_description = "big"; + if (was_heuristic) + endianness_description = "big (heuristic)"; + else + endianness_description = "big"; if (G_BYTE_ORDER == G_LITTLE_ENDIAN) swap_endian = TRUE; break; case OSTREE_DELTA_ENDIAN_LITTLE: - endianness_description = "little"; + if (was_heuristic) + endianness_description = "little (heuristic)"; + else + endianness_description = "little"; if (G_BYTE_ORDER == G_BIG_ENDIAN) swap_endian = TRUE; break; - case OSTREE_DELTA_ENDIAN_UNKNOWN: - endianness_description = "unknown"; - break; case OSTREE_DELTA_ENDIAN_INVALID: endianness_description = "invalid"; break; diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index d9e5c456..41ddad48 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -222,11 +222,10 @@ maybe_swap_endian_u64 (gboolean swap, typedef enum { OSTREE_DELTA_ENDIAN_BIG, OSTREE_DELTA_ENDIAN_LITTLE, - OSTREE_DELTA_ENDIAN_UNKNOWN, OSTREE_DELTA_ENDIAN_INVALID } OstreeDeltaEndianness; -OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock); +OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock, gboolean *out_was_heuristic); gboolean _ostree_delta_needs_byteswap (GVariant *superblock); diff --git a/tests/pre-endian-deltas-repo-big.tar.xz b/tests/pre-endian-deltas-repo-big.tar.xz new file mode 100644 index 0000000000000000000000000000000000000000..05e51d6933fde7853bc773e50e8dedb73a00842e GIT binary patch literal 5760 zcmV-`7JupeH+ooF000E$*0e?f03iVu0001VFXf}*-~SdrT>v;7h{^DB@yqY7q66O5 zH)NzQ(+^~?d1y2q(#UCD_p}~pUXoO}cNFt5j=Of z7oLK-Sd1CSm*46-b!KAqO{!4iU{F6H`?hyii17V;Z@l*YRR~J$+Ms%kMWLsbBmQ~< z{?u?iV~f5r$If}4X%;M=LpdP;Towyx{FV*Zazy~pT`H10Ge^;3@G!}Ny#1m_WB>)E zNeZ`1z(PhMlFU_vqRC$f3ClF$}pQomsnEO8D{~!1+lkL#C0_sl!ez9F`mi zVy0wzYxvTRG%h ziX>rRHO6XwWa`rR_U9aaHm~up1^hP&ix$DuqIHJordSaZXFf?^lVJqS^%%C~K@&%5 zpM28U*KjbANDAE2U<)xldwA^EAD+HRsHP_0OIM^IBO)Hi9j3y0$J=>L3$r&Nts;Y8 zvH`!@UR0Qy?9}qbh1s5_8)t^pR1Rh#|7;yJ*t7p85LHzZW40})6E_DBz3aia_*gZ< zE}aSWl|JVK@ltvNuS-o7I3S#K6RIbO>%ru+A$i6ZVddzM9SOvGq$$3pK(P{b5RS64 z1<|C0hiZl(X@%pZA>~Mpy4(shzd5mI=cURd(gY+urRl?+#N>)p)ik0`SLECqHQ`b= z#({~avf>FHR%Yd%R1lx1e#D(osTY!skq&Mf4u2L{&lKZaVir77sJ*xZ8=U~HX>_Rz zChl*#4S|PSf&C2H=~iNuz#2^e1()e7o~Y6ngmmKr6#O4nU%kG=f0Wp~uTM?#96Bpl zR}Hqu*16m`p0($go1%6^wJ$RJ;+c)0CgzOu9~SNh>F!V1CjNQMArp2h8}+20Alz#; z-&rUM5VwtOJvx80k{y6Hi|3xEFulpHq2j`BbDOe#M_tIQ4_K6*L9OKstnR}0J^5gN z#w2P55QJHurs7Hi_SjQ)Mw`l^G()FBn%o?{ay^2jEkvL8)V)=0zQ!!UHV5xxC$Oa{ z2xYZRnDf2+~hQsPg++aAsGo24tmS z1VAdXwqYVpi-<<~2L`P`mZDsB%+{sdMVL?-$?rdy$(}?}A9j#xXLrqAg-%;U6&Jxvf9_845=H~q@1jquGy__C(sK`C<6&!zMRP8E<_;I zg`+hPD#q`smMn!uc-M$4Yk(xFiMq3EMol8yb6_E~gLay=cEg9J3My^itbuf2^)(5~ zJb9leWt+6`epbpAU(+Zkhsvli34ewCid9CwUQ&u{1oGJ3mP$ret|Uy|L&g*Zp06;e zAxi!;$+X$Ur4f;&2r}gMM3kLkB5= zVce=!AMi?Y&e&mgxuN1wz(d1oI@@1(>1OW`=_aW1xX6l;B!MU{UmgLzoUc%;Y#kNL zq%=YVr#pS|1OWWk`xj!4Y%&pJ?eF@W@hfE_Nq|6we+PtUXAK`QUxywJSj{dU;I{%7 zS}u1dT=$iMEC-T5;a~>F8o`}}0aNkxCi_*I`)JobE5_}@foIgv@V^msaxO;53T|<} zAFM&kH13?LE3G||l4gA#8rVNskiso#l%Pfycz(%2arWLG#w*G2`qSL)EF8;@Vr^}7$$ zOXgW{23m!pqLS5iQtLxZ7)*fdhS;c*0d*y-&hLIj%4JXQ_*xCNTJ81npC#RO8hJD}ctS4{9L?9Av{8 zdJ|%M{;sld?$i~xQnGE*u*x~%#Rmm#JEPX zSW0+;Vas~V0Z*^O_rBX4dQli`4u#kKcl@W+0OOHIEnxr?so!Um?Ru{$7P8M0r6}8K zjXS63I^fb0@7l%j1bgD=WMPHn83M6zqpF+t>6u3KSSw9n+{v3wz`^v~ z>tm@}$qKI3$;vMv3ZKbQK`;z{si&88VxF>($GqR@Q#-ECk3%l3%}0OKfd<>tY?-g2 zM%kQYV-JF|1TQX0RrO1M4N=5!xV0=L!d3k%0chf-&G3(A$Nsf>3$TQHlJ+fBYOG*3 z3v<-jOiuez2%s8^JbL#|CMMyy16p7yzwG|})^x!1nmF0&!{ga;g8b`uilQe{dOVS2 zeILGAl&A*IL;)NBsc~g9%UD-a(XRWSa6CirP9)-L#Jo!V)yNSJ0dL&H)iC@oknrG>nq5$6$>HZ#Oaw^|E2pOVUB3-BKz4)i<%w-s> z5ECt)`F3cPBU-+Y3~v|(sLRPF9B)d<1h|V}*!9qEYLfo@s{B#wQPcm3#az}SdAfet|t zuj}pw7ipWPHGnp;c>doELaB%;N|%@n3z4-yKl=h-z?|~Q)N4?e+e+#f`tJtWCn};G zvw!cW^9ifY>Hqc3Eyyt8mt}5iDjd>w#x6sR%Oe_Per&W_wR*|H_zCC7x}njW%v+ir zI#kNo3wH^Pzl|8jy$wDM(xC88!bNxKGPe&<7?Vkt(#ihS1Mwv-@({y?@~sql$*C-K zc|Oncm4|IYd3D9{?SfE{p*_mf^2ThVtnOE`*thsTK)e6X)<;=EIQF~#I1PxR>f(@j z0u)vf!~Sl(K$EGE3jxu%dJe#qst?%TpCb-HEHw%|nT_uI@!T;AJQC1h4DS71e?_e< z0|aJ#UjAd6`qU+Yd(>L9Xn!1dcv`JDAop681(YWo_OBoP5InVd)SO|z=;#4F{o-^U z<+1H+_#`J8zr@SqjA$-TP3H!!g>#xzi>33bNV~}Q;zxR*m@M|b(O;)RuG45bCT~0q zMo3JQ&bX)vE@mgJWOV!n;ki)OUo71VO*96B#>gN23Xi_dy{nfz^U?c1Vw@$!{)*2jj@8) zIAbKEmez<&s^OM&1DhWG)LPtHpk*fZvzH-NL;8>$0`<95r;)d%XavM|Hdw3AT`L#K z{^NDhM7eM><$9EwpbUPCT`q|105to7^~#Yy0`XJwYYAmWAaDix98pZFqxIS4feXok zW^Cdr40y`3Kg+8mi>@yoq@7}EYj0!zbG#4y%5KAw}27>{DhS?6)(RW6YoKy2>>{l~_|p-wI2D_vs9UmpLyI{fpeJA8LT893%1zPBxkXV#Itf86C$cYSXohv?POeV)B52zcvbWugSG zCj<67&VP<-DM9@ZS>|-)nD?}|Tq<`?R76euR*5R%zd5_an~+9MLS6;C3>2{jeP z>!nNe`Mhr?(ruB}h4oC&E=pE_Vzf#*UXbR#kNB50(3j>e**#G*2$~p3-;ls9;FsV( zKD3X=tHTEd-r=MZznX~*=sW-sYY?juivgFi6cF=#UDTvo7c>eT-WTIDAoTKX-bZhX zsgKm01pJEQTHvitd`u~S5U7vjxvf@dSvTT=#w{QlQLH|LC1^AV)35iJ%iCk`wey^X zaTAxuVNeg^2e+0~&-Y=pp=}nWmR>Wn?2LAnW+zvbK|p6tbUS9Y=MY+-dOe=Y-!xb- z`+CN$dlrB7Htg$>A*_-y;jWDr^hds{miMOc#i~lSxze*EMJ(Q6gg&k0E1?MNLrTDG z4$oUkKboG<3#?^l;C4-|Pninj z`fg)-j3wJV<#$}gtEPtbl4|kXaE+Vx{uNyLV3>94=Y#G;O;VT;bXI9GG{_#>3h4Fz z$@M?r6>JuANMkhrWU(?M=Gv)uo;klQ5gx=ekb+KtBqn{ zOffCS7$m+7Qmf>3)KM7-_s)M9*oD*{{7`>o!y7_>cDpKtkbFF-#|iUEfTL-4{;0Xq zq7&}OXM)qt>7qI&6v4=5`<;Po?E@*k_3k22Ze$lJe%agI#6Qo$g9rqc>)*Z$RSH&) zkqW+>U1W{>$vT&|3=%YY{ua^~ZO=BZ_;~$YOllTnlo(I0KrUICofw`p=jA4!*)S5F z=0bs0JG2DY;;eAfusmteGldO5sVdU@^49@~c~*%XJ;HwV*p%#0laQ8V3Qh3AvLQ^rZUXGoQ+xm)&hq z?xZ~$oSygn6(C$FfHP^78((e&?z5XDl&A{;;TOLFx`q4gz%|G%YU!S8G|z}j>Siz$ zo+0{{>vPJAZ#3QlRFq2ebzMGCo956pjw20_HWcb)eF|-{9*ixK=b1K^b8QV zIt7EB(tVK1#@FDQTc3Ven#dLVDF!w5^kx561*$scZFcXIW6af)LiHfor|o{8S!l(_ z4W>R3qpd9?cUb>!YrAFo{_*J@BItIUqR@7Aj25s|E^>i}cg>vQKNLUoZb-5CZ^GGc zfrm|#QP-CreaSKtzfhuWsL`SfVYK%?CfBWNnk^UUaxa4V28qg26jSuL5s|`Eg+|9X z*&O6Li}eZRjHcB~Dj38@J%q{n{G8OXL_}H&E5Y6r$bD3O!>%w*c8&ermM{HR-j%J# zpk!H+jYS=An(s2^p2L8F$iX##@SN{x<-iGP^bAE}Pd`{I*Cz2yz01>|6GJ{cHh%T?Z|-fkCA)b-o4=3vNu!xsho%kP8yCM z5DS2V1KZqgwm!L#XJ3BVZb_%T+MKyc=idsNpoBZ~Y+sM#-D0|94z;Rld83v>nZ?+& z3U0d-FNxg~Z@Y>==(`b2k{Fg|LB{RrGnJ7J*NgtL+Kx_<_%X z^dDTC7B=dnFjejh)T8L`dkb}mChhZVwfSWgh8~Gy=mL*3g*!qyL=S^=Mo?ja?Y7xK zyTybCKL{PR=<4|#^G2R=D-QwJZ2Qd&^jY1!ydr2>8xA>xzW}PGXBmg7N7;&y_Eia) zF{7Hyez$;zAz(Bss`Zck3Z1u#1Z~UtjBCCqt!C+3CfW*jVn+b-R^?- z?=`s#_4KX85C@$4+R0E85)16~U^YId_pq^zgOr9T5qNwD)eXR-2!vcJPD_jqJ6* z5etvF%4zkOyHd~a9TR9IO{r$lxorIqcoCo^cFVOyWYBdz^2AD3$zI^+{Wmtu@(<5(85eCr^ct~+ zsuxvixn8sv-aY6nHuG@|p(DlkGgwhX7$q`jT#>StpUKmM*BP!XYWeUzh}(Z!>fm$w ySzjGpQfr9eL*Vx^0002=RVvI=atl`g0oyEqzy|<`Oe&1A#Ao{g000001X)_!NJ*mr literal 0 HcmV?d00001 diff --git a/tests/pre-endian-deltas-repo-little.tar.xz b/tests/pre-endian-deltas-repo-little.tar.xz new file mode 100644 index 0000000000000000000000000000000000000000..ef05045c0444ac4c5c04c1230dc13fe00f5623c5 GIT binary patch literal 5752 zcmV-;7KiEmH+ooF000E$*0e?f03iVu0001VFXf}*-~SdjT>v;7h{^DB@yqY7q66O5 zH)NzQ(+^~?d1y2q(#UCD_p}~pUXoO}cNFt5j=Of z7oLK-Sd1CSm*46-b!KAqO{!4iU{F6H`?hyii17V;Z@l*YRR~J$+Ms%kMWLsbBmQ~< z{?u?iV~f5r$If}4X%;M=LpdP;Towyx{FV*Zazy~pT`H10Ge^;3@G!}Ny#1m_WB>)E zNeZ`1z(PhMlFU_vqRC$f3ClF$}pQomsnEO8D{~!1+lkL#C0_sl!ez9F`mi zVy0wzYxvTRG%h ziX>rRHO6XwWa`rR_U9aaHm~up1^hP&ix$DuqIHJordSaZXFf?^lVJqS^%%C~K@&%5 zpM28U*KjbANDAE2U<)xldwA^EAD+HRsHP_0OIM^IBO)Hi9j3y0$J=>L3$r&Nts;Y8 zvH`!@UR0Qy?9}qbh1s5_8)t^pR1Rh#|7;yJ*t7p85LHzZW40})6E_DBz3aia_*gZ< zE}aSWl|JVK@ltvNuS-o7I3S#K6RIbO>%ru+A$i6ZVddzM9SOvGq$$3pK(P{b5RS64 z1<|C0hiZl(X@%pZA>~Mpy4(shzd5mI=cURd(gY+urRl?+#N>)p)ik0`SLECqHQ`b= z#({~avf>FHR%Yd%R1lx1e#D(osTY!skq&Mf4u2L{&lKZaVir77sJ*xZ8=U~HX>_Rz zChl*#4S|PSf&C2H=~iNuz#2^e1()e7o~Y6ngmmKr6#O4nU%kG=f0Wp~uTM?#96Bpl zR}Hqu*16m`p0($go1%6^wJ$RJ;+c)0CgzOu9~SNh>F!V1CjNQMArp2h8}+20Alz#; z-&rUM5VwtOJvx80k{y6Hi|3xEFulpHq2j`BbDOe#M_tIQ4_K6*L9OKstnR}0J^5gN z#w2P55QJHurs7Hi_SjQ)Mw`l^G()FBn%o?{ay^2jEkvL8)V)=0zQ!!UHV5xxC$Oa{ z2xYZRnDf2+~hQsPg++aAsGo24tmS z1VAdXwqYVpi-<<~2L`P`mZDsB%+{sdMVL?-$?rdy$(}?}A9j#xXLrqAg-%;U6&Jxvf9_845=H~q@1jquGy__C(sK`C<6&!zMRP8E<_;I zg`+hPD#q`smMn!uc-M$4Yk(xFiMq3EMol8yb6_E~gLay=cEg9J3My^itbuf2^)(5~ zJb9leWt+6`epbpAU(+Zkhsvli34ewCid9CwUQ&u{1oGJ3mP$ret|Uy|L&g*Zp06;e zAxi!;$+X$Ur4f;&2r}gMM3kLkB5= zVce=!AMi?Y&e&mgxuN1wz(d1oI@@1(>1OW`=_aW1xX6l;B!MU{UmgLzoUc%;Y#kNL zq%=YVr#pS|1OWWk`xj!4Y%&pJ?eF@W@hfE_Nq|6we+PtUXAK`QUxywJSj{dU;I{%7 zS}u1dT=$iMEC-T5;a~>F8o`}}0aNkxCi_*I`)JobE5_}@foIgv@V^msaxO;53T|<} zAFM&kH13?LE3G||l4gA#8rVNskiso#l%Pfycz(%2arWLG#w*G2`qSL)EF8;@Vr^}7$$ zOXgW{23m!pqLS5iQtLxZ7)*fdhS;c*0d*y-&hLIj%4JXQ_*xCNTJ81npC#RO8hJD}ctS4{9L?9Av{8 zdJ|%M{;sld?$i~xQnGE*u*x~%#Rmm#JEPX zSW0+;Vas~V0Z*^O_rBX4dQli`4u#kKcl@W+0OOHIEnxr?so!Um?Ru{$7P8M0r6}8K zjXS63I^fb0@7l%j1bgD=WMPHn83M6zqpF+t>6u3KSSw9n+{v3wz`^v~ z>tm@}$qKI3$;vMv3ZKbQK`;z{si&88VxF>($GqR@Q#-ECk3%l3%}0OKfd<>tY?-g2 zM%kQYV-JF|1TQX0RrO1M4N=5!xV0=L!d3k%0chf-&G3(A$Nsf>3$TQHlJ+fBYOG*3 z3v<-jOiuez2%s8^JbL#|CMMyy16p7yzwG|})^x!1nmF0&!{ga;g8b`uilQe{dOVS2 zeILGAl&A*IL;)NBsc~g9%UD-a(XRWSa6CirP9)-L#Jo!V)yNSJ0dL&H)iC@oknrG>nq5$6$>HZ#Oaw^|E2pOVUB3-BKz4)i<%w-s> z5ECt)`F3cPBU-+Y3~v|(sLRPF9B)d<1h|V}*!9qEYLfo@s{B#wQPcm3#az}SdAfet|t zuj}pw7ipWPHGnp;c>doELaB%;N|%@n3z4-yKl=h-z?|~Q)N4?e+e+#f`tJtWCn};G zvw!cW^9ifY>Hqc3Eyyt8mt}5iDjd>w#x6sR%Oe_Per&W_wR*|H_zCC7x}njW%v+ir zI#kNo3wH^Pzl|8jy$wDM(xC88!bNxKGPe&<7?Vkt(#ihS1Mwv-@({y?@~sql$*C-K zc|Oncm4|IYd3D9{?SfE{p*_mf^2ThVtnOE`*thsTK)e6X)<;=EIQF~#I1PxR>f(@j z0u)vf!~Sl(K$EGE3jxu%dJe#qst?%TpCb-HEHw%|nT_uI@!T;AJQC1h4DS71e?_e< z0|aJ#UjAd6`qU+Yd(>L9Xn!1dcv`JDAop681(YWo_OBoP5InVd)SO|z=;#4F{o-^U z<+1H+_#`J8zr@SqjA$-TP3H!!g>#xzi>33bNV~}Q;zxR*m@M|b(O;)RuG45bCT~0q zMo3JQ&bX)vE@mgJWOV!n;ki)OUo71VO*96B#>gN23Xi_dy{nfz^U?c1Vw@$!{)*2jj@8) zIAbKEmez<&s^OM&1DhWG)LPtHpk*fZvzH-NL;8>$0`<95r;)d%XavM|Hdw3AT`L#K z{^NDhM7eM><$9EwpbUPCT`q|105to7^~#Yy0`XJwYYAmWAaDix98pZFqxIS4feXok zW^Cdr40y`3Kg+8mi>@yoq@7}EYj0!zbG#4y%5KAw}27>{DhS?6)(RW6YoKy2>>{l~_|p-wI2D_vs9UmpLyI{fpeJA8LT893%1zPBxkXV#Itf86C$cYSXohv?POeV)B52zcvbWugSG zCj<67&VP<-DM9@ZS>|-)nD?}|Tq<`?R76euR*5R%zd5_an~+9MLS6;C3>2{jeP z>!nNe`Mhr?(ruB}h4oC&E=pE_Vzf#*UXbR#kNB50(3j>e**#G*2$~p3-;ls9;FsV( zKD3X=tHTEd-r=MZznX~*=sW-sYY?juivgFi6cF=#UDTvo7c>eT-WTIDAoTKX-bZhX zsgKm01pJEQTHvitd`u~S5U7vjxvf@dSvTT=#w{QlQLH|LC1^AV)35iJ%iCk`wey^X zaTAxuVNeg^2e+0~&-Y=pp=}nWmR>Wn?2LAnW+zvbK|p6tbUS9Y=MY+-dOe=Y-!xb- z`+CN$dlrB7Htg$>A*_-y;jWDr^hds{miMOc#i~lSxze*EMJ(Q6gg&k0E1?MNLrTDG z4$oUkKboG<3#?^l;C4-|Pninj z`fg)-j3wJV<#$}gtEPtbl4|kXaE+Vx{uNyLV3>94=Y#G;O;VT;bXI9GG{_#>3h4Fz z$@M?r6>JuANMkhrWU(?M=Gv)uo;klQ5gx=ekb+KtBqn{ zOffCS7$m+7Qmf>3)KM7-_s)M9*oD*{{7`>o!y7_>cDpKtkbFF-#|iUEfTL-4{;0Xq zq7&}OXM)qt>7qI&6v4=5`<;Po?E@*k_3k22Ze$lJe%agI#6UR;yb)frCUnp`bEmj} zuz@P#_|ZoKY8~EL)PRsssiD$sMj0h&O?$yud4kH)|oMX7{0 zk2CC!lgTX}5ZS$dyV>bs={vtUNAjlvL#v_=8tjNl4vdd%b!TzcEi7&$EWj$X-J_b6~_MLKth2#*7bzuJlDf+G10j+EDOnzv5tr<_X6Eq!JnJx{d34Y;QxNiq9y zj#=rApNQ4MEZ^q78)R<GG|Om+ODsQZqqbBY`;mn4(xsrgY%F-EcT> zb}1&tr0MIDp30Lo5E=n{Ovvv{rjV|~A*QtCPSB&X3VBF!2}kN$eS&eDs24*l8zPO) zsV|tTR}R~?T+EweYD>Sf;j6!XxDT#ey7{mxmeF8ZS&93HrGMja6o)my9*)h^%z;P? zVtS!NZ*$FU!uPjwKrt1*g)pm#nC`+;kA1k_-hpF=l9z$p%h|+;9tl0eBRSJLwJwdX zz;AKk?W-gjg4zrUy~W}TzHmTSXG(0WJAD50XW|}{EtESK!KF1czI*-&3@8Q@et%;3 z-#j9-#lw()%(kipo?sAsufDOiBv{>RS7G~v!mStY!z;v9tb?d^VPfPc+r@BdA3qK`A`5PCMWveLvQs!U6oMfnCw#upDGVC8JTy`X@O!JxrRC{p zN}yapy1nfu>kLT04CI6bJyyQ% zD9al$G^9aKnKr)Zw}MB4?G(k;gY(zGF#so7R}^c4$Nc+OB>rDb^- zbZ6<}qmx_~=@~vEM~TJfnVinLfl_Ph=U;g(4)|+eyw_H?n@?mM?D(`MDDFNV9FHL`g@YA)H0@!#8w}IBExL7lo&A=~YI$ zluF7&n)Q5GgmcKWU;qWuEO>ZH3NCgi5(2yA0?9SD-p`YSJFdrPjS3<4r<~2P(&WeS zH6oEHg@STS3k7XG6+s4*3&e=o6ukjiDPWtvF-to1Io|QbrFl`;Op8g=@PQhuxTpWn zGUv!X4n?(iU+TuwmQ7CX2Q^2JwgG7fD5NL=e^Mssn4c)jlPJ+cWRoQDT$1e>BTw)! zM+e@Z^-lb5Wt#U?(UXe&Aoj8kKNP+>|F#FtDkXqzjw)u(@LQQXuoRRV6RuqPyYcLl z`>N&J3D(u0;wbpqlyB@5O$`B4AK)}jZ_KEHRe?_(Xr5Qbh)3SwqXN~=U7ZtNx+SSg z`}bQ|w=$1bFp&QNZ)WMW1kIv>f!ii0MHNyoO*rkxq9pnIe6H;U7Y5}d-|N16Jh}A* zz1rK=4Oa<sryBSBb z%CG2eRaW1e_+#uuc9Pi;aUN{{jmK7eIzFX?h^A4(;_z3BI>H*B4;@>SWh{g1%7#%+ qcQVEtn*acr#WCggtQs@`0n;pizy|>3&Us+5#Ao{g000001X)@eiYm4M literal 0 HcmV?d00001 diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 4679ab8e..ebe35571 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -128,6 +128,27 @@ assert_streq "${totalsize_orig}" "${totalsize_swapped}" echo 'ok generate + show endian swapped' +tar xf ${SRCDIR}/pre-endian-deltas-repo-big.tar.xz +mv pre-endian-deltas-repo{,-big} +tar xf ${SRCDIR}/pre-endian-deltas-repo-little.tar.xz +mv pre-endian-deltas-repo{,-little} +legacy_origrev=$(${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big rev-parse main^) +legacy_newrev=$(${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big rev-parse main) +${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big static-delta show ${legacy_origrev}-${legacy_newrev} > show-legacy-big.txt +totalsize_legacy_big=$(grep 'Total Size:' show-legacy-big.txt) +${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big static-delta show ${legacy_origrev}-${legacy_newrev} > show-legacy-little.txt +totalsize_legacy_little=$(grep 'Total Size:' show-legacy-little.txt) +for f in show-legacy-{big,little}.txt; do + if grep 'Endianness:.*heuristic' $f; then + found_heuristic=yes + break + fi +done +assert_streq "${found_heuristic}" "yes" +assert_streq "${totalsize_legacy_big}" "${totalsize_legacy_little}" + +echo 'ok heuristic endian detection' + mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2 ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev} ${CMD_PREFIX} ostree --repo=repo2 fsck From 5345573470f6c1929f00ff06572cb28b073e1d2f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 25 Feb 2016 11:07:30 -0500 Subject: [PATCH 53/54] deltas: Add a compression size heuristic for endianness detection I see when analyzing a delta here that due to byteswapping a negative compression ratio of 540%, 66%, and 28%. Let's arbitrarily pick 20% as a threshold for detecting byetswapping. --- src/libostree/ostree-repo-static-delta-core.c | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 0669f691..d84f0019 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -705,6 +705,7 @@ _ostree_delta_get_endianness (GVariant *superblock, { g_autoptr(GVariant) meta_entries = NULL; guint n_parts; guint i; + gboolean is_byteswapped = FALSE; g_variant_get_child (superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries); n_parts = g_variant_n_children (meta_entries); @@ -721,15 +722,36 @@ _ostree_delta_get_endianness (GVariant *superblock, total_objects += n_objects; total_size += size; total_usize += usize; + + if (size > usize) + { + double ratio = ((double)size)/((double)usize); + + /* This should really never happen where compressing things makes it more than 50% bigger. + */ + if (ratio > 1.2) + { + is_byteswapped = TRUE; + break; + } + } } - /* If the average object size is greater than 4GiB, let's assume - * we're dealing with opposite endianness. I'm fairly confident - * no one is going to be shipping peta- or exa- byte size ostree - * deltas, period. Past the gigabyte scale you really want - * bittorrent or something. - */ - if ((total_size / total_objects) > G_MAXUINT32) + if (!is_byteswapped) + { + /* If the average object size is greater than 4GiB, let's assume + * we're dealing with opposite endianness. I'm fairly confident + * no one is going to be shipping peta- or exa- byte size ostree + * deltas, period. Past the gigabyte scale you really want + * bittorrent or something. + */ + if ((total_size / total_objects) > G_MAXUINT32) + { + is_byteswapped = TRUE; + } + } + + if (is_byteswapped) { switch (G_BYTE_ORDER) { From 3d7098bc5e451d9c76813168b2c5eef5647b16de Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 26 Feb 2016 11:58:05 -0500 Subject: [PATCH 54/54] Release 2016.3 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 2229b145..0687ecdb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.63]) -AC_INIT([ostree], [2016.2], [walters@verbum.org]) +AC_INIT([ostree], [2016.3], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR([build-aux])