From 298c458fac4f70676ddc64adce2237e7d9a3bd83 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Sat, 14 Jan 2017 14:22:47 -0500 Subject: [PATCH 01/66] trivial-httpd: trivial option help string fixes Add an arg description for -P, otherwise it's not immediately obvious that it takes an argument. Mention that - is supported for --log-file. Closes: #657 Approved by: cgwalters --- src/ostree/ostree-trivial-httpd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index ddcb2fc0..72de8d79 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -59,12 +59,12 @@ typedef struct { static GOptionEntry options[] = { { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &opt_daemonize, "Fork into background when ready", NULL }, { "autoexit", 0, 0, G_OPTION_ARG_NONE, &opt_autoexit, "Automatically exit when directory is deleted", NULL }, - { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", NULL }, + { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", "PORT" }, { "port-file", 'p', 0, G_OPTION_ARG_FILENAME, &opt_port_file, "Write port number to PATH (- for standard output)", "PATH" }, { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL }, { "random-500s", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_percentage, "Generate random HTTP 500 errors approximately for PERCENTAGE requests", "PERCENTAGE" }, { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" }, - { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here", "PATH" }, + { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here (use - for stdout)", "PATH" }, { "expected-cookies", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_cookies, "Expect given cookies in the http request", "KEY=VALUE" }, { "expected-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_headers, "Expect given headers in the http request", "KEY=VALUE" }, { NULL } From 2b3076a8a09607754d913fd9799d5c6e881ed729 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Sun, 22 Jan 2017 19:06:54 -0500 Subject: [PATCH 02/66] docs: update pulp_ostree link Closes: #657 Approved by: cgwalters --- docs/manual/repository-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manual/repository-management.md b/docs/manual/repository-management.md index 64b2f6e1..a9db65ff 100644 --- a/docs/manual/repository-management.md +++ b/docs/manual/repository-management.md @@ -21,7 +21,7 @@ this document. Another example of software which can assist in managing OSTree repositories today is the [Pulp Project](http://www.pulpproject.org/), which has a -[Pulp OSTree plugin](https://pulp-ostree.readthedocs.org/en/latest/). +[Pulp OSTree plugin](https://docs.pulpproject.org/plugins/pulp_ostree/index.html). ## Mirroring repositories From ce10fd926197a102bd44f0faff3d59bb214336aa Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 27 Jan 2017 09:06:39 +0000 Subject: [PATCH 03/66] libostree: Don't distribute generated enumtypes in tarballs They are built at "make" time and cleaned up by "make clean", so there is no need to distribute them. Signed-off-by: Simon McVittie Closes: #665 Approved by: cgwalters --- Makefile-libostree.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 88210c75..7454845a 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -47,11 +47,12 @@ src/libostree/ostree-enumtypes.c: src/libostree/ostree-enumtypes.c.template $(EN --fhead "#include \"ostree-enumtypes.h\"" \ $(ENUM_TYPES) > $@.tmp && mv $@.tmp $@ -ENUM_GENERATED = \ +nodist_libostree_1_la_SOURCES = \ src/libostree/ostree-enumtypes.h \ src/libostree/ostree-enumtypes.c \ $(NULL) -BUILT_SOURCES += $(ENUM_GENERATED) + +BUILT_SOURCES += $(nodist_libostree_1_la_SOURCES) CLEANFILES += $(BUILT_SOURCES) @@ -61,7 +62,6 @@ libbupsplit_la_SOURCES = \ $(NULL) libostree_1_la_SOURCES = \ - $(ENUM_GENERATED) \ src/libostree/ostree-async-progress.c \ src/libostree/ostree-cmdprivate.h \ src/libostree/ostree-cmdprivate.c \ From 1fd05fe840770df2ecadda7c165157032033db54 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Jan 2017 07:44:20 +0100 Subject: [PATCH 04/66] lib: Adjust comments in symbols section for last release 2017.1 was released, move its section above the line. Closes: #661 Approved by: cgwalters --- src/libostree/libostree.sym | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 888a99ba..49a7d3f9 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -366,15 +366,20 @@ global: /* No new symbols in 2016.15 */ +LIBOSTREE_2017.1 { +global: + ostree_repo_prune_from_reachable; +} LIBOSTREE_2016.14; + /* NOTE NOTE NOTE * Versions above here are released. Only add symbols below this line. * NOTE NOTE NOTE */ -LIBOSTREE_2017.1 { +LIBOSTREE_2017.XX { global: - ostree_repo_prune_from_reachable; -} LIBOSTREE_2016.14; + someostree_symbol_deleteme; +} LIBOSTREE_2017.1; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste From a89be1f00fb0d810c6f5c06d4cd776d058074d13 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Jan 2017 10:55:22 +0100 Subject: [PATCH 05/66] lib: Prefix GPG errors with the checksum I was working on https://bugzilla.redhat.com/show_bug.cgi?id=1393545 and it was annoying that I couldn't know what the new (unsigned) commit has was until verification succeeded. I could pull it manually without GPG, but then it'd be sitting in the repo. Now: ``` Updating from: fedora-atomic:fedora-atomic/25/x86_64/docker-host Receiving metadata objects: 0/(estimating) -/s 0 bytes error: Commit 2fb89decd2cb5c3bd73983f0a7b35c7437f23e3aaa91698fab952bb224e46af5: GPG verification enabled, but no signatures found (use gpg-verify=false in remote config to disable) ``` Closes: #663 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 17 +++++++++++++---- src/libostree/ostree-repo.c | 11 ++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 45995a30..0adb65d4 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1035,14 +1035,22 @@ process_verify_result (OtPullData *pull_data, GError **error) { if (result == NULL) - return FALSE; + { + g_prefix_error (error, "Commit %s: ", checksum); + return FALSE; + } /* Allow callers to output the results immediately. */ g_signal_emit_by_name (pull_data->repo, "gpg-verify-result", checksum, result); - return ostree_gpg_verify_result_require_valid_signature (result, error); + if (!ostree_gpg_verify_result_require_valid_signature (result, error)) + { + g_prefix_error (error, "Commit %s: ", checksum); + return FALSE; + } + return TRUE; } static gboolean @@ -1060,8 +1068,9 @@ gpg_verify_unwritten_commit (OtPullData *pull_data, if (!detached_metadata) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No detached metadata found for GPG verification"); + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Commit %s: no detached metadata found for GPG verification", + checksum); return FALSE; } diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 3b3aa664..88e430b8 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4356,8 +4356,8 @@ _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, _OSTREE_METADATA_GPGSIGS_TYPE); if (!signaturedata) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "GPG verification enabled, but no signatures found (use gpg-verify=false in remote config to disable)"); + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "GPG verification enabled, but no signatures found (use gpg-verify=false in remote config to disable)"); return NULL; } @@ -4474,7 +4474,12 @@ ostree_repo_verify_commit (OstreeRepo *self, keyringdir, extra_keyring, cancellable, error); - return ostree_gpg_verify_result_require_valid_signature (result, error); + if (!ostree_gpg_verify_result_require_valid_signature (result, error)) + { + g_prefix_error (error, "Commit %s: ", commit_checksum); + return FALSE; + } + return TRUE; } /** From 95b15afe134cbbcab7f9c331126487889ef3fcb2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Jan 2017 11:21:43 +0100 Subject: [PATCH 06/66] travis: Disable tests (but keep builds) on flaky distros I went through the Travis history a bit, and these seem to be the flaky ones. The ubuntu one is likely no libsoup patches. The other one @smcv has partially traced to a GPGME race condition or something like that. For the libsoup one; as I say in comments, once we have libcurl, I'd like to enable that mostly everywhere, which should (hopefully) be more reliable. Closes: #664 Approved by: cgwalters --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 67bea019..27f84921 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,9 @@ dist: trusty sudo: required env: - - ci_distro=ubuntu ci_suite=trusty + - ci_distro=ubuntu ci_suite=trusty ci_test=no # TODO: use libcurl on this - ci_docker=debian:jessie-slim ci_distro=debian ci_suite=jessie - - ci_docker=debian:stretch-slim ci_distro=debian ci_suite=stretch + - ci_docker=debian:stretch-slim ci_distro=debian ci_suite=stretch ci_test=no # TODO gpgme flake https://github.com/ostreedev/ostree/pull/664#issuecomment-276033383 - ci_docker=ubuntu:xenial ci_distro=ubuntu ci_suite=xenial script: From 5eea1cdad8a5fa8a96c561248d9dabbc0b848016 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 25 Jan 2017 09:25:27 -0500 Subject: [PATCH 07/66] lib: Move the bupsplit selftest into our test framework We weren't running it before. Also I switched it to use GLib. Preparation for some oxidation work (having an implementation of bupsplit in Rust). I exported another function to do the raw rollsum operation which is what this test suite uses. Closes: #655 Approved by: jlebon --- src/libostree/bupsplit.c | 38 ++------------------------------------ src/libostree/bupsplit.h | 7 +++++-- tests/test-rollsum.c | 26 ++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/libostree/bupsplit.c b/src/libostree/bupsplit.c index 067b2e96..79207a65 100644 --- a/src/libostree/bupsplit.c +++ b/src/libostree/bupsplit.c @@ -80,7 +80,8 @@ static uint32_t rollsum_digest(Rollsum *r) } -static uint32_t rollsum_sum(uint8_t *buf, size_t ofs, size_t len) +uint32_t +bupsplit_sum(uint8_t *buf, size_t ofs, size_t len) { size_t count; Rollsum r; @@ -115,38 +116,3 @@ int bupsplit_find_ofs(const unsigned char *buf, int len, int *bits) } return 0; } - - -#ifndef BUP_NO_SELFTEST -#define BUP_SELFTEST_SIZE 100000 - -int bupsplit_selftest() -{ - uint8_t *buf = malloc(BUP_SELFTEST_SIZE); - uint32_t sum1a, sum1b, sum2a, sum2b, sum3a, sum3b; - unsigned count; - - srandom(1); - for (count = 0; count < BUP_SELFTEST_SIZE; count++) - buf[count] = random(); - - sum1a = rollsum_sum(buf, 0, BUP_SELFTEST_SIZE); - sum1b = rollsum_sum(buf, 1, BUP_SELFTEST_SIZE); - sum2a = rollsum_sum(buf, BUP_SELFTEST_SIZE - BUP_WINDOWSIZE*5/2, - BUP_SELFTEST_SIZE - BUP_WINDOWSIZE); - sum2b = rollsum_sum(buf, 0, BUP_SELFTEST_SIZE - BUP_WINDOWSIZE); - sum3a = rollsum_sum(buf, 0, BUP_WINDOWSIZE+3); - sum3b = rollsum_sum(buf, 3, BUP_WINDOWSIZE+3); - - fprintf(stderr, "sum1a = 0x%08x\n", sum1a); - fprintf(stderr, "sum1b = 0x%08x\n", sum1b); - fprintf(stderr, "sum2a = 0x%08x\n", sum2a); - fprintf(stderr, "sum2b = 0x%08x\n", sum2b); - fprintf(stderr, "sum3a = 0x%08x\n", sum3a); - fprintf(stderr, "sum3b = 0x%08x\n", sum3b); - - free(buf); - return sum1a!=sum1b || sum2a!=sum2b || sum3a!=sum3b; -} - -#endif // !BUP_NO_SELFTEST diff --git a/src/libostree/bupsplit.h b/src/libostree/bupsplit.h index e37a56ab..f770ee58 100644 --- a/src/libostree/bupsplit.h +++ b/src/libostree/bupsplit.h @@ -30,6 +30,9 @@ #ifndef __BUPSPLIT_H #define __BUPSPLIT_H +#include +#include + #define BUP_BLOBBITS (13) #define BUP_BLOBSIZE (1< Date: Sun, 29 Jan 2017 14:15:17 -0500 Subject: [PATCH 08/66] tests: Add setup for more realistic repo, change pull-many to use As OSTree has evolved over time, the tests grew with it. We didn't start out with static deltas or a summary file, and the tests reflect this. What I really want to do is change more of the pull tests, from corruption/proxying/mirroring etc. to use this more realistic repo rather than the tiny one the other test creates. We start by using some of the code from `test-pull-many.sh`, and change that test to use `--disable-static-deltas` for pull, since the point of that test is to *not* test deltas. Still TODO is investigate changing other tests to use this. Closes: #658 Approved by: jlebon --- tests/libtest.sh | 96 +++++++++++++++++++++++++++++++++++++++++ tests/test-pull-many.sh | 71 ++---------------------------- 2 files changed, 100 insertions(+), 67 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 927b8b1a..1c603d66 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -270,6 +270,102 @@ setup_fake_remote_repo1() { export OSTREE="${CMD_PREFIX} ostree --repo=repo" } +# Set up a large repository for stress testing. +# Something like the Fedora Atomic Workstation branch which has +# objects: meta: 7497 content: 103541 +# 9443 directories, 7097 symlinks, 112832 regfiles +# So we'll make ~11 files per dir, with one of them a symlink +# Actually, let's cut this down to 1/3 which is still useful. So: +# 3147 dirs, with still ~11 files per dir, for 37610 content objects +setup_exampleos_repo() { + args=${1:-} + cd ${test_tmpdir} + mkdir ostree-srv + mkdir -p ostree-srv/exampleos/{repo,build-repo} + export ORIGIN_REPO=ostree-srv/exampleos/repo + export ORIGIN_BUILD_REPO=ostree-srv/exampleos/build-repo + ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} init --mode=archive + ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} init --mode=bare-user + cd ${test_tmpdir} + rm main -rf + mkdir main + cd main + ndirs=3147 + depth=0 +# set +x # No need to spam the logs for this + echo "$(date): Generating initial content..." + while [ $ndirs -gt 0 ]; do + # 2/3 of the time, recurse a dir, up to a max of 9, otherwise back up + x=$(($ndirs % 3)) + case $x in + 0) if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi ;; + 1|2) if [ $depth -lt 9 ]; then + mkdir dir-${ndirs} + cd dir-${ndirs} + depth=$((depth+1)) + else + if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi + fi ;; + esac + # One symlink - we use somewhat predictable content to have dupes + ln -s $(($x % 20)) link-$ndirs + # 10 files + nfiles=10 + while [ $nfiles -gt 0 ]; do + echo file-$ndirs-$nfiles > f$ndirs-$nfiles + nfiles=$((nfiles-1)) + done + ndirs=$((ndirs-1)) + done + cd ${test_tmpdir} + set -x + + export REF=exampleos/42/standard + + ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} commit -b ${REF} --tree=dir=main + rm main -rf + ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} checkout ${REF} main + + find main > files.txt + nfiles=$(wc -l files.txt | cut -f 1 -d ' ') + # We'll make 5 more commits + for iter in $(seq 5); do + set +x + # Change 10% of files + for fiter in $(seq $(($nfiles / 10))); do + filenum=$(($RANDOM % ${nfiles})) + set +o pipefail + filename=$(tail -n +${filenum} < files.txt | head -1) + set -o pipefail + if test -f $filename; then + rm -f $filename + echo file-${iter}-${fiter} > ${filename} + fi + done + set -x + ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} commit --link-checkout-speedup -b ${REF} --tree=dir=main + done + + ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} pull-local --depth=-1 ${ORIGIN_BUILD_REPO} + + for x in "^^" "^" ""; do + ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} static-delta generate --from="${REF}${x}^" --to="${REF}${x}" + done + ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} summary -u + + cd ${test_tmpdir}/ostree-srv + mkdir httpd + ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd/httpd.log --daemonize -p httpd/port $args + port=$(cat httpd/port) + echo "http://127.0.0.1:${port}" > httpd/address + + cd ${test_tmpdir} + rm repo -rf + ${CMD_PREFIX} ostree --repo=repo init --mode=bare-user + ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat ostree-srv/httpd/address)/exampleos/repo + export OSTREE="${CMD_PREFIX} ostree --repo=repo" +} + setup_os_boot_syslinux() { # Stub syslinux configuration mkdir -p sysroot/boot/loader.0 diff --git a/tests/test-pull-many.sh b/tests/test-pull-many.sh index d90280d5..f39c8e6e 100755 --- a/tests/test-pull-many.sh +++ b/tests/test-pull-many.sh @@ -21,80 +21,17 @@ set -euo pipefail . $(dirname $0)/libtest.sh -setup_fake_remote_repo1 "archive-z2" -cd ${test_tmpdir} -rm ostree-srv/gnomerepo/ -rf -mkdir ostree-srv/gnomerepo/ -${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo init --mode=archive +setup_exampleos_repo echo '1..1' -# Simulate a large archive pull from scratch. Large here is -# something like the Fedora Atomic Workstation branch which has -# objects: meta: 7497 content: 103541 -# 9443 directories, 7097 symlinks, 112832 regfiles -# So we'll make ~11 files per dir, with one of them a symlink - cd ${test_tmpdir} -rm main -rf -mkdir main -cd main -ndirs=9443 -depth=0 -echo "$(date): Generating content..." -set +x # No need to spam the logs for this -while [ $ndirs -gt 0 ]; do - # 2/3 of the time, recurse a dir, up to a max of 9, otherwise back up - x=$(($ndirs % 3)) - case $x in - 0) if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi ;; - 1|2) if [ $depth -lt 9 ]; then - mkdir dir-${ndirs} - cd dir-${ndirs} - depth=$((depth+1)) - else - if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi - fi ;; - esac - - # One symlink - we use somewhat predictable content to have dupes - ln -s $(($x % 20)) link-$ndirs - # 10 files - nfiles=10 - while [ $nfiles -gt 0 ]; do - echo file-$ndirs-$nfiles > f$ndirs-$nfiles - nfiles=$((nfiles-1)) - done - ndirs=$((ndirs-1)) -done set -x -cd ${test_tmpdir} -mkdir build-repo -${CMD_PREFIX} ostree --repo=build-repo init --mode=bare-user -echo "$(date): Committing content..." -${CMD_PREFIX} ostree --repo=build-repo commit -b main -s 'big!' --tree=dir=main -for x in commit dirtree dirmeta file; do - find build-repo/objects -name '*.'${x} |wc -l > ${x}count - echo "$x: " $(cat ${x}count) -done -assert_file_has_content commitcount '^1$' -assert_file_has_content dirmetacount '^1$' -assert_file_has_content filecount '^94433$' -${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo pull-local build-repo echo "$(date): Pulling content..." -rm repo -rf -${CMD_PREFIX} ostree --repo=repo init --mode=archive -${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo - -${CMD_PREFIX} ostree --repo=repo pull --mirror origin main +rev=$(${CMD_PREFIX} ostree --repo=ostree-srv/exampleos/repo rev-parse ${REF}) +${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas --mirror origin ${REF} ${CMD_PREFIX} ostree --repo=repo fsck -for x in commit dirtree dirmeta filez; do - find repo/objects -name '*.'${x} |wc -l > ${x}count - echo "$x: " $(cat ${x}count) -done -assert_file_has_content commitcount '^1$' -assert_file_has_content dirmetacount '^1$' -assert_file_has_content filezcount '^94433$' +assert_streq ${rev} $(${CMD_PREFIX} ostree --repo=repo rev-parse ${REF}) echo "ok" From 7803fe1d60f32f659555acf32a8812a45ab15792 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Jan 2017 05:41:59 +0100 Subject: [PATCH 09/66] Rename to libOSTree There are many motivating factors. The biggest is simply that at a practical level, the command line is not sufficient to build a real system. The docs say that it's a demo for the library. Let's make that more obvious, so people don't try to use `ostree admin upgrade` for their real systems, and also don't use e.g. `ostree commit` on the command line outside of test suites/quick hacking. This change will also help clarify the role of rpm-ostree, which we will likely be renamed to "nts". Then use of the term "ostree" will become much clearer. And similarly for other people writing upgraders, they can say they use libostree. I didn't try to change all of the docs and code at once, because it's going to lead to conflicts. The next big steps are: - Rename the github repo (github will inject a redirect) - Look at supporting a build where we don't do `ostree admin`, or at least it's only built for tests. We may want to split it off as a separate binary or so? That way people with their own upgraders don't need to ship it. Closes: #659 Approved by: jlebon --- README.md | 24 +++++++++++++++--------- configure.ac | 4 ++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 369a8925..f189ebc9 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,25 @@ -OSTree +libOSTree ====== New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ ) ----- -OSTree is a tool that combines a "git-like" model for committing and -downloading bootable filesystem trees, along with a layer for -deploying them and managing the bootloader configuration. +This project is now known as "libOSTree", renamed from "OSTree"; the focus is on +the shared library. However, in most of the rest of the documentation, we will +use the term "OSTree", since it's slightly shorter, and changing all +documentation at once is impractical. We expect to transition to the new name +over time. -OSTree is like git in that it checksums individual files and has a -content-addressed-object store. It's unlike git in that it "checks -out" the files via hardlinks, and they should thus be immutable. -Therefore, another way to think of OSTree is that it's just a more -polished version of +libOSTree is a library and suite of command line tools that combines a +"git-like" model for committing and downloading bootable filesystem trees, along +with a layer for deploying them and managing the bootloader configuration. + +The core OSTree model is like git in that it checksums individual files and has +a content-addressed-object store. It's unlike git in that it "checks out" the +files via hardlinks, and they should thus be immutable. Therefore, another way +to think of OSTree is that it's just a more polished version +of [Linux VServer hardlinks](http://linux-vserver.org/index.php?title=util-vserver:Vhashify&oldid=2285). **Features:** diff --git a/configure.ac b/configure.ac index e3f2d12a..76f23684 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ([2.63]) dnl If incrementing the version here, remember to update libostree.sym too -AC_INIT([ostree], [2017.1], [walters@verbum.org]) +AC_INIT([libostree], [2017.1], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) @@ -334,7 +334,7 @@ src/libostree/ostree-1.pc AC_OUTPUT echo " - OSTree $VERSION + libOSTree $VERSION =============== From d894f609dbacd6d66aaff0402cab60a59901f73e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 24 Jan 2017 21:43:53 -0500 Subject: [PATCH 10/66] oxidation: Add implementation of bupsplit in Rust This is an initial drop of "oxidation", or adding implementation of components in Rust. The bupsplit code is a good target - no dependencies, just computation. Translation into Rust had a few twists - - The C code relies a lot on overflowing unsigned ints, and also on the C promotion rules for e.g. `uint8_t -> int32_t` - There were some odd loops that I introduced bugs in while translating...in particular, the function always returns `len`, but I mistakenly translated to `len+1`, resulting in an OOB read on the C side, which was hard to debug. On the plus side, an off-by-one array indexing in the Rust code paniced nicely. In practice, we'll need a lot more build infrastructure to make this work, such as using `cargo vendor` when producing build artifacts for example. Also, Cargo is yet another thing we need to cache. Where do we go with this? Well, I think we should merge this, it's not a lot of code. We can just have it be an alternative CI target. Should we do a lot more right now? Probably not immediately, but I find the medium/long term prospects pretty exciting! Closes: #656 Approved by: jlebon --- .redhat-ci.yml | 27 +++++++++ Makefile-libostree.am | 22 ++++--- Makefile-tests.am | 4 +- Makefile.am | 15 +++++ configure.ac | 34 +++++++++++ rust/Cargo.toml | 16 ++++++ rust/src/bupsplit.rs | 129 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 238 insertions(+), 9 deletions(-) create mode 100644 rust/Cargo.toml create mode 100644 rust/src/bupsplit.rs diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 5160b9c4..830320ef 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -46,3 +46,30 @@ env: tests: artifacts: + + +--- + +inherit: true + +context: f25-rust + +packages: + - cargo + +build: + config-opts: > + --prefix=/usr + --libdir=/usr/lib64 + --enable-installed-tests + --enable-gtk-doc + --enable-rust + +env: + CC: 'gcc' + +tests: + - make check TESTS=tests/test-rollsum + +artifacts: + - test-suite.log diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 7454845a..fceb846c 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -19,7 +19,19 @@ include Makefile-libostree-defines.am -noinst_LTLIBRARIES += libostree-kernel-args.la libbupsplit.la +noinst_LTLIBRARIES += libostree-kernel-args.la + + +if ENABLE_RUST +bupsplitpath = @abs_top_builddir@/target/@RUST_TARGET_SUBDIR@/libbupsplit_rs.a +.PHONY: $(bupsplitpath) +$(bupsplitpath): Makefile rust/src/bupsplit.rs + cd $(top_srcdir)/rust && CARGO_TARGET_DIR=@abs_top_builddir@/target cargo build --verbose $(CARGO_RELEASE_ARGS) +else +bupsplitpath = libbupsplit.la +noinst_LTLIBRARIES += libbupsplit.la +libbupsplit_la_SOURCES = src/libostree/bupsplit.h src/libostree/bupsplit.c +endif # ENABLE_RUST libostree_kernel_args_la_SOURCES = \ src/libostree/ostree-kernel-args.h \ @@ -56,11 +68,6 @@ BUILT_SOURCES += $(nodist_libostree_1_la_SOURCES) CLEANFILES += $(BUILT_SOURCES) -libbupsplit_la_SOURCES = \ - src/libostree/bupsplit.h \ - src/libostree/bupsplit.c \ - $(NULL) - libostree_1_la_SOURCES = \ src/libostree/ostree-async-progress.c \ src/libostree/ostree-cmdprivate.h \ @@ -142,7 +149,8 @@ libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$( $(OT_INTERNAL_GIO_UNIX_CFLAGS) $(OT_INTERNAL_GPGME_CFLAGS) $(OT_DEP_LZMA_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) \ -fvisibility=hidden '-D_OSTREE_PUBLIC=__attribute__((visibility("default"))) extern' libostree_1_la_LDFLAGS = -version-number 1:0:0 -Bsymbolic-functions -Wl,--version-script=$(top_srcdir)/src/libostree/libostree.sym -libostree_1_la_LIBADD = libotutil.la libbupsplit.la libglnx.la libbsdiff.la libostree-kernel-args.la $(OT_INTERNAL_GIO_UNIX_LIBS) $(OT_INTERNAL_GPGME_LIBS) $(OT_DEP_LZMA_LIBS) $(OT_DEP_ZLIB_LIBS) +libostree_1_la_LIBADD = libotutil.la libglnx.la libbsdiff.la libostree-kernel-args.la $(OT_INTERNAL_GIO_UNIX_LIBS) $(OT_INTERNAL_GPGME_LIBS) $(OT_DEP_LZMA_LIBS) $(OT_DEP_ZLIB_LIBS) +libostree_1_la_LIBADD += $(bupsplitpath) EXTRA_libostree_1_la_DEPENDENCIES = $(top_srcdir)/src/libostree/libostree.sym EXTRA_DIST += src/libostree/libostree.sym diff --git a/Makefile-tests.am b/Makefile-tests.am index a0c05488..8dbd3811 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -200,11 +200,11 @@ TESTS_LDADD = $(common_tests_ldadd) libostreetest.la tests_test_rollsum_cli_SOURCES = src/libostree/ostree-rollsum.c tests/test-rollsum-cli.c tests_test_rollsum_cli_CFLAGS = $(TESTS_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) -tests_test_rollsum_cli_LDADD = libbupsplit.la $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS) +tests_test_rollsum_cli_LDADD = $(bupsplitpath) $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS) tests_test_rollsum_SOURCES = src/libostree/ostree-rollsum.c tests/test-rollsum.c tests_test_rollsum_CFLAGS = $(TESTS_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) -tests_test_rollsum_LDADD = libbupsplit.la $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS) +tests_test_rollsum_LDADD = $(bupsplitpath) $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS) tests_test_mutable_tree_CFLAGS = $(TESTS_CFLAGS) tests_test_mutable_tree_LDADD = $(TESTS_LDADD) diff --git a/Makefile.am b/Makefile.am index 2911a0cf..4660515a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -60,6 +60,21 @@ GIRS = TYPELIBS = $(GIRS:.gir=.typelib) endif +# These bits based on gnome:librsvg/Makefile.am +if ENABLE_RUST +if RUST_DEBUG +CARGO_RELEASE_ARGS= +else +CARGO_RELEASE_ARGS=--release +endif + +check-local: + cd $(srcdir)/rust && CARGO_TARGET_DIR=$(abs_top_builddir)/target cargo test + +clean-local: + cd $(srcdir)/rust && CARGO_TARGET_DIR=$(abs_top_builddir)/target cargo clean +endif # end ENABLE_RUST + libglnx_srcpath := $(srcdir)/libglnx libglnx_cflags := $(OT_DEP_GIO_UNIX_CFLAGS) "-I$(libglnx_srcpath)" libglnx_libs := $(OT_DEP_GIO_UNIX_LIBS) diff --git a/configure.ac b/configure.ac index 76f23684..88e6ea1b 100644 --- a/configure.ac +++ b/configure.ac @@ -166,6 +166,39 @@ AS_IF([test "$enable_man" != no], [ ]) AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) +AC_ARG_ENABLE(rust, + [AS_HELP_STRING([--enable-rust], + [Compile Rust code instead of C [default=no]])],, + [enable_rust=no; rust_debug_release=no]) + +AS_IF([test "$enable_rust" = yes], [ + AC_PATH_PROG([cargo], [cargo]) + AS_IF([test -z "$cargo"], [AC_MSG_ERROR([cargo is required for --enable-rust])]) + AC_PATH_PROG([rustc], [rustc]) + AS_IF([test -z "$rustc"], [AC_MSG_ERROR([rustc is required for --enable-rust])]) + + dnl These bits based on gnome:librsvg/configure.ac + + dnl By default, we build in public release mode. + AC_ARG_ENABLE(rust-debug, + AC_HELP_STRING([--enable-rust-debug], + [Build Rust code with debugging information [default=no]]), + [rust_debug_release=$enableval], + [rust_debug_release=release]) + + AC_MSG_CHECKING(whether to build Rust code with debugging information) + if test "x$rust_debug_release" = "xyes" ; then + rust_debug_release=debug + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT(no) + fi + RUST_TARGET_SUBDIR=${rust_debug_release} + AC_SUBST([RUST_TARGET_SUBDIR]) +]) +AM_CONDITIONAL(RUST_DEBUG, [test "x$rust_debug_release" = "xdebug"]) +AM_CONDITIONAL(ENABLE_RUST, [test "$enable_rust" != no]) + AC_ARG_WITH(libarchive, AS_HELP_STRING([--without-libarchive], [Do not use libarchive]), :, with_libarchive=maybe) @@ -339,6 +372,7 @@ echo " introspection: $found_introspection + Rust (internal oxidation): $rust_debug_release rofiles-fuse: $enable_rofiles_fuse libsoup (retrieve remote HTTP repositories): $with_soup libsoup TLS client certs: $have_libsoup_client_certs diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..4da5ac32 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "bupsplit" +version = "0.0.1" +authors = ["Colin Walters "] + +[dependencies] +libc = "0.2" + +[lib] +name = "bupsplit_rs" +path = "src/bupsplit.rs" +crate-type = ["staticlib"] + +[profile.release] +panic = "abort" +lto = true diff --git a/rust/src/bupsplit.rs b/rust/src/bupsplit.rs new file mode 100644 index 00000000..af97e968 --- /dev/null +++ b/rust/src/bupsplit.rs @@ -0,0 +1,129 @@ +/* + * Copyright 2017 Colin Walters + * Based on original bupsplit.c: + * Copyright 2011 Avery Pennarun. All rights reserved. + * + * (This license applies to bupsplit.c and bupsplit.h only.) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY AVERY PENNARUN ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +extern crate libc; + +use std::slice; + +// According to librsync/rollsum.h: +// "We should make this something other than zero to improve the +// checksum algorithm: tridge suggests a prime number." +// apenwarr: I unscientifically tried 0 and 7919, and they both ended up +// slightly worse than the librsync value of 31 for my arbitrary test data. +const ROLLSUM_CHAR_OFFSET: u32 = 31; + +// Previously in the header file +const BUP_BLOBBITS: u32= 13; +const BUP_BLOBSIZE: u32 = (1< Rollsum { + Rollsum { s1: BUP_WINDOWSIZE * ROLLSUM_CHAR_OFFSET, + s2: BUP_WINDOWSIZE * (BUP_WINDOWSIZE-1) * ROLLSUM_CHAR_OFFSET, + window: [0; 64], + wofs: 0 + } + } + + // These formulas are based on rollsum.h in the librsync project. + pub fn add(&mut self, drop: u8, add: u8) -> () { + let drop_expanded = drop as u32; + let add_expanded = add as u32; + self.s1 = self.s1.wrapping_add(add_expanded.wrapping_sub(drop_expanded)); + self.s2 = self.s2.wrapping_add(self.s1.wrapping_sub(BUP_WINDOWSIZE * (drop_expanded + ROLLSUM_CHAR_OFFSET))); + } + + pub fn roll(&mut self, ch: u8) -> () { + let wofs = self.wofs as usize; + let dval = self.window[wofs]; + self.add(dval, ch); + self.window[wofs] = ch; + self.wofs = (self.wofs + 1) % (BUP_WINDOWSIZE as i32); + } + + pub fn digest(&self) -> u32 { + (self.s1 << 16) | (self.s2 & 0xFFFF) + } +} + +#[no_mangle] +pub extern fn bupsplit_sum(buf: *const u8, ofs: libc::size_t, len: libc::size_t) -> u32 { + let sbuf = unsafe { + assert!(!buf.is_null()); + slice::from_raw_parts(buf.offset(ofs as isize), (len - ofs) as usize) + }; + + let mut r = Rollsum::new(); + for x in sbuf { + r.roll(*x); + } + r.digest() +} + +#[no_mangle] +pub extern fn bupsplit_find_ofs(buf: *const u8, len: libc::size_t, + bits: *mut libc::c_int) -> libc::c_int +{ + let sbuf = unsafe { + assert!(!buf.is_null()); + slice::from_raw_parts(buf, len as usize) + }; + + let mut r = Rollsum::new(); + for x in sbuf { + r.roll(*x); + if (r.s2 & (BUP_BLOBSIZE-1)) == ((u32::max_value()) & (BUP_BLOBSIZE-1)) { + if !bits.is_null() { + let mut sum = r.digest() >> BUP_BLOBBITS; + let mut rbits : libc::c_int = BUP_BLOBBITS as i32; + while sum & 1 != 0 { + sum = sum >> 1; + rbits = rbits + 1; + } + unsafe { + *bits = rbits; + } + } + return len as i32 + } + } + 0 +} From e9b989f02dd8d464abfc5b93d70f84598dc87d96 Mon Sep 17 00:00:00 2001 From: Chen Fan Date: Fri, 3 Feb 2017 15:03:06 +0800 Subject: [PATCH 11/66] doc: fix typo in CONTRIBUTING Signed-off-by: Chen Fan Closes: #667 Approved by: cgwalters --- docs/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 36588113..bbe0d553 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -12,7 +12,7 @@ for more information. Instead, we use an instance of [Homu](https://github.com/servo/homu), currently known as `cgwalters-bot`. -As a review proceeeds, the preferred method is to push `fixup!` +As a review proceeds, the preferred method is to push `fixup!` commits via `git commit --fixup`. Homu knows how to use `--autosquash` when performing the final merge. See the [Git documentation](https://git-scm.com/docs/git-rebase) for more From 0400df75d13d103dd63d7a4ab0b3f17f4bd2f1c9 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 3 Feb 2017 14:10:27 -0500 Subject: [PATCH 12/66] trusted.gpg.d: keep in the same location With the package rename from ostree to libostree, the trusted.gpg.d/ dir changed install location from /usr/share/ostree to /usr/share/libostree. Let's keep the same dir to remain compatible with existing installations that may already have keys there. Closes: #668 Approved by: cgwalters --- Makefile-libostree.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index fceb846c..63f0eb7d 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -206,7 +206,7 @@ endif pkgconfig_DATA += src/libostree/ostree-1.pc gpgreadme_DATA = src/libostree/README-gpg -gpgreadmedir = $(pkgdatadir)/trusted.gpg.d +gpgreadmedir = $(datadir)/ostree/trusted.gpg.d EXTRA_DIST += src/libostree/README-gpg src/libostree/bupsplit.h \ src/libostree/ostree-enumtypes.h.template \ src/libostree/ostree-enumtypes.c.template \ From 200887c7c92032a1d939c1201d1d86813bdf4d80 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 5 Feb 2017 08:25:47 -0500 Subject: [PATCH 13/66] packaging/: Delete This is obsolete for a few reasons. The spec file is out of date (mostly the %files, but also the BRs), and further down the line we'll need to use `make dist` so we pick up vendored Rust sources. So the "git archive" approach won't work for much longer anyways. This came up in https://mail.gnome.org/archives/ostree-list/2017-February/msg00005.html Basically, anyone who wants to build packages should look at the upstream dist-git - until such time as e.g. Fedora learns to support pulling spec files from upstream. Closes: #670 Approved by: giuseppe --- packaging/.gitignore | 2 - packaging/91-ostree.preset | 1 - packaging/Makefile.dist-packaging | 40 ---------- packaging/ostree.spec.in | 120 ------------------------------ packaging/rpmbuild-cwd | 11 --- 5 files changed, 174 deletions(-) delete mode 100644 packaging/.gitignore delete mode 100644 packaging/91-ostree.preset delete mode 100644 packaging/Makefile.dist-packaging delete mode 100644 packaging/ostree.spec.in delete mode 100755 packaging/rpmbuild-cwd diff --git a/packaging/.gitignore b/packaging/.gitignore deleted file mode 100644 index ee5b2283..00000000 --- a/packaging/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -ostree-*/ -x86_64/ diff --git a/packaging/91-ostree.preset b/packaging/91-ostree.preset deleted file mode 100644 index ad0970b4..00000000 --- a/packaging/91-ostree.preset +++ /dev/null @@ -1 +0,0 @@ -enable ostree-remount.service diff --git a/packaging/Makefile.dist-packaging b/packaging/Makefile.dist-packaging deleted file mode 100644 index 096aeaba..00000000 --- a/packaging/Makefile.dist-packaging +++ /dev/null @@ -1,40 +0,0 @@ -# -*- mode: Makefile -*- - -GITREV = $$(git describe --always --tags) -GITREV_FOR_PKG = $(shell echo "$(GITREV)" | sed -e 's,-,\.,g' -e 's,^v,,') - -srcdir=$(shell dirname `pwd`) -PACKAGE=ostree - -PKG_VER = $(PACKAGE)-$(GITREV_FOR_PKG) - -dist-snapshot: - rm -f *.tar.xz - set -x; \ - echo "PACKAGE=$(PACKAGE)"; \ - TARFILE_TMP=$(PKG_VER).tar.tmp; \ - echo "Archiving $(PACKAGE) at $(GITREV)"; \ - (cd ..; git archive --format=tar --prefix=$(PKG_VER)/ $(GITREV)) > $${TARFILE_TMP}; \ - (cd $$(git rev-parse --show-toplevel); git submodule status) | while read line; do \ - rev=$$(echo $$line | cut -f 1 -d ' '); path=$$(echo $$line | cut -f 2 -d ' '); \ - echo "Archiving $${path} at $${rev}"; \ - (cd $(srcdir)/$$path; git archive --format=tar --prefix=$(PKG_VER)/$$path/ $${rev}) > submodule.tar; \ - tar -A -f $${TARFILE_TMP} submodule.tar; \ - rm submodule.tar; \ - done; \ - mv $(PKG_VER).tar.tmp $(PKG_VER).tar; \ - rm -f $(PKG_VER).tar.xz; \ - xz $(PKG_VER).tar - -srpm: dist-snapshot - (tmpd=`pwd`/tmp-packaging; rm -rf "$${tmpd}"; mkdir "$${tmpd}"; \ - sed -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(PACKAGE).spec.in > "$${tmpd}/$(PACKAGE).spec"; \ - cp 91-ostree.preset "$${tmpd}"; ln $(PKG_VER).tar.xz "$${tmpd}"; \ - cd "$${tmpd}" && ../rpmbuild-cwd -bs $(PACKAGE).spec && mv *.src.rpm ..) - -rpm: dist-snapshot - rm -f *.rpm - (tmpd=`pwd`/tmp-packaging; rm -rf "$${tmpd}"; mkdir "$${tmpd}"; \ - sed -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(PACKAGE).spec.in > "$${tmpd}/$(PACKAGE).spec"; \ - cp 91-ostree.preset "$${tmpd}"; ln $(PKG_VER).tar.xz "$${tmpd}"; \ - cd "$${tmpd}" && ../rpmbuild-cwd -bb $(PACKAGE).spec && mv $$(arch)/*.rpm ..) diff --git a/packaging/ostree.spec.in b/packaging/ostree.spec.in deleted file mode 100644 index eff0f353..00000000 --- a/packaging/ostree.spec.in +++ /dev/null @@ -1,120 +0,0 @@ -Summary: Git for operating system binaries -Name: ostree -Version: 2013.7 -Release: 3%{?dist} -#VCS: git:git://git.gnome.org/ostree -Source0: http://ftp.gnome.org/pub/GNOME/sources/ostree/%{version}/ostree-%{version}.tar.xz -Source1: 91-ostree.preset -License: LGPLv2+ -URL: http://live.gnome.org/OSTree - -# We always run autogen.sh -BuildRequires: autoconf automake libtool -# For docs -BuildRequires: gtk-doc -# Core requirements -BuildRequires: pkgconfig(gio-unix-2.0) -BuildRequires: pkgconfig(libsoup-2.4) -BuildRequires: pkgconfig(e2p) -# Extras -BuildRequires: pkgconfig(libarchive) -BuildRequires: pkgconfig(fuse) -BuildRequires: pkgconfig(libselinux) -BuildRequires: libcap-devel -BuildRequires: gpgme-devel -BuildRequires: pkgconfig(systemd) -BuildRequires: /usr/bin/g-ir-scanner -BuildRequires: dracut -BuildRequires: bison - -# Runtime requirements -Requires: dracut -Requires: systemd-units - -%description -OSTree is a tool for managing bootable, immutable, versioned -filesystem trees. While it takes over some of the roles of tradtional -"package managers" like dpkg and rpm, it is not a package system; nor -is it a tool for managing full disk images. Instead, it sits between -those levels, offering a blend of the advantages (and disadvantages) -of both. - -%package devel -Summary: Development headers for %{name} -Group: Development/Libraries -Requires: %{name} = %{version}-%{release} - -%description devel -The %{name}-devel package includes the header files for the %{name} library. - -%package grub2 -Summary: GRUB2 integration for OSTree -Group: Development/Libraries -Requires: grub2 - -%description grub2 -GRUB2 integration for OSTree - -%package fuse -Summary: FUSE utilities for OSTree -Group: Development/Libraries -Requires: fuse - -%description fuse -%{summary} - -%prep -%setup -q -n ostree-%{version} - -%build -env NOCONFIGURE=1 ./autogen.sh -%configure --disable-silent-rules \ - --enable-gtk-doc \ - --with-selinux \ - --with-dracut=yesbutnoconf -make %{?_smp_mflags} - -%install -make install DESTDIR=$RPM_BUILD_ROOT INSTALL="install -p -c" -find $RPM_BUILD_ROOT -name '*.la' -delete -install -D -m 0644 %{SOURCE1} $RPM_BUILD_ROOT/%{_prefix}/lib/systemd/system-preset/91-ostree.preset - -%clean -rm -rf $RPM_BUILD_ROOT - -%post -%systemd_post ostree-remount.service - -%preun -%systemd_preun ostree-remount.service - -%files -%doc COPYING README.md -%{_bindir}/ostree -%{_sbindir}/ostree-prepare-root -%{_sbindir}/ostree-remount -%dir %{_prefix}/lib/dracut/modules.d/98ostree -%{_prefix}/lib/systemd/system/ostree*.service -%{_prefix}/lib/dracut/modules.d/98ostree/* -%{_libdir}/*.so.1* -%{_libdir}/girepository-1.0/OSTree-1.0.typelib -%{_mandir}/man1/*.gz -%{_mandir}/man5/*.gz -%{_prefix}/lib/systemd/system-preset/91-ostree.preset -%dir %{_sysconfdir}/ostree/remotes.d - -%files devel -%{_libdir}/lib*.so -%{_includedir}/* -%{_libdir}/pkgconfig/* -%{_datadir}/ostree -%dir %{_datadir}/gtk-doc/html/ostree -%{_datadir}/gtk-doc/html/ostree -%{_datadir}/gir-1.0/OSTree-1.0.gir - -%files grub2 -%{_sysconfdir}/grub.d/*ostree -%{_libexecdir}/ostree/grub2* - -%files fuse -%{_bindir}/rofiles-fuse diff --git a/packaging/rpmbuild-cwd b/packaging/rpmbuild-cwd deleted file mode 100755 index d0805bb5..00000000 --- a/packaging/rpmbuild-cwd +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# rpmbuild-cwd: -# Run "rpmbuild", defining all RPM variables to use the current directory. -# This matches Fedora's system. -# -# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) -# Copyright (C) 2010 Red Hat, Inc. -# Written by Colin Walters - -pwd=$(pwd) -exec rpmbuild --define "_sourcedir ${pwd}" --define "_specdir ${pwd}" --define "_builddir ${pwd}" --define "_srcrpmdir ${pwd}" --define "_rpmdir ${pwd}" --define "_buildrootdir ${pwd}/.build" "$@" From 9c0af41710fc3800a3b02dd552b96890bf1d432f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Jan 2017 07:43:37 +0100 Subject: [PATCH 14/66] lib: Add ostree_repo_reload_config() For a long time we've cached the remote configs in the repo, which mostly makes sense for the `repo/config` file, but less sense for `/etc/ostree/remotes.d`, because we want to support admins interactively editing them. One can delete the repo instance and create a new one, but that's a bit ugly. Let's introduce an API for this so rpm-ostree can reload remotes after admins/scripts edit them in `/etc`. We also might as well reload any other entries in the config. Structurually now, `ostree_repo_open()` deals with file descriptors, and then calls `ostree_repo_reload_config()`. Except for the uncompressed cache, which is the only thing that deals with FDs that can be configured. But we want to delete that anyways. No tests, since...we don't have a daemon in this codebase, don't want to shave that yak just today. Closes: #662 Approved by: jlebon --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 4 +- src/libostree/ostree-repo.c | 242 +++++++++++++++++++++--------------- src/libostree/ostree-repo.h | 5 + 4 files changed, 147 insertions(+), 105 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index dbd12526..ed606604 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -276,6 +276,7 @@ ostree_repo_remote_get_gpg_verify_summary ostree_repo_remote_gpg_import ostree_repo_remote_fetch_summary ostree_repo_remote_fetch_summary_with_options +ostree_repo_reload_config ostree_repo_get_remote_boolean_option ostree_repo_get_remote_list_option ostree_repo_get_remote_option diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 49a7d3f9..1345b9c1 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -376,9 +376,9 @@ global: * NOTE NOTE NOTE */ -LIBOSTREE_2017.XX { +LIBOSTREE_2017.2 { global: - someostree_symbol_deleteme; + ostree_repo_reload_config; } LIBOSTREE_2017.1; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 88e430b8..fd49f0fc 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2002,14 +2002,125 @@ get_remotes_d_dir (OstreeRepo *self) } static gboolean -append_remotes_d (OstreeRepo *self, - GCancellable *cancellable, - GError **error) +reload_core_config (OstreeRepo *self, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *version = NULL; + g_autofree char *mode = NULL; + g_autofree char *contents = NULL; + g_autofree char *parent_repo_path = NULL; + gboolean is_archive; + gsize len; + + g_clear_pointer (&self->config, (GDestroyNotify)g_key_file_unref); + self->config = g_key_file_new (); + + contents = glnx_file_get_contents_utf8_at (self->repo_dir_fd, "config", &len, + NULL, error); + if (!contents) + return FALSE; + if (!g_key_file_load_from_data (self->config, contents, len, 0, error)) + { + g_prefix_error (error, "Couldn't parse config file: "); + return FALSE; + } + + version = g_key_file_get_value (self->config, "core", "repo_version", error); + if (!version) + return FALSE; + + if (strcmp (version, "1") != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid repository version '%s'", version); + return FALSE; + } + + if (!ot_keyfile_get_boolean_with_default (self->config, "core", "archive", + FALSE, &is_archive, error)) + return FALSE; + if (is_archive) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of OSTree no longer supports \"archive\" repositories; use archive-z2 instead"); + return FALSE; + } + + if (!ot_keyfile_get_value_with_default (self->config, "core", "mode", + "bare", &mode, error)) + return FALSE; + if (!ostree_repo_mode_from_string (mode, &self->mode, error)) + return FALSE; + + if (self->writable) + { + if (!ot_keyfile_get_boolean_with_default (self->config, "core", "enable-uncompressed-cache", + TRUE, &self->enable_uncompressed_cache, error)) + return FALSE; + } + else + self->enable_uncompressed_cache = FALSE; + + { + gboolean do_fsync; + + if (!ot_keyfile_get_boolean_with_default (self->config, "core", "fsync", + TRUE, &do_fsync, error)) + return FALSE; + + if (!do_fsync) + ostree_repo_set_disable_fsync (self, TRUE); + } + + { g_autofree char *tmp_expiry_seconds = NULL; + + /* 86400 secs = one day */ + if (!ot_keyfile_get_value_with_default (self->config, "core", "tmp-expiry-secs", "86400", + &tmp_expiry_seconds, error)) + return FALSE; + + self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10); + } + + if (!ot_keyfile_get_value_with_default (self->config, "core", "parent", + NULL, &parent_repo_path, error)) + return FALSE; + + if (parent_repo_path && parent_repo_path[0]) + { + g_autoptr(GFile) parent_repo_f = g_file_new_for_path (parent_repo_path); + + g_clear_object (&self->parent_repo); + self->parent_repo = ostree_repo_new (parent_repo_f); + + if (!ostree_repo_open (self->parent_repo, cancellable, error)) + { + g_prefix_error (error, "While checking parent repository '%s': ", + gs_file_get_path_cached (parent_repo_f)); + return FALSE; + } + } + + return TRUE; +} + +static gboolean +reload_remote_config (OstreeRepo *self, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; g_autoptr(GFile) remotes_d = NULL; g_autoptr(GFileEnumerator) direnum = NULL; + g_mutex_lock (&self->remotes_lock); + g_hash_table_remove_all (self->remotes); + g_mutex_unlock (&self->remotes_lock); + + if (!add_remotes_from_keyfile (self, self->config, NULL, error)) + goto out; + remotes_d = get_remotes_d_dir (self); if (remotes_d == NULL) return TRUE; @@ -2050,17 +2161,34 @@ append_remotes_d (OstreeRepo *self, return ret; } +/** + * ostree_repo_reload_config: + * @self: repo + * @cancellable: cancellable + * @error: error + * + * By default, an #OstreeRepo will cache the remote configuration and its + * own repo/config data. This API can be used to reload it. + */ +gboolean +ostree_repo_reload_config (OstreeRepo *self, + GCancellable *cancellable, + GError **error) +{ + if (!reload_core_config (self, cancellable, error)) + return FALSE; + if (!reload_remote_config (self, cancellable, error)) + return FALSE; + return TRUE; +} + gboolean ostree_repo_open (OstreeRepo *self, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - gboolean is_archive; struct stat stbuf; - g_autofree char *version = NULL; - g_autofree char *mode = NULL; - g_autofree char *parent_repo_path = NULL; g_return_val_if_fail (error == NULL || *error == NULL, FALSE); @@ -2126,102 +2254,6 @@ ostree_repo_open (OstreeRepo *self, self->target_owner_uid = self->target_owner_gid = -1; } - self->config = g_key_file_new (); - - { g_autofree char *contents = NULL; - gsize len; - - contents = glnx_file_get_contents_utf8_at (self->repo_dir_fd, "config", &len, - NULL, error); - if (!contents) - goto out; - if (!g_key_file_load_from_data (self->config, contents, len, 0, error)) - { - g_prefix_error (error, "Couldn't parse config file: "); - goto out; - } - } - if (!add_remotes_from_keyfile (self, self->config, NULL, error)) - goto out; - - version = g_key_file_get_value (self->config, "core", "repo_version", error); - if (!version) - goto out; - - if (strcmp (version, "1") != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid repository version '%s'", version); - goto out; - } - - if (!ot_keyfile_get_boolean_with_default (self->config, "core", "archive", - FALSE, &is_archive, error)) - goto out; - if (is_archive) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "This version of OSTree no longer supports \"archive\" repositories; use archive-z2 instead"); - goto out; - } - - if (!ot_keyfile_get_value_with_default (self->config, "core", "mode", - "bare", &mode, error)) - goto out; - if (!ostree_repo_mode_from_string (mode, &self->mode, error)) - goto out; - - if (!ot_keyfile_get_value_with_default (self->config, "core", "parent", - NULL, &parent_repo_path, error)) - goto out; - - if (parent_repo_path && parent_repo_path[0]) - { - g_autoptr(GFile) parent_repo_f = g_file_new_for_path (parent_repo_path); - - self->parent_repo = ostree_repo_new (parent_repo_f); - - if (!ostree_repo_open (self->parent_repo, cancellable, error)) - { - g_prefix_error (error, "While checking parent repository '%s': ", - gs_file_get_path_cached (parent_repo_f)); - goto out; - } - } - - if (self->writable) - { - if (!ot_keyfile_get_boolean_with_default (self->config, "core", "enable-uncompressed-cache", - TRUE, &self->enable_uncompressed_cache, error)) - goto out; - } - else - self->enable_uncompressed_cache = FALSE; - - { - gboolean do_fsync; - - if (!ot_keyfile_get_boolean_with_default (self->config, "core", "fsync", - TRUE, &do_fsync, error)) - goto out; - - if (!do_fsync) - ostree_repo_set_disable_fsync (self, TRUE); - } - - { g_autofree char *tmp_expiry_seconds = NULL; - - /* 86400 secs = one day */ - if (!ot_keyfile_get_value_with_default (self->config, "core", "tmp-expiry-secs", "86400", - &tmp_expiry_seconds, error)) - goto out; - - self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10); - } - - if (!append_remotes_d (self, cancellable, error)) - goto out; - if (!glnx_opendirat (self->repo_dir_fd, "tmp", TRUE, &self->tmp_dir_fd, error)) goto out; @@ -2234,6 +2266,10 @@ ostree_repo_open (OstreeRepo *self, goto out; } + if (!ostree_repo_reload_config (self, cancellable, error)) + goto out; + + /* TODO - delete this */ if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && self->enable_uncompressed_cache) { if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, "uncompressed-objects-cache", 0755, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 341a4d9c..648bd129 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -101,6 +101,11 @@ GKeyFile * ostree_repo_get_config (OstreeRepo *self); _OSTREE_PUBLIC GKeyFile * ostree_repo_copy_config (OstreeRepo *self); +_OSTREE_PUBLIC +gboolean ostree_repo_reload_config (OstreeRepo *self, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_remote_add (OstreeRepo *self, const char *name, From 9169268c31df31cc09495e2a04c30cd251f22b5d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 4 Feb 2017 11:29:20 -0500 Subject: [PATCH 15/66] rust: Support `make dist` -> cargo vendor What we do here basically is set things up in a `dist-hook` so that our Rust sources are vendored at `dist` time. This gives us a single tarball still, and ideally should be transparent to downstream builders, as long as they have the `cargo/rust` toolchain. Closes: #669 Approved by: jlebon --- Makefile-libostree.am | 4 +++- Makefile.am | 9 +++++++++ rust/cargo-vendor-config | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 rust/cargo-vendor-config diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 63f0eb7d..d40196d4 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -25,7 +25,9 @@ noinst_LTLIBRARIES += libostree-kernel-args.la if ENABLE_RUST bupsplitpath = @abs_top_builddir@/target/@RUST_TARGET_SUBDIR@/libbupsplit_rs.a .PHONY: $(bupsplitpath) -$(bupsplitpath): Makefile rust/src/bupsplit.rs +BUPSPLIT_RUST_SOURCES = rust/src/bupsplit.rs +EXTRA_DIST += $(BUPSPLIT_RUST_SOURCES) +$(bupsplitpath): Makefile $(BUPSPLIT_RUST_SOURCES) cd $(top_srcdir)/rust && CARGO_TARGET_DIR=@abs_top_builddir@/target cargo build --verbose $(CARGO_RELEASE_ARGS) else bupsplitpath = libbupsplit.la diff --git a/Makefile.am b/Makefile.am index 4660515a..31c41251 100644 --- a/Makefile.am +++ b/Makefile.am @@ -73,6 +73,15 @@ check-local: clean-local: cd $(srcdir)/rust && CARGO_TARGET_DIR=$(abs_top_builddir)/target cargo clean + +dist-hook: + (cd $(distdir)/rust && \ + cp $(abs_top_srcdir)/rust/Cargo.lock . && \ + cargo vendor -q && \ + mkdir .cargo && \ + cp cargo-vendor-config .cargo/config) + +EXTRA_DIST += $(srcdir)/rust/Cargo.toml $(srcdir)/rust/cargo-vendor-config endif # end ENABLE_RUST libglnx_srcpath := $(srcdir)/libglnx diff --git a/rust/cargo-vendor-config b/rust/cargo-vendor-config new file mode 100644 index 00000000..5407266e --- /dev/null +++ b/rust/cargo-vendor-config @@ -0,0 +1,8 @@ +# This is used after `cargo vendor` is run from `make dist` + +[source.crates-io] +registry = 'https://github.com/rust-lang/crates.io-index' +replace-with = 'vendored-sources' + +[source.vendored-sources] +directory = './vendor' From 3d38f03e4fd4cc20f754bb787feb0d109387f4f8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 7 Feb 2017 08:59:32 -0500 Subject: [PATCH 16/66] repo: Add archive/zlib-level option, drop default compression to 6 The gzip default is 6. When I was writing this code, I chose 9 under the assumption that for long-term archival, the extra compression was worth it. Turns out level 9 is really, really not worth it. Here's run at level 9 compressing the current Fedora Atomic Host into archive: ``` ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host real 2m38.115s user 2m31.210s sys 0m3.114s 617M repo ``` And here's the new default level of 6: ``` ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host real 0m53.712s user 0m43.727s sys 0m3.601s 619M repo 619M total ``` As you can see, we run almost *three times* faster, and we take up *less than one percent* more space. Conclusion: Using level 9 is dumb. And here's a run at compression level 1: ``` ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host real 0m24.073s user 0m17.574s sys 0m2.636s 643M repo 643M total ``` I would argue actually many people would prefer even this for "devel" repos. For production repos, you want static deltas anyways. (However, perhaps we should support a model where generating a delta involves re-compressing fallback objects with a bit stronger compression level). Anyways, let's make everyone's life better and switch the default to 6. Closes: #671 Approved by: jlebon --- src/libostree/ostree-core-private.h | 15 +++++++++ src/libostree/ostree-core.c | 48 +++++++++++++++++++---------- src/libostree/ostree-repo-commit.c | 2 +- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 13 ++++++++ 5 files changed, 61 insertions(+), 18 deletions(-) diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 0c5fb0eb..cfd8a998 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -24,6 +24,9 @@ G_BEGIN_DECLS +/* It's what gzip does, 9 is too slow */ +#define OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL (6) + /* This file contains private implementation data format definitions * read by multiple implementation .c files. */ @@ -143,4 +146,16 @@ _ostree_detached_metadata_append_gpg_sig (GVariant *existing_metadata, GFile * _ostree_get_default_sysroot_path (void); +_OSTREE_PUBLIC +gboolean +_ostree_raw_file_to_archive_stream (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + guint compression_level, + GInputStream **out_input, + GCancellable *cancellable, + GError **error); + + + G_END_DECLS diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index e3f0a771..af36d98b 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -453,6 +453,34 @@ header_and_input_to_stream (GVariant *file_header, return TRUE; } +gboolean +_ostree_raw_file_to_archive_stream (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + guint compression_level, + GInputStream **out_input, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariant) file_header = NULL; + g_autoptr(GInputStream) zlib_input = NULL; + + file_header = _ostree_zlib_file_header_new (file_info, xattrs); + if (input != NULL) + { + g_autoptr(GConverter) zlib_compressor = NULL; + + zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, compression_level)); + zlib_input = g_converter_input_stream_new (input, zlib_compressor); + } + return header_and_input_to_stream (file_header, + zlib_input, + out_input, + NULL, + cancellable, + error); +} + /** * ostree_raw_file_to_archive_z2_stream: * @input: File raw content stream @@ -473,23 +501,9 @@ ostree_raw_file_to_archive_z2_stream (GInputStream *input, GCancellable *cancellable, GError **error) { - g_autoptr(GVariant) file_header = NULL; - g_autoptr(GInputStream) zlib_input = NULL; - - file_header = _ostree_zlib_file_header_new (file_info, xattrs); - if (input != NULL) - { - g_autoptr(GConverter) zlib_compressor = NULL; - - zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9)); - zlib_input = g_converter_input_stream_new (input, zlib_compressor); - } - return header_and_input_to_stream (file_header, - zlib_input, - out_input, - NULL, - cancellable, - error); + return _ostree_raw_file_to_archive_stream (input, file_info, xattrs, + OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL, + out_input, cancellable, error); } /** diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index eb9733e8..07940f48 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -719,7 +719,7 @@ write_object (OstreeRepo *self, if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { - zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9); + zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, self->zlib_compression_level); compressed_out_stream = g_converter_output_stream_new (temp_out, zlib_compressor); /* Don't close the base; we'll do that later */ g_filter_output_stream_set_close_base_stream ((GFilterOutputStream*)compressed_out_stream, FALSE); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index a4e59e44..cfc178f3 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -89,6 +89,7 @@ struct OstreeRepo { GError *writable_error; gboolean in_transaction; gboolean disable_fsync; + guint zlib_compression_level; GHashTable *loose_object_devino_hash; GHashTable *updated_uncompressed_dirs; GHashTable *object_sizes; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index fd49f0fc..4ac39d11 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2083,6 +2083,19 @@ reload_core_config (OstreeRepo *self, self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10); } + { g_autofree char *compression_level_str = NULL; + + /* gzip defaults to 6 */ + (void)ot_keyfile_get_value_with_default (self->config, "archive", "zlib-level", NULL, + &compression_level_str, NULL); + + if (compression_level_str) + /* Ensure level is in [1,9] */ + self->zlib_compression_level = MAX (1, MIN (9, g_ascii_strtoull (compression_level_str, NULL, 10))); + else + self->zlib_compression_level = OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL; + } + if (!ot_keyfile_get_value_with_default (self->config, "core", "parent", NULL, &parent_repo_path, error)) return FALSE; From c18628ecb8b2ef591db0444fa57052cba60807a8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 18 Jan 2017 21:57:07 -0500 Subject: [PATCH 17/66] pull: Add queuing into the higher level logic Working on the libcurl backend, I didn't want to reimplement another queue. I think the queue logic is really better done at the high level, since the fetcher knows how we want to prioritize metadata over content, etc. Adding another queue here is duplication, but things will look nicer when we can actually delete the libsoup one in the next commit. Closes: #654 Approved by: jlebon --- src/libostree/ostree-repo-private.h | 3 + src/libostree/ostree-repo-pull.c | 237 +++++++++++++++++++++++----- 2 files changed, 198 insertions(+), 42 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index cfc178f3..73e02446 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -32,6 +32,9 @@ G_BEGIN_DECLS #define _OSTREE_SUMMARY_CACHE_DIR "summaries" #define _OSTREE_CACHE_DIR "cache" +#define _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS 8 +#define _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS 2 + typedef enum { OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0) } OstreeRepoTestErrorFlags; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0adb65d4..21f3007e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -81,6 +81,9 @@ typedef struct { GHashTable *scanned_metadata; /* Maps object name to itself */ GHashTable *requested_metadata; /* Maps object name to itself */ GHashTable *requested_content; /* Maps checksum to itself */ + GHashTable *pending_fetch_metadata; /* Map */ + GHashTable *pending_fetch_content; /* Map */ + GHashTable *pending_fetch_deltaparts; /* Set */ guint n_outstanding_metadata_fetches; guint n_outstanding_metadata_write_requests; guint n_outstanding_content_fetches; @@ -133,6 +136,10 @@ typedef struct { OtPullData *pull_data; GVariant *objects; char *expected_checksum; + char *from_revision; + char *to_revision; + guint i; + guint64 size; } FetchStaticDeltaData; typedef struct { @@ -142,6 +149,10 @@ typedef struct { guint recursion_depth; } ScanObjectQueueData; +static void start_fetch (OtPullData *pull_data, FetchObjectData *fetch); +static void start_fetch_deltapart (OtPullData *pull_data, + FetchStaticDeltaData *fetch); +static gboolean fetcher_queue_is_full (OtPullData *pull_data); static void queue_scan_one_metadata_object (OtPullData *pull_data, const char *csum, OstreeObjectType objtype, @@ -271,6 +282,77 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_error_free (error); } } + else + { + GHashTableIter hiter; + gpointer key, value; + + /* We may have just completed an async fetch operation. Now we look at + * possibly enqueuing more requests. The goal of queuing is to both avoid + * overloading the fetcher backend with HTTP requests, but also to + * prioritize metadata fetches over content, so we have accurate + * reporting. Hence here, we process metadata fetches first. + */ + + /* Try filling the queue with metadata we need to fetch */ + g_hash_table_iter_init (&hiter, pull_data->pending_fetch_metadata); + while (!fetcher_queue_is_full (pull_data) && + g_hash_table_iter_next (&hiter, &key, &value)) + { + GVariant *objname = key; + FetchObjectData *fetch = value; + + /* Steal both key and value */ + g_hash_table_iter_steal (&hiter); + + /* This takes ownership of the value */ + start_fetch (pull_data, fetch); + /* And unref the key */ + g_variant_unref (objname); + } + + /* Now, process deltapart requests */ + g_hash_table_iter_init (&hiter, pull_data->pending_fetch_deltaparts); + while (!fetcher_queue_is_full (pull_data) && + g_hash_table_iter_next (&hiter, &key, &value)) + { + FetchStaticDeltaData *fetch = key; + g_hash_table_iter_steal (&hiter); + /* Takes ownership */ + start_fetch_deltapart (pull_data, fetch); + } + + /* Next, fill the queue with content */ + g_hash_table_iter_init (&hiter, pull_data->pending_fetch_content); + while (!fetcher_queue_is_full (pull_data) && + g_hash_table_iter_next (&hiter, &key, &value)) + { + char *checksum = key; + FetchObjectData *fetch = value; + + /* Steal both key and value */ + g_hash_table_iter_steal (&hiter); + + /* This takes ownership of the value */ + start_fetch (pull_data, fetch); + /* And unref the key */ + g_free (checksum); + } + + } +} + +/* We have a total-request limit, as well has a hardcoded max of 2 for delta + * parts. The logic for the delta one is that processing them is expensive, and + * doing multiple simultaneously could risk space/memory on smaller devices. + */ +static gboolean +fetcher_queue_is_full (OtPullData *pull_data) +{ + return (pull_data->n_outstanding_metadata_fetches + + pull_data->n_outstanding_content_fetches + + pull_data->n_outstanding_deltapart_fetches) == _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS || + pull_data->n_outstanding_deltapart_fetches == _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS; } static gboolean @@ -942,6 +1024,8 @@ fetch_static_delta_data_free (gpointer data) FetchStaticDeltaData *fetch_data = data; g_free (fetch_data->expected_checksum); g_variant_unref (fetch_data->objects); + g_free (fetch_data->from_revision); + g_free (fetch_data->to_revision); g_free (fetch_data); } @@ -1343,41 +1427,11 @@ enqueue_one_object_request (OtPullData *pull_data, gboolean is_detached_meta, gboolean object_is_stored) { - g_autofree char *obj_subpath = NULL; gboolean is_meta; FetchObjectData *fetch_data; - guint64 *expected_max_size_p; - guint64 expected_max_size; - GPtrArray *mirrorlist = NULL; - - g_debug ("queuing fetch of %s.%s%s", checksum, - ostree_object_type_to_string (objtype), - is_detached_meta ? " (detached)" : ""); - - if (is_detached_meta) - { - char buf[_OSTREE_LOOSE_PATH_MAX]; - _ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode); - obj_subpath = g_build_filename ("objects", buf, NULL); - mirrorlist = pull_data->meta_mirrorlist; - } - else - { - obj_subpath = _ostree_get_relative_object_path (checksum, objtype, TRUE); - mirrorlist = pull_data->content_mirrorlist; - } is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); - if (is_meta) - { - pull_data->n_outstanding_metadata_fetches++; - pull_data->n_requested_metadata++; - } - else - { - pull_data->n_outstanding_content_fetches++; - pull_data->n_requested_content++; - } + fetch_data = g_new0 (FetchObjectData, 1); fetch_data->pull_data = pull_data; fetch_data->object = ostree_object_name_serialize (checksum, objtype); @@ -1385,10 +1439,80 @@ enqueue_one_object_request (OtPullData *pull_data, fetch_data->is_detached_meta = is_detached_meta; fetch_data->object_is_stored = object_is_stored; - expected_max_size_p = is_detached_meta ? NULL : g_hash_table_lookup (pull_data->expected_commit_sizes, checksum); + if (is_meta) + pull_data->n_requested_metadata++; + else + pull_data->n_requested_content++; + + /* Are too many requests are in flight? */ + if (fetcher_queue_is_full (pull_data)) + { + g_debug ("queuing fetch of %s.%s%s", checksum, + ostree_object_type_to_string (objtype), + is_detached_meta ? " (detached)" : ""); + + if (is_meta) + { + GVariant *objname = ostree_object_name_serialize (checksum, objtype); + g_hash_table_insert (pull_data->pending_fetch_metadata, objname, fetch_data); + } + else + { + g_hash_table_insert (pull_data->pending_fetch_content, g_strdup (checksum), fetch_data); + } + } + else + { + start_fetch (pull_data, fetch_data); + } +} + +static void +start_fetch (OtPullData *pull_data, + FetchObjectData *fetch) +{ + gboolean is_meta; + g_autofree char *obj_subpath = NULL; + guint64 *expected_max_size_p; + guint64 expected_max_size; + const char *expected_checksum; + OstreeObjectType objtype; + GPtrArray *mirrorlist = NULL; + + ostree_object_name_deserialize (fetch->object, &expected_checksum, &objtype); + is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); + + g_debug ("starting fetch of %s.%s%s", expected_checksum, + ostree_object_type_to_string (objtype), + fetch->is_detached_meta ? " (detached)" : ""); + + is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); + if (is_meta) + pull_data->n_outstanding_metadata_fetches++; + else + pull_data->n_outstanding_content_fetches++; + + /* Override the path if we're trying to fetch the .commitmeta file first */ + if (fetch->is_detached_meta) + { + char buf[_OSTREE_LOOSE_PATH_MAX]; + _ostree_loose_path (buf, expected_checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode); + obj_subpath = g_build_filename ("objects", buf, NULL); + mirrorlist = pull_data->meta_mirrorlist; + } + else + { + obj_subpath = _ostree_get_relative_object_path (expected_checksum, objtype, TRUE); + mirrorlist = pull_data->content_mirrorlist; + } + + /* We may have determined maximum sizes from the summary file content; if so, + * honor it. Otherwise, metadata has a baseline max size. + */ + expected_max_size_p = fetch->is_detached_meta ? NULL : g_hash_table_lookup (pull_data->expected_commit_sizes, expected_checksum); if (expected_max_size_p) expected_max_size = *expected_max_size_p; - else if (is_meta) + else if (OSTREE_OBJECT_TYPE_IS_META (objtype)) expected_max_size = OSTREE_MAX_METADATA_SIZE; else expected_max_size = 0; @@ -1398,7 +1522,7 @@ enqueue_one_object_request (OtPullData *pull_data, is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY : OSTREE_REPO_PULL_CONTENT_PRIORITY, pull_data->cancellable, - is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data); + is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch); } static gboolean @@ -1502,6 +1626,22 @@ process_one_static_delta_fallback (OtPullData *pull_data, return ret; } +static void +start_fetch_deltapart (OtPullData *pull_data, + FetchStaticDeltaData *fetch) +{ + g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (fetch->from_revision, fetch->to_revision, fetch->i); + pull_data->n_outstanding_deltapart_fetches++; + g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS); + _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, + pull_data->content_mirrorlist, + deltapart_path, fetch->size, + OSTREE_FETCHER_DEFAULT_PRIORITY, + pull_data->cancellable, + static_deltapart_fetch_on_complete, + fetch); +} + static gboolean process_one_static_delta (OtPullData *pull_data, const char *from_revision, @@ -1652,9 +1792,13 @@ process_one_static_delta (OtPullData *pull_data, continue; fetch_data = g_new0 (FetchStaticDeltaData, 1); + fetch_data->from_revision = g_strdup (from_revision); + fetch_data->to_revision = g_strdup (to_revision); fetch_data->pull_data = pull_data; fetch_data->objects = g_variant_ref (objects); fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v); + fetch_data->size = size; + fetch_data->i = i; if (inline_part_bytes != NULL) { @@ -1678,14 +1822,12 @@ process_one_static_delta (OtPullData *pull_data, } else { - _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, - pull_data->content_mirrorlist, - deltapart_path, size, - OSTREE_FETCHER_DEFAULT_PRIORITY, - pull_data->cancellable, - static_deltapart_fetch_on_complete, - fetch_data); - pull_data->n_outstanding_deltapart_fetches++; + if (!fetcher_queue_is_full (pull_data)) + start_fetch_deltapart (pull_data, fetch_data); + else + { + g_hash_table_add (pull_data->pending_fetch_deltaparts, fetch_data); + } } } @@ -2446,6 +2588,14 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)g_free, NULL); pull_data->requested_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, NULL); + pull_data->pending_fetch_content = g_hash_table_new_full (g_str_hash, g_str_equal, + (GDestroyNotify)g_free, + (GDestroyNotify)fetch_object_data_free); + pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, + (GDestroyNotify)g_variant_unref, + (GDestroyNotify)fetch_object_data_free); + pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL); + if (dir_to_pull != NULL || dirs_to_pull != NULL) { pull_data->dirs = g_ptr_array_new_with_free_func (g_free); @@ -3157,6 +3307,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->idle_src, (GDestroyNotify) g_source_destroy); g_clear_pointer (&pull_data->dirs, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref); From f4d1334e19ce3ab2f8872b1e28da52044f559401 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 19 Jan 2017 05:34:20 -0500 Subject: [PATCH 18/66] fetcher: Drop the libsoup queue Now that we have queuing in the higher level pull logic, we don't need to do this anymore. It's tempting to keep it since the code diff is so small (without completely rewriting things), but dropping it here will make it easier to see when things go wrong at a higher level. Note that I kept an assertion. Closes: #654 Approved by: jlebon --- src/libostree/ostree-fetcher.c | 80 ++++++++++------------------------ 1 file changed, 22 insertions(+), 58 deletions(-) diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index a178abfe..bc6c14c9 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -62,8 +62,7 @@ typedef struct { GVariant *extra_headers; int max_outstanding; - /* Queue for libsoup, see bgo#708591 */ - GQueue pending_queue; + /* Our active HTTP requests */ GHashTable *outstanding; /* Shared across threads; be sure to lock. */ @@ -77,9 +76,6 @@ typedef struct { } ThreadClosure; -static void -session_thread_process_pending_queue (ThreadClosure *thread_closure); - typedef struct { volatile int ref_count; @@ -187,18 +183,6 @@ idle_closure_free (IdleClosure *idle_closure) g_slice_free (IdleClosure, idle_closure); } -static int -pending_task_compare (gconstpointer a, - gconstpointer b, - gpointer unused) -{ - gint priority_a = g_task_get_priority (G_TASK (a)); - gint priority_b = g_task_get_priority (G_TASK (b)); - - return (priority_a == priority_b) ? 0 : - (priority_a < priority_b) ? -1 : 1; -} - static OstreeFetcherPendingURI * pending_uri_ref (OstreeFetcherPendingURI *pending) { @@ -403,30 +387,23 @@ static void on_request_sent (GObject *object, GAsyncResult *result, gpointer user_data); static void -session_thread_process_pending_queue (ThreadClosure *thread_closure) +start_pending_request (ThreadClosure *thread_closure, + GTask *task) { - while (g_queue_peek_head (&thread_closure->pending_queue) != NULL && - g_hash_table_size (thread_closure->outstanding) < thread_closure->max_outstanding) - { - GTask *task; - OstreeFetcherPendingURI *pending; - GCancellable *cancellable; + OstreeFetcherPendingURI *pending; + GCancellable *cancellable; - task = g_queue_pop_head (&thread_closure->pending_queue); + g_assert_cmpint (g_hash_table_size (thread_closure->outstanding), <, thread_closure->max_outstanding); - pending = g_task_get_task_data (task); - cancellable = g_task_get_cancellable (task); + pending = g_task_get_task_data (task); + cancellable = g_task_get_cancellable (task); - g_hash_table_add (thread_closure->outstanding, pending_uri_ref (pending)); - - soup_request_send_async (pending->request, - cancellable, - on_request_sent, - g_object_ref (task)); - - g_object_unref (task); - } + g_hash_table_add (thread_closure->outstanding, pending_uri_ref (pending)); + soup_request_send_async (pending->request, + cancellable, + on_request_sent, + g_object_ref (task)); } static void @@ -547,10 +524,7 @@ session_thread_request_uri (ThreadClosure *thread_closure, pending->out_tmpfile = tmpfile; tmpfile = NULL; /* Transfer ownership */ - g_queue_insert_sorted (&thread_closure->pending_queue, - g_object_ref (task), - pending_task_compare, NULL); - session_thread_process_pending_queue (thread_closure); + start_pending_request (thread_closure, task); } } @@ -600,8 +574,6 @@ ostree_fetcher_session_thread (gpointer data) * unreference all data related to the SoupSession ourself to ensure * it's freed in the same thread where it was created. */ g_clear_pointer (&closure->outstanding, g_hash_table_unref); - while (!g_queue_is_empty (&closure->pending_queue)) - g_object_unref (g_queue_pop_head (&closure->pending_queue)); g_clear_pointer (&closure->session, g_object_unref); thread_closure_unref (closure); @@ -903,11 +875,6 @@ finish_stream (OstreeFetcherPendingURI *pending, pending->state = OSTREE_FETCHER_STATE_COMPLETE; - /* Now that we've finished downloading, continue with other queued - * requests. - */ - session_thread_process_pending_queue (pending->thread_closure); - if (!pending->is_membuf) { if (stbuf.st_size < pending->content_length) @@ -935,14 +902,13 @@ on_stream_read (GObject *object, gpointer user_data); static void -remove_pending_rerun_queue (OstreeFetcherPendingURI *pending) +remove_pending (OstreeFetcherPendingURI *pending) { /* Hold a temporary ref to ensure the reference to * pending->thread_closure is valid. */ pending_uri_ref (pending); g_hash_table_remove (pending->thread_closure->outstanding, pending); - session_thread_process_pending_queue (pending->thread_closure); pending_uri_unref (pending); } @@ -976,7 +942,7 @@ on_out_splice_complete (GObject *object, if (local_error) { g_task_return_error (task, local_error); - remove_pending_rerun_queue (pending); + remove_pending (pending); } g_object_unref (task); @@ -1018,7 +984,7 @@ on_stream_read (GObject *object, g_strdup (pending->out_tmpfile), (GDestroyNotify) g_free); } - remove_pending_rerun_queue (pending); + remove_pending (pending); } else { @@ -1057,7 +1023,7 @@ on_stream_read (GObject *object, if (local_error) { g_task_return_error (task, local_error); - remove_pending_rerun_queue (pending); + remove_pending (pending); } g_object_unref (task); @@ -1096,7 +1062,7 @@ on_request_sent (GObject *object, g_task_return_pointer (task, g_strdup (pending->out_tmpfile), (GDestroyNotify) g_free); - remove_pending_rerun_queue (pending); + remove_pending (pending); goto out; } else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code)) @@ -1110,10 +1076,8 @@ on_request_sent (GObject *object, goto out; (void) g_input_stream_close (pending->request_body, NULL, NULL); - g_queue_insert_sorted (&pending->thread_closure->pending_queue, - g_object_ref (task), pending_task_compare, - NULL); - remove_pending_rerun_queue (pending); + + start_pending_request (pending->thread_closure, task); } else { @@ -1204,7 +1168,7 @@ on_request_sent (GObject *object, if (pending->request_body) (void) g_input_stream_close (pending->request_body, NULL, NULL); g_task_return_error (task, local_error); - remove_pending_rerun_queue (pending); + remove_pending (pending); } g_object_unref (task); From 425ccc0a33610c71f0258423c83701dc4e273ee7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 19 Jan 2017 05:49:23 -0500 Subject: [PATCH 19/66] pull: Show Estimating if we're scanning too The libcurl backend does all the work in the main thread/loop, which seems to starve the idle scanning worker more. With the libcurl backend, we're a lot more likely to have at least one outstanding metadata request. But it can more easily transiently happen with libcurl that all of our current fetches are content. To be accurate here, just show Estimating if we're scanning too. Closes: #654 Approved by: jlebon --- src/libostree/ostree-fetcher.c | 4 ++-- src/libostree/ostree-repo-pull.c | 1 + src/libostree/ostree-repo.c | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index bc6c14c9..bb98023d 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -552,11 +552,11 @@ ostree_fetcher_session_thread (gpointer data) /* XXX: Now that we have mirrorlist support, we could make this even smarter * by spreading requests across mirrors. */ g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL); - if (max_conns < 8) + if (max_conns < _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS) { /* We download a lot of small objects in ostree, so this * helps a lot. Also matches what most modern browsers do. */ - max_conns = 8; + max_conns = _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS; g_object_set (closure->session, "max-conns-per-host", max_conns, NULL); diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 21f3007e..79da7809 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -210,6 +210,7 @@ update_progress (gpointer user_data) ostree_async_progress_set_uint (pull_data->progress, "outstanding-writes", outstanding_writes); ostree_async_progress_set_uint (pull_data->progress, "fetched", fetched); ostree_async_progress_set_uint (pull_data->progress, "requested", requested); + ostree_async_progress_set_uint (pull_data->progress, "scanning", g_queue_is_empty (&pull_data->scan_object_queue) ? 0 : 1); ostree_async_progress_set_uint (pull_data->progress, "scanned-metadata", n_scanned_metadata); ostree_async_progress_set_uint64 (pull_data->progress, "bytes-transferred", bytes_transferred); ostree_async_progress_set_uint64 (pull_data->progress, "start-time", start_time); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 4ac39d11..47e95ae4 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3876,6 +3876,7 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress { GString *buf; g_autofree char *status = NULL; + gboolean scanning; guint outstanding_fetches; guint outstanding_metadata_fetches; guint outstanding_writes; @@ -3889,6 +3890,7 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress outstanding_fetches = ostree_async_progress_get_uint (progress, "outstanding-fetches"); outstanding_metadata_fetches = ostree_async_progress_get_uint (progress, "outstanding-metadata-fetches"); outstanding_writes = ostree_async_progress_get_uint (progress, "outstanding-writes"); + scanning = ostree_async_progress_get_uint (progress, "scanning") == 1; n_scanned_metadata = ostree_async_progress_get_uint (progress, "scanned-metadata"); fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts"); total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts"); @@ -3937,7 +3939,7 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress formatted_bytes_sec, formatted_bytes_transferred, formatted_total, formatted_est_time_remaining); } - else if (outstanding_metadata_fetches) + else if (scanning || outstanding_metadata_fetches) { g_string_append_printf (buf, "Receiving metadata objects: %u/(estimating) %s/s %s", metadata_fetched, formatted_bytes_sec, formatted_bytes_transferred); From 361aa449fbb43f8c9ab91e9d56c2d73e01c732e0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Dec 2016 21:02:30 -0500 Subject: [PATCH 20/66] libcurl backend For rpm-ostree, we already link to libcurl indirectly via librepo, and only having one HTTP library in process makes sense. Further, libcurl is (I think) more popular in the embedded space. It also supports HTTP/2.0 today, which is a *very* nice to have for OSTree. This seems to be working fairly well for me in my local testing, but it's obviously brand new nontrivial code, so it's going to need some soak time. The ugliest part of this is having to vendor in the soup-url code. With Oxidation we could follow the path of Firefox and use the [Servo URL parser](https://github.com/servo/rust-url). Having to redo cookie parsing also sucked, and that would also be a good oxidation target. But that's for the future. Closes: #641 Approved by: jlebon --- .redhat-ci.yml | 22 + .travis.yml | 1 + Makefile-libostree.am | 17 +- Makefile-ostree.am | 30 +- configure.ac | 30 +- src/libostree/ostree-fetcher-curl.c | 922 ++++++++++ ...ostree-fetcher.c => ostree-fetcher-soup.c} | 81 - src/libostree/ostree-fetcher-uri.c | 118 ++ src/libostree/ostree-repo-pull.c | 6 +- src/libostree/ostree-soup-form.c | 140 ++ src/libostree/ostree-soup-uri.c | 1483 +++++++++++++++++ src/libostree/ostree-soup-uri.h | 147 ++ src/ostree/ot-remote-builtin-add-cookie.c | 16 +- src/ostree/ot-remote-builtin-delete-cookie.c | 34 +- src/ostree/ot-remote-builtin-list-cookies.c | 27 +- src/ostree/ot-remote-cookie-util.c | 333 ++++ src/ostree/ot-remote-cookie-util.h | 42 + tests/ci-build.sh | 6 + tests/ci-install.sh | 4 + tests/test-remote-cookies.sh | 13 +- 20 files changed, 3301 insertions(+), 171 deletions(-) create mode 100644 src/libostree/ostree-fetcher-curl.c rename src/libostree/{ostree-fetcher.c => ostree-fetcher-soup.c} (95%) create mode 100644 src/libostree/ostree-fetcher-uri.c create mode 100644 src/libostree/ostree-soup-form.c create mode 100644 src/libostree/ostree-soup-uri.c create mode 100644 src/libostree/ostree-soup-uri.h create mode 100644 src/ostree/ot-remote-cookie-util.c create mode 100644 src/ostree/ot-remote-cookie-util.h diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 830320ef..11e5a9d6 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -71,5 +71,27 @@ env: tests: - make check TESTS=tests/test-rollsum +--- + +inherit: true +required: true + +context: curl + +packages: + - pkgconfig(libcurl) + +build: + config-opts: > + --prefix=/usr + --libdir=/usr/lib64 + --enable-installed-tests + --enable-gtk-doc + --with-curl + +tests: + - make check + - gnome-desktop-testing-runner -p 0 ostree + artifacts: - test-suite.log diff --git a/.travis.yml b/.travis.yml index 27f84921..a021592c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ sudo: required env: - ci_distro=ubuntu ci_suite=trusty ci_test=no # TODO: use libcurl on this - ci_docker=debian:jessie-slim ci_distro=debian ci_suite=jessie + - ci_docker=debian:jessie-slim ci_distro=debian ci_suite=jessie ci_configopts="--with-curl" - ci_docker=debian:stretch-slim ci_distro=debian ci_suite=stretch ci_test=no # TODO gpgme flake https://github.com/ostreedev/ostree/pull/664#issuecomment-276033383 - ci_docker=ubuntu:xenial ci_distro=ubuntu ci_suite=xenial diff --git a/Makefile-libostree.am b/Makefile-libostree.am index d40196d4..79fddec7 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -167,18 +167,31 @@ libostree_1_la_CFLAGS += $(LIBSYSTEMD_CFLAGS) libostree_1_la_LIBADD += $(LIBSYSTEMD_LIBS) endif -if USE_LIBSOUP +if USE_CURL_OR_SOUP libostree_1_la_SOURCES += \ src/libostree/ostree-fetcher.h \ - src/libostree/ostree-fetcher.c \ src/libostree/ostree-fetcher-util.h \ src/libostree/ostree-fetcher-util.c \ + src/libostree/ostree-fetcher-uri.c \ src/libostree/ostree-metalink.h \ src/libostree/ostree-metalink.c \ $(NULL) +endif + +if USE_CURL +libostree_1_la_SOURCES += src/libostree/ostree-fetcher-curl.c \ + src/libostree/ostree-soup-uri.h src/libostree/ostree-soup-uri.c \ + src/libostree/ostree-soup-form.c \ + $(NULL) +libostree_1_la_CFLAGS += $(OT_DEP_CURL_CFLAGS) +libostree_1_la_LIBADD += $(OT_DEP_CURL_LIBS) +else +if USE_LIBSOUP +libostree_1_la_SOURCES += src/libostree/ostree-fetcher-soup.c libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) libostree_1_la_LIBADD += $(OT_INTERNAL_SOUP_LIBS) endif +endif if USE_LIBMOUNT libostree_1_la_CFLAGS += $(OT_DEP_LIBMOUNT_CFLAGS) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 05fec155..0b520c68 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -88,12 +88,14 @@ ostree_SOURCES += \ src/ostree/ot-remote-builtin-summary.c \ $(NULL) -if USE_LIBSOUP -ostree_SOURCES += \ - src/ostree/ot-remote-builtin-add-cookie.c \ - src/ostree/ot-remote-builtin-delete-cookie.c \ - src/ostree/ot-remote-builtin-list-cookies.c \ - $(NULL) + +if USE_CURL_OR_SOUP +ostree_SOURCES += src/ostree/ot-remote-builtin-add-cookie.c \ + src/ostree/ot-remote-builtin-delete-cookie.c \ + src/ostree/ot-remote-builtin-list-cookies.c \ + src/ostree/ot-remote-cookie-util.h \ + src/ostree/ot-remote-cookie-util.c \ + $(NULL) endif src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile @@ -112,15 +114,23 @@ ostree_CFLAGS = $(ostree_bin_shared_cflags) ostree_LDADD = $(ostree_bin_shared_ldadd) libbsdiff.la libostree-kernel-args.la $(LIBSYSTEMD_LIBS) -if USE_LIBSOUP -ostree_SOURCES += src/ostree/ot-builtin-pull.c src/ostree/ot-builtin-trivial-httpd.c -ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) -ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS) +if USE_CURL_OR_SOUP +ostree_SOURCES += src/ostree/ot-builtin-pull.c +endif +if USE_LIBSOUP +# Eventually once we stop things from using this, we should support disabling this +ostree_SOURCES += src/ostree/ot-builtin-trivial-httpd.c pkglibexec_PROGRAMS += ostree-trivial-httpd ostree_trivial_httpd_SOURCES = src/ostree/ostree-trivial-httpd.c ostree_trivial_httpd_CFLAGS = $(ostree_bin_shared_cflags) $(OT_INTERNAL_SOUP_CFLAGS) ostree_trivial_httpd_LDADD = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_SOUP_LIBS) + +if !USE_CURL +# This is necessary for the cookie jar bits +ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) +ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS) +endif endif if USE_LIBARCHIVE diff --git a/configure.ac b/configure.ac index 88e6ea1b..596ee040 100644 --- a/configure.ac +++ b/configure.ac @@ -79,14 +79,29 @@ PKG_CHECK_MODULES(OT_DEP_ZLIB, zlib) dnl We're not actually linking to this, just using the header PKG_CHECK_MODULES(OT_DEP_E2P, e2p) +dnl Arbitrary version that's in CentOS7.2 now +CURL_DEPENDENCY=7.29.0 +AC_ARG_WITH(curl, + AS_HELP_STRING([--with-curl], [Use libcurl @<:@default=no@:>@]), + [], [with_curl=no]) +AS_IF([test x$with_curl != xno ], [ + PKG_CHECK_MODULES(OT_DEP_CURL, libcurl >= $CURL_DEPENDENCY) + with_curl=yes + AC_DEFINE([HAVE_LIBCURL], 1, [Define if we have libcurl.pc]) + dnl Currently using libcurl requires soup for trivial-httpd for tests + with_soup_default=yes +], [with_soup_default=check]) +AM_CONDITIONAL(USE_CURL, test x$with_curl != xno) +if test x$with_curl = xyes; then OSTREE_FEATURES="$OSTREE_FEATURES +libcurl"; fi + dnl When bumping the libsoup-2.4 dependency, remember to bump dnl SOUP_VERSION_MIN_REQUIRED and SOUP_VERSION_MAX_ALLOWED in dnl Makefile.am SOUP_DEPENDENCY="libsoup-2.4 >= 2.39.1" AC_ARG_WITH(soup, AS_HELP_STRING([--with-soup], [Use libsoup @<:@default=yes@:>@]), - [], [with_soup=check]) -AS_IF([test x$with_soup != xno ], [ + [], [with_soup=$with_soup_default]) +AS_IF([test x$with_soup != xno], [ AC_ARG_ENABLE(libsoup_client_certs, AS_HELP_STRING([--enable-libsoup-client-certs], [Require availability of new enough libsoup TLS client cert API (default: auto)]),, @@ -120,6 +135,14 @@ if test x$with_soup != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +libsoup"; fi AM_CONDITIONAL(USE_LIBSOUP, test x$with_soup != xno) AM_CONDITIONAL(HAVE_LIBSOUP_CLIENT_CERTS, test x$have_libsoup_client_certs = xyes) +AS_IF([test x$with_curl = xyes && test x$with_soup = xno], [ + AC_MSG_ERROR([Curl enabled, but libsoup is not; libsoup is needed for tests]) +]) +AM_CONDITIONAL(USE_CURL_OR_SOUP, test x$with_curl != xno || test x$with_soup != xno) +AS_IF([test x$with_curl != xno || test x$with_soup != xno], + [AC_DEFINE([HAVE_LIBCURL_OR_LIBSOUP], 1, [Define if we have soup or curl])]) +AS_IF([test x$with_curl = xyes], [fetcher_backend=curl], [test x$with_soup = xyes], [fetcher_backend=libsoup], [fetcher_backend=none]) + m4_ifdef([GOBJECT_INTROSPECTION_CHECK], [ GOBJECT_INTROSPECTION_CHECK([1.34.0]) ]) @@ -374,8 +397,7 @@ echo " introspection: $found_introspection Rust (internal oxidation): $rust_debug_release rofiles-fuse: $enable_rofiles_fuse - libsoup (retrieve remote HTTP repositories): $with_soup - libsoup TLS client certs: $have_libsoup_client_certs + HTTP backend: $fetcher_backend SELinux: $with_selinux systemd: $have_libsystemd libmount: $with_libmount diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c new file mode 100644 index 00000000..be3250fb --- /dev/null +++ b/src/libostree/ostree-fetcher-curl.c @@ -0,0 +1,922 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2016 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include +#include +#include +#include + +/* These macros came from 7.43.0, but we want to check + * for versions a bit earlier than that (to work on CentOS 7), + * so define them here if we're using an older version. + */ +#ifndef CURL_VERSION_BITS +#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z) +#endif +#ifndef CURL_AT_LEAST_VERSION +#define CURL_AT_LEAST_VERSION(x,y,z) (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z)) +#endif + +#include "ostree-fetcher.h" +#include "ostree-enumtypes.h" +#include "ostree-repo-private.h" +#include "otutil.h" + +#include "ostree-soup-uri.h" + +typedef struct FetcherRequest FetcherRequest; +typedef struct SockInfo SockInfo; + +static int sock_cb (CURL *e, curl_socket_t s, int what, void *cbp, void *sockp); +static gboolean timer_cb (gpointer data); +static void sock_unref (SockInfo *f); +static int update_timeout_cb (CURLM *multi, long timeout_ms, void *userp); +static void request_unref (FetcherRequest *req); +static void initiate_next_curl_request (FetcherRequest *req, GTask *task); +static void destroy_and_unref_source (GSource *source); + +struct OstreeFetcher +{ + GObject parent_instance; + + OstreeFetcherConfigFlags config_flags; + char *tls_ca_db_path; + char *tls_client_cert_path; + char *tls_client_key_path; + char *cookie_jar_path; + char *proxy; + struct curl_slist *extra_headers; + int tmpdir_dfd; + + GMainContext *mainctx; + CURLM *multi; + GSource *timer_event; + int curl_running; + GHashTable *outstanding_requests; /* Set */ + GHashTable *sockets; /* Set */ + + guint64 bytes_transferred; +}; + +/* Information associated with a request */ +struct FetcherRequest { + guint refcount; + GPtrArray *mirrorlist; + guint idx; + + char *filename; + guint64 current_size; + guint64 max_size; + OstreeFetcherRequestFlags flags; + gboolean is_membuf; + GError *caught_write_error; + char *out_tmpfile; + int out_tmpfile_fd; + GString *output_buf; + + CURL *easy; + char error[CURL_ERROR_SIZE]; + + OstreeFetcher *fetcher; +}; + +/* Information associated with a specific socket */ +struct SockInfo { + guint refcount; + curl_socket_t sockfd; + int action; + long timeout; + GSource *ch; + OstreeFetcher *fetcher; +}; + +enum { + PROP_0, + PROP_CONFIG_FLAGS +}; + +G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT) + +static void +_ostree_fetcher_set_property (GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + self->config_flags = g_value_get_flags (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_get_property (GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + g_value_set_flags (value, self->config_flags); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_finalize (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + g_free (self->cookie_jar_path); + g_free (self->proxy); + g_assert_cmpint (g_hash_table_size (self->outstanding_requests), ==, 0); + g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all); + g_hash_table_unref (self->outstanding_requests); + g_hash_table_unref (self->sockets); + g_clear_pointer (&self->timer_event, (GDestroyNotify)destroy_and_unref_source); + if (self->mainctx) + g_main_context_unref (self->mainctx); + curl_multi_cleanup (self->multi); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object); +} + +static void +_ostree_fetcher_constructed (GObject *object) +{ + // OstreeFetcher *self = OSTREE_FETCHER (object); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object); +} + +static void +_ostree_fetcher_class_init (OstreeFetcherClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = _ostree_fetcher_set_property; + gobject_class->get_property = _ostree_fetcher_get_property; + gobject_class->finalize = _ostree_fetcher_finalize; + gobject_class->constructed = _ostree_fetcher_constructed; + + g_object_class_install_property (gobject_class, + PROP_CONFIG_FLAGS, + g_param_spec_flags ("config-flags", + "", + "", + OSTREE_TYPE_FETCHER_CONFIG_FLAGS, + OSTREE_FETCHER_FLAGS_NONE, + G_PARAM_READWRITE | + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS)); +} + +static void +_ostree_fetcher_init (OstreeFetcher *self) +{ + self->multi = curl_multi_init(); + self->outstanding_requests = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)g_object_unref, NULL); + self->sockets = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)sock_unref, NULL); + curl_multi_setopt (self->multi, CURLMOPT_SOCKETFUNCTION, sock_cb); + curl_multi_setopt (self->multi, CURLMOPT_SOCKETDATA, self); + curl_multi_setopt (self->multi, CURLMOPT_TIMERFUNCTION, update_timeout_cb); + curl_multi_setopt (self->multi, CURLMOPT_TIMERDATA, self); +#if CURL_AT_LEAST_VERSION(7, 30, 0) + /* Let's do something reasonable here. */ + curl_multi_setopt (self->multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, 8); +#endif +} + + +OstreeFetcher * +_ostree_fetcher_new (int tmpdir_dfd, + OstreeFetcherConfigFlags flags) +{ + OstreeFetcher *fetcher = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); + fetcher->tmpdir_dfd = tmpdir_dfd; + return fetcher; +} + +static void +destroy_and_unref_source (GSource *source) +{ + g_source_destroy (source); + g_source_unref (source); +} + +static char * +request_get_uri (FetcherRequest *req, guint idx) +{ + SoupURI *baseuri = req->mirrorlist->pdata[idx]; + if (!req->filename) + return soup_uri_to_string (baseuri, FALSE); + { g_autofree char *uristr = soup_uri_to_string (baseuri, FALSE); + return g_build_filename (uristr, req->filename, NULL); + } +} + +static gboolean +ensure_tmpfile (FetcherRequest *req, GError **error) +{ + if (req->out_tmpfile_fd == -1) + { + if (!glnx_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".", + O_WRONLY, &req->out_tmpfile_fd, + &req->out_tmpfile, + error)) + return FALSE; + } + return TRUE; +} +/* Check for completed transfers, and remove their easy handles */ +static void +check_multi_info (OstreeFetcher *fetcher) +{ + CURLMsg *msg; + int msgs_left; + + while ((msg = curl_multi_info_read (fetcher->multi, &msgs_left)) != NULL) + { + long response; + CURL *easy = msg->easy_handle; + CURLcode curlres = msg->data.result; + GTask *task; + FetcherRequest *req; + const char *eff_url; + gboolean is_file; + gboolean continued_request = FALSE; + + if (msg->msg != CURLMSG_DONE) + continue; + + curl_easy_getinfo (easy, CURLINFO_PRIVATE, &task); + curl_easy_getinfo (easy, CURLINFO_EFFECTIVE_URL, &eff_url); + /* We should have limited the protocols; this is what + * curl's tool_operate.c does. + */ + is_file = g_str_has_prefix (eff_url, "file:"); + g_assert (is_file || g_str_has_prefix (eff_url, "http")); + + req = g_task_get_task_data (task); + + if (req->caught_write_error) + g_task_return_error (task, g_steal_pointer (&req->caught_write_error)); + else if (curlres != CURLE_OK) + { + if (is_file && curlres == CURLE_FILE_COULDNT_READ_FILE) + { + /* Handle file not found */ + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "%s", + curl_easy_strerror (curlres)); + } + else + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED, "[%u] %s", + curlres, + curl_easy_strerror (curlres)); + } + else + { + curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response); + if (!is_file && !(response >= 200 && response < 300)) + { + GIOErrorEnum giocode; + + /* TODO - share with soup */ + switch (response) + { + case 404: + case 403: + case 410: + giocode = G_IO_ERROR_NOT_FOUND; + break; + default: + giocode = G_IO_ERROR_FAILED; + } + + if (req->idx + 1 == req->mirrorlist->len) + { + g_task_return_new_error (task, G_IO_ERROR, giocode, + "Server returned HTTP %lu", response); + } + else + { + continued_request = TRUE; + } + } + else if (req->is_membuf) + { + GBytes *ret; + if ((req->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0) + g_string_append_c (req->output_buf, '\0'); + ret = g_string_free_to_bytes (req->output_buf); + req->output_buf = NULL; + g_task_return_pointer (task, ret, (GDestroyNotify)g_bytes_unref); + } + else + { + g_autoptr(GError) local_error = NULL; + GError **error = &local_error; + + /* TODO - share file naming with soup, and fix it */ + g_autofree char *tmpfile_path = + g_compute_checksum_for_string (G_CHECKSUM_SHA256, + eff_url, strlen (eff_url)); + if (!ensure_tmpfile (req, error)) + { + g_task_return_error (task, g_steal_pointer (&local_error)); + } + else if (fchmod (req->out_tmpfile_fd, 0644) < 0) + { + glnx_set_error_from_errno (error); + g_task_return_error (task, g_steal_pointer (&local_error)); + } + else if (!glnx_link_tmpfile_at (fetcher->tmpdir_dfd, + GLNX_LINK_TMPFILE_REPLACE, + req->out_tmpfile_fd, + req->out_tmpfile, + fetcher->tmpdir_dfd, + tmpfile_path, + error)) + g_task_return_error (task, g_steal_pointer (&local_error)); + else + { + g_task_return_pointer (task, g_steal_pointer (&tmpfile_path), g_free); + } + } + } + + curl_multi_remove_handle (fetcher->multi, easy); + if (continued_request) + { + req->idx++; + initiate_next_curl_request (req, task); + } + else + { + g_hash_table_remove (fetcher->outstanding_requests, task); + if (g_hash_table_size (fetcher->outstanding_requests) == 0) + { + g_clear_pointer (&fetcher->mainctx, g_main_context_unref); + } + } + } +} + +/* Called by glib when our timeout expires */ +static gboolean +timer_cb (gpointer data) +{ + OstreeFetcher *fetcher = data; + CURLMcode rc; + + fetcher->timer_event = NULL; + rc = curl_multi_socket_action (fetcher->multi, CURL_SOCKET_TIMEOUT, 0, &fetcher->curl_running); + g_assert (rc == CURLM_OK); + check_multi_info (fetcher); + + return FALSE; +} + +/* Update the event timer after curl_multi library calls */ +static int +update_timeout_cb (CURLM *multi, long timeout_ms, void *userp) +{ + OstreeFetcher *fetcher = userp; + + g_clear_pointer (&fetcher->timer_event, (GDestroyNotify)destroy_and_unref_source); + + if (timeout_ms != -1) + { + fetcher->timer_event = g_timeout_source_new (timeout_ms); + g_source_set_callback (fetcher->timer_event, timer_cb, fetcher, NULL); + g_source_attach (fetcher->timer_event, fetcher->mainctx); + } + + return 0; +} + +/* Called by glib when we get action on a multi socket */ +static gboolean +event_cb (int fd, GIOCondition condition, gpointer data) +{ + OstreeFetcher *fetcher = data; + CURLMcode rc; + + int action = + (condition & G_IO_IN ? CURL_CSELECT_IN : 0) | + (condition & G_IO_OUT ? CURL_CSELECT_OUT : 0); + + rc = curl_multi_socket_action (fetcher->multi, fd, action, &fetcher->curl_running); + g_assert (rc == CURLM_OK); + + check_multi_info (fetcher); + if (fetcher->curl_running > 0) + { + return TRUE; + } + else + { + return FALSE; + } +} + +/* Clean up the SockInfo structure */ +static void +sock_unref (SockInfo *f) +{ + if (!f) + return; + if (--f->refcount != 0) + return; + g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source); + g_free (f); +} + +/* Assign information to a SockInfo structure */ +static void +setsock (SockInfo*f, curl_socket_t s, int act, OstreeFetcher *fetcher) +{ + GIOCondition kind = + (act&CURL_POLL_IN?G_IO_IN:0)|(act&CURL_POLL_OUT?G_IO_OUT:0); + + f->sockfd = s; + f->action = act; + g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source); + /* TODO - investigate new g_source_modify_unix_fd() so changing the poll + * flags involves less allocation. + */ + f->ch = g_unix_fd_source_new (f->sockfd, kind); + g_source_set_callback (f->ch, (GSourceFunc) event_cb, fetcher, NULL); + g_source_attach (f->ch, fetcher->mainctx); +} + +/* Initialize a new SockInfo structure */ +static void +addsock (curl_socket_t s, CURL *easy, int action, OstreeFetcher *fetcher) +{ + SockInfo *fdp = g_new0 (SockInfo, 1); + + fdp->refcount = 1; + fdp->fetcher = fetcher; + setsock (fdp, s, action, fetcher); + curl_multi_assign (fetcher->multi, s, fdp); + g_hash_table_add (fetcher->sockets, fdp); +} + +/* CURLMOPT_SOCKETFUNCTION */ +static int +sock_cb (CURL *easy, curl_socket_t s, int what, void *cbp, void *sockp) +{ + OstreeFetcher *fetcher = cbp; + SockInfo *fdp = (SockInfo*) sockp; + + if (what == CURL_POLL_REMOVE) + { + if (!g_hash_table_remove (fetcher->sockets, fdp)) + g_assert_not_reached (); + } + else + { + if (!fdp) + { + addsock (s, easy, what, fetcher); + } + else + { + setsock (fdp, s, what, fetcher); + } + } + return 0; +} + +/* CURLOPT_WRITEFUNCTION */ +static size_t +write_cb (void *ptr, size_t size, size_t nmemb, void *data) +{ + const size_t realsize = size * nmemb; + GTask *task = data; + FetcherRequest *req; + + req = g_task_get_task_data (task); + + if (req->caught_write_error) + return -1; + + if (req->max_size > 0) + { + if (realsize > req->max_size || + (realsize + req->current_size) > req->max_size) + { + const char *eff_url; + curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url); + req->caught_write_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED, + "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes", + eff_url, req->max_size); + return -1; + } + } + + if (req->is_membuf) + g_string_append_len (req->output_buf, ptr, realsize); + else + { + if (!ensure_tmpfile (req, &req->caught_write_error)) + return -1; + g_assert (req->out_tmpfile_fd >= 0); + if (glnx_loop_write (req->out_tmpfile_fd, ptr, realsize) < 0) + { + glnx_set_error_from_errno (&req->caught_write_error); + return -1; + } + } + + req->current_size += realsize; + req->fetcher->bytes_transferred += realsize; + + return realsize; +} + +/* CURLOPT_PROGRESSFUNCTION */ +static int +prog_cb (void *p, double dltotal, double dlnow, double ult, double uln) +{ + GTask *task = p; + FetcherRequest *req; + char *eff_url; + req = g_task_get_task_data (task); + curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url); + g_printerr ("Progress: %s (%g/%g)\n", eff_url, dlnow, dltotal); + return 0; +} + +static void +request_unref (FetcherRequest *req) +{ + if (--req->refcount != 0) + return; + + g_ptr_array_unref (req->mirrorlist); + g_free (req->filename); + g_clear_error (&req->caught_write_error); + if (req->out_tmpfile_fd != -1) + (void) close (req->out_tmpfile_fd); + g_free (req->out_tmpfile); + if (req->output_buf) + g_string_free (req->output_buf, TRUE); + curl_easy_cleanup (req->easy); + + g_free (req); +} + +int +_ostree_fetcher_get_dfd (OstreeFetcher *fetcher) +{ + return fetcher->tmpdir_dfd; +} + +void +_ostree_fetcher_set_proxy (OstreeFetcher *self, + const char *http_proxy) +{ + g_free (self->proxy); + self->proxy = g_strdup (http_proxy); +} + +void +_ostree_fetcher_set_cookie_jar (OstreeFetcher *self, + const char *jar_path) +{ + g_free (self->cookie_jar_path); + self->cookie_jar_path = g_strdup (jar_path); +} + +void +_ostree_fetcher_set_client_cert (OstreeFetcher *self, + const char *cert_path, + const char *key_path) +{ + g_assert ((cert_path == NULL && key_path == NULL) + || (cert_path != NULL && key_path != NULL)); + g_free (self->tls_client_cert_path); + self->tls_client_cert_path = g_strdup (cert_path); + g_free (self->tls_client_key_path); + self->tls_client_key_path = g_strdup (key_path); +} + +void +_ostree_fetcher_set_tls_database (OstreeFetcher *self, + const char *dbpath) +{ + g_free (self->tls_ca_db_path); + self->tls_ca_db_path = g_strdup (dbpath); +} + +void +_ostree_fetcher_set_extra_headers (OstreeFetcher *self, + GVariant *extra_headers) +{ + GVariantIter viter; + const char *key; + const char *value; + + g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all); + + g_variant_iter_init (&viter, extra_headers); + while (g_variant_iter_loop (&viter, "(&s&s)", &key, &value)) + { + g_autofree char *header = g_strdup_printf ("%s: %s", key, value); + self->extra_headers = curl_slist_append (self->extra_headers, header); + } +} + +/* Re-bind all of the outstanding curl items to our new main context */ +static void +adopt_steal_mainctx (OstreeFetcher *self, + GMainContext *mainctx) +{ + GHashTableIter hiter; + gpointer key, value; + + g_assert (self->mainctx == NULL); + self->mainctx = mainctx; /* Transfer */ + + if (self->timer_event != NULL) + { + guint64 readytime = g_source_get_ready_time (self->timer_event); + guint64 curtime = g_source_get_time (self->timer_event); + guint64 timeout_micros = curtime - readytime; + if (timeout_micros < 0) + timeout_micros = 0; + update_timeout_cb (self->multi, timeout_micros / 1000, self); + } + + g_hash_table_iter_init (&hiter, self->sockets); + while (g_hash_table_iter_next (&hiter, &key, &value)) + { + SockInfo *fdp = key; + setsock (fdp, fdp->sockfd, fdp->action, self); + } +} + +static void +initiate_next_curl_request (FetcherRequest *req, + GTask *task) +{ + CURLMcode rc; + OstreeFetcher *self = req->fetcher; + + req->easy = curl_easy_init (); + g_assert (req->easy); + + g_assert_cmpint (req->idx, <, req->mirrorlist->len); + + { g_autofree char *uri = request_get_uri (req, req->idx); + curl_easy_setopt (req->easy, CURLOPT_URL, uri); + } + + curl_easy_setopt (req->easy, CURLOPT_USERAGENT, "ostree "); + if (self->extra_headers) + curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers); + + if (self->cookie_jar_path) + { + rc = curl_easy_setopt (req->easy, CURLOPT_COOKIEFILE, self->cookie_jar_path); + g_assert_cmpint (rc, ==, CURLM_OK); + rc = curl_easy_setopt (req->easy, CURLOPT_COOKIELIST, "RELOAD"); + g_assert_cmpint (rc, ==, CURLM_OK); + } + + if (self->proxy) + { + rc = curl_easy_setopt (req->easy, CURLOPT_PROXY, self->proxy); + g_assert_cmpint (rc, ==, CURLM_OK); + } + + if (self->tls_ca_db_path) + curl_easy_setopt (req->easy, CURLOPT_CAINFO, self->tls_ca_db_path); + + if ((self->config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0) + curl_easy_setopt (req->easy, CURLOPT_SSL_VERIFYPEER, 0L); + + if (self->tls_client_cert_path) + { + curl_easy_setopt (req->easy, CURLOPT_SSLCERT, self->tls_client_cert_path); + curl_easy_setopt (req->easy, CURLOPT_SSLKEY, self->tls_client_key_path); + } + + /* We should only speak HTTP; TODO: only enable file if specified */ + curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE)); + /* Picked the current version in F25 as of 20170127, since + * there are numerous HTTP/2 fixes since the original version in + * libcurl 7.43.0. + */ +#if CURL_AT_LEAST_VERSION(7, 51, 0) + curl_easy_setopt (req->easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); +#endif + curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb); + if (g_getenv ("OSTREE_DEBUG_HTTP")) + curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L); + curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error); + /* Note that the "easy" object's privdata is the task */ + curl_easy_setopt (req->easy, CURLOPT_NOPROGRESS, 1L); + curl_easy_setopt (req->easy, CURLOPT_PROGRESSFUNCTION, prog_cb); + curl_easy_setopt (req->easy, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt (req->easy, CURLOPT_CONNECTTIMEOUT, 30L); + curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_TIME, 30L); + + /* closure bindings -> task */ + curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task); + curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task); + curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task); + + rc = curl_multi_add_handle (self->multi, req->easy); + g_assert (rc == CURLM_OK); +} + +static void +_ostree_fetcher_request_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + gboolean is_membuf, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + FetcherRequest *req; + g_autoptr(GMainContext) mainctx = g_main_context_ref_thread_default (); + + /* We don't support multiple concurrent main contexts; take + * a ref to the first one, and require that later invocations + * share it. + */ + if (g_hash_table_size (self->outstanding_requests) == 0 + && mainctx != self->mainctx) + { + adopt_steal_mainctx (self, g_steal_pointer (&mainctx)); + } + else + { + g_assert (self->mainctx == mainctx); + } + + req = g_new0 (FetcherRequest, 1); + req->refcount = 1; + req->error[0]='\0'; + req->fetcher = self; + req->mirrorlist = g_ptr_array_ref (mirrorlist); + req->filename = g_strdup (filename); + req->max_size = max_size; + req->flags = flags; + req->is_membuf = is_membuf; + /* We'll allocate the tmpfile on demand, so we handle + * file I/O errors just in the write func. + */ + req->out_tmpfile_fd = -1; + if (req->is_membuf) + req->output_buf = g_string_new (""); + + task = g_task_new (self, cancellable, callback, user_data); + /* We'll use the GTask priority for our own priority queue. */ + g_task_set_priority (task, priority); + g_task_set_source_tag (task, _ostree_fetcher_request_async); + g_task_set_task_data (task, req, (GDestroyNotify) request_unref); + + initiate_next_curl_request (req, task); + + g_hash_table_add (self->outstanding_requests, g_steal_pointer (&task)); + + /* Sanity check, I added * 2 just so we don't abort if something odd happens, + * but we do want to abort if we're asked to do obviously too many requests. + */ + g_assert_cmpint (g_hash_table_size (self->outstanding_requests), <, + _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS * 2); +} + +void +_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, 0, FALSE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, + GAsyncResult *result, + char **out_filename, + GError **error) +{ + GTask *task; + FetcherRequest *req; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + req = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (!req->is_membuf); + g_assert (out_filename); + *out_filename = ret; + + return TRUE; +} + +void +_ostree_fetcher_request_to_membuf (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, + GAsyncResult *result, + GBytes **out_buf, + GError **error) +{ + GTask *task; + FetcherRequest *req; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + req = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (req->is_membuf); + g_assert (out_buf); + *out_buf = ret; + + return TRUE; +} + +guint64 +_ostree_fetcher_bytes_transferred (OstreeFetcher *self) +{ + return self->bytes_transferred; +} diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher-soup.c similarity index 95% rename from src/libostree/ostree-fetcher.c rename to src/libostree/ostree-fetcher-soup.c index bb98023d..fcdf8e0e 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -1332,84 +1332,3 @@ _ostree_fetcher_bytes_transferred (OstreeFetcher *self) return ret; } - -void -_ostree_fetcher_uri_free (OstreeFetcherURI *uri) -{ - if (uri) - soup_uri_free ((SoupURI*)uri); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_parse (const char *str, - GError **error) -{ - SoupURI *soupuri = soup_uri_new (str); - if (soupuri == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Failed to parse uri: %s", str); - return NULL; - } - return (OstreeFetcherURI*)soupuri; -} - -static OstreeFetcherURI * -_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri, - gboolean extend, - const char *path) -{ - SoupURI *newuri = soup_uri_copy ((SoupURI*)uri); - if (path) - { - if (extend) - { - const char *origpath = soup_uri_get_path ((SoupURI*)uri); - g_autofree char *newpath = g_build_filename (origpath, path, NULL); - soup_uri_set_path (newuri, newpath); - } - else - { - soup_uri_set_path (newuri, path); - } - } - return (OstreeFetcherURI*)newuri; -} - -OstreeFetcherURI * -_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri, - const char *path) -{ - return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri, - const char *subpath) -{ - return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_clone (OstreeFetcherURI *uri) -{ - return _ostree_fetcher_uri_new_subpath (uri, NULL); -} - -char * -_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri) -{ - return g_strdup (soup_uri_get_scheme ((SoupURI*)uri)); -} - -char * -_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri) -{ - return g_strdup (soup_uri_get_path ((SoupURI*)uri)); -} - -char * -_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri) -{ - return soup_uri_to_string ((SoupURI*)uri, FALSE); -} diff --git a/src/libostree/ostree-fetcher-uri.c b/src/libostree/ostree-fetcher-uri.c new file mode 100644 index 00000000..7ef42eca --- /dev/null +++ b/src/libostree/ostree-fetcher-uri.c @@ -0,0 +1,118 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2011,2017 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Author: Colin Walters + */ + +#include "config.h" + + +#ifdef HAVE_LIBCURL +#include "ostree-soup-uri.h" +#else +#define LIBSOUP_USE_UNSTABLE_REQUEST_API +#include +#include +#include +#endif + +#include "ostree-fetcher.h" + +#include "libglnx.h" + +void +_ostree_fetcher_uri_free (OstreeFetcherURI *uri) +{ + if (uri) + soup_uri_free ((SoupURI*)uri); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_parse (const char *str, + GError **error) +{ + SoupURI *soupuri = soup_uri_new (str); + if (soupuri == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to parse uri: %s", str); + return NULL; + } + return (OstreeFetcherURI*)soupuri; +} + +static OstreeFetcherURI * +_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri, + gboolean extend, + const char *path) +{ + SoupURI *newuri = soup_uri_copy ((SoupURI*)uri); + if (path) + { + if (extend) + { + const char *origpath = soup_uri_get_path ((SoupURI*)uri); + g_autofree char *newpath = g_build_filename (origpath, path, NULL); + soup_uri_set_path (newuri, newpath); + } + else + { + soup_uri_set_path (newuri, path); + } + } + return (OstreeFetcherURI*)newuri; +} + +OstreeFetcherURI * +_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri, + const char *path) +{ + return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri, + const char *subpath) +{ + return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_clone (OstreeFetcherURI *uri) +{ + return _ostree_fetcher_uri_new_subpath (uri, NULL); +} + +char * +_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri) +{ + return g_strdup (soup_uri_get_scheme ((SoupURI*)uri)); +} + +char * +_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri) +{ + return g_strdup (soup_uri_get_path ((SoupURI*)uri)); +} + +char * +_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri) +{ + return soup_uri_to_string ((SoupURI*)uri, FALSE); +} diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 79da7809..295973ec 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -26,7 +26,7 @@ #include "ostree.h" #include "otutil.h" -#ifdef HAVE_LIBSOUP +#ifdef HAVE_LIBCURL_OR_LIBSOUP #include "ostree-core-private.h" #include "ostree-repo-private.h" @@ -3405,7 +3405,7 @@ out: return ret; } -#else /* HAVE_LIBSOUP */ +#else /* HAVE_LIBCURL_OR_LIBSOUP */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -3434,4 +3434,4 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, return FALSE; } -#endif /* HAVE_LIBSOUP */ +#endif /* HAVE_LIBCURL_OR_LIBSOUP */ diff --git a/src/libostree/ostree-soup-form.c b/src/libostree/ostree-soup-form.c new file mode 100644 index 00000000..74f9c7bb --- /dev/null +++ b/src/libostree/ostree-soup-form.c @@ -0,0 +1,140 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* soup-form.c : utility functions for HTML forms */ + +/* + * Copyright 2008 Red Hat, Inc. + */ + +/* This one is stripped down to only have soup_form_encode_hash() + * and soup_form_encode_valist() which are the only bits that soup-uri.c + * calls. + */ + +#include + +#include + +#include "ostree-soup-uri.h" + +/** + * SECTION:soup-form + * @short_description: HTML form handling + * @see_also: #SoupMultipart + * + * libsoup contains several help methods for processing HTML forms as + * defined by the + * HTML 4.01 specification. + **/ + +/** + * SOUP_FORM_MIME_TYPE_URLENCODED: + * + * A macro containing the value + * "application/x-www-form-urlencoded"; the default + * MIME type for POSTing HTML form data. + * + * Since: 2.26 + **/ + +/** + * SOUP_FORM_MIME_TYPE_MULTIPART: + * + * A macro containing the value + * "multipart/form-data"; the MIME type used for + * posting form data that contains files to be uploaded. + * + * Since: 2.26 + **/ + +#define XDIGIT(c) ((c) <= '9' ? (c) - '0' : ((c) & 0x4F) - 'A' + 10) +#define HEXCHAR(s) ((XDIGIT (s[1]) << 4) + XDIGIT (s[2])) + +static void +append_form_encoded (GString *str, const char *in) +{ + const unsigned char *s = (const unsigned char *)in; + + while (*s) { + if (*s == ' ') { + g_string_append_c (str, '+'); + s++; + } else if (!g_ascii_isalnum (*s) && (*s != '-') && (*s != '_') + && (*s != '.')) + g_string_append_printf (str, "%%%02X", (int)*s++); + else + g_string_append_c (str, *s++); + } +} + +static void +encode_pair (GString *str, const char *name, const char *value) +{ + g_return_if_fail (name != NULL); + g_return_if_fail (value != NULL); + + if (str->len) + g_string_append_c (str, '&'); + append_form_encoded (str, name); + g_string_append_c (str, '='); + append_form_encoded (str, value); +} + +/** + * soup_form_encode_hash: + * @form_data_set: (element-type utf8 utf8): a hash table containing + * name/value pairs (as strings) + * + * Encodes @form_data_set into a value of type + * "application/x-www-form-urlencoded", as defined in the HTML 4.01 + * spec. + * + * Note that the HTML spec states that "The control names/values are + * listed in the order they appear in the document." Since this method + * takes a hash table, it cannot enforce that; if you care about the + * ordering of the form fields, use soup_form_encode_datalist(). + * + * Return value: the encoded form + **/ +char * +soup_form_encode_hash (GHashTable *form_data_set) +{ + GString *str = g_string_new (NULL); + GHashTableIter iter; + gpointer name, value; + + g_hash_table_iter_init (&iter, form_data_set); + while (g_hash_table_iter_next (&iter, &name, &value)) + encode_pair (str, name, value); + return g_string_free (str, FALSE); +} + +/** + * soup_form_encode_valist: + * @first_field: name of the first form field + * @args: pointer to additional values, as in soup_form_encode() + * + * See soup_form_encode(). This is mostly an internal method, used by + * various other methods such as soup_uri_set_query_from_fields() and + * soup_form_request_new(). + * + * Return value: the encoded form + **/ +char * +soup_form_encode_valist (const char *first_field, va_list args) +{ + GString *str = g_string_new (NULL); + const char *name, *value; + + name = first_field; + value = va_arg (args, const char *); + while (name && value) { + encode_pair (str, name, value); + + name = va_arg (args, const char *); + if (name) + value = va_arg (args, const char *); + } + + return g_string_free (str, FALSE); +} diff --git a/src/libostree/ostree-soup-uri.c b/src/libostree/ostree-soup-uri.c new file mode 100644 index 00000000..97f74636 --- /dev/null +++ b/src/libostree/ostree-soup-uri.c @@ -0,0 +1,1483 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* soup-uri.c : utility functions to parse URLs */ + +/* + * Copyright 1999-2003 Ximian, Inc. + */ + +#include "config.h" + +#include +#include + +#include "ostree-soup-uri.h" + +/* OSTREECHANGE: definitions from soup-misc-private.h */ +char *soup_uri_decoded_copy (const char *str, int length, int *decoded_length); +char *soup_uri_to_string_internal (SoupURI *uri, gboolean just_path_and_query, + gboolean force_port); +gboolean soup_uri_is_http (SoupURI *uri, char **aliases); +gboolean soup_uri_is_https (SoupURI *uri, char **aliases); + +/* OSTREECHANGE: import soup-misc's char helpers */ +#define SOUP_CHAR_URI_PERCENT_ENCODED 0x01 +#define SOUP_CHAR_URI_GEN_DELIMS 0x02 +#define SOUP_CHAR_URI_SUB_DELIMS 0x04 +#define SOUP_CHAR_HTTP_SEPARATOR 0x08 +#define SOUP_CHAR_HTTP_CTL 0x10 + +/* 00 URI_UNRESERVED + * 01 URI_PCT_ENCODED + * 02 URI_GEN_DELIMS + * 04 URI_SUB_DELIMS + * 08 HTTP_SEPARATOR + * 10 HTTP_CTL + */ +const char soup_char_attributes[] = { + /* 0x00 - 0x07 */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x08 - 0x0f */ + 0x11, 0x19, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x10 - 0x17 */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x18 - 0x1f */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* !"#$%&' */ + 0x09, 0x04, 0x09, 0x02, 0x04, 0x01, 0x04, 0x04, + /* ()*+,-./ */ + 0x0c, 0x0c, 0x04, 0x04, 0x0c, 0x00, 0x00, 0x0a, + /* 01234567 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 89:;<=>? */ + 0x00, 0x00, 0x0a, 0x0c, 0x09, 0x0a, 0x09, 0x0a, + /* @ABCDEFG */ + 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* HIJKLMNO */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* PQRSTUVW */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* XYZ[\]^_ */ + 0x00, 0x00, 0x00, 0x0a, 0x09, 0x0a, 0x01, 0x00, + /* `abcdefg */ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* hijklmno */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* pqrstuvw */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* xyz{|}~ */ + 0x00, 0x00, 0x00, 0x09, 0x01, 0x09, 0x00, 0x11, + /* 0x80 - 0xFF */ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 +}; + +#define soup_char_is_uri_percent_encoded(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_PERCENT_ENCODED) +#define soup_char_is_uri_gen_delims(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_GEN_DELIMS) +#define soup_char_is_uri_sub_delims(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_SUB_DELIMS) +#define soup_char_is_uri_unreserved(ch) (!(soup_char_attributes[(guchar)ch] & (SOUP_CHAR_URI_PERCENT_ENCODED | SOUP_CHAR_URI_GEN_DELIMS | SOUP_CHAR_URI_SUB_DELIMS))) +#define soup_char_is_token(ch) (!(soup_char_attributes[(guchar)ch] & (SOUP_CHAR_HTTP_SEPARATOR | SOUP_CHAR_HTTP_CTL))) + +/** + * soup_str_case_hash: + * @key: ASCII string to hash + * + * Hashes @key in a case-insensitive manner. + * + * Return value: the hash code. + **/ +static guint +soup_str_case_hash (gconstpointer key) +{ + const char *p = key; + guint h = g_ascii_toupper(*p); + + if (h) + for (p += 1; *p != '\0'; p++) + h = (h << 5) - h + g_ascii_toupper(*p); + + return h; +} + +/** + * SECTION:soup-uri + * @short_description: URIs + * + * A #SoupURI represents a (parsed) URI. + * + * Many applications will not need to use #SoupURI directly at all; on + * the client side, soup_message_new() takes a stringified URI, and on + * the server side, the path and query components are provided for you + * in the server callback. + **/ + +/** + * SoupURI: + * @scheme: the URI scheme (eg, "http") + * @user: a username, or %NULL + * @password: a password, or %NULL + * @host: the hostname or IP address + * @port: the port number on @host + * @path: the path on @host + * @query: a query for @path, or %NULL + * @fragment: a fragment identifier within @path, or %NULL + * + * A #SoupURI represents a (parsed) URI. #SoupURI supports RFC 3986 + * (URI Generic Syntax), and can parse any valid URI. However, libsoup + * only uses "http" and "https" URIs internally; You can use + * SOUP_URI_VALID_FOR_HTTP() to test if a #SoupURI is a valid HTTP + * URI. + * + * @scheme will always be set in any URI. It is an interned string and + * is always all lowercase. (If you parse a URI with a non-lowercase + * scheme, it will be converted to lowercase.) The macros + * %SOUP_URI_SCHEME_HTTP and %SOUP_URI_SCHEME_HTTPS provide the + * interned values for "http" and "https" and can be compared against + * URI @scheme values. + * + * @user and @password are parsed as defined in the older URI specs + * (ie, separated by a colon; RFC 3986 only talks about a single + * "userinfo" field). Note that @password is not included in the + * output of soup_uri_to_string(). libsoup does not normally use these + * fields; authentication is handled via #SoupSession signals. + * + * @host contains the hostname, and @port the port specified in the + * URI. If the URI doesn't contain a hostname, @host will be %NULL, + * and if it doesn't specify a port, @port may be 0. However, for + * "http" and "https" URIs, @host is guaranteed to be non-%NULL + * (trying to parse an http URI with no @host will return %NULL), and + * @port will always be non-0 (because libsoup knows the default value + * to use when it is not specified in the URI). + * + * @path is always non-%NULL. For http/https URIs, @path will never be + * an empty string either; if the input URI has no path, the parsed + * #SoupURI will have a @path of "/". + * + * @query and @fragment are optional for all URI types. + * soup_form_decode() may be useful for parsing @query. + * + * Note that @path, @query, and @fragment may contain + * %-encoded characters. soup_uri_new() calls + * soup_uri_normalize() on them, but not soup_uri_decode(). This is + * necessary to ensure that soup_uri_to_string() will generate a URI + * that has exactly the same meaning as the original. (In theory, + * #SoupURI should leave @user, @password, and @host partially-encoded + * as well, but this would be more annoying than useful.) + **/ + +/** + * SOUP_URI_IS_VALID: + * @uri: a #SoupURI + * + * Tests whether @uri is a valid #SoupURI; that is, that it is non-%NULL + * and its @scheme and @path members are also non-%NULL. + * + * This macro does not check whether http and https URIs have a non-%NULL + * @host member. + * + * Return value: %TRUE if @uri is valid for use. + * + * Since: 2.38 + **/ + +/** + * SOUP_URI_VALID_FOR_HTTP: + * @uri: a #SoupURI + * + * Tests if @uri is a valid #SoupURI for HTTP communication; that is, if + * it can be used to construct a #SoupMessage. + * + * Return value: %TRUE if @uri is a valid "http" or "https" URI. + * + * Since: 2.24 + **/ + +/** + * SOUP_URI_SCHEME_HTTP: + * + * "http" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + */ +/** + * SOUP_URI_SCHEME_HTTPS: + * + * "https" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + */ +/** + * SOUP_URI_SCHEME_FTP: + * + * "ftp" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_FILE: + * + * "file" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_DATA: + * + * "data" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_RESOURCE: + * + * "data" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.42 + */ +/** + * SOUP_URI_SCHEME_WS: + * + * "ws" (WebSocket) as an interned string; you can compare this + * directly to a #SoupURI's scheme field using + * ==. + * + * Since: 2.50 + */ +/** + * SOUP_URI_SCHEME_WSS: + * + * "wss" (WebSocket over TLS) as an interned string; you can compare + * this directly to a #SoupURI's scheme field using + * ==. + * + * Since: 2.50 + */ + +struct _SoupURI { + const char *scheme; + + char *user; + char *password; + + char *host; + guint port; + + char *path; + char *query; + + char *fragment; +}; + +static void append_uri_encoded (GString *str, const char *in, const char *extra_enc_chars); +static char *uri_normalized_copy (const char *str, int length, const char *unescape_extra); + +gpointer _SOUP_URI_SCHEME_HTTP, _SOUP_URI_SCHEME_HTTPS; +gpointer _SOUP_URI_SCHEME_WS, _SOUP_URI_SCHEME_WSS; +gpointer _SOUP_URI_SCHEME_FTP; +gpointer _SOUP_URI_SCHEME_FILE, _SOUP_URI_SCHEME_DATA, _SOUP_URI_SCHEME_RESOURCE; + +static inline const char * +soup_uri_parse_scheme (const char *scheme, int len) +{ + if (len == 4 && !g_ascii_strncasecmp (scheme, "http", len)) { + return SOUP_URI_SCHEME_HTTP; + } else if (len == 5 && !g_ascii_strncasecmp (scheme, "https", len)) { + return SOUP_URI_SCHEME_HTTPS; + } else if (len == 8 && !g_ascii_strncasecmp (scheme, "resource", len)) { + return SOUP_URI_SCHEME_RESOURCE; + } else if (len == 2 && !g_ascii_strncasecmp (scheme, "ws", len)) { + return SOUP_URI_SCHEME_WS; + } else if (len == 3 && !g_ascii_strncasecmp (scheme, "wss", len)) { + return SOUP_URI_SCHEME_WSS; + } else { + char *lower_scheme; + + lower_scheme = g_ascii_strdown (scheme, len); + scheme = g_intern_static_string (lower_scheme); + if (scheme != (const char *)lower_scheme) + g_free (lower_scheme); + return scheme; + } +} + +static inline guint +soup_scheme_default_port (const char *scheme) +{ + if (scheme == SOUP_URI_SCHEME_HTTP || scheme == SOUP_URI_SCHEME_WS) + return 80; + else if (scheme == SOUP_URI_SCHEME_HTTPS || scheme == SOUP_URI_SCHEME_WSS) + return 443; + else if (scheme == SOUP_URI_SCHEME_FTP) + return 21; + else + return 0; +} + +/** + * soup_uri_new_with_base: + * @base: a base URI + * @uri_string: the URI + * + * Parses @uri_string relative to @base. + * + * Return value: a parsed #SoupURI. + **/ +SoupURI * +soup_uri_new_with_base (SoupURI *base, const char *uri_string) +{ + SoupURI *uri, fixed_base; + const char *end, *hash, *colon, *at, *path, *question; + const char *p, *hostend; + gboolean remove_dot_segments = TRUE; + int len; + + g_return_val_if_fail (uri_string != NULL, NULL); + + /* Allow a %NULL path in @base, for compatibility */ + if (base && base->scheme && !base->path) { + g_warn_if_fail (SOUP_URI_IS_VALID (base)); + + memcpy (&fixed_base, base, sizeof (SoupURI)); + fixed_base.path = ""; + base = &fixed_base; + } + + g_return_val_if_fail (base == NULL || SOUP_URI_IS_VALID (base), NULL); + + /* First some cleanup steps (which are supposed to all be no-ops, + * but...). Skip initial whitespace, strip out internal tabs and + * line breaks, and ignore trailing whitespace. + */ + while (g_ascii_isspace (*uri_string)) + uri_string++; + + len = strcspn (uri_string, "\t\n\r"); + if (uri_string[len]) { + char *clean = g_malloc (strlen (uri_string) + 1), *d; + const char *s; + + for (s = uri_string, d = clean; *s; s++) { + if (*s != '\t' && *s != '\n' && *s != '\r') + *d++ = *s; + } + *d = '\0'; + + uri = soup_uri_new_with_base (base, clean); + g_free (clean); + return uri; + } + end = uri_string + len; + while (end > uri_string && g_ascii_isspace (end[-1])) + end--; + + uri = g_slice_new0 (SoupURI); + + /* Find fragment. */ + hash = strchr (uri_string, '#'); + if (hash) { + uri->fragment = uri_normalized_copy (hash + 1, end - hash + 1, + NULL); + end = hash; + } + + /* Find scheme */ + p = uri_string; + while (p < end && (g_ascii_isalpha (*p) || + (p > uri_string && (g_ascii_isdigit (*p) || + *p == '.' || + *p == '+' || + *p == '-')))) + p++; + + if (p > uri_string && *p == ':') { + uri->scheme = soup_uri_parse_scheme (uri_string, p - uri_string); + uri_string = p + 1; + } + + if (uri_string == end && !base && !uri->fragment) { + uri->path = g_strdup (""); + return uri; + } + + /* Check for authority */ + if (strncmp (uri_string, "//", 2) == 0) { + uri_string += 2; + + path = uri_string + strcspn (uri_string, "/?#"); + if (path > end) + path = end; + at = strchr (uri_string, '@'); + if (at && at < path) { + colon = strchr (uri_string, ':'); + if (colon && colon < at) { + uri->password = soup_uri_decoded_copy (colon + 1, + at - colon - 1, NULL); + } else { + uri->password = NULL; + colon = at; + } + + uri->user = soup_uri_decoded_copy (uri_string, + colon - uri_string, NULL); + uri_string = at + 1; + } else + uri->user = uri->password = NULL; + + /* Find host and port. */ + if (*uri_string == '[') { + const char *pct; + + uri_string++; + hostend = strchr (uri_string, ']'); + if (!hostend || hostend > path) { + soup_uri_free (uri); + return NULL; + } + if (*(hostend + 1) == ':') + colon = hostend + 1; + else + colon = NULL; + + pct = memchr (uri_string, '%', hostend - uri_string); + if (!pct || (pct[1] == '2' && pct[2] == '5')) { + uri->host = soup_uri_decoded_copy (uri_string, + hostend - uri_string, NULL); + } else + uri->host = g_strndup (uri_string, hostend - uri_string); + } else { + colon = memchr (uri_string, ':', path - uri_string); + hostend = colon ? colon : path; + uri->host = soup_uri_decoded_copy (uri_string, + hostend - uri_string, NULL); + } + + if (colon && colon != path - 1) { + char *portend; + uri->port = strtoul (colon + 1, &portend, 10); + if (portend != (char *)path) { + soup_uri_free (uri); + return NULL; + } + } + + uri_string = path; + } + + /* Find query */ + question = memchr (uri_string, '?', end - uri_string); + if (question) { + uri->query = uri_normalized_copy (question + 1, + end - (question + 1), + NULL); + end = question; + } + + if (end != uri_string) { + uri->path = uri_normalized_copy (uri_string, end - uri_string, + NULL); + } + + /* Apply base URI. This is spelled out in RFC 3986. */ + if (base && !uri->scheme && uri->host) + uri->scheme = base->scheme; + else if (base && !uri->scheme) { + uri->scheme = base->scheme; + uri->user = g_strdup (base->user); + uri->password = g_strdup (base->password); + uri->host = g_strdup (base->host); + uri->port = base->port; + + if (!uri->path) { + uri->path = g_strdup (base->path); + if (!uri->query) + uri->query = g_strdup (base->query); + remove_dot_segments = FALSE; + } else if (*uri->path != '/') { + char *newpath, *last; + + last = strrchr (base->path, '/'); + if (last) { + newpath = g_strdup_printf ("%.*s%s", + (int)(last + 1 - base->path), + base->path, + uri->path); + } else + newpath = g_strdup_printf ("/%s", uri->path); + + g_free (uri->path); + uri->path = newpath; + } + } + + if (remove_dot_segments && uri->path && *uri->path) { + char *p, *q; + + /* Remove "./" where "." is a complete segment. */ + for (p = uri->path + 1; *p; ) { + if (*(p - 1) == '/' && + *p == '.' && *(p + 1) == '/') + memmove (p, p + 2, strlen (p + 2) + 1); + else + p++; + } + /* Remove "." at end. */ + if (p > uri->path + 2 && + *(p - 1) == '.' && *(p - 2) == '/') + *(p - 1) = '\0'; + + /* Remove "/../" where != ".." */ + for (p = uri->path + 1; *p; ) { + if (!strncmp (p, "../", 3)) { + p += 3; + continue; + } + q = strchr (p + 1, '/'); + if (!q) + break; + if (strncmp (q, "/../", 4) != 0) { + p = q + 1; + continue; + } + memmove (p, q + 4, strlen (q + 4) + 1); + p = uri->path + 1; + } + /* Remove "/.." at end where != ".." */ + q = strrchr (uri->path, '/'); + if (q && !strcmp (q, "/..")) { + p = q - 1; + while (p > uri->path && *p != '/') + p--; + if (strncmp (p, "/../", 4) != 0) + *(p + 1) = 0; + } + + /* Remove extraneous initial "/.."s */ + while (!strncmp (uri->path, "/../", 4)) + memmove (uri->path, uri->path + 3, strlen (uri->path) - 2); + if (!strcmp (uri->path, "/..")) + uri->path[1] = '\0'; + } + + /* HTTP-specific stuff */ + if (uri->scheme == SOUP_URI_SCHEME_HTTP || + uri->scheme == SOUP_URI_SCHEME_HTTPS) { + if (!uri->path) + uri->path = g_strdup ("/"); + if (!SOUP_URI_VALID_FOR_HTTP (uri)) { + soup_uri_free (uri); + return NULL; + } + } + + if (uri->scheme == SOUP_URI_SCHEME_FTP) { + if (!uri->host) { + soup_uri_free (uri); + return NULL; + } + } + + if (!uri->port) + uri->port = soup_scheme_default_port (uri->scheme); + if (!uri->path) + uri->path = g_strdup (""); + + return uri; +} + +/** + * soup_uri_new: + * @uri_string: (allow-none): a URI + * + * Parses an absolute URI. + * + * You can also pass %NULL for @uri_string if you want to get back an + * "empty" #SoupURI that you can fill in by hand. (You will need to + * call at least soup_uri_set_scheme() and soup_uri_set_path(), since + * those fields are required.) + * + * Return value: (nullable): a #SoupURI, or %NULL if the given string + * was found to be invalid. + **/ +SoupURI * +soup_uri_new (const char *uri_string) +{ + SoupURI *uri; + + if (!uri_string) + return g_slice_new0 (SoupURI); + + uri = soup_uri_new_with_base (NULL, uri_string); + if (!uri) + return NULL; + if (!SOUP_URI_IS_VALID (uri)) { + soup_uri_free (uri); + return NULL; + } + + return uri; +} + + +char * +soup_uri_to_string_internal (SoupURI *uri, gboolean just_path_and_query, + gboolean force_port) +{ + GString *str; + char *return_result; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + str = g_string_sized_new (40); + + if (uri->scheme && !just_path_and_query) + g_string_append_printf (str, "%s:", uri->scheme); + if (uri->host && !just_path_and_query) { + g_string_append (str, "//"); + if (uri->user) { + append_uri_encoded (str, uri->user, ":;@?/"); + g_string_append_c (str, '@'); + } + if (strchr (uri->host, ':')) { + const char *pct; + + g_string_append_c (str, '['); + pct = strchr (uri->host, '%'); + if (pct) { + g_string_append_printf (str, "%.*s%%25%s", + (int) (pct - uri->host), + uri->host, pct + 1); + } else + g_string_append (str, uri->host); + g_string_append_c (str, ']'); + } else + append_uri_encoded (str, uri->host, ":/"); + if (uri->port && (force_port || uri->port != soup_scheme_default_port (uri->scheme))) + g_string_append_printf (str, ":%u", uri->port); + if (!uri->path && (uri->query || uri->fragment)) + g_string_append_c (str, '/'); + else if ((!uri->path || !*uri->path) && + (uri->scheme == SOUP_URI_SCHEME_HTTP || + uri->scheme == SOUP_URI_SCHEME_HTTPS)) + g_string_append_c (str, '/'); + } + + if (uri->path && *uri->path) + g_string_append (str, uri->path); + else if (just_path_and_query) + g_string_append_c (str, '/'); + + if (uri->query) { + g_string_append_c (str, '?'); + g_string_append (str, uri->query); + } + if (uri->fragment && !just_path_and_query) { + g_string_append_c (str, '#'); + g_string_append (str, uri->fragment); + } + + return_result = str->str; + g_string_free (str, FALSE); + + return return_result; +} + +/** + * soup_uri_to_string: + * @uri: a #SoupURI + * @just_path_and_query: if %TRUE, output just the path and query portions + * + * Returns a string representing @uri. + * + * If @just_path_and_query is %TRUE, this concatenates the path and query + * together. That is, it constructs the string that would be needed in + * the Request-Line of an HTTP request for @uri. + * + * Note that the output will never contain a password, even if @uri + * does. + * + * Return value: a string representing @uri, which the caller must free. + **/ +char * +soup_uri_to_string (SoupURI *uri, gboolean just_path_and_query) +{ + return soup_uri_to_string_internal (uri, just_path_and_query, FALSE); +} + +/** + * soup_uri_copy: + * @uri: a #SoupURI + * + * Copies @uri + * + * Return value: a copy of @uri, which must be freed with soup_uri_free() + **/ +SoupURI * +soup_uri_copy (SoupURI *uri) +{ + SoupURI *dup; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + dup = g_slice_new0 (SoupURI); + dup->scheme = uri->scheme; + dup->user = g_strdup (uri->user); + dup->password = g_strdup (uri->password); + dup->host = g_strdup (uri->host); + dup->port = uri->port; + dup->path = g_strdup (uri->path); + dup->query = g_strdup (uri->query); + dup->fragment = g_strdup (uri->fragment); + + return dup; +} + +static inline gboolean +parts_equal (const char *one, const char *two, gboolean insensitive) +{ + if (!one && !two) + return TRUE; + if (!one || !two) + return FALSE; + return insensitive ? !g_ascii_strcasecmp (one, two) : !strcmp (one, two); +} + +/** + * soup_uri_equal: + * @uri1: a #SoupURI + * @uri2: another #SoupURI + * + * Tests whether or not @uri1 and @uri2 are equal in all parts + * + * Return value: %TRUE or %FALSE + **/ +gboolean +soup_uri_equal (SoupURI *uri1, SoupURI *uri2) +{ + g_return_val_if_fail (uri1 != NULL, FALSE); + g_return_val_if_fail (uri2 != NULL, FALSE); + g_warn_if_fail (SOUP_URI_IS_VALID (uri1)); + g_warn_if_fail (SOUP_URI_IS_VALID (uri2)); + + if (uri1->scheme != uri2->scheme || + uri1->port != uri2->port || + !parts_equal (uri1->user, uri2->user, FALSE) || + !parts_equal (uri1->password, uri2->password, FALSE) || + !parts_equal (uri1->host, uri2->host, TRUE) || + !parts_equal (uri1->path, uri2->path, FALSE) || + !parts_equal (uri1->query, uri2->query, FALSE) || + !parts_equal (uri1->fragment, uri2->fragment, FALSE)) + return FALSE; + + return TRUE; +} + +/** + * soup_uri_free: + * @uri: a #SoupURI + * + * Frees @uri. + **/ +void +soup_uri_free (SoupURI *uri) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->user); + g_free (uri->password); + g_free (uri->host); + g_free (uri->path); + g_free (uri->query); + g_free (uri->fragment); + + g_slice_free (SoupURI, uri); +} + +static void +append_uri_encoded (GString *str, const char *in, const char *extra_enc_chars) +{ + const unsigned char *s = (const unsigned char *)in; + + while (*s) { + if (soup_char_is_uri_percent_encoded (*s) || + soup_char_is_uri_gen_delims (*s) || + (extra_enc_chars && strchr (extra_enc_chars, *s))) + g_string_append_printf (str, "%%%02X", (int)*s++); + else + g_string_append_c (str, *s++); + } +} + +/** + * soup_uri_encode: + * @part: a URI part + * @escape_extra: (allow-none): additional reserved characters to + * escape (or %NULL) + * + * This %-encodes the given URI part and returns the escaped + * version in allocated memory, which the caller must free when it is + * done. + * + * Return value: the encoded URI part + **/ +char * +soup_uri_encode (const char *part, const char *escape_extra) +{ + GString *str; + char *encoded; + + g_return_val_if_fail (part != NULL, NULL); + + str = g_string_new (NULL); + append_uri_encoded (str, part, escape_extra); + encoded = str->str; + g_string_free (str, FALSE); + + return encoded; +} + +#define XDIGIT(c) ((c) <= '9' ? (c) - '0' : ((c) & 0x4F) - 'A' + 10) +#define HEXCHAR(s) ((XDIGIT (s[1]) << 4) + XDIGIT (s[2])) + +char * +soup_uri_decoded_copy (const char *part, int length, int *decoded_length) +{ + unsigned char *s, *d; + char *decoded; + + g_return_val_if_fail (part != NULL, NULL); + + decoded = g_strndup (part, length); + s = d = (unsigned char *)decoded; + do { + if (*s == '%') { + if (!g_ascii_isxdigit (s[1]) || + !g_ascii_isxdigit (s[2])) { + *d++ = *s; + continue; + } + *d++ = HEXCHAR (s); + s += 2; + } else + *d++ = *s; + } while (*s++); + + if (decoded_length) + *decoded_length = d - (unsigned char *)decoded - 1; + + return decoded; +} + +/** + * soup_uri_decode: + * @part: a URI part + * + * Fully %-decodes @part. + * + * In the past, this would return %NULL if @part contained invalid + * percent-encoding, but now it just ignores the problem (as + * soup_uri_new() already did). + * + * Return value: the decoded URI part. + */ +char * +soup_uri_decode (const char *part) +{ + g_return_val_if_fail (part != NULL, NULL); + + return soup_uri_decoded_copy (part, strlen (part), NULL); +} + +static char * +uri_normalized_copy (const char *part, int length, + const char *unescape_extra) +{ + unsigned char *s, *d, c; + char *normalized = g_strndup (part, length); + gboolean need_fixup = FALSE; + + if (!unescape_extra) + unescape_extra = ""; + + s = d = (unsigned char *)normalized; + while (*s) { + if (*s == '%') { + if (!g_ascii_isxdigit (s[1]) || + !g_ascii_isxdigit (s[2])) { + *d++ = *s++; + continue; + } + + c = HEXCHAR (s); + if (soup_char_is_uri_unreserved (c) || + (c && strchr (unescape_extra, c))) { + *d++ = c; + s += 3; + } else { + /* We leave it unchanged. We used to uppercase percent-encoded + * triplets but we do not do it any more as RFC3986 Section 6.2.2.1 + * says that they only SHOULD be case normalized. + */ + *d++ = *s++; + *d++ = *s++; + *d++ = *s++; + } + } else { + if (!g_ascii_isgraph (*s) && + !strchr (unescape_extra, *s)) + need_fixup = TRUE; + *d++ = *s++; + } + } + *d = '\0'; + + if (need_fixup) { + GString *fixed; + + fixed = g_string_new (NULL); + s = (guchar *)normalized; + while (*s) { + if (g_ascii_isgraph (*s) || + strchr (unescape_extra, *s)) + g_string_append_c (fixed, *s); + else + g_string_append_printf (fixed, "%%%02X", (int)*s); + s++; + } + g_free (normalized); + normalized = g_string_free (fixed, FALSE); + } + + return normalized; +} + +/** + * soup_uri_normalize: + * @part: a URI part + * @unescape_extra: (allow-none): reserved characters to unescape (or %NULL) + * + * %-decodes any "unreserved" characters (or characters in + * @unescape_extra) in @part, and %-encodes any non-ASCII + * characters, spaces, and non-printing characters in @part. + * + * "Unreserved" characters are those that are not allowed to be used + * for punctuation according to the URI spec. For example, letters are + * unreserved, so soup_uri_normalize() will turn + * http://example.com/foo/b%61r into + * http://example.com/foo/bar, which is guaranteed + * to mean the same thing. However, "/" is "reserved", so + * http://example.com/foo%2Fbar would not + * be changed, because it might mean something different to the + * server. + * + * In the past, this would return %NULL if @part contained invalid + * percent-encoding, but now it just ignores the problem (as + * soup_uri_new() already did). + * + * Return value: the normalized URI part + */ +char * +soup_uri_normalize (const char *part, const char *unescape_extra) +{ + g_return_val_if_fail (part != NULL, NULL); + + return uri_normalized_copy (part, strlen (part), unescape_extra); +} + + +/** + * soup_uri_uses_default_port: + * @uri: a #SoupURI + * + * Tests if @uri uses the default port for its scheme. (Eg, 80 for + * http.) (This only works for http, https and ftp; libsoup does not know + * the default ports of other protocols.) + * + * Return value: %TRUE or %FALSE + **/ +gboolean +soup_uri_uses_default_port (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, FALSE); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + return uri->port == soup_scheme_default_port (uri->scheme); +} + +/** + * soup_uri_get_scheme: + * @uri: a #SoupURI + * + * Gets @uri's scheme. + * + * Return value: @uri's scheme. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_scheme (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->scheme; +} + +/** + * soup_uri_set_scheme: + * @uri: a #SoupURI + * @scheme: the URI scheme + * + * Sets @uri's scheme to @scheme. This will also set @uri's port to + * the default port for @scheme, if known. + **/ +void +soup_uri_set_scheme (SoupURI *uri, const char *scheme) +{ + g_return_if_fail (uri != NULL); + g_return_if_fail (scheme != NULL); + + uri->scheme = soup_uri_parse_scheme (scheme, strlen (scheme)); + uri->port = soup_scheme_default_port (uri->scheme); +} + +/** + * soup_uri_get_user: + * @uri: a #SoupURI + * + * Gets @uri's user. + * + * Return value: @uri's user. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_user (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->user; +} + +/** + * soup_uri_set_user: + * @uri: a #SoupURI + * @user: (allow-none): the username, or %NULL + * + * Sets @uri's user to @user. + **/ +void +soup_uri_set_user (SoupURI *uri, const char *user) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->user); + uri->user = g_strdup (user); +} + +/** + * soup_uri_get_password: + * @uri: a #SoupURI + * + * Gets @uri's password. + * + * Return value: @uri's password. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_password (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->password; +} + +/** + * soup_uri_set_password: + * @uri: a #SoupURI + * @password: (allow-none): the password, or %NULL + * + * Sets @uri's password to @password. + **/ +void +soup_uri_set_password (SoupURI *uri, const char *password) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->password); + uri->password = g_strdup (password); +} + +/** + * soup_uri_get_host: + * @uri: a #SoupURI + * + * Gets @uri's host. + * + * Return value: @uri's host. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_host (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->host; +} + +/** + * soup_uri_set_host: + * @uri: a #SoupURI + * @host: (allow-none): the hostname or IP address, or %NULL + * + * Sets @uri's host to @host. + * + * If @host is an IPv6 IP address, it should not include the brackets + * required by the URI syntax; they will be added automatically when + * converting @uri to a string. + * + * http and https URIs should not have a %NULL @host. + **/ +void +soup_uri_set_host (SoupURI *uri, const char *host) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->host); + uri->host = g_strdup (host); +} + +/** + * soup_uri_get_port: + * @uri: a #SoupURI + * + * Gets @uri's port. + * + * Return value: @uri's port. + * + * Since: 2.32 + **/ +guint +soup_uri_get_port (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, 0); + + return uri->port; +} + +/** + * soup_uri_set_port: + * @uri: a #SoupURI + * @port: the port, or 0 + * + * Sets @uri's port to @port. If @port is 0, @uri will not have an + * explicitly-specified port. + **/ +void +soup_uri_set_port (SoupURI *uri, guint port) +{ + g_return_if_fail (uri != NULL); + + uri->port = port; +} + +/** + * soup_uri_get_path: + * @uri: a #SoupURI + * + * Gets @uri's path. + * + * Return value: @uri's path. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_path (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->path; +} + +/** + * soup_uri_set_path: + * @uri: a #SoupURI + * @path: the non-%NULL path + * + * Sets @uri's path to @path. + **/ +void +soup_uri_set_path (SoupURI *uri, const char *path) +{ + g_return_if_fail (uri != NULL); + + /* We allow a NULL path for compatibility, but warn about it. */ + if (!path) { + g_warn_if_fail (path != NULL); + path = ""; + } + + g_free (uri->path); + uri->path = g_strdup (path); +} + +/** + * soup_uri_get_query: + * @uri: a #SoupURI + * + * Gets @uri's query. + * + * Return value: @uri's query. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_query (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->query; +} + +/** + * soup_uri_set_query: + * @uri: a #SoupURI + * @query: (allow-none): the query + * + * Sets @uri's query to @query. + **/ +void +soup_uri_set_query (SoupURI *uri, const char *query) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->query); + uri->query = g_strdup (query); +} + +/** + * soup_uri_set_query_from_form: + * @uri: a #SoupURI + * @form: (element-type utf8 utf8): a #GHashTable containing HTML form + * information + * + * Sets @uri's query to the result of encoding @form according to the + * HTML form rules. See soup_form_encode_hash() for more information. + **/ +void +soup_uri_set_query_from_form (SoupURI *uri, GHashTable *form) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->query); + uri->query = soup_form_encode_hash (form); +} + +/** + * soup_uri_set_query_from_fields: + * @uri: a #SoupURI + * @first_field: name of the first form field to encode into query + * @...: value of @first_field, followed by additional field names + * and values, terminated by %NULL. + * + * Sets @uri's query to the result of encoding the given form fields + * and values according to the * HTML form rules. See + * soup_form_encode() for more information. + **/ +void +soup_uri_set_query_from_fields (SoupURI *uri, + const char *first_field, + ...) +{ + va_list args; + + g_return_if_fail (uri != NULL); + + g_free (uri->query); + va_start (args, first_field); + uri->query = soup_form_encode_valist (first_field, args); + va_end (args); +} + +/** + * soup_uri_get_fragment: + * @uri: a #SoupURI + * + * Gets @uri's fragment. + * + * Return value: @uri's fragment. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_fragment (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->fragment; +} + +/** + * soup_uri_set_fragment: + * @uri: a #SoupURI + * @fragment: (allow-none): the fragment + * + * Sets @uri's fragment to @fragment. + **/ +void +soup_uri_set_fragment (SoupURI *uri, const char *fragment) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->fragment); + uri->fragment = g_strdup (fragment); +} + +/** + * soup_uri_copy_host: + * @uri: a #SoupURI + * + * Makes a copy of @uri, considering only the protocol, host, and port + * + * Return value: the new #SoupURI + * + * Since: 2.28 + **/ +SoupURI * +soup_uri_copy_host (SoupURI *uri) +{ + SoupURI *dup; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + dup = soup_uri_new (NULL); + dup->scheme = uri->scheme; + dup->host = g_strdup (uri->host); + dup->port = uri->port; + dup->path = g_strdup (""); + + return dup; +} + +/** + * soup_uri_host_hash: + * @key: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * + * Hashes @key, considering only the scheme, host, and port. + * + * Return value: a hash + * + * Since: 2.28 + **/ +guint +soup_uri_host_hash (gconstpointer key) +{ + const SoupURI *uri = key; + + g_return_val_if_fail (uri != NULL && uri->host != NULL, 0); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + return GPOINTER_TO_UINT (uri->scheme) + uri->port + + soup_str_case_hash (uri->host); +} + +/** + * soup_uri_host_equal: + * @v1: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * @v2: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * + * Compares @v1 and @v2, considering only the scheme, host, and port. + * + * Return value: whether or not the URIs are equal in scheme, host, + * and port. + * + * Since: 2.28 + **/ +gboolean +soup_uri_host_equal (gconstpointer v1, gconstpointer v2) +{ + const SoupURI *one = v1; + const SoupURI *two = v2; + + g_return_val_if_fail (one != NULL && two != NULL, one == two); + g_return_val_if_fail (one->host != NULL && two->host != NULL, one->host == two->host); + g_warn_if_fail (SOUP_URI_IS_VALID (one)); + g_warn_if_fail (SOUP_URI_IS_VALID (two)); + + if (one->scheme != two->scheme) + return FALSE; + if (one->port != two->port) + return FALSE; + + return g_ascii_strcasecmp (one->host, two->host) == 0; +} + +gboolean +soup_uri_is_http (SoupURI *uri, char **aliases) +{ + int i; + + if (uri->scheme == SOUP_URI_SCHEME_HTTP) + return TRUE; + else if (uri->scheme == SOUP_URI_SCHEME_HTTPS) + return FALSE; + else if (!aliases) + return FALSE; + + for (i = 0; aliases[i]; i++) { + if (uri->scheme == aliases[i]) + return TRUE; + } + + if (!aliases[1] && !strcmp (aliases[0], "*")) + return TRUE; + else + return FALSE; +} + +gboolean +soup_uri_is_https (SoupURI *uri, char **aliases) +{ + int i; + + if (uri->scheme == SOUP_URI_SCHEME_HTTPS) + return TRUE; + else if (uri->scheme == SOUP_URI_SCHEME_HTTP) + return FALSE; + else if (!aliases) + return FALSE; + + for (i = 0; aliases[i]; i++) { + if (uri->scheme == aliases[i]) + return TRUE; + } + + return FALSE; +} + +/* OSTREECHANGE: drop boxed type definition */ +/* G_DEFINE_BOXED_TYPE (SoupURI, soup_uri, soup_uri_copy, soup_uri_free) */ diff --git a/src/libostree/ostree-soup-uri.h b/src/libostree/ostree-soup-uri.h new file mode 100644 index 00000000..650b7efc --- /dev/null +++ b/src/libostree/ostree-soup-uri.h @@ -0,0 +1,147 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ + +/* + * Copyright 1999-2002 Ximian, Inc. + */ + +/* NOTE - taken from the libsoup codebase for use by the ostree curl backend + * (yes, ironically enough). + * + * Please watch for future changes in libsoup. + */ + + +#ifndef SOUP_URI_H +#define SOUP_URI_H 1 + +/* OSTREECHANGE: make struct private + * Only include gio, and skip available definitions. + */ +#include +#define SOUP_AVAILABLE_IN_2_4 +#define SOUP_AVAILABLE_IN_2_28 +#define SOUP_AVAILABLE_IN_2_32 + +G_BEGIN_DECLS + +/* OSTREECHANGE: make struct private */ +typedef struct _SoupURI SoupURI; + +/* OSTREECHANGE: import soup-misc's interning */ +#define SOUP_VAR extern +#define _SOUP_ATOMIC_INTERN_STRING(variable, value) ((const char *)(g_atomic_pointer_get (&(variable)) ? (variable) : (g_atomic_pointer_set (&(variable), (gpointer)g_intern_static_string (value)), (variable)))) +#define SOUP_URI_SCHEME_HTTP _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_HTTP, "http") +#define SOUP_URI_SCHEME_HTTPS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_HTTPS, "https") +#define SOUP_URI_SCHEME_FTP _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_FTP, "ftp") +#define SOUP_URI_SCHEME_FILE _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_FILE, "file") +#define SOUP_URI_SCHEME_DATA _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_DATA, "data") +#define SOUP_URI_SCHEME_RESOURCE _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_RESOURCE, "resource") +#define SOUP_URI_SCHEME_WS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_WS, "ws") +#define SOUP_URI_SCHEME_WSS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_WSS, "wss") + +/* OSTREECHANGE: import soup-form bits */ +SOUP_AVAILABLE_IN_2_4 +char *soup_form_encode_hash (GHashTable *form_data_set); +SOUP_AVAILABLE_IN_2_4 +char *soup_form_encode_valist (const char *first_field, + va_list args); + +SOUP_VAR gpointer _SOUP_URI_SCHEME_HTTP, _SOUP_URI_SCHEME_HTTPS; +SOUP_VAR gpointer _SOUP_URI_SCHEME_FTP; +SOUP_VAR gpointer _SOUP_URI_SCHEME_FILE, _SOUP_URI_SCHEME_DATA, _SOUP_URI_SCHEME_RESOURCE; +SOUP_VAR gpointer _SOUP_URI_SCHEME_WS, _SOUP_URI_SCHEME_WSS; + +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_new_with_base (SoupURI *base, + const char *uri_string); +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_new (const char *uri_string); + +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_to_string (SoupURI *uri, + gboolean just_path_and_query); + +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_copy (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_4 +gboolean soup_uri_equal (SoupURI *uri1, + SoupURI *uri2); + +SOUP_AVAILABLE_IN_2_4 +void soup_uri_free (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_encode (const char *part, + const char *escape_extra); +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_decode (const char *part); +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_normalize (const char *part, + const char *unescape_extra); + +SOUP_AVAILABLE_IN_2_4 +gboolean soup_uri_uses_default_port (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_scheme (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_scheme (SoupURI *uri, + const char *scheme); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_user (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_user (SoupURI *uri, + const char *user); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_password (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_password (SoupURI *uri, + const char *password); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_host (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_host (SoupURI *uri, + const char *host); +SOUP_AVAILABLE_IN_2_32 +guint soup_uri_get_port (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_port (SoupURI *uri, + guint port); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_path (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_path (SoupURI *uri, + const char *path); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_query (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query (SoupURI *uri, + const char *query); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query_from_form (SoupURI *uri, + GHashTable *form); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query_from_fields (SoupURI *uri, + const char *first_field, + ...) G_GNUC_NULL_TERMINATED; +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_fragment (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_fragment (SoupURI *uri, + const char *fragment); + +SOUP_AVAILABLE_IN_2_28 +SoupURI *soup_uri_copy_host (SoupURI *uri); +SOUP_AVAILABLE_IN_2_28 +guint soup_uri_host_hash (gconstpointer key); +SOUP_AVAILABLE_IN_2_28 +gboolean soup_uri_host_equal (gconstpointer v1, + gconstpointer v2); + +#define SOUP_URI_IS_VALID(uri) ((uri) && (uri)->scheme && (uri)->path) +#define SOUP_URI_VALID_FOR_HTTP(uri) ((uri) && ((uri)->scheme == SOUP_URI_SCHEME_HTTP || (uri)->scheme == SOUP_URI_SCHEME_HTTPS) && (uri)->host && (uri)->path) + +G_END_DECLS + +#endif /*SOUP_URI_H*/ diff --git a/src/ostree/ot-remote-builtin-add-cookie.c b/src/ostree/ot-remote-builtin-add-cookie.c index 509c9c7a..e4156172 100644 --- a/src/ostree/ot-remote-builtin-add-cookie.c +++ b/src/ostree/ot-remote-builtin-add-cookie.c @@ -21,13 +21,12 @@ #include "config.h" -#include - #include "otutil.h" #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { @@ -46,8 +45,6 @@ ot_remote_builtin_add_cookie (int argc, char **argv, GCancellable *cancellable, const char *value; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - SoupCookie *cookie; context = g_option_context_new ("NAME DOMAIN PATH COOKIE_NAME VALUE - Add a cookie to remote"); @@ -70,15 +67,8 @@ ot_remote_builtin_add_cookie (int argc, char **argv, GCancellable *cancellable, cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (gs_file_get_path_cached (repo->repodir), cookie_file, NULL); - jar = soup_cookie_jar_text_new (jar_path, FALSE); - - /* Pick a silly long expire time, we're just storing the cookies in the - * jar and on pull the jar is read-only so expiry has little actual value */ - cookie = soup_cookie_new (cookie_name, value, domain, path, - SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); - - /* jar takes ownership of cookie */ - soup_cookie_jar_add_cookie (jar, cookie); + if (!ot_add_cookie_at (AT_FDCWD, jar_path, domain, path, cookie_name, value, error)) + return FALSE; return TRUE; } diff --git a/src/ostree/ot-remote-builtin-delete-cookie.c b/src/ostree/ot-remote-builtin-delete-cookie.c index d974dd8d..6d1b85ad 100644 --- a/src/ostree/ot-remote-builtin-delete-cookie.c +++ b/src/ostree/ot-remote-builtin-delete-cookie.c @@ -21,14 +21,13 @@ #include "config.h" -#include - #include "otutil.h" +#include #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" - +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { { NULL } @@ -45,9 +44,6 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl const char *cookie_name; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - GSList *cookies; - gboolean found = FALSE; context = g_option_context_new ("NAME DOMAIN PATH COOKIE_NAME- Remote one cookie from remote"); @@ -69,28 +65,8 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (gs_file_get_path_cached (repo->repodir), cookie_file, NULL); - jar = soup_cookie_jar_text_new (jar_path, FALSE); - cookies = soup_cookie_jar_all_cookies (jar); + if (!ot_delete_cookie_at (AT_FDCWD, jar_path, domain, path, cookie_name, error)) + return FALSE; - while (cookies != NULL) - { - SoupCookie *cookie = cookies->data; - - if (!strcmp (domain, soup_cookie_get_domain (cookie)) && - !strcmp (path, soup_cookie_get_path (cookie)) && - !strcmp (cookie_name, soup_cookie_get_name (cookie))) - { - soup_cookie_jar_delete_cookie (jar, cookie); - - found = TRUE; - } - - soup_cookie_free (cookie); - cookies = g_slist_delete_link (cookies, cookies); - } - - if (!found) - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Cookie not found in jar"); - - return found; + return TRUE; } diff --git a/src/ostree/ot-remote-builtin-list-cookies.c b/src/ostree/ot-remote-builtin-list-cookies.c index 1865fb07..1c3924af 100644 --- a/src/ostree/ot-remote-builtin-list-cookies.c +++ b/src/ostree/ot-remote-builtin-list-cookies.c @@ -21,14 +21,12 @@ #include "config.h" -#include - #include "otutil.h" #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" - +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { { NULL } @@ -42,8 +40,6 @@ ot_remote_builtin_list_cookies (int argc, char **argv, GCancellable *cancellable const char *remote_name; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - GSList *cookies; context = g_option_context_new ("NAME - Show remote repository cookies"); @@ -62,25 +58,8 @@ ot_remote_builtin_list_cookies (int argc, char **argv, GCancellable *cancellable cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (g_file_get_path (repo->repodir), cookie_file, NULL); - jar = soup_cookie_jar_text_new (jar_path, TRUE); - cookies = soup_cookie_jar_all_cookies (jar); - - while (cookies != NULL) - { - SoupCookie *cookie = cookies->data; - SoupDate *expiry = soup_cookie_get_expires (cookie); - - g_print ("--\n"); - g_print ("Domain: %s\n", soup_cookie_get_domain (cookie)); - g_print ("Path: %s\n", soup_cookie_get_path (cookie)); - g_print ("Name: %s\n", soup_cookie_get_name (cookie)); - g_print ("Secure: %s\n", soup_cookie_get_secure (cookie) ? "yes" : "no"); - g_print ("Expires: %s\n", soup_date_to_string (expiry, SOUP_DATE_COOKIE)); - g_print ("Value: %s\n", soup_cookie_get_value (cookie)); - - soup_cookie_free (cookie); - cookies = g_slist_delete_link (cookies, cookies); - } + if (!ot_list_cookies_at (AT_FDCWD, jar_path, error)) + return FALSE; return TRUE; } diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c new file mode 100644 index 00000000..a96038aa --- /dev/null +++ b/src/ostree/ot-remote-cookie-util.c @@ -0,0 +1,333 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2015 Red Hat, Inc. + * Copyright (C) 2016 Sjoerd Simons + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "ot-remote-cookie-util.h" + +#ifndef HAVE_LIBCURL +#include +#endif + +#include "otutil.h" +#include "ot-main.h" +#include "ot-remote-builtins.h" +#include "ostree-repo-private.h" + +typedef struct OtCookieParser OtCookieParser; +struct OtCookieParser { + char *buf; + char *iter; + + char *line; + char *domain; + char *flag; + char *path; + char *secure; + long long unsigned int expiration; + char *name; + char *value; +}; +void ot_cookie_parser_free (OtCookieParser *parser); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtCookieParser, ot_cookie_parser_free) + +gboolean +ot_parse_cookies_at (int dfd, const char *path, + OtCookieParser **out_parser, + GCancellable *cancellable, + GError **error); +gboolean +ot_parse_cookies_next (OtCookieParser *parser); + +static void +ot_cookie_parser_clear (OtCookieParser *parser) +{ + g_clear_pointer (&parser->domain, (GDestroyNotify)g_free); + g_clear_pointer (&parser->flag, (GDestroyNotify)g_free); + g_clear_pointer (&parser->path, (GDestroyNotify)g_free); + g_clear_pointer (&parser->secure, (GDestroyNotify)g_free); + g_clear_pointer (&parser->name, (GDestroyNotify)g_free); + g_clear_pointer (&parser->value, (GDestroyNotify)g_free); +} + +void +ot_cookie_parser_free (OtCookieParser *parser) +{ + ot_cookie_parser_clear (parser); + g_free (parser->buf); + g_free (parser); +} + +gboolean +ot_parse_cookies_at (int dfd, const char *path, + OtCookieParser **out_parser, + GCancellable *cancellable, + GError **error) +{ + OtCookieParser *parser; + g_autofree char *cookies_content = NULL; + glnx_fd_close int infd = -1; + + infd = openat (dfd, path, O_RDONLY | O_CLOEXEC); + if (infd < 0) + { + if (errno != ENOENT) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + else + { + cookies_content = glnx_fd_readall_utf8 (infd, NULL, cancellable, error); + if (!cookies_content) + return FALSE; + } + + parser = *out_parser = g_new0 (OtCookieParser, 1); + parser->buf = g_steal_pointer (&cookies_content); + parser->iter = parser->buf; + return TRUE; +} + +gboolean +ot_parse_cookies_next (OtCookieParser *parser) +{ + while (parser->iter) + { + char *iter = parser->iter; + char *next = strchr (iter, '\n'); + + if (next) + { + *next = '\0'; + parser->iter = next + 1; + } + else + parser->iter = NULL; + + ot_cookie_parser_clear (parser); + if (sscanf (iter, "%ms\t%ms\t%ms\t%ms\t%llu\t%ms\t%ms", + &parser->domain, + &parser->flag, + &parser->path, + &parser->secure, + &parser->expiration, + &parser->name, + &parser->value) != 7) + continue; + + parser->line = iter; + return TRUE; + } + + return FALSE; +} + +gboolean +ot_add_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, const char *value, + GError **error) +{ +#ifdef HAVE_LIBCURL + glnx_fd_close int fd = openat (AT_FDCWD, jar_path, O_WRONLY | O_APPEND | O_CREAT, 0644); + g_autofree char *buf = NULL; + g_autoptr(GDateTime) now = NULL; + g_autoptr(GDateTime) expires = NULL; + + if (fd < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + + now = g_date_time_new_now_utc (); + expires = g_date_time_add_years (now, 25); + + /* Adapted from soup-cookie-jar-text.c:write_cookie() */ + buf = g_strdup_printf ("%s\t%s\t%s\t%s\t%llu\t%s\t%s\n", + domain, + *domain == '.' ? "TRUE" : "FALSE", + path, + "FALSE", + (long long unsigned)g_date_time_to_unix (expires), + name, + value); + if (glnx_loop_write (fd, buf, strlen (buf)) < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } +#else + glnx_unref_object SoupCookieJar *jar = NULL; + SoupCookie *cookie; + + jar = soup_cookie_jar_text_new (jar_path, FALSE); + + /* Pick a silly long expire time, we're just storing the cookies in the + * jar and on pull the jar is read-only so expiry has little actual value */ + cookie = soup_cookie_new (name, value, domain, path, + SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); + + /* jar takes ownership of cookie */ + soup_cookie_jar_add_cookie (jar, cookie); +#endif + return TRUE; +} + +gboolean +ot_delete_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, + GError **error) +{ + gboolean found = FALSE; +#ifdef HAVE_LIBCURL + glnx_fd_close int tempfile_fd = -1; + g_autofree char *tempfile_path = NULL; + g_autofree char *dnbuf = NULL; + const char *dn = NULL; + g_autoptr(OtCookieParser) parser = NULL; + + if (!ot_parse_cookies_at (dfd, jar_path, &parser, NULL, error)) + return FALSE; + + dnbuf = g_strdup (jar_path); + dn = dirname (dnbuf); + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tempfile_fd, &tempfile_path, + error)) + return FALSE; + + while (ot_parse_cookies_next (parser)) + { + if (strcmp (domain, parser->domain) == 0 && + strcmp (path, parser->path) == 0 && + strcmp (name, parser->name) == 0) + { + found = TRUE; + /* Match, skip writing this one */ + continue; + } + + if (glnx_loop_write (tempfile_fd, parser->line, strlen (parser->line)) < 0 || + glnx_loop_write (tempfile_fd, "\n", 1) < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + + if (!glnx_link_tmpfile_at (AT_FDCWD, GLNX_LINK_TMPFILE_REPLACE, + tempfile_fd, + tempfile_path, + AT_FDCWD, jar_path, + error)) + return FALSE; +#else + GSList *cookies; + glnx_unref_object SoupCookieJar *jar = NULL; + + jar = soup_cookie_jar_text_new (jar_path, FALSE); + cookies = soup_cookie_jar_all_cookies (jar); + + while (cookies != NULL) + { + SoupCookie *cookie = cookies->data; + + if (!strcmp (domain, soup_cookie_get_domain (cookie)) && + !strcmp (path, soup_cookie_get_path (cookie)) && + !strcmp (name, soup_cookie_get_name (cookie))) + { + soup_cookie_jar_delete_cookie (jar, cookie); + + found = TRUE; + } + + soup_cookie_free (cookie); + cookies = g_slist_delete_link (cookies, cookies); + } +#endif + + if (!found) + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Cookie not found in jar"); + + return TRUE; +} + + +gboolean +ot_list_cookies_at (int dfd, const char *jar_path, GError **error) +{ +#ifdef HAVE_LIBCURL + glnx_fd_close int tempfile_fd = -1; + g_autofree char *tempfile_path = NULL; + g_autofree char *dnbuf = NULL; + const char *dn = NULL; + g_autoptr(OtCookieParser) parser = NULL; + + if (!ot_parse_cookies_at (AT_FDCWD, jar_path, &parser, NULL, error)) + return FALSE; + + dnbuf = dirname (g_strdup (jar_path)); + dn = dnbuf; + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tempfile_fd, &tempfile_path, + error)) + return FALSE; + + while (ot_parse_cookies_next (parser)) + { + g_autoptr(GDateTime) expires = g_date_time_new_from_unix_utc (parser->expiration); + g_autofree char *expires_str = g_date_time_format (expires, "%Y-%m-%d %H:%M:%S +0000"); + + g_print ("--\n"); + g_print ("Domain: %s\n", parser->domain); + g_print ("Path: %s\n", parser->path); + g_print ("Name: %s\n", parser->name); + g_print ("Secure: %s\n", parser->secure); + g_print ("Expires: %s\n", expires_str); + g_print ("Value: %s\n", parser->value); + } +#else + glnx_unref_object SoupCookieJar *jar = soup_cookie_jar_text_new (jar_path, TRUE); + GSList *cookies = soup_cookie_jar_all_cookies (jar); + + while (cookies != NULL) + { + SoupCookie *cookie = cookies->data; + SoupDate *expiry = soup_cookie_get_expires (cookie); + + g_print ("--\n"); + g_print ("Domain: %s\n", soup_cookie_get_domain (cookie)); + g_print ("Path: %s\n", soup_cookie_get_path (cookie)); + g_print ("Name: %s\n", soup_cookie_get_name (cookie)); + g_print ("Secure: %s\n", soup_cookie_get_secure (cookie) ? "yes" : "no"); + g_print ("Expires: %s\n", soup_date_to_string (expiry, SOUP_DATE_COOKIE)); + g_print ("Value: %s\n", soup_cookie_get_value (cookie)); + + soup_cookie_free (cookie); + cookies = g_slist_delete_link (cookies, cookies); + } +#endif + return TRUE; +} diff --git a/src/ostree/ot-remote-cookie-util.h b/src/ostree/ot-remote-cookie-util.h new file mode 100644 index 00000000..1bcc0e87 --- /dev/null +++ b/src/ostree/ot-remote-cookie-util.h @@ -0,0 +1,42 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2017 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#pragma once + +#include "libglnx.h" + +G_BEGIN_DECLS + +gboolean +ot_add_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, const char *value, + GError **error); + +gboolean +ot_delete_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, + GError **error); + +gboolean +ot_list_cookies_at (int dfd, const char *jar_path, GError **error); + +G_END_DECLS diff --git a/tests/ci-build.sh b/tests/ci-build.sh index 4b736a3c..23eacf07 100755 --- a/tests/ci-build.sh +++ b/tests/ci-build.sh @@ -52,6 +52,10 @@ NULL= # If yes, test failures break the build; if no, they are reported but ignored : "${ci_test_fatal:=yes}" +# ci_configopts: +# Additional args for configure +: "${ci_configopts:=}" + if [ -n "$ci_docker" ]; then exec docker run \ --env=ci_docker="" \ @@ -59,6 +63,7 @@ if [ -n "$ci_docker" ]; then --env=ci_sudo=yes \ --env=ci_test="${ci_test}" \ --env=ci_test_fatal="${ci_test_fatal}" \ + --env=ci_configopts="${ci_configopts}" \ --privileged \ ci-image \ tests/ci-build.sh @@ -81,6 +86,7 @@ make="make -j${ci_parallel} V=1 VERBOSE=1" ../configure \ --enable-always-build-tests \ --enable-installed-tests \ + ${ci_configopts} "$@" ${make} diff --git a/tests/ci-install.sh b/tests/ci-install.sh index dbc86c69..92f802d9 100755 --- a/tests/ci-install.sh +++ b/tests/ci-install.sh @@ -50,6 +50,9 @@ NULL= # Typical values for ci_distro=fedora might be 25, rawhide : "${ci_suite:=jessie}" +# ci_configopts: Additional arguments for configure +: "${ci_configopts:=}" + if [ $(id -u) = 0 ]; then sudo= else @@ -104,6 +107,7 @@ case "$ci_distro" in libmount-dev \ libselinux1-dev \ libsoup2.4-dev \ + libcurl4-openssl-dev \ procps \ zlib1g-dev \ ${NULL} diff --git a/tests/test-remote-cookies.sh b/tests/test-remote-cookies.sh index 11c201f1..ab2bf263 100755 --- a/tests/test-remote-cookies.sh +++ b/tests/test-remote-cookies.sh @@ -28,12 +28,9 @@ setup_fake_remote_repo1 "archive-z2" "" \ "--expected-cookies foo=bar --expected-cookies baz=badger" assert_fail (){ - set +e - $@ - if [ $? = 0 ] ; then - echo 1>&2 "$@ did not fail"; exit 1 + if $@; then + (echo 1>&2 "$@ did not fail"; exit 1) fi - set -euo pipefail } cd ${test_tmpdir} @@ -50,12 +47,16 @@ echo "ok, setup done" # Add 2 cookies, pull should succeed now ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / foo bar ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_file_has_content repo/origin.cookies.txt baz.*badger ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, initial cookie pull succeeded" # Delete one cookie, if successful pulls will fail again ${CMD_PREFIX} ostree --repo=repo remote delete-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_not_file_has_content repo/origin.cookies.txt baz.*badger assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, delete succeeded" @@ -63,6 +64,8 @@ echo "ok, delete succeeded" # Re-add the removed cooking and things succeed again, verified the removal # removed exactly one cookie ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_file_has_content repo/origin.cookies.txt baz.*badger ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, second cookie pull succeeded" From 40a1d32067aeed836858da87c1f03ea1dbb3515e Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 8 Feb 2017 22:06:05 -0500 Subject: [PATCH 21/66] fetcher queue: also throttle on outstanding writes When fetching over a fast enough connection, we can be receiving files faster than we write them. This can then lead to EMFILE when we have enough files open. This was made very easy to notice with the upcoming libcurl backend, which makes use of pipelining. Closes: #675 Approved by: cgwalters --- src/libostree/ostree-repo-private.h | 8 ++++++++ src/libostree/ostree-repo-pull.c | 21 ++++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 73e02446..f1e00f27 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -35,6 +35,14 @@ G_BEGIN_DECLS #define _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS 8 #define _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS 2 +/* In most cases, writing to disk should be much faster than + * fetching from the network, so we shouldn't actually hit + * this. But if using pipelining and e.g. pulling over LAN + * (or writing to slow media), we can have a runaway + * situation towards EMFILE. + * */ +#define _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS 16 + typedef enum { OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0) } OstreeRepoTestErrorFlags; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 295973ec..0e5128d9 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -345,15 +345,26 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, /* We have a total-request limit, as well has a hardcoded max of 2 for delta * parts. The logic for the delta one is that processing them is expensive, and - * doing multiple simultaneously could risk space/memory on smaller devices. + * doing multiple simultaneously could risk space/memory on smaller devices. We + * also throttle on outstanding writes in case fetches are faster. */ static gboolean fetcher_queue_is_full (OtPullData *pull_data) { - return (pull_data->n_outstanding_metadata_fetches + - pull_data->n_outstanding_content_fetches + - pull_data->n_outstanding_deltapart_fetches) == _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS || - pull_data->n_outstanding_deltapart_fetches == _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS; + const gboolean fetch_full = + ((pull_data->n_outstanding_metadata_fetches + + pull_data->n_outstanding_content_fetches + + pull_data->n_outstanding_deltapart_fetches) == + _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS); + const gboolean deltas_full = + (pull_data->n_outstanding_deltapart_fetches == + _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS); + const gboolean writes_full = + ((pull_data->n_outstanding_metadata_write_requests + + pull_data->n_outstanding_content_write_requests + + pull_data->n_outstanding_deltapart_write_requests) >= + _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS); + return fetch_full || deltas_full || writes_full; } static gboolean From b13ead1c5bacc7b8bfb12fb858f8699fac3bd7bb Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 10 Feb 2017 14:14:27 +0200 Subject: [PATCH 22/66] libostree: added empty ot_cleanup_{read,write}_archive macros. Added empty macros for ot_cleanup_{read,write}_archive to fix errors when compiling without libarchive. Signed-off-by: Krisztian Litkey Closes: #677 Approved by: cgwalters --- src/libostree/ostree-libarchive-private.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libostree/ostree-libarchive-private.h b/src/libostree/ostree-libarchive-private.h index 177c6568..865a4b8d 100644 --- a/src/libostree/ostree-libarchive-private.h +++ b/src/libostree/ostree-libarchive-private.h @@ -37,6 +37,9 @@ GLNX_DEFINE_CLEANUP_FUNCTION (void *, flatpak_local_free_write_archive, archive_ GLNX_DEFINE_CLEANUP_FUNCTION (void *, flatpak_local_free_read_archive, archive_read_free) #define ot_cleanup_read_archive __attribute__((cleanup (flatpak_local_free_read_archive))) +#else +#define ot_cleanup_write_archive +#define ot_cleanup_read_archive #endif G_END_DECLS From e6a8979e05894f78bedcc71bb1baf3a8edfdf08a Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 9 Feb 2017 19:49:06 +0000 Subject: [PATCH 23/66] ostree-repo: Clarify error behaviour of remote option getters Clarify the documentation for functions like ostree_repo_get_remote_boolean_option(), stating what out_value will be set to on error. Signed-off-by: Philip Withnall Closes: #676 Approved by: cgwalters --- src/libostree/ostree-repo.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 47e95ae4..e7cbb16d 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -284,7 +284,7 @@ _ostree_repo_remote_name_is_file (const char *remote_name) * OSTree remotes are represented by keyfile groups, formatted like: * `[remote "remotename"]`. This function returns a value named @option_name * underneath that group, or @default_value if the remote exists but not the - * option name. + * option name. If an error is returned, @out_value will be set to %NULL. * * Returns: %TRUE on success, otherwise %FALSE with @error set */ @@ -360,8 +360,9 @@ ostree_repo_get_remote_option (OstreeRepo *self, * * OSTree remotes are represented by keyfile groups, formatted like: * `[remote "remotename"]`. This function returns a value named @option_name - * underneath that group, and returns it as an zero terminated array of strings. - * If the option is not set, @out_value will be set to %NULL. + * underneath that group, and returns it as a zero terminated array of strings. + * If the option is not set, or if an error is returned, @out_value will be set + * to %NULL. * * Returns: %TRUE on success, otherwise %FALSE with @error set */ @@ -435,7 +436,8 @@ ostree_repo_get_remote_list_option (OstreeRepo *self, * OSTree remotes are represented by keyfile groups, formatted like: * `[remote "remotename"]`. This function returns a value named @option_name * underneath that group, and returns it as a boolean. - * If the option is not set, @out_value will be set to @default_value. + * If the option is not set, @out_value will be set to @default_value. If an + * error is returned, @out_value will be set to %FALSE. * * Returns: %TRUE on success, otherwise %FALSE with @error set */ From b8f4465b50ba9ff87fb6f44c2f93d7b574754c0a Mon Sep 17 00:00:00 2001 From: Anton Gerasimov Date: Mon, 13 Feb 2017 17:34:18 +0100 Subject: [PATCH 24/66] admin-switch: Don't segfault if there's no remote Switching between local branches should be supported too. Signed-off-by: Anton Gerasimov Closes: #683 Approved by: cgwalters --- src/ostree/ot-admin-builtin-switch.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/ostree/ot-admin-builtin-switch.c b/src/ostree/ot-admin-builtin-switch.c index 877cbe96..d72aeebd 100644 --- a/src/ostree/ot-admin-builtin-switch.c +++ b/src/ostree/ot-admin-builtin-switch.c @@ -53,6 +53,7 @@ ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GErro g_autofree char *new_remote = NULL; g_autofree char *new_ref = NULL; g_autofree char *new_refspec = NULL; + const char* remote; glnx_unref_object OstreeSysrootUpgrader *upgrader = NULL; glnx_unref_object OstreeAsyncProgress *progress = NULL; gboolean changed; @@ -101,12 +102,17 @@ ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GErro if (!ostree_parse_refspec (new_provided_refspec, &new_remote, &new_ref, error)) goto out; } - + if (!new_remote) - new_refspec = g_strconcat (origin_remote, ":", new_ref, NULL); + remote = origin_remote; else - new_refspec = g_strconcat (new_remote, ":", new_ref, NULL); - + remote = new_remote; + + if (remote) + new_refspec = g_strconcat (remote, ":", new_ref, NULL); + else + new_refspec = g_strdup (new_ref); + if (strcmp (origin_refspec, new_refspec) == 0) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, From 5a73a366b966f5917bf962e447744ae58a74a890 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 10 Feb 2017 17:18:31 -0500 Subject: [PATCH 25/66] libglnx: Bump Pulls in the xattr fixes and the tempname perf improvement. Closes: #680 Approved by: jlebon --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index abd37a47..7a703638 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit abd37a4790f86f53bfb442e6d80e1710f50bff92 +Subproject commit 7a703638d12410bb58ca6d85d6556d03c2a5a7ce From 46544f5b4da4d33967794136094d3d6f14d599f1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 11 Feb 2017 11:24:15 -0500 Subject: [PATCH 26/66] commit: Support -F/--body-file, like git This is more convenient to script for projects which haven't yet made the leap to using the API. Closes: https://github.com/ostreedev/ostree/issues/674 Closes: #681 Approved by: jlebon --- src/ostree/ot-builtin-commit.c | 18 +++++++++++++++--- tests/basic-test.sh | 18 +++++++++++++++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index d3b634f9..218fb701 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -32,6 +32,7 @@ static char *opt_subject; static char *opt_body; +static char *opt_body_file; static gboolean opt_editor; static char *opt_parent; static gboolean opt_orphan; @@ -74,6 +75,7 @@ static GOptionEntry options[] = { { "parent", 0, 0, G_OPTION_ARG_STRING, &opt_parent, "Parent ref, or \"none\"", "REF" }, { "subject", 's', 0, G_OPTION_ARG_STRING, &opt_subject, "One line subject", "SUBJECT" }, { "body", 'm', 0, G_OPTION_ARG_STRING, &opt_body, "Full description", "BODY" }, + { "body-file", 'F', 0, G_OPTION_ARG_STRING, &opt_body_file, "Commit message from FILE path", "FILE" }, { "editor", 'e', 0, G_OPTION_ARG_NONE, &opt_editor, "Use an editor to write the commit message", NULL }, { "branch", 'b', 0, G_OPTION_ARG_STRING, &opt_branch, "Branch", "BRANCH" }, { "orphan", 0, 0, G_OPTION_ARG_NONE, &opt_orphan, "Create a commit without writing a ref", NULL }, @@ -349,6 +351,7 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError OstreeRepoCommitModifier *modifier = NULL; OstreeRepoTransactionStats stats; struct CommitFilterData filter_data = { 0, }; + g_autofree char *commit_body = NULL; context = g_option_context_new ("[PATH] - Commit a new revision"); @@ -441,9 +444,18 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError if (opt_editor) { - if (!commit_editor (repo, opt_branch, &opt_subject, &opt_body, cancellable, error)) + if (!commit_editor (repo, opt_branch, &opt_subject, &commit_body, cancellable, error)) goto out; } + else if (opt_body_file) + { + commit_body = glnx_file_get_contents_utf8_at (AT_FDCWD, opt_body_file, NULL, + cancellable, error); + if (!commit_body) + goto out; + } + else if (opt_body) + commit_body = g_strdup (opt_body); if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error)) goto out; @@ -576,7 +588,7 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError timestamp = g_date_time_to_unix (now); g_date_time_unref (now); - if (!ostree_repo_write_commit (repo, parent, opt_subject, opt_body, metadata, + if (!ostree_repo_write_commit (repo, parent, opt_subject, commit_body, metadata, OSTREE_REPO_FILE (root), &commit_checksum, cancellable, error)) goto out; @@ -592,7 +604,7 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError } timestamp = ts.tv_sec; - if (!ostree_repo_write_commit_with_time (repo, parent, opt_subject, opt_body, metadata, + if (!ostree_repo_write_commit_with_time (repo, parent, opt_subject, commit_body, metadata, OSTREE_REPO_FILE (root), timestamp, &commit_checksum, cancellable, error)) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 5cad8ab3..d9308138 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..60" +echo "1..61" $OSTREE checkout test2 checkout-test2 echo "ok checkout" @@ -115,6 +115,22 @@ omitted_rev=$($OSTREE commit -b test2-no-subject-2 --timestamp="2005-10-29 12:43 assert_streq $empty_rev $omitted_rev echo "ok commit no subject" +cd ${test_tmpdir} +cat >commitmsg.txt < log.txt +assert_file_has_content log.txt '^ *This is a long$' +assert_file_has_content log.txt '^ *Build-Host:.*example.com$' +assert_file_has_content log.txt '^ *Crunchy-With.*true$' +$OSTREE refs --delete branch-with-commitmsg +echo "ok commit body file" + cd ${test_tmpdir} $OSTREE commit -b test2-custom-parent -s '' $test_tmpdir/checkout-test2-4 $OSTREE commit -b test2-custom-parent -s '' $test_tmpdir/checkout-test2-4 From ba350982e89660ecf0caf1f4422d4266cadedf86 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 11 Feb 2017 15:33:57 -0500 Subject: [PATCH 27/66] build: Remove .PHONY for Rust shared library I have no idea why I made the lib `.PHONY` originally; it's clearly wrong, and I noticed because when I was doing `sudo make install`, we were doing a rebuild, which in turn triggered other things to be built, and they'd be owned by root. Closes: #682 Approved by: jlebon --- Makefile-libostree.am | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 79fddec7..70452f1a 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -24,7 +24,6 @@ noinst_LTLIBRARIES += libostree-kernel-args.la if ENABLE_RUST bupsplitpath = @abs_top_builddir@/target/@RUST_TARGET_SUBDIR@/libbupsplit_rs.a -.PHONY: $(bupsplitpath) BUPSPLIT_RUST_SOURCES = rust/src/bupsplit.rs EXTRA_DIST += $(BUPSPLIT_RUST_SOURCES) $(bupsplitpath): Makefile $(BUPSPLIT_RUST_SOURCES) From 4e908f867d3577a418fb54adb356f396a1181259 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 14 Feb 2017 14:36:28 +0100 Subject: [PATCH 28/66] rofiles-fuse: Support write/read_buf() These allow us to avoid copying a lot of data around in userspace. Instead we splice the data directly from the fd to the destination fd. Closes: #684 Approved by: cgwalters --- src/rofiles-fuse/main.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/rofiles-fuse/main.c b/src/rofiles-fuse/main.c index ac44a438..3f0832f5 100644 --- a/src/rofiles-fuse/main.c +++ b/src/rofiles-fuse/main.c @@ -344,6 +344,26 @@ callback_create(const char *path, mode_t mode, struct fuse_file_info *finfo) return do_open (path, mode, finfo); } +static int +callback_read_buf (const char *path, struct fuse_bufvec **bufp, + size_t size, off_t offset, struct fuse_file_info *finfo) +{ + struct fuse_bufvec *src; + + src = malloc (sizeof (struct fuse_bufvec)); + if (src == NULL) + return -ENOMEM; + + *src = FUSE_BUFVEC_INIT (size); + + src->buf[0].flags = FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK; + src->buf[0].fd = finfo->fh; + src->buf[0].pos = offset; + *bufp = src; + + return 0; +} + static int callback_read (const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *finfo) @@ -355,6 +375,19 @@ callback_read (const char *path, char *buf, size_t size, off_t offset, return r; } +static int +callback_write_buf (const char *path, struct fuse_bufvec *buf, off_t offset, + struct fuse_file_info *finfo) +{ + struct fuse_bufvec dst = FUSE_BUFVEC_INIT (fuse_buf_size (buf)); + + dst.buf[0].flags = FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK; + dst.buf[0].fd = finfo->fh; + dst.buf[0].pos = offset; + + return fuse_buf_copy (&dst, buf, FUSE_BUF_SPLICE_NONBLOCK); +} + static int callback_write (const char *path, const char *buf, size_t size, off_t offset, struct fuse_file_info *finfo) @@ -454,7 +487,9 @@ struct fuse_operations callback_oper = { .utime = callback_utime, .create = callback_create, .open = callback_open, + .read_buf = callback_read_buf, .read = callback_read, + .write_buf = callback_write_buf, .write = callback_write, .statfs = callback_statfs, .release = callback_release, From da21d7350e1655f9cd575e79464e53fe58e4e23b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 14 Feb 2017 10:14:25 -0500 Subject: [PATCH 29/66] Release 2017.2 We should get a release out to try to keep with at least a once-a-month cadence. This one has some exciting stuff like libcurl and Rust, and various bugfixes. Also importantly I want to cut this *before* we land some other bigger stuff, so rpm-ostree can start using the reload_config API etc. Closes: #685 Approved by: jlebon --- configure.ac | 2 +- src/libostree/libostree.sym | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/configure.ac b/configure.ac index 596ee040..3448ab6e 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ([2.63]) dnl If incrementing the version here, remember to update libostree.sym too -AC_INIT([libostree], [2017.1], [walters@verbum.org]) +AC_INIT([libostree], [2017.2], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 1345b9c1..a07e5d3b 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -371,21 +371,29 @@ global: ostree_repo_prune_from_reachable; } LIBOSTREE_2016.14; -/* NOTE NOTE NOTE - * Versions above here are released. Only add symbols below this line. - * NOTE NOTE NOTE - */ - LIBOSTREE_2017.2 { global: ostree_repo_reload_config; } LIBOSTREE_2017.1; -/* Stub section for the stable release *after* this development one; don't - * edit this other than to update the last number. This is just a copy/paste - * source. -LIBOSTREE_2016.XX { +/* NOTE NOTE NOTE + * Versions above here are released. Only add symbols below this line. + * NOTE NOTE NOTE + */ + +/* Uncomment this when adding the first new symbol for 2 +LIBOSTREE_2017.XX { global: someostree_symbol_deleteme; -} LIBOSTREE_2016.14; +} LIBOSTREE_2017.2; +*/ + +/* Stub section for the stable release *after* this development one; don't + * edit this other than to update the last number. This is just a copy/paste + * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION + * with whatever the next version with new symbols will be. +LIBOSTREE_2017.$NEWVERSION { +global: + someostree_symbol_deleteme; +} LIBOSTREE_2017.$LASTSTABLE; */ From a1805d6101eb8efeaa72459c22d88ed08ff7a065 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 16 Feb 2017 20:52:09 +0100 Subject: [PATCH 30/66] contrib/golang: rm directory rm -r the directory since we are keeping the Go bindings separately under https://github.com/ostreedev/ostree-go Signed-off-by: Giuseppe Scrivano Closes: #690 Approved by: cgwalters --- contrib/golang/COPYING | 17 --- contrib/golang/README.md | 2 - contrib/golang/glibobject.go | 201 --------------------------------- contrib/golang/glibobject.go.h | 17 --- contrib/golang/ostree.go | 94 --------------- contrib/golang/ostree.go.h | 21 ---- contrib/golang/ostree_test.go | 55 --------- libglnx | 2 +- 8 files changed, 1 insertion(+), 408 deletions(-) delete mode 100644 contrib/golang/COPYING delete mode 100644 contrib/golang/README.md delete mode 100644 contrib/golang/glibobject.go delete mode 100644 contrib/golang/glibobject.go.h delete mode 100644 contrib/golang/ostree.go delete mode 100644 contrib/golang/ostree.go.h delete mode 100644 contrib/golang/ostree_test.go diff --git a/contrib/golang/COPYING b/contrib/golang/COPYING deleted file mode 100644 index aa93b4da..00000000 --- a/contrib/golang/COPYING +++ /dev/null @@ -1,17 +0,0 @@ -Portions of this code are derived from: - -https://github.com/dradtke/gotk3 - -Copyright (c) 2013 Conformal Systems LLC. - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/contrib/golang/README.md b/contrib/golang/README.md deleted file mode 100644 index 60a4856d..00000000 --- a/contrib/golang/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This file contains demonstration FFI bindings for using `-lostree-1` -and `-larchive` from golang. diff --git a/contrib/golang/glibobject.go b/contrib/golang/glibobject.go deleted file mode 100644 index 0b62b443..00000000 --- a/contrib/golang/glibobject.go +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright (c) 2013 Conformal Systems - * - * This file originated from: http://opensource.conformal.com/ - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package ostree - -// #cgo pkg-config: glib-2.0 gobject-2.0 -// #include -// #include -// #include -// #include "glibobject.go.h" -// #include -import "C" -import ( - "unsafe" - "runtime" - "fmt" - "errors" -) - -func GBool(b bool) C.gboolean { - if b { - return C.gboolean(1) - } - return C.gboolean(0) -} - -func GoBool(b C.gboolean) bool { - if b != 0 { - return true - } - return false -} - -type GError struct { - ptr unsafe.Pointer -} - -func NewGError() GError { - return GError{nil} -} - -func (e *GError) Native() *C.GError { - if e == nil { - return nil - } - return (*C.GError)(e.ptr) -} - -func ConvertGError(e *C.GError) error { - defer C.g_error_free(e) - return errors.New(C.GoString((*C.char)(C._g_error_get_message(e)))) -} - -type GType uint - -func (t GType) Name() string { - return C.GoString((*C.char)(C.g_type_name(C.GType(t)))) -} - -type GVariant struct { - ptr unsafe.Pointer -} - -func GVariantNew(p unsafe.Pointer) *GVariant { - o := &GVariant{p} - runtime.SetFinalizer(o, (*GVariant).Unref) - return o; -} - -func GVariantNewSink(p unsafe.Pointer) *GVariant { - o := &GVariant{p} - runtime.SetFinalizer(o, (*GVariant).Unref) - o.RefSink() - return o; -} - -func (v *GVariant) native() *C.GVariant { - return (*C.GVariant)(v.ptr); -} - -func (v *GVariant) Ref() { - C.g_variant_ref(v.native()) -} - -func (v *GVariant) Unref() { - C.g_variant_unref(v.native()) -} - -func (v *GVariant) RefSink() { - C.g_variant_ref_sink(v.native()) -} - -func (v *GVariant) TypeString() string { - cs := (*C.char)(C.g_variant_get_type_string(v.native())) - return C.GoString(cs) -} - -func (v *GVariant) GetChildValue(i int) *GVariant { - cchild := C.g_variant_get_child_value(v.native(), C.gsize(i)) - return GVariantNew(unsafe.Pointer(cchild)); -} - -func (v *GVariant) LookupString(key string) (string, error) { - ckey := C.CString(key) - defer C.free(unsafe.Pointer(ckey)) - // TODO: Find a way to have constant C strings in golang - cstr := C._g_variant_lookup_string(v.native(), ckey) - if cstr == nil { - return "", fmt.Errorf("No such key: %s", key) - } - return C.GoString(cstr), nil -} - -/* - * GObject - */ - -// IObject is an interface type implemented by Object and all types which embed -// an Object. It is meant to be used as a type for function arguments which -// require GObjects or any subclasses thereof. -type IObject interface { - toGObject() *C.GObject - ToObject() *GObject -} - -// Object is a representation of GLib's GObject. -type GObject struct { - ptr unsafe.Pointer -} - -func GObjectNew(p unsafe.Pointer) *GObject { - o := &GObject{p} - runtime.SetFinalizer(o, (*GObject).Unref) - return o; -} - -func (v *GObject) Ptr() unsafe.Pointer { - return v.ptr -} - -func (v *GObject) Native() *C.GObject { - if v == nil || v.ptr == nil { - return nil - } - return (*C.GObject)(v.ptr) -} - -func (v *GObject) toGObject() *C.GObject { - if v == nil { - return nil - } - return v.Native() -} - -func (v *GObject) Ref() { - C.g_object_ref(C.gpointer(v.ptr)) -} - -func (v *GObject) Unref() { - C.g_object_unref(C.gpointer(v.ptr)) -} - -func (v *GObject) RefSink() { - C.g_object_ref_sink(C.gpointer(v.ptr)) -} - -func (v *GObject) IsFloating() bool { - c := C.g_object_is_floating(C.gpointer(v.ptr)) - return GoBool(c) -} - -func (v *GObject) ForceFloating() { - C.g_object_force_floating((*C.GObject)(v.ptr)) -} - -// GIO types - -type GCancellable struct { - *GObject -} - -func (self *GCancellable) native() *C.GCancellable { - return (*C.GCancellable)(self.ptr) -} - -// At the moment, no cancellable API, just pass nil diff --git a/contrib/golang/glibobject.go.h b/contrib/golang/glibobject.go.h deleted file mode 100644 index a55bd242..00000000 --- a/contrib/golang/glibobject.go.h +++ /dev/null @@ -1,17 +0,0 @@ -#include - -static char * -_g_error_get_message (GError *error) -{ - g_assert (error != NULL); - return error->message; -} - -static const char * -_g_variant_lookup_string (GVariant *v, const char *key) -{ - const char *r; - if (g_variant_lookup (v, key, "&s", &r)) - return r; - return NULL; -} diff --git a/contrib/golang/ostree.go b/contrib/golang/ostree.go deleted file mode 100644 index 8891319c..00000000 --- a/contrib/golang/ostree.go +++ /dev/null @@ -1,94 +0,0 @@ -// +build linux - -// Public API specification for libostree Go bindings - -package ostree - -import ( - "unsafe" -) - -// #cgo pkg-config: ostree-1 -// #include -// #include -// #include -// #include "ostree.go.h" -import "C" - -type Repo struct { - *GObject -} - -func RepoGetType() GType { - return GType(C.ostree_repo_get_type()) -} - -func (r *Repo) native() *C.OstreeRepo { - return (*C.OstreeRepo)(r.ptr) -} - -func repoFromNative(p *C.OstreeRepo) *Repo { - if p == nil { - return nil - } - o := GObjectNew(unsafe.Pointer(p)) - r := &Repo{o} - return r -} - -func RepoNewOpen(path string) (*Repo, error) { - var cerr *C.GError = nil - cpath := C.CString(path) - pathc := C.g_file_new_for_path(cpath); - defer C.g_object_unref(C.gpointer(pathc)) - crepo := C.ostree_repo_new(pathc) - repo := repoFromNative(crepo); - r := GoBool(C.ostree_repo_open(repo.native(), nil, &cerr)) - if !r { - return nil, ConvertGError(cerr) - } - return repo, nil -} - -func (r *Repo) GetParent() *Repo { - return repoFromNative(C.ostree_repo_get_parent(r.native())) -} - -type ObjectType int - -const ( - OBJECT_TYPE_FILE ObjectType = C.OSTREE_OBJECT_TYPE_FILE - OBJECT_TYPE_DIR_TREE = C.OSTREE_OBJECT_TYPE_DIR_TREE - OBJECT_TYPE_DIR_META = C.OSTREE_OBJECT_TYPE_DIR_META - OBJECT_TYPE_COMMIT = C.OSTREE_OBJECT_TYPE_COMMIT - OBJECT_TYPE_TOMBSTONE_COMMIT = C.OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT -) - -func (repo *Repo) LoadVariant(t ObjectType, checksum string) (*GVariant, error) { - var cerr *C.GError = nil - var cvariant *C.GVariant = nil - - r := GoBool(C.ostree_repo_load_variant(repo.native(), C.OstreeObjectType(t), C.CString(checksum), &cvariant, &cerr)) - if !r { - return nil, ConvertGError(cerr) - } - variant := GVariantNew(unsafe.Pointer(cvariant)) - return variant, nil -} - -func (repo *Repo) ResolveRev(ref string) (string, error) { - var cerr *C.GError = nil - var crev *C.char = nil - - r := GoBool(C.ostree_repo_resolve_rev(repo.native(), C.CString(ref), GBool(true), &crev, &cerr)) - if !r { - return "", ConvertGError(cerr) - } - defer C.free(unsafe.Pointer(crev)) - return C.GoString(crev), nil -} - -func (commit *GVariant) CommitGetMetadataKeyString(key string) (string, error) { - cmeta := GVariantNew(unsafe.Pointer(C.g_variant_get_child_value(commit.native(), 0))) - return cmeta.LookupString(key) -} diff --git a/contrib/golang/ostree.go.h b/contrib/golang/ostree.go.h deleted file mode 100644 index b1a15517..00000000 --- a/contrib/golang/ostree.go.h +++ /dev/null @@ -1,21 +0,0 @@ -#include -#include - -static void -_ostree_repo_checkout_options_init_docker_union (OstreeRepoCheckoutOptions *opts) -{ - memset (opts, 0, sizeof (*opts)); - opts->mode = OSTREE_REPO_CHECKOUT_MODE_USER; - opts->overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; - opts->disable_fsync = 1; - opts->process_whiteouts = 1; -} - -static const char * -_g_variant_lookup_string (GVariant *v, const char *key) -{ - const char *r; - if (g_variant_lookup (v, key, "&s", &r)) - return r; - return NULL; -} diff --git a/contrib/golang/ostree_test.go b/contrib/golang/ostree_test.go deleted file mode 100644 index 0ffcb858..00000000 --- a/contrib/golang/ostree_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// +build linux - -// Public API specification for libostree Go bindings - -package ostree - -import ( - "testing" -) - -func TestTypeName(t *testing.T) { - name := RepoGetType().Name(); - if name != "OstreeRepo" { - t.Errorf("%s != OstreeRepo"); - } -} - -func TestRepoNew(t *testing.T) { - r, err := RepoNewOpen("/ostree/repo") - if err != nil { - t.Errorf("%s", err); - return - } - parent := r.GetParent() - if parent != nil { - t.Errorf("Expected no parent") - return - } -} - -func TestRepoGetMetadataVersion(t *testing.T) { - r, err := RepoNewOpen("/ostree/repo") - if err != nil { - t.Errorf("%s", err); - return - } - commit,err := r.ResolveRev("rhel-atomic-host/7/x86_64/standard") - if err != nil { - t.Errorf("%s", err) - return - } - commitv,err := r.LoadVariant(OBJECT_TYPE_COMMIT, commit) - if err != nil { - t.Errorf("%s", err) - return - } - ver, err := commitv.CommitGetMetadataKeyString("version") - if err != nil { - t.Errorf("%s", err) - return - } - if ver != "7.1.3" { - t.Errorf("expected 7.1.3") - } -} diff --git a/libglnx b/libglnx index 7a703638..4ae5e3be 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 7a703638d12410bb58ca6d85d6556d03c2a5a7ce +Subproject commit 4ae5e3beaaa674abfabf7404ab6fafcc4ec547db From 98a45475107d507d85d946e934be9064fe2efcf7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 10 Feb 2017 16:05:17 -0500 Subject: [PATCH 31/66] deltas: Don't put unreadable *from* objects in fallback In https://github.com/ostreedev/ostree/pull/634 we introduced a subtle regression - the unreadable object was added to the *new* reachable objects, when it shouldn't have been. Because it was a *from* object, clients already had it. This became more obvious now that I'm working on fixing delta progress - I noticed my deltas were always starting out with 40MB fetched, which turned out to be a non-world-readable initramfs object. This code should simply *skip* the unreadable object, and the delta processing below properly iterates over "new objects", so we'll pick it up from there. Closes: #678 Approved by: giuseppe --- src/libostree/ostree-repo-static-delta-compilation.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 657e8676..47fae9f5 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -1003,11 +1003,7 @@ generate_delta_lowlatency (OstreeRepo *repo, if (!check_object_world_readable (repo, from_checksum, &from_world_readable, cancellable, error)) goto out; if (!from_world_readable) - { - g_hash_table_iter_steal (&hashiter); - g_hash_table_add (new_reachable_regfile_content, (char*)from_checksum); - continue; - } + continue; if (!try_content_rollsum (repo, opts, from_checksum, to_checksum, &rollsum, cancellable, error)) From 0142e5ff393d4fc9e781d22cfea2e68a95a69dae Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 10 Feb 2017 16:10:20 -0500 Subject: [PATCH 32/66] delta-show: Don't dump whole superblock, do show fallback checksums Doing `g_variant_print (superblock)` is unreadable and not very useful, since we show the checksums as byte arrays. However, do show the checksums for fallback objects. This makes it easier to see which objects are fallbacks (and inspect why). Closes: #678 Approved by: giuseppe --- src/libostree/ostree-repo-static-delta-core.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index edbf965b..1ab11ed7 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -884,10 +884,6 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, OT_VARIANT_MAP_TRUSTED, &delta_superblock, error)) goto out; - { g_autofree char *variant_string = g_variant_print (delta_superblock, 1); - g_print ("%s\n", variant_string); - } - g_print ("Delta: %s\n", delta_id); { const char *endianness_description; gboolean was_heuristic; @@ -940,9 +936,13 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, for (i = 0; i < n_fallback; i++) { guint64 size, usize; - g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize); + g_autoptr(GVariant) checksum_v = NULL; + char checksum[65]; + g_variant_get_child (fallback, i, "(y@aytt)", NULL, &checksum_v, &size, &usize); + ostree_checksum_inplace_from_bytes (ostree_checksum_bytes_peek (checksum_v), checksum); size = maybe_swap_endian_u64 (swap_endian, size); usize = maybe_swap_endian_u64 (swap_endian, usize); + g_print (" %s\n", checksum); total_fallback_size += size; total_fallback_usize += usize; } From e1118e320d6c71a9b678b1dee395687889d51422 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 10 Feb 2017 14:29:54 -0500 Subject: [PATCH 33/66] repo: Fix static delta progress display There were a few bugs here. - We need to keep track of the size of the delta parts we've already processed, in order to make progress reliable at all in the face of interruptions. Add a new `fetched-delta-part-size` async progress variable for this. - The total before disregarded what we'd already downloaded, which was confusing. Now, a progress percentage is `fetched/total`. - Correctly handle "unknown bytes/sec" in the progress display. However, to be fully correct we need to show the fallback objects too. That would require tracking in the pull code when we fetch an object as a fallback versus "normally". This would be simpler really if we could assume in a run we were *only* processing a delta, but currently we don't do that. Related: https://github.com/ostreedev/ostree/issues/475 Closes: #678 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 29 ++++++++++++++++-------- src/libostree/ostree-repo.c | 39 ++++++++++++++++++++++---------- src/ostree/ot-builtin-pull.c | 12 ++++++---- 3 files changed, 55 insertions(+), 25 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0e5128d9..129e25da 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -91,6 +91,8 @@ typedef struct { guint n_outstanding_deltapart_fetches; guint n_outstanding_deltapart_write_requests; guint n_total_deltaparts; + guint n_total_delta_fallbacks; + guint64 fetched_deltapart_size; /* How much of the delta we have now */ guint64 total_deltapart_size; guint64 total_deltapart_usize; gint n_requested_metadata; @@ -220,6 +222,10 @@ update_progress (gpointer user_data) pull_data->n_fetched_deltaparts); ostree_async_progress_set_uint (pull_data->progress, "total-delta-parts", pull_data->n_total_deltaparts); + ostree_async_progress_set_uint (pull_data->progress, "total-delta-fallbacks", + pull_data->n_total_delta_fallbacks); + ostree_async_progress_set_uint64 (pull_data->progress, "fetched-delta-part-size", + pull_data->fetched_deltapart_size); ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size", pull_data->total_deltapart_size); ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize", @@ -1590,15 +1596,10 @@ process_one_static_delta_fallback (OtPullData *pull_data, compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size); uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size); + pull_data->n_total_delta_fallbacks += 1; pull_data->total_deltapart_size += compressed_size; pull_data->total_deltapart_usize += uncompressed_size; - if (pull_data->dry_run) - { - ret = TRUE; - goto out; - } - objtype = (OstreeObjectType)objtype_y; checksum = ostree_checksum_from_bytes_v (csum_v); @@ -1607,6 +1608,15 @@ process_one_static_delta_fallback (OtPullData *pull_data, cancellable, error)) goto out; + if (is_stored) + pull_data->fetched_deltapart_size += compressed_size; + + if (pull_data->dry_run) + { + ret = TRUE; + goto out; + } + if (!is_stored) { if (OSTREE_OBJECT_TYPE_IS_META (objtype)) @@ -1779,11 +1789,15 @@ process_one_static_delta (OtPullData *pull_data, cancellable, error)) goto out; + pull_data->total_deltapart_size += size; + pull_data->total_deltapart_usize += usize; + if (have_all) { g_debug ("Have all objects from static delta %s-%s part %u", from_revision ? from_revision : "empty", to_revision, i); + pull_data->fetched_deltapart_size += size; pull_data->n_fetched_deltaparts++; continue; } @@ -1797,9 +1811,6 @@ process_one_static_delta (OtPullData *pull_data, inline_part_bytes = g_variant_get_data_as_bytes (part_datav); } - pull_data->total_deltapart_size += size; - pull_data->total_deltapart_usize += usize; - if (pull_data->dry_run) continue; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index e7cbb16d..98abd926 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3848,8 +3848,7 @@ _formatted_time_remaining_from_seconds (guint64 seconds_remaining) if (minutes_remaining) g_string_append_printf (description, "%" G_GUINT64_FORMAT " minutes ", minutes_remaining % 60); - if (seconds_remaining) - g_string_append_printf (description, "%" G_GUINT64_FORMAT " seconds ", seconds_remaining % 60); + g_string_append_printf (description, "%" G_GUINT64_FORMAT " seconds ", seconds_remaining % 60); return g_string_free (description, FALSE); } @@ -3913,33 +3912,49 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress g_autofree char *formatted_bytes_transferred = g_format_size_full (bytes_transferred, 0); g_autofree char *formatted_bytes_sec = NULL; - g_autofree char *formatted_est_time_remaining = NULL; + guint64 bytes_sec; /* Ignore the first second, or when we haven't transferred any * data, since those could cause divide by zero below. */ if ((current_time - start_time) < G_USEC_PER_SEC || bytes_transferred == 0) { + bytes_sec = 0; formatted_bytes_sec = g_strdup ("-"); - formatted_est_time_remaining = g_strdup ("- "); } else { - guint64 bytes_sec = bytes_transferred / ((current_time - start_time) / G_USEC_PER_SEC); - guint64 est_time_remaining = (total_delta_part_size - bytes_transferred) / bytes_sec; + bytes_sec = bytes_transferred / ((current_time - start_time) / G_USEC_PER_SEC); formatted_bytes_sec = g_format_size (bytes_sec); - formatted_est_time_remaining = _formatted_time_remaining_from_seconds (est_time_remaining); } + /* Are we doing deltas? If so, we can be more accurate */ if (total_delta_parts > 0) { + guint64 fetched_delta_part_size = ostree_async_progress_get_uint64 (progress, "fetched-delta-part-size"); + g_autofree char *formatted_fetched = + g_format_size (fetched_delta_part_size); g_autofree char *formatted_total = g_format_size (total_delta_part_size); - /* No space between %s and remaining, since formatted_est_time_remaining has a trailing space */ - g_string_append_printf (buf, "Receiving delta parts: %u/%u %s/s %s/%s %sremaining", - fetched_delta_parts, total_delta_parts, - formatted_bytes_sec, formatted_bytes_transferred, - formatted_total, formatted_est_time_remaining); + + if (bytes_sec > 0) + { + /* MAX(0, value) here just to be defensive */ + guint64 est_time_remaining = MAX(0, (total_delta_part_size - fetched_delta_part_size)) / bytes_sec; + g_autofree char *formatted_est_time_remaining = _formatted_time_remaining_from_seconds (est_time_remaining); + /* No space between %s and remaining, since formatted_est_time_remaining has a trailing space */ + g_string_append_printf (buf, "Receiving delta parts: %u/%u %s/%s %s/s %sremaining", + fetched_delta_parts, total_delta_parts, + formatted_fetched, formatted_total, + formatted_bytes_sec, + formatted_est_time_remaining); + } + else + { + g_string_append_printf (buf, "Receiving delta parts: %u/%u %s/%s", + fetched_delta_parts, total_delta_parts, + formatted_fetched, formatted_total); + } } else if (scanning || outstanding_metadata_fetches) { diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index f370ca01..7df48002 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -80,7 +80,7 @@ dry_run_console_progress_changed (OstreeAsyncProgress *progress, gpointer user_data) { guint fetched_delta_parts, total_delta_parts; - guint64 total_delta_part_size, total_delta_part_usize; + guint64 fetched_delta_part_size, total_delta_part_size, total_delta_part_usize; GString *buf; g_assert (!printed_console_progress); @@ -88,19 +88,23 @@ dry_run_console_progress_changed (OstreeAsyncProgress *progress, fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts"); total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts"); + fetched_delta_part_size = ostree_async_progress_get_uint64 (progress, "fetched-delta-part-size"); total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); total_delta_part_usize = ostree_async_progress_get_uint64 (progress, "total-delta-part-usize"); buf = g_string_new (""); - { g_autofree char *formatted_size = + { g_autofree char *formatted_fetched = + g_format_size (fetched_delta_part_size); + g_autofree char *formatted_size = g_format_size (total_delta_part_size); g_autofree char *formatted_usize = g_format_size (total_delta_part_usize); - g_string_append_printf (buf, "Delta update: %u/%u parts, %s to transfer, %s uncompressed", + g_string_append_printf (buf, "Delta update: %u/%u parts, %s/%s, %s total uncompressed", fetched_delta_parts, total_delta_parts, - formatted_size, formatted_usize); + formatted_fetched, formatted_size, + formatted_usize); } g_print ("%s\n", buf->str); g_string_free (buf, TRUE); From 693f7c5f609bbbddd84de054d6edd04a56b5c8d1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 10 Feb 2017 17:35:23 -0500 Subject: [PATCH 34/66] pull: Explicitly error out if metadata objects are fallbacks I don't know why I added support for this; it makes no sense really. If we have large metadata objects something has gone badly wrong. The delta compiler has always only processed fallbacks for regular content files. Dropping support in the fetcher for this will simplify later handling of fallback progress accounting. Closes: #678 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 129e25da..6246011f 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1618,19 +1618,14 @@ process_one_static_delta_fallback (OtPullData *pull_data, } if (!is_stored) - { + { + /* The delta compiler never did this, there's no reason to support it */ if (OSTREE_OBJECT_TYPE_IS_META (objtype)) { - g_autoptr(GVariant) objname = ostree_object_name_serialize (checksum, objtype); - if (!g_hash_table_lookup (pull_data->requested_metadata, objname)) - { - gboolean do_fetch_detached; - g_hash_table_add (pull_data->requested_metadata, g_variant_ref (objname)); - - do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT); - enqueue_one_object_request (pull_data, checksum, objtype, NULL, do_fetch_detached, FALSE); - checksum = NULL; /* Transfer ownership */ - } + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Found metadata object as fallback: %s.%s", checksum, + ostree_object_type_to_string (objtype)); + goto out; } else { From b5c5003ff6d199c9edd3237658765927597190bf Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 10 Feb 2017 18:09:16 -0500 Subject: [PATCH 35/66] pull: Fold together deltapart+fallback count for display It's just simpler, and I'm not sure people are going to care much about the difference by default. We already folded in the fallback sizes into the download totals, so folding in the count makes things consistent; previously you could see e.g. `3/3 parts, 100MB/150MB` and be confused. Closes: #678 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 17 ++++++++++++++++- src/libostree/ostree-repo-static-delta-core.c | 2 +- src/libostree/ostree-repo.c | 17 +++++++++++++---- src/ostree/ot-builtin-pull.c | 10 ++++++++++ tests/libtest.sh | 4 ++++ tests/pull-test.sh | 3 ++- tests/test-pull-many.sh | 19 +++++++++++++++++-- 7 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 6246011f..3452fbe9 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -81,6 +81,7 @@ typedef struct { GHashTable *scanned_metadata; /* Maps object name to itself */ GHashTable *requested_metadata; /* Maps object name to itself */ GHashTable *requested_content; /* Maps checksum to itself */ + GHashTable *requested_fallback_content; /* Maps checksum to itself */ GHashTable *pending_fetch_metadata; /* Map */ GHashTable *pending_fetch_content; /* Map */ GHashTable *pending_fetch_deltaparts; /* Set */ @@ -98,6 +99,7 @@ typedef struct { gint n_requested_metadata; gint n_requested_content; guint n_fetched_deltaparts; + guint n_fetched_deltapart_fallbacks; guint n_fetched_metadata; guint n_fetched_content; @@ -222,6 +224,8 @@ update_progress (gpointer user_data) pull_data->n_fetched_deltaparts); ostree_async_progress_set_uint (pull_data->progress, "total-delta-parts", pull_data->n_total_deltaparts); + ostree_async_progress_set_uint (pull_data->progress, "fetched-delta-fallbacks", + pull_data->n_fetched_deltapart_fallbacks); ostree_async_progress_set_uint (pull_data->progress, "total-delta-fallbacks", pull_data->n_total_delta_fallbacks); ostree_async_progress_set_uint64 (pull_data->progress, "fetched-delta-part-size", @@ -785,6 +789,9 @@ content_fetch_on_write_complete (GObject *object, } pull_data->n_fetched_content++; + /* Was this a delta fallback? */ + if (g_hash_table_remove (pull_data->requested_fallback_content, expected_checksum)) + pull_data->n_fetched_deltapart_fallbacks++; out: pull_data->n_outstanding_content_write_requests--; check_outstanding_requests_handle_error (pull_data, local_error); @@ -1631,9 +1638,14 @@ process_one_static_delta_fallback (OtPullData *pull_data, { if (!g_hash_table_lookup (pull_data->requested_content, checksum)) { + /* Mark this as requested, like we do in the non-delta path */ g_hash_table_add (pull_data->requested_content, checksum); + /* But also record it's a delta fallback object, so we can account + * for it as logically part of the delta fetch. + */ + g_hash_table_add (pull_data->requested_fallback_content, g_strdup (checksum)); enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, NULL, FALSE, FALSE); - checksum = NULL; /* Transfer ownership */ + checksum = NULL; /* We transferred ownership to the requested_content hash */ } } } @@ -2604,6 +2616,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)g_variant_unref, NULL); pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); + pull_data->requested_fallback_content = g_hash_table_new_full (g_str_hash, g_str_equal, + (GDestroyNotify)g_free, NULL); pull_data->requested_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, NULL); pull_data->pending_fetch_content = g_hash_table_new_full (g_str_hash, g_str_equal, @@ -3324,6 +3338,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->requested_fallback_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref); diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 1ab11ed7..21ed081d 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -937,7 +937,7 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, { guint64 size, usize; g_autoptr(GVariant) checksum_v = NULL; - char checksum[65]; + char checksum[OSTREE_SHA256_STRING_LEN+1]; g_variant_get_child (fallback, i, "(y@aytt)", NULL, &checksum_v, &size, &usize); ostree_checksum_inplace_from_bytes (ostree_checksum_bytes_peek (checksum_v), checksum); size = maybe_swap_endian_u64 (swap_endian, size); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 98abd926..dc0eb575 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3884,6 +3884,8 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress guint n_scanned_metadata; guint fetched_delta_parts; guint total_delta_parts; + guint fetched_delta_part_fallbacks; + guint total_delta_part_fallbacks; buf = g_string_new (""); @@ -3895,6 +3897,8 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress n_scanned_metadata = ostree_async_progress_get_uint (progress, "scanned-metadata"); fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts"); total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts"); + fetched_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "fetched-delta-fallbacks"); + total_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "total-delta-fallbacks"); if (status) { @@ -3932,10 +3936,15 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress if (total_delta_parts > 0) { guint64 fetched_delta_part_size = ostree_async_progress_get_uint64 (progress, "fetched-delta-part-size"); - g_autofree char *formatted_fetched = - g_format_size (fetched_delta_part_size); - g_autofree char *formatted_total = - g_format_size (total_delta_part_size); + g_autofree char *formatted_fetched = NULL; + g_autofree char *formatted_total = NULL; + + /* Here we merge together deltaparts + fallbacks to avoid bloating the text UI */ + fetched_delta_parts += fetched_delta_part_fallbacks; + total_delta_parts += total_delta_part_fallbacks; + + formatted_fetched = g_format_size (fetched_delta_part_size); + formatted_total = g_format_size (total_delta_part_size); if (bytes_sec > 0) { diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 7df48002..170a3a91 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -80,14 +80,24 @@ dry_run_console_progress_changed (OstreeAsyncProgress *progress, gpointer user_data) { guint fetched_delta_parts, total_delta_parts; + guint fetched_delta_part_fallbacks, total_delta_part_fallbacks; guint64 fetched_delta_part_size, total_delta_part_size, total_delta_part_usize; GString *buf; g_assert (!printed_console_progress); printed_console_progress = TRUE; + /* Number of parts */ fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts"); total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts"); + fetched_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "fetched-delta-fallbacks"); + total_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "total-delta-fallbacks"); + /* Fold the count of deltaparts + fallbacks for simplicity; if changing this, + * please change ostree_repo_pull_default_console_progress_changed() first. + */ + fetched_delta_parts += fetched_delta_part_fallbacks; + total_delta_parts += total_delta_part_fallbacks; + /* Size variables */ fetched_delta_part_size = ostree_async_progress_get_uint64 (progress, "fetched-delta-part-size"); total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); total_delta_part_usize = ostree_async_progress_get_uint64 (progress, "total-delta-part-usize"); diff --git a/tests/libtest.sh b/tests/libtest.sh index 1c603d66..ce41668f 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -313,6 +313,10 @@ setup_exampleos_repo() { nfiles=10 while [ $nfiles -gt 0 ]; do echo file-$ndirs-$nfiles > f$ndirs-$nfiles + # Make an unreadable file to trigger https://github.com/ostreedev/ostree/pull/634 + if [ $(($x % 10)) -eq 0 ]; then + chmod 0600 f$ndirs-$nfiles + fi nfiles=$((nfiles-1)) done ndirs=$((ndirs-1)) diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 56b24a0c..693eac06 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -144,7 +144,8 @@ cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >dry-run-pull.txt -assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts' +# Compression can vary, so we support 400-699 +assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts, 0 bytes/[456][0-9][0-9] bytes, 455 bytes total uncompressed' rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) assert_streq "${prev_rev}" "${rev}" ${CMD_PREFIX} ostree --repo=repo fsck diff --git a/tests/test-pull-many.sh b/tests/test-pull-many.sh index f39c8e6e..73245a6b 100755 --- a/tests/test-pull-many.sh +++ b/tests/test-pull-many.sh @@ -23,15 +23,30 @@ set -euo pipefail setup_exampleos_repo -echo '1..1' +echo '1..3' cd ${test_tmpdir} set -x echo "$(date): Pulling content..." rev=$(${CMD_PREFIX} ostree --repo=ostree-srv/exampleos/repo rev-parse ${REF}) -${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas --mirror origin ${REF} +${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas origin ${REF} ${CMD_PREFIX} ostree --repo=repo fsck assert_streq ${rev} $(${CMD_PREFIX} ostree --repo=repo rev-parse ${REF}) +echo "ok without deltas" + +previous=$(${CMD_PREFIX} ostree --repo=repo rev-parse ${rev}^) +rm repo/refs/{heads,remotes}/* -rf +${CMD_PREFIX} ostree --repo=repo prune --refs-only +${CMD_PREFIX} ostree --repo=repo pull origin ${REF}@${previous} +${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin ${REF} > output.txt +assert_file_has_content output.txt 'Delta update: 0/1 parts, 0 bytes/1.[012] MB, 1.[345] MB total uncompressed' + +echo "ok delta dry-run" + +${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${REF} +assert_streq $(${CMD_PREFIX} ostree --repo=repo rev-parse ${REF}) ${rev} +${CMD_PREFIX} ostree --repo=repo fsck + echo "ok" From 6b93cb3173bd7f5e39dddb8ebc01bd216b6a4792 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Feb 2017 15:59:20 -0500 Subject: [PATCH 36/66] ci: Install PyYAML We'll use it for https://github.com/ostreedev/ostree/pull/691 Closes: #692 Approved by: jlebon --- .redhat-ci.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.redhat-ci.Dockerfile b/.redhat-ci.Dockerfile index 4970c77d..d5a2e255 100644 --- a/.redhat-ci.Dockerfile +++ b/.redhat-ci.Dockerfile @@ -12,6 +12,7 @@ RUN dnf install -y \ libubsan \ libasan \ libtsan \ + PyYAML \ gnome-desktop-testing \ redhat-rpm-config \ elfutils \ From c9356a50b860aacace0ca819d0849854de9edd4d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 18 Feb 2017 10:15:39 -0500 Subject: [PATCH 37/66] lib: Ensure an error is set in ensure_unlinked() if errno != ENOENT We hit this with: ``` 27411 unlink("/boot/efi/EFI/fedora/grub.cfg.new") = -1 EROFS (Read-only file system) ``` from the grub2 code. https://github.com/projectatomic/rpm-ostree/issues/633 Closes: #694 Approved by: giuseppe --- src/libotutil/ot-gio-utils.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libotutil/ot-gio-utils.c b/src/libotutil/ot-gio-utils.c index ba21b467..da32653e 100644 --- a/src/libotutil/ot-gio-utils.c +++ b/src/libotutil/ot-gio-utils.c @@ -309,7 +309,10 @@ ot_gfile_ensure_unlinked (GFile *path, if (unlink (gs_file_get_path_cached (path)) != 0) { if (errno != ENOENT) - return FALSE; + { + glnx_set_error_from_errno (error); + return FALSE; + } } return TRUE; } From f02dcc4997b228dcd551f2c3f01102e794599931 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 21 Feb 2017 10:15:03 -0500 Subject: [PATCH 38/66] libtest: Re-enable quiet mode for building fs tree I think I commented this out while debugging something, and forgot to re-enable it. Reading the log files should be a better again after this. Closes: #699 Approved by: giuseppe --- tests/libtest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index ce41668f..54108268 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -292,7 +292,7 @@ setup_exampleos_repo() { cd main ndirs=3147 depth=0 -# set +x # No need to spam the logs for this + set +x # No need to spam the logs for this echo "$(date): Generating initial content..." while [ $ndirs -gt 0 ]; do # 2/3 of the time, recurse a dir, up to a max of 9, otherwise back up From 7d7ab92a0580d4332b5df82a200cc37b07432cb7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 21 Feb 2017 11:20:52 -0500 Subject: [PATCH 39/66] README.md: Add more/clean up links to consuming projects Add meta-updater and QtOTA, and delete OpenEmbedded since it's implied by the first two. Merge rpm-ostree + Atomic Host since they're close. Clarify gnome-continuous a bit. Closes: #700 Approved by: jlebon --- README.md | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f189ebc9..c985cd14 100644 --- a/README.md +++ b/README.md @@ -38,26 +38,36 @@ use OSTree as a "deduplicating hardlink store". Projects using OSTree --------------------- -[rpm-ostree](https://github.com/projectatomic/rpm-ostree) is a tool -that uses OSTree as a shared library, and supports committing RPMs -into an OSTree repository, and deploying them on the client. This is -appropriate for "fixed purpose" systems. There is in progress work -for more sophisticated hybrid models, deeply integrating the RPM -packaging with OSTree. +[meta-updater](https://github.com/advancedtelematic/meta-updater) is +a layer available for [OpenEmbedded](http://www.openembedded.org/wiki/Main_Page) +systems. -[Project Atomic](http://www.projectatomic.io/) uses rpm-ostree to -provide a minimal host for Docker formatted Linux containers. -Replicating a base immutable OS, then using Docker for applications -meshes together two different tools with different tradeoffs. +[QtOTA](http://doc.qt.io/QtOTA/) is Qt's over-the-air update framework +which uses libostree. -[flatpak](https://github.com/alexlarsson/xdg-app) uses OSTree -for desktop application containers. +[rpm-ostree](https://github.com/projectatomic/rpm-ostree) is a next-generation +hybrid package/image system for [Fedora](https://getfedora.org/) and [CentOS](https://www.centos.org/), +used by the [Atomic Host](http://www.projectatomic.io/) project. +By default it uses libostree to atomically replicate a base OS (all dependency +resolution is done on the server), but it supports "package layering", where +additional RPMs can be layered on top of the base. This brings a "best of both worlds"" +model for image and package systems. + +[flatpak](https://github.com/flatpak/flatpak) uses libostree for desktop +application containers. Unlike most of the other systems here, flatpak does not +use the "libostree host system" aspects (e.g. bootloader management), just the +"git-like hardlink dedup". For example, flatpak supports a per-user OSTree +repository. + +[Endless OS](https://endlessos.com/) uses libostree for their host system as +well as flatpak. See +their [eos-updater](https://github.com/endlessm/eos-updater) +and [deb-ostree-builder](https://github.com/dbnicholson/deb-ostree-builder) +projects. [GNOME Continuous](https://wiki.gnome.org/Projects/GnomeContinuous) is -a custom build system designed for OSTree, using -[OpenEmbedded](http://www.openembedded.org/wiki/Main_Page) in concert -with a custom build system to do continuous delivery from hundreds of -git repositories. +where OSTree was born - as a high performance continuous delivery/testing +system for GNOME. Building -------- From 6d7e85bb38a0613f20b0df3cc2d1a2fd5b1bdc4b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Feb 2017 16:58:26 -0500 Subject: [PATCH 40/66] libglnx: Re-bump to master due to accidental reversion Commit a1805d6101eb8efeaa72459c22d88ed08ff7a065 reverted this unintentionally. We should have some CI check that requires a commit message has something like "libglnx bump" or something? Closes: #693 Approved by: jlebon --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index 4ae5e3be..0c1603de 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 4ae5e3beaaa674abfabf7404ab6fafcc4ec547db +Subproject commit 0c1603debac440978c1d55c46cb11059f8350b35 From 515f83206785bd9fb50b77df17b48bbed2827cff Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Feb 2017 16:59:21 -0500 Subject: [PATCH 41/66] ci: Hard error on all -fsanitize=undefined warnings I saw in a recent test log a ton of spam ``` libglnx/glnx-dirfd.c:253:3: runtime error: null pointer passed as argument 1, which is declared to never be null ``` which actually turned out to be libglnx getting reverted. But let's be sure now we actually bomb out quickly on UBSAN warnings in general. Closes: #693 Approved by: jlebon --- .redhat-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 11e5a9d6..74778f8a 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -12,7 +12,7 @@ packages: - libasan env: - CFLAGS: '-fsanitize=undefined -fsanitize=address' + CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address' ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc # TODO when we're doing leak checks: G_SLICE: "always-malloc" From 3ec509c89b07efbfc4b4d007327609b6dee77dd3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 21 Feb 2017 13:58:40 -0500 Subject: [PATCH 42/66] build: Add --with-smack, use it to reset contexts for writing objects At some point we'll want to follow what systemd is doing and add better support for smack, along the lines of `OstreeSePolicy`. However, short term this patch fixes AGL which uses Smack. See: https://jira.automotivelinux.org/browse/SPEC-386 See: https://github.com/ostreedev/ostree/pull/698 Closes: #698 Approved by: OYTIS --- configure.ac | 8 ++++++++ src/libostree/ostree-repo-commit.c | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/configure.ac b/configure.ac index 3448ab6e..c5940c9f 100644 --- a/configure.ac +++ b/configure.ac @@ -273,6 +273,14 @@ AS_IF([ test x$with_selinux != xno ], [ if test x$with_selinux != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +selinux"; fi AM_CONDITIONAL(USE_SELINUX, test $with_selinux != no) +AC_ARG_WITH(smack, +AS_HELP_STRING([--with-smack], [Enable smack]), +:, with_smack=no) +AS_IF([ test x$with_smack = xyes], [ + AC_DEFINE([WITH_SMACK], 1, [Define if we have smack.pc]) +]) +AM_CONDITIONAL(USE_SMACK, test $with_smack != no) + dnl This is what is in RHEL7.2 right now, picking it arbitrarily LIBMOUNT_DEPENDENCY="mount >= 2.23.0" diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 07940f48..f6129ca6 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -110,6 +110,30 @@ write_file_metadata_to_xattr (int fd, return TRUE; } +/* See https://github.com/ostreedev/ostree/pull/698 */ +#ifdef WITH_SMACK +#define XATTR_NAME_SMACK "security.SMACK64" +#endif + +static void +ot_security_smack_reset_dfd_name (int dfd, const char *name) +{ +#ifdef WITH_SMACK + char buf[PATH_MAX]; + /* See glnx-xattrs.c */ + snprintf (buf, sizeof (buf), "/proc/self/fd/%d/%s", dfd, name); + (void) lremovexattr (buf, XATTR_NAME_SMACK); +#endif +} + +static void +ot_security_smack_reset_fd (int fd) +{ +#ifdef WITH_SMACK + (void) fremovexattr (fd, XATTR_NAME_SMACK); +#endif +} + gboolean _ostree_repo_commit_loose_final (OstreeRepo *self, const char *checksum, @@ -221,6 +245,7 @@ commit_loose_object_trusted (OstreeRepo *self, if (xattrs != NULL) { + ot_security_smack_reset_dfd_name (self->tmp_dir_fd, temp_filename); if (!glnx_dfd_name_set_all_xattrs (self->tmp_dir_fd, temp_filename, xattrs, cancellable, error)) goto out; @@ -252,6 +277,7 @@ commit_loose_object_trusted (OstreeRepo *self, if (xattrs) { + ot_security_smack_reset_fd (fd); if (!glnx_fd_set_all_xattrs (fd, xattrs, cancellable, error)) goto out; } From 09b392675aa7ede9eb108cce478cc491080ac1f0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Feb 2017 10:22:43 -0500 Subject: [PATCH 43/66] main: Make ostree --version output YAML (and add gitrev) I learned today that `docker version` does this and I really like the idea. While we have the patient open, also add the gitrev with code taken from https://github.com/projectatomic/rpm-ostree/pull/584 Closes: #691 Approved by: giuseppe --- Makefile.am | 3 +++ configure.ac | 12 ++++++------ src/ostree/ot-main.c | 11 ++++++++++- tests/basic-test.sh | 6 +++++- tests/ci-install.sh | 1 + tests/libtest.sh | 5 ++++- tests/test-libarchive.sh | 2 +- 7 files changed, 30 insertions(+), 10 deletions(-) diff --git a/Makefile.am b/Makefile.am index 31c41251..cc0e76f5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,6 +19,8 @@ include Makefile-decls.am shortened_sysconfdir = $$(echo "$(sysconfdir)" | sed -e 's|^$(prefix)||' -e 's|^/||') +OSTREE_GITREV=$(shell if command -v git >/dev/null 2>&1 && test -d $(srcdir)/.git; then git describe --abbrev=42 --tags --always HEAD; fi) + ACLOCAL_AMFLAGS = -I buildutil -I libglnx ${ACLOCAL_FLAGS} AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DLOCALEDIR=\"$(datadir)/locale\" -DSYSCONFDIR=\"$(sysconfdir)\" \ @@ -26,6 +28,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DOSTREE_FEATURES='"$(OSTREE_FEATURES)"' \ -DOSTREE_COMPILATION \ -DG_LOG_DOMAIN=\"OSTree\" \ + -DOSTREE_GITREV='"$(OSTREE_GITREV)"' \ -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40 \ -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 -DSOUP_VERSION_MAX_ALLOWED=SOUP_VERSION_2_48 AM_CFLAGS += -std=gnu99 $(WARN_CFLAGS) diff --git a/configure.ac b/configure.ac index c5940c9f..979d8ccb 100644 --- a/configure.ac +++ b/configure.ac @@ -92,7 +92,7 @@ AS_IF([test x$with_curl != xno ], [ with_soup_default=yes ], [with_soup_default=check]) AM_CONDITIONAL(USE_CURL, test x$with_curl != xno) -if test x$with_curl = xyes; then OSTREE_FEATURES="$OSTREE_FEATURES +libcurl"; fi +if test x$with_curl = xyes; then OSTREE_FEATURES="$OSTREE_FEATURES libcurl"; fi dnl When bumping the libsoup-2.4 dependency, remember to bump dnl SOUP_VERSION_MIN_REQUIRED and SOUP_VERSION_MAX_ALLOWED in @@ -131,7 +131,7 @@ AS_IF([test x$with_soup != xno], [ with_soup=no ]) ], [ with_soup=no ]) -if test x$with_soup != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +libsoup"; fi +if test x$with_soup != xno; then OSTREE_FEATURES="$OSTREE_FEATURES libsoup"; fi AM_CONDITIONAL(USE_LIBSOUP, test x$with_soup != xno) AM_CONDITIONAL(HAVE_LIBSOUP_CLIENT_CERTS, test x$have_libsoup_client_certs = xyes) @@ -158,7 +158,7 @@ PKG_CHECK_MODULES(OT_DEP_GPGME, gpgme-pthread >= $LIBGPGME_DEPENDENCY, have_gpgm AS_IF([ test x$have_gpgme = xno ], [ AC_MSG_ERROR([Need GPGME_PTHREAD version $LIBGPGME_DEPENDENCY or later]) ]) -OSTREE_FEATURES="$OSTREE_FEATURES +gpgme" +OSTREE_FEATURES="$OSTREE_FEATURES gpgme" LIBARCHIVE_DEPENDENCY="libarchive >= 2.8.0" # What's in RHEL7.2. @@ -245,7 +245,7 @@ AS_IF([ test x$with_libarchive != xno ], [ with_libarchive=no ]) ], [ with_libarchive=no ]) -if test x$with_libarchive != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +libarchive"; fi +if test x$with_libarchive != xno; then OSTREE_FEATURES="$OSTREE_FEATURES libarchive"; fi AM_CONDITIONAL(USE_LIBARCHIVE, test $with_libarchive != no) dnl This is what is in RHEL7 anyways @@ -270,7 +270,7 @@ AS_IF([ test x$with_selinux != xno ], [ with_selinux=no ]) ], [ with_selinux=no ]) -if test x$with_selinux != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +selinux"; fi +if test x$with_selinux != xno; then OSTREE_FEATURES="$OSTREE_FEATURES selinux"; fi AM_CONDITIONAL(USE_SELINUX, test $with_selinux != no) AC_ARG_WITH(smack, @@ -303,7 +303,7 @@ AS_IF([ test x$with_libmount != xno ], [ with_libmount=no ]) ], [ with_libmount=no ]) -if test x$with_libmount != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +libmount"; fi +if test x$with_libmount != xno; then OSTREE_FEATURES="$OSTREE_FEATURES libmount"; fi AM_CONDITIONAL(USE_LIBMOUNT, test $with_libmount != no) # Enabled by default because I think people should use it. diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index fb782275..c6a2b6dd 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -241,7 +241,16 @@ ostree_option_context_parse (GOptionContext *context, if (opt_version) { - g_print ("%s\n %s\n", PACKAGE_STRING, OSTREE_FEATURES); + /* This should now be YAML, like `docker version`, so it's both nice to read + * possible to parse */ + g_auto(GStrv) features = g_strsplit (OSTREE_FEATURES, " ", -1); + g_print ("%s:\n", PACKAGE_NAME); + g_print (" Version: %s\n", PACKAGE_VERSION); + if (strlen (OSTREE_GITREV) > 0) + g_print (" Git: %s\n", OSTREE_GITREV); + g_print (" Features:\n"); + for (char **iter = features; iter && *iter; iter++) + g_print (" - %s\n", *iter); exit (EXIT_SUCCESS); } diff --git a/tests/basic-test.sh b/tests/basic-test.sh index d9308138..26982796 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,11 @@ set -euo pipefail -echo "1..61" +echo "1..62" + +$CMD_PREFIX ostree --version > version.yaml +python -c 'import yaml; yaml.safe_load(open("version.yaml"))' +echo "ok yaml version" $OSTREE checkout test2 checkout-test2 echo "ok checkout" diff --git a/tests/ci-install.sh b/tests/ci-install.sh index 92f802d9..d927d962 100755 --- a/tests/ci-install.sh +++ b/tests/ci-install.sh @@ -110,6 +110,7 @@ case "$ci_distro" in libcurl4-openssl-dev \ procps \ zlib1g-dev \ + python-yaml \ ${NULL} if [ "$ci_in_docker" = yes ]; then diff --git a/tests/libtest.sh b/tests/libtest.sh index 54108268..0126827e 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -538,7 +538,10 @@ skip_without_fuse () { } has_gpgme () { - ${CMD_PREFIX} ostree --version | grep -q -e '\+gpgme' + ${CMD_PREFIX} ostree --version > version.txt + assert_file_has_content version.txt '- gpgme' + rm -f version.txt + true } libtest_cleanup_gpg () { diff --git a/tests/test-libarchive.sh b/tests/test-libarchive.sh index 0c579459..6540b94b 100755 --- a/tests/test-libarchive.sh +++ b/tests/test-libarchive.sh @@ -19,7 +19,7 @@ set -euo pipefail -if ! ostree --version | grep -q -e '\+libarchive'; then +if ! ostree --version | grep -q -e '- libarchive'; then echo "1..0 #SKIP no libarchive support compiled in" exit 0 fi From 0817be61a17cc8b770cad54196182ac9c3109caf Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 23 Feb 2017 09:40:17 -0500 Subject: [PATCH 44/66] deploy: Correctly use libmount unref() calls rather than free() We saw a random ostree SEGV start popping up in our CI environment: https://github.com/projectatomic/rpm-ostree/pull/641#issuecomment-281870424 Looking at this code more and comparing it to what util-linux does, I noticed we had a write-after-free, since `mnt_unref_table()` will invoke `mnt_unref_cache()` on its cache, and that function does: ``` if (cache) { cache->rfcount--; ``` unconditionally. Fix this by using `unref()`. Closes: #705 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index cb5a4615..5a3f6d85 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1692,8 +1692,8 @@ is_ro_mount (const char *path) fs = mnt_table_find_target(tb, path, MNT_ITER_BACKWARD); is_mount = fs && mnt_fs_get_target (fs); - mnt_free_cache (cache); - mnt_free_table (tb); + mnt_unref_cache (cache); + mnt_unref_table (tb); if (!is_mount) return FALSE; From a71d550860e68be2b177d9016fe0227f1a547269 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 23 Feb 2017 18:01:28 -0500 Subject: [PATCH 45/66] man/repo-config: Document mirrorlist We should get more strict about docs. Add some text about `contenturl` and mirrorlists. Closes: #709 Approved by: jlebon --- man/ostree.repo-config.xml | 40 ++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index b4c276df..d187f89f 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -127,10 +127,16 @@ Boston, MA 02111-1307, USA. url - Must be present; declares URL for accessing - this remote. The only supported schemes are the moment are - file, http, and - https. + Must be present; declares URL for accessing metadata and + content for remote. See also contenturl. The + supported schemes are documented below. + + + + contenturl + Declares URL for accessing content (filez, static delta + parts). When specified, url is used just for + metadata: summary, static delta "superblocks". @@ -201,6 +207,32 @@ Boston, MA 02111-1307, USA. + + Repository url/contenturl + + Originally, OSTree had just a url option + for remotes. Since then, the contenturl + option was introduced. Both of these support + file, http, and + https schemes. + + + Additionally, both of these can be prefixed with the string + mirrorlist=, which instructs the client + that the target url is a "mirrorlist" format, which is + a plain text file of newline-separated URLs. Earlier + URLs will be given precedence. + + + Note that currently, the tls-ca-path and + tls-client-cert-path options apply to every HTTP + request, even when contenturl and/or + mirrorlist are in use. This may change in the future to + only apply to metadata (i.e. url, not + contenturl) fetches. + + + Per-remote GPG keyrings and verification From 877a27da0fecc0b9e4ff3f79ae50efa32bbffd16 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 24 Feb 2017 09:33:20 -0500 Subject: [PATCH 46/66] tree-wide: Squash noncritical compiler warnings Should fix everything from Anything that uses autocleanups should *always* be initialized directly I think, even if a few lines down we directly assign, since this way it's more robust against refactoring. And the `freopen()` warnings are right - IMO we should *always* check return values. Closes: #711 Approved by: jlebon --- src/ostree/ostree-trivial-httpd.c | 11 +++++++---- src/ostree/ot-builtin-admin.c | 2 +- src/ostree/ot-builtin-remote.c | 2 +- src/ostree/ot-main.c | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index 72de8d79..d6f0c4d4 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -520,7 +520,7 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) } else { - g_autoptr(GFile) log_file; + g_autoptr(GFile) log_file = NULL; GFileOutputStream* log_stream; log_file = g_file_new_for_path (opt_log); @@ -601,9 +601,12 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) if (setsid () < 0) err (1, "setsid"); /* Daemonising: close stdout/stderr so $() et al work on us */ - freopen("/dev/null", "r", stdin); - freopen("/dev/null", "w", stdout); - freopen("/dev/null", "w", stderr); + if (freopen("/dev/null", "r", stdin) == NULL) + err (1, "freopen"); + if (freopen("/dev/null", "w", stdout) == NULL) + err (1, "freopen"); + if (freopen("/dev/null", "w", stderr) == NULL) + err (1, "freopen"); } else { diff --git a/src/ostree/ot-builtin-admin.c b/src/ostree/ot-builtin-admin.c index 4760e532..0d8290a8 100644 --- a/src/ostree/ot-builtin-admin.c +++ b/src/ostree/ot-builtin-admin.c @@ -125,7 +125,7 @@ ostree_builtin_admin (int argc, char **argv, GCancellable *cancellable, GError * if (!subcommand->name) { g_autoptr(GOptionContext) context = NULL; - g_autofree char *help; + g_autofree char *help = NULL; context = ostree_admin_option_context_new_with_commands (); diff --git a/src/ostree/ot-builtin-remote.c b/src/ostree/ot-builtin-remote.c index 4f01cac2..f0667a42 100644 --- a/src/ostree/ot-builtin-remote.c +++ b/src/ostree/ot-builtin-remote.c @@ -114,7 +114,7 @@ ostree_builtin_remote (int argc, char **argv, GCancellable *cancellable, GError if (!subcommand->name) { g_autoptr(GOptionContext) context = NULL; - g_autofree char *help; + g_autofree char *help = NULL; context = remote_option_context_new_with_commands (); diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index c6a2b6dd..3484b18e 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -171,7 +171,7 @@ ostree_run (int argc, if (!command->fn) { g_autoptr(GOptionContext) context = NULL; - g_autofree char *help; + g_autofree char *help = NULL; context = ostree_option_context_new_with_commands (commands); From cee57a0268334d51cd312c6cdcf367bedfd3e30d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 24 Feb 2017 10:18:27 -0500 Subject: [PATCH 47/66] deploy/libmount: Fix build with old util-linux 2.23 (CentOS7) https://github.com/ostreedev/ostree/pull/705 broke the build on CentOS 7 which only has util-linux 2.23. When I was thinking about this, I realized that there must really be a way to make this safe even for older versions. Looking at that version of util-linux, all we need to do is invert the order of frees so we `mnt_free_table()` *before* `mnt_free_cache()`, like util-linux does: https://github.com/karelzak/util-linux/blob/stable/v2.23/sys-utils/eject.c#L1131 We still use the `_unref()` versions if available. I also fixed the ordering there too for double plus redundant safety. Closes: #712 Approved by: jlebon --- configure.ac | 4 ++++ src/libostree/ostree-sysroot-deploy.c | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 979d8ccb..48d8949c 100644 --- a/configure.ac +++ b/configure.ac @@ -299,6 +299,10 @@ AS_IF([ test x$with_libmount != xno ], [ AC_DEFINE([HAVE_LIBMOUNT], 1, [Define if we have libmount.pc]) PKG_CHECK_MODULES(OT_DEP_LIBMOUNT, $LIBMOUNT_DEPENDENCY) with_libmount=yes + save_LIBS=$LIBS + LIBS=$OT_DEP_LIBMOUNT_LIBS + AC_CHECK_FUNCS(mnt_unref_cache) + LIBS=$save_LIBS ], [ with_libmount=no ]) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 5a3f6d85..1cfe6ab1 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1692,8 +1692,13 @@ is_ro_mount (const char *path) fs = mnt_table_find_target(tb, path, MNT_ITER_BACKWARD); is_mount = fs && mnt_fs_get_target (fs); - mnt_unref_cache (cache); +#ifdef HAVE_MNT_UNREF_CACHE mnt_unref_table (tb); + mnt_unref_cache (cache); +#else + mnt_free_table (tb); + mnt_free_cache (cache); +#endif if (!is_mount) return FALSE; From 2c326d705eec5f379a01ba2b584986548473d6c2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 23 Feb 2017 09:24:58 -0500 Subject: [PATCH 48/66] fetcher: Log failures into journal Particularly when HTTP requests fail, I really want a lot more information. We could theoretically stuff it into the `GError` message field, but that gets ugly *fast*. Using the systemd journal allows us to log things in a structured fashion. Right now e.g. rpm-ostree won't be aware of this additional information, but I think we could teach it to be down the line. In the short term, users can learn to find it from `systemctl status rpm-ostreed` or `journalctl -b -r -u rpm-ostreed`, etc. One thing I'd like to do next is log successful fetches of e.g. commit objects as well with more information about the originating server (things like the final URL if we were redirected, did we use TLS pinning, what was the negotiated TLS version+cipher, etc). Closes: #708 Approved by: jlebon --- src/libostree/ostree-fetcher-curl.c | 23 +++++++++++++++++++---- src/libostree/ostree-fetcher-soup.c | 14 +++++++++++++- src/libostree/ostree-fetcher-util.c | 24 ++++++++++++++++++++++++ src/libostree/ostree-fetcher-util.h | 6 ++++++ src/libostree/ostree-fetcher.h | 1 + src/libostree/ostree-repo-pull.c | 2 +- 6 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index be3250fb..36bd917b 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -37,6 +37,7 @@ #endif #include "ostree-fetcher.h" +#include "ostree-fetcher-util.h" #include "ostree-enumtypes.h" #include "ostree-repo-private.h" #include "otutil.h" @@ -59,6 +60,7 @@ struct OstreeFetcher GObject parent_instance; OstreeFetcherConfigFlags config_flags; + char *remote_name; char *tls_ca_db_path; char *tls_client_cert_path; char *tls_client_key_path; @@ -159,6 +161,7 @@ _ostree_fetcher_finalize (GObject *object) { OstreeFetcher *self = OSTREE_FETCHER (object); + g_free (self->remote_name); g_free (self->cookie_jar_path); g_free (self->proxy); g_assert_cmpint (g_hash_table_size (self->outstanding_requests), ==, 0); @@ -222,9 +225,11 @@ _ostree_fetcher_init (OstreeFetcher *self) OstreeFetcher * _ostree_fetcher_new (int tmpdir_dfd, + const char *remote_name, OstreeFetcherConfigFlags flags) { OstreeFetcher *fetcher = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); + fetcher->remote_name = g_strdup (remote_name); fetcher->tmpdir_dfd = tmpdir_dfd; return fetcher; } @@ -303,9 +308,14 @@ check_multi_info (OstreeFetcher *fetcher) curl_easy_strerror (curlres)); } else - g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED, "[%u] %s", - curlres, - curl_easy_strerror (curlres)); + { + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED, "[%u] %s", + curlres, + curl_easy_strerror (curlres)); + if (req->fetcher->remote_name) + _ostree_fetcher_journal_failure (req->fetcher->remote_name, + eff_url, curl_easy_strerror (curlres)); + } } else { @@ -328,8 +338,13 @@ check_multi_info (OstreeFetcher *fetcher) if (req->idx + 1 == req->mirrorlist->len) { + g_autofree char *msg = g_strdup_printf ("Server returned HTTP %lu", response); g_task_return_new_error (task, G_IO_ERROR, giocode, - "Server returned HTTP %lu", response); + "%s", msg); + if (req->fetcher->remote_name) + _ostree_fetcher_journal_failure (req->fetcher->remote_name, + eff_url, msg); + } else { diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c index fcdf8e0e..b130b48c 100644 --- a/src/libostree/ostree-fetcher-soup.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -32,6 +32,7 @@ #include "libglnx.h" #include "ostree-fetcher.h" +#include "ostree-fetcher-util.h" #ifdef HAVE_LIBSOUP_CLIENT_CERTS #include "ostree-tls-cert-interaction.h" #endif @@ -55,6 +56,7 @@ typedef struct { GError *initialization_error; /* Any failure to load the db */ int tmpdir_dfd; + char *remote_name; char *tmpdir_name; GLnxLockFile tmpdir_lock; int base_tmpdir_dfd; @@ -168,6 +170,8 @@ thread_closure_unref (ThreadClosure *thread_closure) g_clear_pointer (&thread_closure->oob_error, g_error_free); + g_free (thread_closure->remote_name); + g_slice_free (ThreadClosure, thread_closure); } } @@ -725,12 +729,13 @@ _ostree_fetcher_init (OstreeFetcher *self) OstreeFetcher * _ostree_fetcher_new (int tmpdir_dfd, + const char *remote_name, OstreeFetcherConfigFlags flags) { OstreeFetcher *self; self = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); - + self->thread_closure->remote_name = g_strdup (remote_name); self->thread_closure->base_tmpdir_dfd = tmpdir_dfd; return self; @@ -1081,6 +1086,9 @@ on_request_sent (GObject *object, } else { + g_autofree char *uristring + = soup_uri_to_string (soup_request_get_uri (pending->request), FALSE); + GIOErrorEnum code; switch (msg->status_code) { @@ -1115,6 +1123,10 @@ on_request_sent (GObject *object, g_prefix_error (&local_error, "All %u mirrors failed. Last error was: ", pending->mirrorlist->len); + if (pending->thread_closure->remote_name) + _ostree_fetcher_journal_failure (pending->thread_closure->remote_name, + uristring, local_error->message); + } goto out; } diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c index b8af972a..408b8bcb 100644 --- a/src/libostree/ostree-fetcher-util.c +++ b/src/libostree/ostree-fetcher-util.c @@ -23,6 +23,10 @@ #include #include +#ifdef HAVE_LIBSYSTEMD +#include +#endif + #include "ostree-fetcher-util.h" #include "otutil.h" @@ -122,3 +126,23 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, out_contents, max_size, cancellable, error); } + +#define OSTREE_HTTP_FAILURE_ID SD_ID128_MAKE(f0,2b,ce,89,a5,4e,4e,fa,b3,a9,4a,79,7d,26,20,4a) + +void +_ostree_fetcher_journal_failure (const char *remote_name, + const char *url, + const char *msg) +{ +#ifdef HAVE_LIBSYSTEMD + /* Sanity - we don't want to log this when doing local/file pulls */ + if (!remote_name) + return; + sd_journal_send ("MESSAGE=libostree HTTP error from remote %s for <%s>: %s", + remote_name, url, msg, + "MESSAGE_ID=" SD_ID128_FORMAT_STR, SD_ID128_FORMAT_VAL(OSTREE_HTTP_FAILURE_ID), + "OSTREE_REMOTE=%s", remote_name, + "OSTREE_URL=%s", url, + NULL); +#endif +} diff --git a/src/libostree/ostree-fetcher-util.h b/src/libostree/ostree-fetcher-util.h index 0f25dc30..fe0921cd 100644 --- a/src/libostree/ostree-fetcher-util.h +++ b/src/libostree/ostree-fetcher-util.h @@ -44,6 +44,12 @@ gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, guint64 max_size, GCancellable *cancellable, GError **error); + +void _ostree_fetcher_journal_failure (const char *remote_name, + const char *url, + const char *msg); + + G_END_DECLS #endif diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h index f19eb73b..78b29fae 100644 --- a/src/libostree/ostree-fetcher.h +++ b/src/libostree/ostree-fetcher.h @@ -82,6 +82,7 @@ _ostree_fetcher_uri_to_string (OstreeFetcherURI *uri); GType _ostree_fetcher_get_type (void) G_GNUC_CONST; OstreeFetcher *_ostree_fetcher_new (int tmpdir_dfd, + const char *remote_name, OstreeFetcherConfigFlags flags); int _ostree_fetcher_get_dfd (OstreeFetcher *fetcher); diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 3452fbe9..7cbe8f92 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2107,7 +2107,7 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, if (tls_permissive) fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE; - fetcher = _ostree_fetcher_new (self->tmp_dir_fd, fetcher_flags); + fetcher = _ostree_fetcher_new (self->tmp_dir_fd, remote_name, fetcher_flags); { g_autofree char *tls_client_cert_path = NULL; From 36b28cb4d2126320d07029432abd2327b442a126 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 5 Jan 2017 11:29:41 -0500 Subject: [PATCH 49/66] upgrade: Add support for --pull-only and --deploy-only This makes it easier to script downloading updates in the background, and only do deployments just before rebooting. Partially addresses https://github.com/ostreedev/ostree/issues/640 Closes: #642 Approved by: jlebon --- Makefile-tests.am | 1 + man/ostree-admin-upgrade.xml | 23 +++++++++- src/libostree/ostree-sysroot-upgrader.c | 3 +- src/libostree/ostree-sysroot-upgrader.h | 3 +- src/ostree/ot-admin-builtin-upgrade.c | 43 ++++++++++++++++-- tests/test-admin-pull-deploy-split.sh | 59 +++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 7 deletions(-) create mode 100755 tests/test-admin-pull-deploy-split.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 8dbd3811..fd755ee1 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -83,6 +83,7 @@ dist_test_scripts = \ tests/test-admin-instutil-set-kargs.sh \ tests/test-admin-upgrade-not-backwards.sh \ tests/test-admin-pull-deploy-commit.sh \ + tests/test-admin-pull-deploy-split.sh \ tests/test-admin-locking.sh \ tests/test-admin-deploy-clean.sh \ tests/test-repo-checkout-subpath.sh \ diff --git a/man/ostree-admin-upgrade.xml b/man/ostree-admin-upgrade.xml index ddd33dc6..7766c66e 100644 --- a/man/ostree-admin-upgrade.xml +++ b/man/ostree-admin-upgrade.xml @@ -57,7 +57,10 @@ Boston, MA 02111-1307, USA. Description - Downloads the latest version of the current ref from the build server and deploys it, if it changed. Reboot the machine for the changes to take effect. + Downloads the latest version of the current ref from the build + server and deploys it, if it changed. Reboot the machine for the + changes to take effect. These phases can be split via + and . @@ -73,6 +76,24 @@ Boston, MA 02111-1307, USA. + + + + Only perform a pull into the repository; do not + create a deployment. This option can hence safely be used in a + background scheduled job with the assurance of not changing + system state. + + + + + + Create a new deployment from the latest commit + in the tracked origin refspec. This option is intended to be used + by a scheduled system that detected changes via , + and is ready to deploy them. + + , diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index daf2445c..232f8435 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -552,7 +552,8 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, g_assert (self->merge_deployment); from_revision = ostree_deployment_get_csum (self->merge_deployment); - if (self->origin_remote) + if (self->origin_remote && + (upgrader_flags & OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_SYNTHETIC) == 0) { if (!ostree_repo_pull_one_dir (repo, self->origin_remote, dir_to_pull, refs_to_fetch, flags, progress, diff --git a/src/libostree/ostree-sysroot-upgrader.h b/src/libostree/ostree-sysroot-upgrader.h index 77bc8a1d..83c4ad32 100644 --- a/src/libostree/ostree-sysroot-upgrader.h +++ b/src/libostree/ostree-sysroot-upgrader.h @@ -85,7 +85,8 @@ gboolean ostree_sysroot_upgrader_check_timestamps (OstreeRepo *repo, typedef enum { OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_NONE = 0, - OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER = (1 << 0) + OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER = (1 << 0), + OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_SYNTHETIC = (1 << 1) /* Don't actually do a pull, just check timestamps/changed */ } OstreeSysrootUpgraderPullFlags; _OSTREE_PUBLIC diff --git a/src/ostree/ot-admin-builtin-upgrade.c b/src/ostree/ot-admin-builtin-upgrade.c index 394b339a..7c710ffc 100644 --- a/src/ostree/ot-admin-builtin-upgrade.c +++ b/src/ostree/ot-admin-builtin-upgrade.c @@ -34,6 +34,8 @@ static gboolean opt_reboot; static gboolean opt_allow_downgrade; +static gboolean opt_pull_only; +static gboolean opt_deploy_only; static char *opt_osname; static char *opt_override_commit; @@ -42,6 +44,8 @@ static GOptionEntry options[] = { { "reboot", 'r', 0, G_OPTION_ARG_NONE, &opt_reboot, "Reboot after a successful upgrade", NULL }, { "allow-downgrade", 0, 0, G_OPTION_ARG_NONE, &opt_allow_downgrade, "Permit deployment of chronologically older trees", NULL }, { "override-commit", 0, 0, G_OPTION_ARG_STRING, &opt_override_commit, "Deploy CHECKSUM instead of the latest tree", "CHECKSUM" }, + { "pull-only", 0, 0, G_OPTION_ARG_NONE, &opt_pull_only, "Do not create a deployment, just download", NULL }, + { "deploy-only", 0, 0, G_OPTION_ARG_NONE, &opt_deploy_only, "Do not pull, only deploy", NULL }, { NULL } }; @@ -64,6 +68,19 @@ ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GErr &sysroot, cancellable, error)) goto out; + if (opt_pull_only && opt_deploy_only) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Cannot simultaneously specify --pull-only and --deploy-only"); + goto out; + } + else if (opt_pull_only && opt_reboot) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Cannot simultaneously specify --pull-only and --reboot"); + goto out; + } + if (!ostree_sysroot_load (sysroot, cancellable, error)) goto out; @@ -104,6 +121,9 @@ ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GErr } } + if (opt_deploy_only) + upgraderpullflags |= OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_SYNTHETIC; + { g_auto(GLnxConsoleRef) console = { 0, }; glnx_console_lock (&console); @@ -112,11 +132,23 @@ ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GErr if (opt_allow_downgrade) upgraderpullflags |= OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER; - + if (!ostree_sysroot_upgrader_pull (upgrader, 0, upgraderpullflags, progress, &changed, cancellable, error)) - goto out; + { + /* In the pull-only case, we do a cleanup here to ensure that if + * multiple commits were pulled, we garbage collect any old + * partially-pulled intermediate commits before pulling more. This is + * really a best practice in general, but for maximum compatiblity, we + * only do cleanup if a user specifies the new --pull-only option. + * Otherwise, we would break the case of trying to deploy a commit that + * isn't directly referenced. + */ + if (opt_pull_only) + (void) ostree_sysroot_cleanup (sysroot, NULL, NULL); + goto out; + } if (progress) ostree_async_progress_finish (progress); @@ -128,8 +160,11 @@ ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GErr } else { - if (!ostree_sysroot_upgrader_deploy (upgrader, cancellable, error)) - goto out; + if (!opt_pull_only) + { + if (!ostree_sysroot_upgrader_deploy (upgrader, cancellable, error)) + goto out; + } if (opt_reboot) { diff --git a/tests/test-admin-pull-deploy-split.sh b/tests/test-admin-pull-deploy-split.sh new file mode 100755 index 00000000..7a6750e2 --- /dev/null +++ b/tests/test-admin-pull-deploy-split.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Copyright (C) 2017 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +# See https://github.com/ostreedev/ostree/pull/642 + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo "1..1" + +setup_os_repository "archive-z2" "syslinux" + +cd ${test_tmpdir} +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo remote add --set=gpg-verify=false testos $(cat httpd-address)/ostree/testos-repo +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull testos testos/buildmaster/x86_64-runtime +rev=$(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse testos/buildmaster/x86_64-runtime) +parent_rev=$(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse ${rev}^) +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull testos testos/buildmaster/x86_64-runtime@${parent_rev} +${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=MOO --karg=quiet --os=testos testos:testos/buildmaster/x86_64-runtime +assert_has_dir sysroot/ostree/deploy/testos/deploy/${parent_rev}.0 +assert_not_has_dir sysroot/ostree/deploy/testos/deploy/${rev}.0 +# Do a pull, this one should get us new content +${CMD_PREFIX} ostree admin upgrade --os=testos --pull-only --os=testos > out.txt +assert_not_file_has_content out.txt 'No update available' +# And pull again should still tell us we have new content +${CMD_PREFIX} ostree admin upgrade --os=testos --pull-only --os=testos > out.txt +assert_not_file_has_content out.txt 'No update available' +assert_has_dir sysroot/ostree/deploy/testos/deploy/${parent_rev}.0 +assert_not_has_dir sysroot/ostree/deploy/testos/deploy/${rev}.0 +assert_file_has_content sysroot/boot/loader/entries/ostree-testos-0.conf 'TestOS 42 1.0.9' +assert_streq "${rev}" $(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse testos/buildmaster/x86_64-runtime) +# Now, generate new content upstream; we shouldn't pull it +os_repository_new_commit +${CMD_PREFIX} ostree admin upgrade --os=testos --deploy-only --os=testos > out.txt +assert_not_file_has_content out.txt 'No update available' +assert_file_has_content sysroot/boot/loader/entries/ostree-testos-0.conf 'TestOS 42 1.0.10' +assert_has_dir sysroot/ostree/deploy/testos/deploy/${parent_rev}.0 +assert_has_dir sysroot/ostree/deploy/testos/deploy/${rev}.0 +${CMD_PREFIX} ostree admin upgrade --os=testos --deploy-only --os=testos > out.txt +assert_file_has_content out.txt 'No update available' + +echo 'ok upgrade --pull-only + --deploy-only' From 9695c476846684098ad454fcbe06a370fe92d63e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 22 Feb 2017 11:33:38 -0500 Subject: [PATCH 50/66] grub2: Use g_spawn_sync() rather than GSubprocess to avoid SIGCHLD Due to the async nature of `GSubprocess` it grabs `SIGCHLD` which affects other software which might be using libostree, such as QtOTA. Closes: https://github.com/ostreedev/ostree/issues/696 Closes: #702 Approved by: jlebon --- src/libostree/ostree-bootloader-grub2.c | 64 ++++++++++++++----------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/src/libostree/ostree-bootloader-grub2.c b/src/libostree/ostree-bootloader-grub2.c index 0fbb098e..5830660b 100644 --- a/src/libostree/ostree-bootloader-grub2.c +++ b/src/libostree/ostree-bootloader-grub2.c @@ -239,12 +239,26 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot return ret; } +typedef struct { + const char *root; + const char *bootversion_str; + gboolean is_efi; +} Grub2ChildSetupData; + static void grub2_child_setup (gpointer user_data) { - const char *root = user_data; + Grub2ChildSetupData *cdata = user_data; - if (chdir (root) != 0) + setenv ("_OSTREE_GRUB2_BOOTVERSION", cdata->bootversion_str, TRUE); + /* We have to pass our state to the child */ + if (cdata->is_efi) + setenv ("_OSTREE_GRUB2_IS_EFI", "1", TRUE); + + if (!cdata->root) + return; + + if (chdir (cdata->root) != 0) { perror ("chdir"); _exit (1); @@ -268,7 +282,7 @@ grub2_child_setup (gpointer user_data) _exit (1); } - if (mount (root, "/", NULL, MS_MOVE, NULL) < 0) + if (mount (cdata->root, "/", NULL, MS_MOVE, NULL) < 0) { perror ("failed to MS_MOVE to /"); _exit (1); @@ -290,14 +304,15 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader *bootloader, OstreeBootloaderGrub2 *self = OSTREE_BOOTLOADER_GRUB2 (bootloader); gboolean ret = FALSE; g_autoptr(GFile) new_config_path = NULL; - GSubprocessFlags subp_flags = 0; - glnx_unref_object GSubprocessLauncher *launcher = NULL; - glnx_unref_object GSubprocess *proc = NULL; g_autofree char *bootversion_str = g_strdup_printf ("%u", (guint)bootversion); g_autoptr(GFile) config_path_efi_dir = NULL; g_autofree char *grub2_mkconfig_chroot = NULL; gboolean use_system_grub2_mkconfig = TRUE; const gchar *grub_exec = NULL; + const char *grub_argv[4] = { NULL, "-o", NULL, NULL}; + GSpawnFlags grub_spawnflags = G_SPAWN_SEARCH_PATH; + int grub2_estatus; + Grub2ChildSetupData cdata = { NULL, }; #ifdef USE_BUILTIN_GRUB2_MKCONFIG use_system_grub2_mkconfig = FALSE; @@ -354,22 +369,15 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader *bootloader, bootversion); } - if (!g_getenv ("OSTREE_DEBUG_GRUB2")) - subp_flags |= (G_SUBPROCESS_FLAGS_STDOUT_SILENCE | G_SUBPROCESS_FLAGS_STDERR_SILENCE); - - launcher = g_subprocess_launcher_new (subp_flags); - g_subprocess_launcher_setenv (launcher, "_OSTREE_GRUB2_BOOTVERSION", bootversion_str, TRUE); - /* We have to pass our state to the child */ - if (self->is_efi) - g_subprocess_launcher_setenv (launcher, "_OSTREE_GRUB2_IS_EFI", "1", TRUE); - - /* We need to chroot() if we're not in /. This assumes our caller has - * set up the bind mounts outside. - */ - if (grub2_mkconfig_chroot != NULL) - g_subprocess_launcher_set_child_setup (launcher, grub2_child_setup, grub2_mkconfig_chroot, NULL); + grub_argv[0] = grub_exec; + grub_argv[2] = gs_file_get_path_cached (new_config_path); - /* In the current Fedora grub2 package, this script doesn't even try + if (!g_getenv ("OSTREE_DEBUG_GRUB2")) + grub_spawnflags |= G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL; + cdata.root = grub2_mkconfig_chroot; + cdata.bootversion_str = bootversion_str; + cdata.is_efi = self->is_efi; + /* Note in older versions of the grub2 package, this script doesn't even try to be atomic; it just does: cat ${grub_cfg}.new > ${grub_cfg} @@ -377,13 +385,15 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader *bootloader, Upstream is fixed though. */ - proc = g_subprocess_launcher_spawn (launcher, error, - grub_exec, "-o", - gs_file_get_path_cached (new_config_path), - NULL); - - if (!g_subprocess_wait_check (proc, cancellable, error)) + if (!g_spawn_sync (NULL, (char**)grub_argv, NULL, grub_spawnflags, + grub2_child_setup, &cdata, NULL, NULL, + &grub2_estatus, error)) goto out; + if (!g_spawn_check_exit_status (grub2_estatus, error)) + { + g_prefix_error (error, "%s: ", grub_argv[0]); + goto out; + } /* Now let's fdatasync() for the new file */ { glnx_fd_close int new_config_fd = open (gs_file_get_path_cached (new_config_path), O_RDONLY | O_CLOEXEC); From 64422a7d0beb5243d051320cc94c14324b845049 Mon Sep 17 00:00:00 2001 From: Gatis Paeglis Date: Thu, 24 Nov 2016 15:59:36 +0100 Subject: [PATCH 51/66] deltas: Expose the filename parameter The C API (ostree_repo_static_delta_generate) knows what to do with it, but this parameter was never exposed via command line tool. Closes: https://github.com/ostreedev/ostree/issues/695 Closes: #703 Approved by: jlebon --- src/ostree/ot-builtin-static-delta.c | 5 +++++ tests/test-delta.sh | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index c8843a65..1019f06f 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -33,6 +33,7 @@ static char *opt_min_fallback_size; static char *opt_max_bsdiff_size; static char *opt_max_chunk_size; static char *opt_endianness; +static char *opt_filename; static gboolean opt_empty; static gboolean opt_swap_endianness; static gboolean opt_inline; @@ -71,6 +72,7 @@ static GOptionEntry generate_options[] = { { "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in megabytes for individual HTTP request", NULL}, { "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL}, { "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL}, + { "filename", 0, 0, G_OPTION_ARG_STRING, &opt_filename, "Write the delta content to PATH (a directory). If not specified, the OSTree repository is used", "PATH"}, { NULL } }; @@ -322,6 +324,9 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab if (opt_inline) g_variant_builder_add (parambuilder, "{sv}", "inline-parts", g_variant_new_boolean (TRUE)); + if (opt_filename) + g_variant_builder_add (parambuilder, "{sv}", + "filename", g_variant_new_bytestring (opt_filename)); g_variant_builder_add (parambuilder, "{sv}", "verbose", g_variant_new_boolean (TRUE)); if (opt_endianness || opt_swap_endianness) diff --git a/tests/test-delta.sh b/tests/test-delta.sh index bb523b47..6d041f72 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -121,6 +121,14 @@ if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded" fi +${CMD_PREFIX} ostree --repo=temp-repo init --mode=archive +${CMD_PREFIX} ostree --repo=temp-repo pull-local repo +${CMD_PREFIX} ostree --repo=temp-repo static-delta generate --empty --to=${newrev} --filename=some.delta +assert_has_file some.delta +${CMD_PREFIX} ostree --repo=temp-repo static-delta list > delta-list.txt +assert_file_has_content delta-list.txt 'No static deltas' +rm temp-repo -rf + echo 'ok generate' ${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show.txt From 199646ccfa59f919d92af956195e3997d370d236 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 2 Mar 2017 12:11:17 -0500 Subject: [PATCH 52/66] pull: don't use static deltas if archive repo In https://github.com/ostreedev/ostree/pull/408, we disabled the use of static deltas when mirroring. Later, https://github.com/ostreedev/ostree/pull/506 loosened this up again so that we could use static deltas when mirroring into bare{-user} repos. However, the issue which originally spurrred #408 is even more generic than that: we want to avoid static deltas for any archive repo, not just when doing a mirror pull. This patch tightens this up, and also relocates the decision code to make it easier to read. Closes: #715 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 23 +++++++++++++++++++---- tests/pull-test.sh | 12 +++++++++++- tests/test-delta.sh | 4 ++-- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 7cbe8f92..60eb214e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2542,7 +2542,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; const char *url_override = NULL; - gboolean mirroring_into_archive; gboolean inherit_transaction = FALSE; int i; @@ -2581,6 +2580,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_return_val_if_fail (dirs_to_pull[i][0] == '/', FALSE); g_return_val_if_fail (!(disable_static_deltas && pull_data->require_static_deltas), FALSE); + /* We only do dry runs with static deltas, because we don't really have any * in-advance information for bare fetches. */ @@ -2591,8 +2591,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->is_untrusted = (flags & OSTREE_REPO_PULL_FLAGS_UNTRUSTED) > 0; pull_data->cancellable = cancellable ? g_object_ref (cancellable) : NULL; - mirroring_into_archive = pull_data->is_mirror && self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2; - if (error) pull_data->async_error = &pull_data->cached_async_error; else @@ -2862,6 +2860,23 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->remote_repo_local && !pull_data->require_static_deltas) disable_static_deltas = TRUE; + /* We can't use static deltas if pulling into an archive-z2 repo. */ + if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2) + { + if (pull_data->require_static_deltas) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't use static deltas in an archive repo"); + goto out; + } + disable_static_deltas = TRUE; + } + + /* It's not efficient to use static deltas if all we want is the commit + * metadata. */ + if (pull_data->is_commit_only) + disable_static_deltas = TRUE; + pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); { @@ -3148,7 +3163,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, &from_revision, error)) goto out; - if (!(disable_static_deltas || mirroring_into_archive || pull_data->is_commit_only) && + if (!disable_static_deltas && (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0)) { g_autofree char *delta_name = diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 693eac06..f6176079 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -35,7 +35,7 @@ function verify_initial_contents() { assert_file_has_content baz/cow '^moo$' } -echo "1..14" +echo "1..15" # Try both syntaxes repo_init @@ -69,6 +69,16 @@ ${CMD_PREFIX} ostree --repo=mirrorrepo pull --mirror origin main ${CMD_PREFIX} ostree --repo=mirrorrepo fsck echo "ok pull mirror (should not apply deltas)" +cd ${test_tmpdir} +if ${CMD_PREFIX} ostree --repo=mirrorrepo \ + pull origin main --require-static-deltas 2>err.txt; then + assert_not_reached "--require-static-deltas unexpectedly succeeded" +fi +assert_file_has_content err.txt "Can't use static deltas in an archive repo" +${CMD_PREFIX} ostree --repo=mirrorrepo pull origin main +${CMD_PREFIX} ostree --repo=mirrorrepo fsck +echo "ok pull (refuses deltas)" + cd ${test_tmpdir} rm mirrorrepo/refs/remotes/* -rf ${CMD_PREFIX} ostree --repo=mirrorrepo prune --refs-only diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 6d041f72..46af2582 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -168,7 +168,7 @@ echo 'ok heuristic endian detection' ${CMD_PREFIX} ostree --repo=repo summary -u -mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2 +mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=bare-user ${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${newrev} ${CMD_PREFIX} ostree --repo=repo2 fsck ${CMD_PREFIX} ostree --repo=repo2 ls ${newrev} >/dev/null @@ -236,7 +236,7 @@ echo 'ok generate + show empty delta part' ${CMD_PREFIX} ostree --repo=repo summary -u rm -rf repo2 -mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2 +mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=bare-user ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev} ${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev} ${CMD_PREFIX} ostree --repo=repo2 fsck From c4f65228295e34ef25e8a66464120a87f9c3b192 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 2 Mar 2017 14:50:24 -0500 Subject: [PATCH 53/66] libglnx: bump for -Wmaybe-uninitialized fix https://github.com/GNOME/libglnx/pull/37 Closes: #715 Approved by: cgwalters --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index 0c1603de..5309e363 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 0c1603debac440978c1d55c46cb11059f8350b35 +Subproject commit 5309e363aa30d2108a264ae35d8d870ee3e0c443 From b41f150a722ded3914a38b2f8a123d1aca7f8260 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 2 Mar 2017 21:30:10 -0500 Subject: [PATCH 54/66] grub2: Use "linux16" only on x86/x86_64 Got a report that a Fedora Atomic Host built for ppc64le didn't work with the `linux16`, it needed `linux`. See the comments for more links. Closes: #716 Approved by: vathpela --- src/libostree/ostree-bootloader-grub2.c | 30 +++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-bootloader-grub2.c b/src/libostree/ostree-bootloader-grub2.c index 5830660b..6f245fb7 100644 --- a/src/libostree/ostree-bootloader-grub2.c +++ b/src/libostree/ostree-bootloader-grub2.c @@ -29,6 +29,20 @@ #include +/* I only did some cursory research here, but it appears + * that we only want to use "linux16" for x86 platforms. + * At least, I got a report that "linux16" is definitely wrong + * for ppc64. See + * http://pkgs.fedoraproject.org/cgit/rpms/grub2.git/tree/0036-Use-linux16-when-appropriate-880840.patch?h=f25 + * https://bugzilla.redhat.com/show_bug.cgi?id=1108296 + * among others. + */ +#if defined(__i386__) || defined(__x86_64__) +#define GRUB2_USES_16 1 +#else +#define GRUB2_USES_16 0 +#endif + struct _OstreeBootloaderGrub2 { GObject parent_instance; @@ -203,7 +217,13 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot if (is_efi) g_string_append (output, "linuxefi "); else - g_string_append (output, "linux16 "); + { +#if GRUB2_USES_16 + g_string_append (output, "linux16 "); +#else + g_string_append (output, "linux "); +#endif + } g_string_append (output, kernel); options = ostree_bootconfig_parser_get (config, "options"); @@ -220,7 +240,13 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot if (is_efi) g_string_append (output, "initrdefi "); else - g_string_append (output, "initrd16 "); + { +#if GRUB2_USES_16 + g_string_append (output, "initrd16 "); +#else + g_string_append (output, "initrd "); +#endif + } g_string_append (output, initrd); g_string_append_c (output, '\n'); } From a787e0c072756863136e1951ce3a99012bda40d2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 23 Feb 2017 21:49:21 -0500 Subject: [PATCH 55/66] pull: Use all available commits for delta sources The previous logic for static deltas was to use as a FROM revision the current branch tip. However, we want to support deltas between branches in an automatic fashion. If a summary file is available, we already have an enumerated list of deltas - so the logic introduced here is to search it, and find the newest commit we have locally that matches the TO revision target. This builds on some thoughts from https://github.com/ostreedev/ostree/pull/151#issuecomment-232390232 Closes: https://github.com/ostreedev/ostree/pull/151 Closes: #710 Approved by: giuseppe --- src/libostree/ostree-repo-pull.c | 99 +++++++++++++++++++++++++++++++- tests/test-delta.sh | 13 ++++- 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 60eb214e..d7097320 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1866,6 +1866,87 @@ process_one_static_delta (OtPullData *pull_data, return ret; } +/* Loop over the static delta data we got from the summary, + * and find the newest commit for @out_from_revision that + * goes to @to_revision. + */ +static gboolean +get_best_static_delta_start_for (OtPullData *pull_data, + const char *to_revision, + char **out_from_revision, + GCancellable *cancellable, + GError **error) +{ + GHashTableIter hiter; + gpointer hkey, hvalue; + /* Array of possible from checksums */ + g_autoptr(GPtrArray) candidates = g_ptr_array_new_with_free_func (g_free); + const char *newest_candidate = NULL; + guint64 newest_candidate_timestamp; + + g_assert (pull_data->summary_deltas_checksums != NULL); + g_hash_table_iter_init (&hiter, pull_data->summary_deltas_checksums); + + /* Loop over all deltas known from the summary file, + * finding ones which go to to_revision */ + while (g_hash_table_iter_next (&hiter, &hkey, &hvalue)) + { + const char *delta_name = hkey; + g_autofree char *cur_from_rev = NULL; + g_autofree char *cur_to_rev = NULL; + + /* Gracefully handle corrupted (or malicious) summary files */ + if (!_ostree_parse_delta_name (delta_name, &cur_from_rev, &cur_to_rev, error)) + return FALSE; + + /* Is this the checksum we want? */ + if (strcmp (cur_to_rev, to_revision) != 0) + continue; + + if (cur_from_rev) + g_ptr_array_add (candidates, g_steal_pointer (&cur_from_rev)); + } + + /* Loop over our candidates, find the newest one */ + for (guint i = 0; i < candidates->len; i++) + { + const char *candidate = candidates->pdata[i]; + guint64 candidate_ts; + g_autoptr(GVariant) commit = NULL; + OstreeRepoCommitState state; + gboolean have_candidate; + + /* Do we have this commit at all? If not, skip it */ + if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, + candidate, &have_candidate, + NULL, error)) + return FALSE; + if (!have_candidate) + continue; + + /* Load it */ + if (!ostree_repo_load_commit (pull_data->repo, candidate, + &commit, &state, error)) + return FALSE; + + /* Ignore partial commits, we can't use them */ + if (state & OSTREE_REPO_COMMIT_STATE_PARTIAL) + continue; + + /* Is it newer? */ + candidate_ts = ostree_commit_get_timestamp (commit); + if (newest_candidate == NULL || + candidate_ts > newest_candidate_timestamp) + { + newest_candidate = candidate; + newest_candidate_timestamp = candidate_ts; + } + } + + *out_from_revision = g_strdup (newest_candidate); + return TRUE; +} + typedef struct { OtPullData *pull_data; char *from_revision; @@ -3159,9 +3240,21 @@ ostree_repo_pull_with_options (OstreeRepo *self, const char *ref = key; const char *to_revision = value; - if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE, - &from_revision, error)) - goto out; + /* If we have a summary, find the latest local commit we have + * to use as a from revision for static deltas. + */ + if (pull_data->summary) + { + if (!get_best_static_delta_start_for (pull_data, to_revision, &from_revision, + cancellable, error)) + goto out; + } + else + { + if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE, + &from_revision, error)) + goto out; + } if (!disable_static_deltas && (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0)) diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 46af2582..b9883830 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -26,7 +26,7 @@ skip_without_user_xattrs bindatafiles="bash true ostree" morebindatafiles="false ls" -echo '1..11' +echo '1..12' mkdir repo ${CMD_PREFIX} ostree --repo=repo init --mode=archive-z2 @@ -244,6 +244,17 @@ ${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null echo 'ok pull empty delta part' +# Make a new branch to test "rebase deltas" +echo otherbranch-content > files/otherbranch-content +${CMD_PREFIX} ostree --repo=repo commit -b otherbranch --tree=dir=files +samerev=$(${CMD_PREFIX} ostree --repo=repo rev-parse test) +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=test --to=otherbranch +${CMD_PREFIX} ostree --repo=repo summary -u +${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo otherbranch + +echo 'ok rebase deltas' + +${CMD_PREFIX} ostree --repo=repo summary -u if ${CMD_PREFIX} ostree --repo=repo static-delta show GARBAGE 2> err.txt; then assert_not_reached "static-delta show GARBAGE unexpectedly succeeded" fi From f667a82fc10a0e006c4aa634e4ffbffe1f9c537d Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 6 Mar 2017 12:19:28 +0000 Subject: [PATCH 56/66] build: Fix disabling --enable-man if xsltproc is not available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If --enable-man=maybe (the default), and xsltproc is not available, the configuration code would check for it, correctly set enable_man=no, then incorrectly overwrite that with enable_man=yes, which would result in later trying to execute $(XSLTPROC) when it’s empty. Signed-off-by: Philip Withnall Closes: #720 Approved by: cgwalters --- configure.ac | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 48d8949c..83b8f48f 100644 --- a/configure.ac +++ b/configure.ac @@ -184,8 +184,9 @@ AS_IF([test "$enable_man" != no], [ AC_MSG_ERROR([xsltproc is required for --enable-man]) ]) enable_man=no + ],[ + enable_man=yes ]) - enable_man=yes ]) AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) From 72336f1c4885f7916be58c7bcf2501655b33e5aa Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 4 Mar 2017 10:11:15 -0500 Subject: [PATCH 57/66] fetcher/curl: Fix leaks caught by ASAN I had to rebuild `glib` with `-fsanitize=address` in order to get a stack trace to finally get this one. However, *installing* that glib "system wide" in my container breaks everything (including `rpm-ostree`, `dnf`, `pkg-config` etc.) that wasn't built with ASAN. So my test scenario right now is to extract the libs and do e.g.: ``` make && env LD_LIBRARY_PATH=$HOME/src/distgit/fedora/glib2/asan-libs make check TESTS=tests/test-basic.sh ``` Closes: #719 Approved by: jlebon --- src/libostree/ostree-fetcher-curl.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index 36bd917b..be0e4b41 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -416,11 +416,13 @@ timer_cb (gpointer data) { OstreeFetcher *fetcher = data; CURLMcode rc; + GSource *orig_src = fetcher->timer_event; - fetcher->timer_event = NULL; rc = curl_multi_socket_action (fetcher->multi, CURL_SOCKET_TIMEOUT, 0, &fetcher->curl_running); g_assert (rc == CURLM_OK); check_multi_info (fetcher); + if (fetcher->timer_event == orig_src) + fetcher->timer_event = NULL; return FALSE; } @@ -713,6 +715,8 @@ initiate_next_curl_request (FetcherRequest *req, CURLMcode rc; OstreeFetcher *self = req->fetcher; + if (req->easy) + curl_easy_cleanup (req->easy); req->easy = curl_easy_init (); g_assert (req->easy); From 574c3ea6f935c392da99a721907a98086eeaad44 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 6 Mar 2017 17:48:36 +0000 Subject: [PATCH 58/66] libostree: Allow compression level to be set for archive-z2 stream Add a ostree_raw_file_to_archive_z2_stream_with_options() variant of ostree_raw_file_to_archive_z2_stream(), to allow a compression-level option to be passed in and passed through to zlib. This is useful when building archive-z2 files on the fly for transmission over a non-bandwidth-limited channel, such as a local network. In this case, CPU time is more valuable than bandwidth, so we want a low compression level. Signed-off-by: Philip Withnall Closes: #721 Approved by: cgwalters --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 6 ++---- src/libostree/ostree-core.c | 40 +++++++++++++++++++++++++++++++++++++ src/libostree/ostree-core.h | 10 ++++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index ed606604..787664b3 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -117,6 +117,7 @@ ostree_content_stream_parse ostree_content_file_parse ostree_content_file_parse_at ostree_raw_file_to_archive_z2_stream +ostree_raw_file_to_archive_z2_stream_with_options ostree_raw_file_to_content_stream ostree_checksum_file_from_input ostree_checksum_file diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index a07e5d3b..7c7c3aef 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -381,12 +381,10 @@ global: * NOTE NOTE NOTE */ -/* Uncomment this when adding the first new symbol for 2 -LIBOSTREE_2017.XX { +LIBOSTREE_2017.3 { global: - someostree_symbol_deleteme; + ostree_raw_file_to_archive_z2_stream_with_options; } LIBOSTREE_2017.2; -*/ /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index af36d98b..cf238b41 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -506,6 +506,46 @@ ostree_raw_file_to_archive_z2_stream (GInputStream *input, out_input, cancellable, error); } +/** + * ostree_raw_file_to_archive_z2_stream_with_options: + * @input: File raw content stream + * @file_info: A file info + * @xattrs: (allow-none): Optional extended attributes + * @options: (nullable): A GVariant `a{sv}` with an extensible set of flags + * @out_input: (out): Serialized object stream + * @cancellable: Cancellable + * @error: Error + * + * Like ostree_raw_file_to_archive_z2_stream(), but supports an extensible set + * of flags. The following flags are currently defined: + * + * - `compression-level` (`i`): Level of compression to use, 0–9, with 0 being + * the least compression, and <0 giving the default level (currently 6). + * + * Since: 2017.3 + */ +gboolean +ostree_raw_file_to_archive_z2_stream_with_options (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + GVariant *options, + GInputStream **out_input, + GCancellable *cancellable, + GError **error) +{ + gint compression_level = -1; + + if (options) + (void) g_variant_lookup (options, "compression-level", "i", &compression_level); + + if (compression_level < 0) + compression_level = OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL; + + return _ostree_raw_file_to_archive_stream (input, file_info, xattrs, + compression_level, + out_input, cancellable, error); +} + /** * ostree_raw_file_to_content_stream: * @input: File raw content stream diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index a3419949..bd3d5f2c 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -311,6 +311,16 @@ ostree_raw_file_to_archive_z2_stream (GInputStream *input, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean +ostree_raw_file_to_archive_z2_stream_with_options (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + GVariant *options, + GInputStream **out_input, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_raw_file_to_content_stream (GInputStream *input, GFileInfo *file_info, From 3219a5d0ee4bf10479363690e05400ebdab18be3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 3 Mar 2017 12:10:46 -0500 Subject: [PATCH 59/66] Allow and start using C99 declaration-after-statement I've seen code in a few places that I think on balance is definitely better this way. Some of our functions have huge variable declaration sections. This change includes one small example where we could start using declarations after statements. A concern I had was - how does this interact with `__attribute__((cleanup))` and early returns? I tested it, and AFAICS the behavior is what you'd expect - the cleanup function isn't called if its variable isn't reachable. Closes: #718 Approved by: jlebon --- configure.ac | 1 - src/libostree/ostree-repo.c | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 83b8f48f..7fac081b 100644 --- a/configure.ac +++ b/configure.ac @@ -33,7 +33,6 @@ CC_CHECK_FLAGS_APPEND([WARN_CFLAGS], [CFLAGS], [\ -Werror=incompatible-pointer-types \ -Werror=misleading-indentation \ -Werror=missing-include-dirs -Werror=aggregate-return \ - -Werror=declaration-after-statement \ ]) AC_SUBST(WARN_CFLAGS) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index dc0eb575..71604480 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4309,16 +4309,14 @@ find_keyring (OstreeRepo *self, OstreeRemote *remote, GCancellable *cancellable) { - g_autoptr(GFile) remotes_d = NULL; - g_autoptr(GFile) file = NULL; - file = g_file_get_child (self->repodir, remote->keyring); + g_autoptr(GFile) file = g_file_get_child (self->repodir, remote->keyring); if (g_file_query_exists (file, cancellable)) { return g_steal_pointer (&file); } - remotes_d = get_remotes_d_dir (self); + g_autoptr(GFile) remotes_d = get_remotes_d_dir (self); if (remotes_d) { g_autoptr(GFile) file2 = g_file_get_child (remotes_d, remote->keyring); From ff34810097460a4cac7965213db54150e4e7ae06 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 6 Mar 2017 14:51:14 -0500 Subject: [PATCH 60/66] repo/checkout: Verify early if src/destination are on same device At least in all Linux kernels up to today, one can never `link()` across devices, so we might as well verify that up front. This will help for a future patch to add a new type of union-add checkout, since Linux checks for `EEXIST` before `EXDEV`. Closes: #714 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 21 +++++++++++++++++++++ tests/test-rofiles-fuse.sh | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 9a151646..50bc7030 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -641,6 +641,8 @@ checkout_tree_at (OstreeRepo *self, gboolean did_exist = FALSE; glnx_fd_close int destination_dfd = -1; int res; + struct stat repo_dfd_stat; + struct stat destination_stat; g_autoptr(GVariant) xattrs = NULL; g_autoptr(GFileEnumerator) dir_enum = NULL; @@ -666,6 +668,25 @@ checkout_tree_at (OstreeRepo *self, &destination_dfd, error)) goto out; + if (fstat (self->repo_dir_fd, &repo_dfd_stat) < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + if (fstat (destination_dfd, &destination_stat) < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + if (options->no_copy_fallback && repo_dfd_stat.st_dev != destination_stat.st_dev) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unable to do hardlink checkout across devices (src=%lu destination=%lu)", + repo_dfd_stat.st_dev, destination_stat.st_dev); + goto out; + } + /* Set the xattrs now, so any derived labeling works */ if (!did_exist && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) { diff --git a/tests/test-rofiles-fuse.sh b/tests/test-rofiles-fuse.sh index 4dfec514..56045c61 100755 --- a/tests/test-rofiles-fuse.sh +++ b/tests/test-rofiles-fuse.sh @@ -78,6 +78,6 @@ assert_file_has_content mnt/test2-checkout-copy-fallback/anewfile-for-fuse anewf if ${CMD_PREFIX} ostree --repo=repo checkout -UH test2 mnt/test2-checkout-copy-hardlinked 2>err.txt; then assert_not_reached "Checking out via hardlinks across mountpoint unexpectedly succeeded!" fi -assert_file_has_content err.txt "Invalid cross-device link" +assert_file_has_content err.txt "Unable to do hardlink checkout across devices" echo "ok checkout copy fallback" From 94948e3522b56b2e80a7cae636aef06c1b372fc8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 1 Mar 2017 22:42:07 -0500 Subject: [PATCH 61/66] checkout: Support a "pure addition" mode I plan to use this for `rpm-ostree livefs`. https://github.com/projectatomic/rpm-ostree/issues/639 Closes: #714 Approved by: jlebon --- man/ostree-checkout.xml | 8 +++ src/libostree/ostree-repo-checkout.c | 74 ++++++++++++++++++---------- src/libostree/ostree-repo.h | 6 ++- src/ostree/ot-builtin-checkout.c | 17 +++++-- tests/basic-test.sh | 20 +++++++- 5 files changed, 94 insertions(+), 31 deletions(-) diff --git a/man/ostree-checkout.xml b/man/ostree-checkout.xml index 67d6469e..c8585878 100644 --- a/man/ostree-checkout.xml +++ b/man/ostree-checkout.xml @@ -89,6 +89,14 @@ Boston, MA 02111-1307, USA. + + + + + Keep existing directories and files. + + + diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 50bc7030..53409529 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -201,8 +201,16 @@ checkout_file_from_input_at (OstreeRepo *self, while (G_UNLIKELY (res == -1 && errno == EINTR)); if (res == -1) { - glnx_set_error_from_errno (error); - goto out; + if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) + { + ret = TRUE; + goto out; + } + else + { + glnx_set_error_from_errno (error); + goto out; + } } if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) @@ -240,6 +248,11 @@ checkout_file_from_input_at (OstreeRepo *self, while (G_UNLIKELY (fd == -1 && errno == EINTR)); if (fd == -1) { + if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) + { + ret = TRUE; + goto out; + } glnx_set_error_from_errno (error); goto out; } @@ -332,6 +345,12 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo, return ret; } +typedef enum { + HARDLINK_RESULT_NOT_SUPPORTED, + HARDLINK_RESULT_SKIP_EXISTED, + HARDLINK_RESULT_LINKED +} HardlinkResult; + static gboolean checkout_file_hardlink (OstreeRepo *self, OstreeRepoCheckoutAtOptions *options, @@ -339,28 +358,32 @@ checkout_file_hardlink (OstreeRepo *self, int destination_dfd, const char *destination_name, gboolean allow_noent, - gboolean *out_was_supported, + HardlinkResult *out_result, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - gboolean ret_was_supported = FALSE; + HardlinkResult ret_result = HARDLINK_RESULT_NOT_SUPPORTED; int srcfd = (self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER) ? self->objects_dir_fd : self->uncompressed_objects_dir_fd; again: - if (linkat (srcfd, loose_path, destination_dfd, destination_name, 0) != -1) - ret_was_supported = TRUE; + if (linkat (srcfd, loose_path, destination_dfd, destination_name, 0) == 0) + ret_result = HARDLINK_RESULT_LINKED; else if (!options->no_copy_fallback && (errno == EMLINK || errno == EXDEV || errno == EPERM)) { /* EMLINK, EXDEV and EPERM shouldn't be fatal; we just can't do the * optimization of hardlinking instead of copying. */ - ret_was_supported = FALSE; } else if (allow_noent && errno == ENOENT) { - ret_was_supported = FALSE; + } + else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) + { + /* In this mode, we keep existing content. Distinguish this case though to + * avoid inserting into the devino cache. + */ + ret_result = HARDLINK_RESULT_SKIP_EXISTED; } else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES) { @@ -370,7 +393,7 @@ checkout_file_hardlink (OstreeRepo *self, * the same file, then rename() does nothing, and returns a * success status." * - * So we can't make this atomic. + * So we can't make this atomic. */ (void) unlinkat (destination_dfd, destination_name, 0); goto again; @@ -379,14 +402,12 @@ checkout_file_hardlink (OstreeRepo *self, { g_prefix_error (error, "Hardlinking %s to %s: ", loose_path, destination_name); glnx_set_error_from_errno (error); - goto out; + return FALSE; } - ret = TRUE; - if (out_was_supported) - *out_was_supported = ret_was_supported; - out: - return ret; + if (out_result) + *out_result = ret_result; + return TRUE; } static gboolean @@ -439,7 +460,7 @@ checkout_one_file_at (OstreeRepo *repo, } else { - gboolean did_hardlink = FALSE; + HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED; /* Try to do a hardlink first, if it's a regular file. This also * traverses all parent repos. */ @@ -469,11 +490,11 @@ checkout_one_file_at (OstreeRepo *repo, options, loose_path_buf, destination_dfd, destination_name, - TRUE, &did_hardlink, + TRUE, &hardlink_res, cancellable, error)) goto out; - if (did_hardlink && options->devino_to_csum_cache) + if (hardlink_res == HARDLINK_RESULT_LINKED && options->devino_to_csum_cache) { struct stat stbuf; OstreeDevIno *key; @@ -492,13 +513,13 @@ checkout_one_file_at (OstreeRepo *repo, g_hash_table_add ((GHashTable*)options->devino_to_csum_cache, key); } - if (did_hardlink) + if (hardlink_res != HARDLINK_RESULT_NOT_SUPPORTED) break; } current_repo = current_repo->parent_repo; } - need_copy = !did_hardlink; + need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED); } can_cache = (options->enable_uncompressed_cache @@ -514,7 +535,7 @@ checkout_one_file_at (OstreeRepo *repo, && repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) { - gboolean did_hardlink; + HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED; if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL, cancellable, error)) @@ -560,19 +581,20 @@ checkout_one_file_at (OstreeRepo *repo, if (!checkout_file_hardlink (repo, options, loose_path_buf, destination_dfd, destination_name, - FALSE, &did_hardlink, + FALSE, &hardlink_res, cancellable, error)) { g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name); goto out; } - need_copy = !did_hardlink; + need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED); } /* Fall back to copy if we couldn't hardlink */ if (need_copy) { + g_assert (!options->no_copy_fallback); if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs, cancellable, error)) goto out; @@ -655,7 +677,9 @@ checkout_tree_at (OstreeRepo *self, while (G_UNLIKELY (res == -1 && errno == EINTR)); if (res == -1) { - if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES) + if (errno == EEXIST && + (options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES + || options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)) did_exist = TRUE; else { diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 648bd129..34685cc6 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -722,11 +722,13 @@ typedef enum { /** * OstreeRepoCheckoutOverwriteMode: * @OSTREE_REPO_CHECKOUT_OVERWRITE_NONE: No special options - * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, overwrite earlier files, but keep earlier directories + * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, unlink() and replace existing files, but do not modify existing directories + * @OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES: Only add new files/directories */ typedef enum { OSTREE_REPO_CHECKOUT_OVERWRITE_NONE = 0, - OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1 + OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1, + OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES = 2, /* Since: 2017.3 */ } OstreeRepoCheckoutOverwriteMode; _OSTREE_PUBLIC diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index 95172f8b..74e27cfb 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -36,6 +36,7 @@ static gboolean opt_allow_noent; static gboolean opt_disable_cache; static char *opt_subpath; static gboolean opt_union; +static gboolean opt_union_add; static gboolean opt_whiteouts; static gboolean opt_from_stdin; static char *opt_from_file; @@ -63,6 +64,7 @@ static GOptionEntry options[] = { { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Do not update or use the internal repository uncompressed object cache", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, { "union", 0, 0, G_OPTION_ARG_NONE, &opt_union, "Keep existing directories, overwrite existing files", NULL }, + { "union-add", 0, 0, G_OPTION_ARG_NONE, &opt_union_add, "Keep existing files/directories, only add new", NULL }, { "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL }, { "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL }, { "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL }, @@ -87,14 +89,23 @@ process_one_checkout (OstreeRepo *repo, * `ostree_repo_checkout_at` until such time as we have a more * convenient infrastructure for testing C APIs with data. */ - if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks) + if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks || opt_union_add) { OstreeRepoCheckoutAtOptions options = { 0, }; - + if (opt_user_mode) options.mode = OSTREE_REPO_CHECKOUT_MODE_USER; - if (opt_union) + /* Can't union these */ + if (opt_union && opt_union_add) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Cannot specify both --union and --union-add"); + goto out; + } + else if (opt_union) options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; + else if (opt_union_add) + options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES; if (opt_whiteouts) options.process_whiteouts = TRUE; if (subpath) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 26982796..045e4217 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..62" +echo "1..63" $CMD_PREFIX ostree --version > version.yaml python -c 'import yaml; yaml.safe_load(open("version.yaml"))' @@ -279,6 +279,24 @@ cd checkout-test2-union assert_file_has_content ./yet/another/tree/green "leaf" echo "ok checkout union 1" +cd ${test_tmpdir} +$OSTREE commit -b test-union-add --tree=ref=test2 +$OSTREE checkout test-union-add checkout-test-union-add +echo 'file for union add testing' > checkout-test-union-add/union-add-test +echo 'another file for union add testing' > checkout-test-union-add/union-add-test2 +$OSTREE commit -b test-union-add --tree=dir=checkout-test-union-add +rm checkout-test-union-add -rf +# Check out previous +$OSTREE checkout test-union-add^ checkout-test-union-add +assert_not_has_file checkout-test-union-add/union-add-test +assert_not_has_file checkout-test-union-add/union-add-test2 +# Now create a file we don't want overwritten +echo 'existing file for union add' > checkout-test-union-add/union-add-test +$OSTREE checkout --union-add test-union-add checkout-test-union-add +assert_file_has_content checkout-test-union-add/union-add-test 'existing file for union add' +assert_file_has_content checkout-test-union-add/union-add-test2 'another file for union add testing' +echo "ok checkout union add" + cd ${test_tmpdir} rm -rf shadow-repo mkdir shadow-repo From 031d7898ccb4ab4aa5fe994b016cb785dad2b895 Mon Sep 17 00:00:00 2001 From: Christian Hergert Date: Tue, 7 Mar 2017 22:48:51 -0800 Subject: [PATCH 62/66] repo/checkout: fix 32-bit builds __dev_t is 64-bit even on 32-bit Linux systems such as i386. Closes: #724 Approved by: cgwalters --- src/libostree/ostree-repo-checkout.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 53409529..77bda09a 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -706,8 +706,8 @@ checkout_tree_at (OstreeRepo *self, if (options->no_copy_fallback && repo_dfd_stat.st_dev != destination_stat.st_dev) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unable to do hardlink checkout across devices (src=%lu destination=%lu)", - repo_dfd_stat.st_dev, destination_stat.st_dev); + "Unable to do hardlink checkout across devices (src=%"G_GUINT64_FORMAT" destination=%"G_GUINT64_FORMAT")", + (guint64)repo_dfd_stat.st_dev, (guint64)destination_stat.st_dev); goto out; } From d8ac9f75cdb567cade86c51c4f7b65151fc4273f Mon Sep 17 00:00:00 2001 From: Georges Basile Stavracas Neto Date: Wed, 8 Mar 2017 14:14:15 -0300 Subject: [PATCH 63/66] repo-pull: add option to set the async update frequency When using Flatpak with GNOME Software, it is important to show the progress of the download and install as close as possible to the real progress. However, OSTree forces the frequency to call the async progress callback to 1 second, which causes an unpleasant effect on the UI, specially when the download size is so small that everything happens in less than 1 second. Fix that by adding making OSTree read a custom 'update-frequency' option and set the timeout source timeout to that. If no custom frequency is passed, we assume the default 1 second timeout, maintaining the current behavior. Closes: #725 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index d7097320..5ee43918 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2589,6 +2589,7 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, GError **e * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options * * inherit-transaction (b): Don't initiate, finish or abort a transaction, usefult to do mutliple pulls in one transaction. * * http-headers (a(ss)): Additional headers to add to all HTTP requests + * * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -2613,6 +2614,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, char **configured_branches = NULL; guint64 bytes_transferred; guint64 end_time; + guint update_frequency = 0; OstreeRepoPullFlags flags = 0; const char *dir_to_pull = NULL; g_autofree char **dirs_to_pull = NULL; @@ -2648,6 +2650,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "override-url", "&s", &url_override); (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction); (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers); + (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); @@ -3283,7 +3286,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->progress) { - update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1); + /* Setup a custom frequency if set */ + if (update_frequency > 0) + update_timeout = g_timeout_source_new (pull_data->dry_run ? 0 : update_frequency); + else + update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1); + g_source_set_priority (update_timeout, G_PRIORITY_HIGH); g_source_set_callback (update_timeout, update_progress, pull_data, NULL); g_source_attach (update_timeout, pull_data->main_context); From bb3a0e3fa4a3569882b8e4ad9aceae5271bd0c63 Mon Sep 17 00:00:00 2001 From: Georges Basile Stavracas Neto Date: Wed, 8 Mar 2017 14:19:29 -0300 Subject: [PATCH 64/66] ostree: allow setting update frequency from command line After commit 80b3edc64731a5f0 introducing the option to set a custom timeout, adapt the ostree program to be able to update that. Closes: #725 Approved by: jlebon --- src/ostree/ot-builtin-pull.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 170a3a91..4287afcd 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -38,6 +38,7 @@ static char** opt_subpaths; static char** opt_http_headers; static char* opt_cache_dir; static int opt_depth = 0; +static int opt_frequency = 0; static char* opt_url; static GOptionEntry options[] = { @@ -53,6 +54,7 @@ static GOptionEntry options[] = { { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL }, { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" }, + { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" }, { NULL } }; @@ -252,6 +254,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_variant_builder_add (&builder, "{s@v}", "depth", g_variant_new_variant (g_variant_new_int32 (opt_depth))); + g_variant_builder_add (&builder, "{s@v}", "update-frequency", + g_variant_new_variant (g_variant_new_uint32 (opt_frequency))); + g_variant_builder_add (&builder, "{s@v}", "disable-static-deltas", g_variant_new_variant (g_variant_new_boolean (opt_disable_static_deltas))); From 3e32d5c4b6b082ad5d0cbd0ae540c6e6d478458e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 6 Mar 2017 15:27:22 -0500 Subject: [PATCH 65/66] repo/checkout: Convert a few functions to new "stmt-decl/FALSE" style Just testing the waters a bit more. Yeah, definitely nicer. Closes: #722 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 43 ++++++++++++---------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 77bda09a..7ae7cd55 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -947,10 +947,6 @@ ostree_repo_checkout_at (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GFile) commit_root = NULL; - g_autoptr(GFile) target_dir = NULL; - g_autoptr(GFileInfo) target_info = NULL; OstreeRepoCheckoutAtOptions default_options = { 0, }; if (!options) @@ -959,33 +955,33 @@ ostree_repo_checkout_at (OstreeRepo *self, options = &default_options; } - commit_root = (GFile*) _ostree_repo_file_new_for_commit (self, commit, error); + g_autoptr(GFile) commit_root = (GFile*) _ostree_repo_file_new_for_commit (self, commit, error); if (!commit_root) - goto out; + return FALSE; if (!ostree_repo_file_ensure_resolved ((OstreeRepoFile*)commit_root, error)) - goto out; + return FALSE; + g_autoptr(GFile) target_dir = NULL; if (options->subpath && strcmp (options->subpath, "/") != 0) target_dir = g_file_get_child (commit_root, options->subpath); else target_dir = g_object_ref (commit_root); - target_info = g_file_query_info (target_dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); + g_autoptr(GFileInfo) target_info = + g_file_query_info (target_dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); if (!target_info) - goto out; + return FALSE; if (!checkout_tree_at (self, options, destination_dfd, destination_path, (OstreeRepoFile*)target_dir, target_info, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } static guint @@ -1039,16 +1035,15 @@ ostree_repo_checkout_gc (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GHashTable) to_clean_dirs = NULL; - GHashTableIter iter; - gpointer key, value; g_mutex_lock (&self->cache_lock); to_clean_dirs = self->updated_uncompressed_dirs; self->updated_uncompressed_dirs = g_hash_table_new (NULL, NULL); g_mutex_unlock (&self->cache_lock); + GHashTableIter iter; + gpointer key, value; if (to_clean_dirs) g_hash_table_iter_init (&iter, to_clean_dirs); while (to_clean_dirs && g_hash_table_iter_next (&iter, &key, &value)) @@ -1058,7 +1053,7 @@ ostree_repo_checkout_gc (OstreeRepo *self, if (!glnx_dirfd_iterator_init_at (self->uncompressed_objects_dir_fd, objdir_name, FALSE, &dfd_iter, error)) - goto out; + return FALSE; while (TRUE) { @@ -1066,14 +1061,14 @@ ostree_repo_checkout_gc (OstreeRepo *self, struct stat stbuf; if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error)) - goto out; + return FALSE; if (dent == NULL) break; if (fstatat (dfd_iter.fd, dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW) != 0) { glnx_set_error_from_errno (error); - goto out; + return FALSE; } if (stbuf.st_nlink == 1) @@ -1081,13 +1076,11 @@ ostree_repo_checkout_gc (OstreeRepo *self, if (unlinkat (dfd_iter.fd, dent->d_name, 0) != 0) { glnx_set_error_from_errno (error); - goto out; + return FALSE; } } } } - ret = TRUE; - out: - return ret; + return TRUE; } From e02e90020663b8629e7d3d0ef8801d3af4ee1dd4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 8 Mar 2017 21:47:00 -0500 Subject: [PATCH 66/66] Release 2017.3 It's been almost a month, I think the current git is working well and not too risky. We have some new API additions which I think the various consumers of them are going to want. Closes: #726 Approved by: jlebon --- configure.ac | 2 +- src/libostree/libostree.sym | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 7fac081b..08894480 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ([2.63]) dnl If incrementing the version here, remember to update libostree.sym too -AC_INIT([libostree], [2017.2], [walters@verbum.org]) +AC_INIT([libostree], [2017.3], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 7c7c3aef..b7cfc454 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -376,15 +376,22 @@ global: ostree_repo_reload_config; } LIBOSTREE_2017.1; +LIBOSTREE_2017.3 { +global: + ostree_raw_file_to_archive_z2_stream_with_options; +} LIBOSTREE_2017.2; + /* NOTE NOTE NOTE * Versions above here are released. Only add symbols below this line. * NOTE NOTE NOTE */ -LIBOSTREE_2017.3 { +/* Stub section for new version, uncomment when the first symbol is added +LIBOSTREE_2017.$NEWVERSION { global: - ostree_raw_file_to_archive_z2_stream_with_options; + someostree_symbol_deleteme; } LIBOSTREE_2017.2; +*/ /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste