From dc92b09efb5b19866a060151ae7893a95a499eaf Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 24 Oct 2019 15:18:45 +0000 Subject: [PATCH 01/17] Post-release version bump --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 4b022e36..2f9579f5 100644 --- a/configure.ac +++ b/configure.ac @@ -7,10 +7,10 @@ dnl Seed the release notes with `git-shortlog-with-prs ..`. Th dnl `git-evtag` to create the tag and push it. Finally, create a GitHub release and attach dnl the tarball from `make dist`. m4_define([year_version], [2019]) -m4_define([release_version], [5]) +m4_define([release_version], [6]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) From 4d17cd917fe4fb67d2f9e4881b6693ac7289f423 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Wed, 30 Oct 2019 07:55:41 +0000 Subject: [PATCH 02/17] tests/core: Fallback to en_US.UTF-8 locale A number of tests expect explicit left/right single quotes in their messages, which will never happen in the C locale. Change so we pick a likely UTF-8 locale, or fail if we can't find one. Signed-off-by: Alex Kiernan --- tests/libtest-core.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh index f6f94288..bc191332 100644 --- a/tests/libtest-core.sh +++ b/tests/libtest-core.sh @@ -35,15 +35,11 @@ assert_not_reached () { } # Some tests look for specific English strings. Use a UTF-8 version -# of the C (POSIX) locale if we have one, or fall back to POSIX +# of the C (POSIX) locale if we have one, or fall back to en_US.UTF-8 # (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8) -if locale -a | grep '^C.UTF-8$' >/dev/null; then - export LC_ALL=C.UTF-8 -elif locale -a | grep '^C.utf8$' >/dev/null; then - export LC_ALL=C.utf8 -else - export LC_ALL=C -fi +export LC_ALL=$(locale -a | grep -Ee '\.(UTF-8|utf8)' | grep -iEe '^(C|en_US)' | head -1 || true) +if [ -z "${LC_ALL}" ]; then fatal "Can't find suitable UTF-8 locale"; fi + # A GNU extension, used whenever LC_ALL is not C unset LANGUAGE From fb519f0361565cad0d340095263a1e8866ec4f89 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Thu, 31 Oct 2019 11:30:00 +0000 Subject: [PATCH 03/17] tests: Handle EPIPE failures when head terminates When using musl, it appears that the default is line buffered output, so when `head -1` reads from a pipe we have to handle the source end of the pipe getting EPIPE. Signed-off-by: Alex Kiernan --- tests/pull-test.sh | 2 +- tests/test-pull-mirrorlist.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 0a97a119..2cfd8e02 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -177,7 +177,7 @@ if ! skip_one_without_user_xattrs; then ${CMD_PREFIX} ostree --repo=cacherepo pull-local ostree-srv/gnomerepo main rev=$(ostree --repo=cacherepo rev-parse main) ${CMD_PREFIX} ostree --repo=cacherepo ls -R -C main > ls.txt - regfile_hash=$(grep -E -e '^-0' ls.txt | head -1 | awk '{ print $5 }') + regfile_hash=$((grep -E -e '^-0' ls.txt || true) | head -1 | awk '{ print $5 }') ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false corruptrepo $(cat httpd-address)/ostree/corruptrepo # Make this a loop so in the future we can add more object types like commit etc. for object in ${regfile_hash}.file; do diff --git a/tests/test-pull-mirrorlist.sh b/tests/test-pull-mirrorlist.sh index 4e44852d..85ff66e9 100755 --- a/tests/test-pull-mirrorlist.sh +++ b/tests/test-pull-mirrorlist.sh @@ -47,12 +47,12 @@ setup_mirror content_mirror3 # Let's delete a file from 1 so that it falls back on 2 cd ${test_tmpdir}/content_mirror1/ostree/gnomerepo -filez=$(find objects/ -name '*.filez' | head -n 1) +filez=$((find objects/ -name '*.filez' || true) | head -n 1) rm ${filez} # Let's delete a file from 1 and 2 so that it falls back on 3 cd ${test_tmpdir}/content_mirror1/ostree/gnomerepo -filez=$(find objects/ -name '*.filez' | head -n 1) +filez=$((find objects/ -name '*.filez' || true) | head -n 1) rm ${filez} cd ${test_tmpdir}/content_mirror2/ostree/gnomerepo rm ${filez} From 3d48021fb0892721fab8359b49470e521aaed493 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Thu, 31 Oct 2019 11:28:07 +0000 Subject: [PATCH 04/17] tests/core: Assume C.UTF-8 if locale isn't found When building with musl there's no locale command, also its default locale is C.UTF-8, so just get C.UTF-8 if we can't find locale. Signed-off-by: Alex Kiernan --- tests/libtest-core.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh index bc191332..46aafab0 100644 --- a/tests/libtest-core.sh +++ b/tests/libtest-core.sh @@ -37,9 +37,15 @@ assert_not_reached () { # Some tests look for specific English strings. Use a UTF-8 version # of the C (POSIX) locale if we have one, or fall back to en_US.UTF-8 # (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8) -export LC_ALL=$(locale -a | grep -Ee '\.(UTF-8|utf8)' | grep -iEe '^(C|en_US)' | head -1 || true) -if [ -z "${LC_ALL}" ]; then fatal "Can't find suitable UTF-8 locale"; fi - +# +# If we can't find the locale command assume we have support for C.UTF-8 +# (e.g. musl based systems) +if type -p locale >/dev/null; then + export LC_ALL=$(locale -a | grep -Ee '\.(UTF-8|utf8)' | grep -iEe '^(C|en_US)' | head -1 || true) + if [ -z "${LC_ALL}" ]; then fatal "Can't find suitable UTF-8 locale"; fi +else + export LC_ALL=C.UTF-8 +fi # A GNU extension, used whenever LC_ALL is not C unset LANGUAGE From abf1a7392b2fa1945286a39a35190d74fe141e52 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Thu, 31 Oct 2019 17:09:36 +0000 Subject: [PATCH 05/17] tests: Avoid musl failure with `cp -a` When copying the tree, using musl and GNU coreutils, something gets confused when setting the ownership of symlinks and the copy fails with: cp: failed to preserve ownership for osdata-devel/bin: Not supported Rework using tar to avoid the problem. Signed-off-by: Alex Kiernan --- tests/libtest.sh | 4 +++- tests/test-admin-gpg.sh | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index ba00073a..3f5fd931 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -440,7 +440,9 @@ EOF ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string version=1.0.10 -b testos/buildmaster/x86_64-runtime -s "Build" cd ${test_tmpdir} - cp -a osdata osdata-devel + rm -rf osdata-devel + mkdir osdata-devel + tar -C osdata -cf - . | tar -C osdata-devel -xf - cd osdata-devel mkdir -p usr/include echo "a development header" > usr/include/foo.h diff --git a/tests/test-admin-gpg.sh b/tests/test-admin-gpg.sh index 1f50c8a9..dc776743 100755 --- a/tests/test-admin-gpg.sh +++ b/tests/test-admin-gpg.sh @@ -90,7 +90,9 @@ EOF ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string version=1.0.10 -b testos/buildmaster/x86_64-runtime -s "Build" --gpg-sign=$keyid --gpg-homedir=${test_tmpdir}/gpghome cd ${test_tmpdir} - cp -a osdata osdata-devel + rm -rf osdata-devel + mkdir osdata-devel + tar -C osdata -cf - . | tar -C osdata-devel -xf - cd osdata-devel mkdir -p usr/include echo "a development header" > usr/include/foo.h From de8e0765a4a54f0367999f682ce4cc53bfaa60b0 Mon Sep 17 00:00:00 2001 From: Ricardo Salveti Date: Tue, 5 Nov 2019 21:16:07 -0300 Subject: [PATCH 06/17] Makefile: declare ostree_boot_SCRIPTS and append values ostree_boot_SCRIPTS was being set on both Makefile-boot.am and Makefile-switchroot.am, causing the first one to be replaced by the other at the final Makefile, so declare as empty and append on both places instead. Signed-off-by: Ricardo Salveti --- Makefile-boot.am | 2 +- Makefile-decls.am | 1 + Makefile-switchroot.am | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile-boot.am b/Makefile-boot.am index b4d4a1af..b51928f0 100644 --- a/Makefile-boot.am +++ b/Makefile-boot.am @@ -60,7 +60,7 @@ grub2configdir = $(sysconfdir)/grub.d INSTALL_DATA_HOOKS += install-grub2-config-hook else # We're using our internal generator -ostree_boot_SCRIPTS = src/boot/grub2/ostree-grub-generator +ostree_boot_SCRIPTS += src/boot/grub2/ostree-grub-generator endif EXTRA_DIST += src/boot/dracut/module-setup.sh \ diff --git a/Makefile-decls.am b/Makefile-decls.am index 115c19fb..086ee138 100644 --- a/Makefile-decls.am +++ b/Makefile-decls.am @@ -47,6 +47,7 @@ typelibdir = $(libdir)/girepository-1.0 typelib_DATA = gsettings_SCHEMAS = ostree_bootdir = $(prefix)/lib/ostree +ostree_boot_SCRIPTS = ostree_boot_PROGRAMS = # This initializes some more variables diff --git a/Makefile-switchroot.am b/Makefile-switchroot.am index ff44d4bc..b81b843f 100644 --- a/Makefile-switchroot.am +++ b/Makefile-switchroot.am @@ -42,7 +42,7 @@ if BUILDOPT_USE_STATIC_COMPILER # to get autotools to install this as an executable but without generating rules # to make it itself which we have specified manually. See # https://lists.gnu.org/archive/html/help-gnu-utils/2007-01/msg00007.html -ostree_boot_SCRIPTS = ostree-prepare-root +ostree_boot_SCRIPTS += ostree-prepare-root ostree-prepare-root : $(ostree_prepare_root_SOURCES) $(STATIC_COMPILER) -o $@ -static $(top_srcdir)/src/switchroot/ostree-prepare-root.c $(ostree_prepare_root_CPPFLAGS) $(AM_CFLAGS) $(DEFAULT_INCLUDES) -DOSTREE_PREPARE_ROOT_STATIC=1 From 58980a717a7854b27f48a0753dfbe5b8b94c8164 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 7 Nov 2019 16:06:39 +0000 Subject: [PATCH 07/17] lib/keyfile: Treat "group not found" the same as "key not found" Prep for fsverity, where I want to create a new group `[fsverity]` in the keyfile that has default values. We should treat the absence of a group the same as absence of a key in these "with defaults" APIs. --- src/libotutil/ot-keyfile-utils.c | 14 ++++++++++---- tests/test-keyfile-utils.c | 26 +++++++++++++------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/libotutil/ot-keyfile-utils.c b/src/libotutil/ot-keyfile-utils.c index 2050e969..3e028b20 100644 --- a/src/libotutil/ot-keyfile-utils.c +++ b/src/libotutil/ot-keyfile-utils.c @@ -27,6 +27,13 @@ #include +static gboolean +is_notfound (GError *error) +{ + return g_error_matches (error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND) + || g_error_matches (error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_GROUP_NOT_FOUND); +} + gboolean ot_keyfile_get_boolean_with_default (GKeyFile *keyfile, const char *section, @@ -43,7 +50,7 @@ ot_keyfile_get_boolean_with_default (GKeyFile *keyfile, gboolean ret_bool = g_key_file_get_boolean (keyfile, section, value, &temp_error); if (temp_error) { - if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) + if (is_notfound (temp_error)) { g_clear_error (&temp_error); ret_bool = default_value; @@ -75,7 +82,7 @@ ot_keyfile_get_value_with_default (GKeyFile *keyfile, g_autofree char *ret_value = g_key_file_get_value (keyfile, section, value, &temp_error); if (temp_error) { - if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) + if (is_notfound (temp_error)) { g_clear_error (&temp_error); g_assert (ret_value == NULL); @@ -206,8 +213,7 @@ ot_keyfile_get_string_list_with_default (GKeyFile *keyfile, if (temp_error) { - if (g_error_matches (temp_error, G_KEY_FILE_ERROR, - G_KEY_FILE_ERROR_KEY_NOT_FOUND)) + if (is_notfound (temp_error)) { g_clear_error (&temp_error); ret_value = g_strdupv (default_value); diff --git a/tests/test-keyfile-utils.c b/tests/test-keyfile-utils.c index c580f81c..3014cf16 100644 --- a/tests/test-keyfile-utils.c +++ b/tests/test-keyfile-utils.c @@ -60,12 +60,12 @@ test_get_boolean_with_default (void) g_assert_true (out); g_clear_error (&error); - g_assert_false (ot_keyfile_get_boolean_with_default (g_keyfile, - "a_fake_section", - "a_boolean_true", - FALSE, - &out, - &error)); + g_assert (ot_keyfile_get_boolean_with_default (g_keyfile, + "a_fake_section", + "a_boolean_true", + FALSE, + &out, + &error)); } static void @@ -122,13 +122,13 @@ test_get_value_with_default (void) g_assert_cmpstr (out, ==, "correct"); g_clear_pointer (&out, g_free); - g_assert_false (ot_keyfile_get_value_with_default (g_keyfile, - "a_fake_section", - "a_value_true", - "no value", - &out, - &error)); - g_clear_error (&error); + g_assert (ot_keyfile_get_value_with_default (g_keyfile, + "a_fake_section", + "a_value_true", + "no value", + &out, + &error)); + g_assert_cmpstr (out, ==, "no value"); g_clear_pointer (&out, g_free); } From aadc4db012ed32455416864c6841e68a8b8ebf58 Mon Sep 17 00:00:00 2001 From: Robert Fairley Date: Thu, 29 Aug 2019 12:14:26 -0400 Subject: [PATCH 08/17] lib/kernel-args: Store kernel args as key/value entries Define an `OstreeKernelArgsEntry` structure, which holds both the key and the value. The kargs order array stores entries for each key/value pair, instead of just the keys. The hash table is used to locate entries, by storing entries in a pointer array for each key. The same public interface is preserved, while maintaining ordering information of each key/value pair when appending/replacing/deleting kargs. Fixes: #1859 --- src/libostree/ostree-kernel-args.c | 264 ++++++++++++++++++++--------- src/libostree/ostree-kernel-args.h | 29 ++++ tests/test-admin-deploy-karg.sh | 4 + tests/test-kargs.c | 39 +++-- 4 files changed, 244 insertions(+), 92 deletions(-) diff --git a/src/libostree/ostree-kernel-args.c b/src/libostree/ostree-kernel-args.c index dbf2ec8a..c6300823 100644 --- a/src/libostree/ostree-kernel-args.c +++ b/src/libostree/ostree-kernel-args.c @@ -30,6 +30,111 @@ struct _OstreeKernelArgs { GHashTable *table; }; +struct _OstreeKernelArgsEntry { + char *key; + char *value; +}; + +char * +_ostree_kernel_args_entry_get_key (const OstreeKernelArgsEntry *e) +{ + return e->key; +} + +char * +_ostree_kernel_args_entry_get_value (const OstreeKernelArgsEntry *e) +{ + return e->value; +} + +void +_ostree_kernel_args_entry_set_key (OstreeKernelArgsEntry *e, + char *key) +{ + e->key = key; +} + +void +_ostree_kernel_args_entry_set_value (OstreeKernelArgsEntry *e, + char *value) +{ + e->value = value; +} + +char * +_ostree_kernel_args_get_key_index (const OstreeKernelArgs *kargs, + int i) +{ + OstreeKernelArgsEntry *e = kargs->order->pdata[i]; + return e->key; +} + +char * +_ostree_kernel_args_get_value_index (const OstreeKernelArgs *kargs, + int i) +{ + OstreeKernelArgsEntry *e = kargs->order->pdata[i]; + return e->value; +} + +OstreeKernelArgsEntry * +_ostree_kernel_args_entry_new (void) +{ + return g_new0 (OstreeKernelArgsEntry, 1); +} + +void +_ostree_kernel_args_entry_value_free (OstreeKernelArgsEntry *e) +{ + g_clear_pointer (&e->value, g_free); +} + +/* Free the value field, and the entry. This should be set as the free + * function, for all pointer arrays stored in the hash table. + */ +static void +kernel_args_entry_free_from_table (gpointer data) +{ + OstreeKernelArgsEntry *e = data; + // The hash table owns the key; do not free it here. + g_free (_ostree_kernel_args_entry_get_value (e)); + g_free (e); +} + +static gboolean +kernel_args_entry_value_equal (gconstpointer data, + gconstpointer value) +{ + const OstreeKernelArgsEntry *e = data; + return g_strcmp0 (_ostree_kernel_args_entry_get_value (e), value) == 0; +} + +static gboolean +kernel_args_entry_key_equal (gconstpointer data, + gconstpointer key) +{ + const OstreeKernelArgsEntry *e = data; + return g_strcmp0 (_ostree_kernel_args_entry_get_key (e), key) == 0; +} + +static void +kernel_args_entry_replace_value (OstreeKernelArgsEntry *e, + const char *value) +{ + g_assert (e); + _ostree_kernel_args_entry_value_free (e); + _ostree_kernel_args_entry_set_value (e, g_strdup (value)); +} + +static void +kernel_args_remove_entries_from_order (GPtrArray *order, + GPtrArray *entries) +{ + g_assert (entries); + for (int i = 0; i < entries->len; i++) + g_assert (g_ptr_array_remove (order, entries->pdata[i])); +} + static char * split_keyeq (char *arg) { @@ -82,9 +187,11 @@ ostree_kernel_args_new (void) { OstreeKernelArgs *ret; ret = g_new0 (OstreeKernelArgs, 1); - ret->order = g_ptr_array_new_with_free_func (g_free); + /* Hash table owns the kernel args entries, since it uses keys to index, + * and its values are used to locate entries in the order array. */ ret->table = g_hash_table_new_full (g_str_hash, g_str_equal, - NULL, (GDestroyNotify)g_ptr_array_unref); + g_free, (GDestroyNotify)g_ptr_array_unref); + ret->order = g_ptr_array_new_with_free_func (NULL); return ret; } @@ -194,10 +301,10 @@ ostree_kernel_args_new_replace (OstreeKernelArgs *kargs, const char *key = arg_owned; const char *val = split_keyeq (arg_owned); - GPtrArray *values = g_hash_table_lookup (kargs->table, key); - if (!values) + GPtrArray *entries = g_hash_table_lookup (kargs->table, key); + if (!entries) return glnx_throw (error, "No key '%s' found", key); - g_assert_cmpuint (values->len, >, 0); + g_assert_cmpuint (entries->len, >, 0); /* first handle the case where the user just wants to replace an old value */ if (val && strchr (val, '=')) @@ -207,20 +314,18 @@ ostree_kernel_args_new_replace (OstreeKernelArgs *kargs, g_assert (new_val); guint i = 0; - if (!ot_ptr_array_find_with_equal_func (values, old_val, strcmp0_equal, &i)) + if (!ot_ptr_array_find_with_equal_func (entries, old_val, kernel_args_entry_value_equal, &i)) return glnx_throw (error, "No karg '%s=%s' found", key, old_val); - g_clear_pointer (&values->pdata[i], g_free); - values->pdata[i] = g_strdup (new_val); + kernel_args_entry_replace_value (entries->pdata[i], new_val); return TRUE; } /* can't know which val to replace without the old_val=new_val syntax */ - if (values->len > 1) + if (entries->len > 1) return glnx_throw (error, "Multiple values for key '%s' found", key); - g_clear_pointer (&values->pdata[0], g_free); - values->pdata[0] = g_strdup (val); + kernel_args_entry_replace_value (entries->pdata[0], val); return TRUE; } @@ -246,6 +351,13 @@ ostree_kernel_args_delete_key_entry (OstreeKernelArgs *kargs, const char *key, GError **error) { + GPtrArray *entries = g_hash_table_lookup (kargs->table, key); + if (!entries) + return glnx_throw (error, "No key '%s' found", key); + g_assert_cmpuint (entries->len, >, 0); + + kernel_args_remove_entries_from_order (kargs->order, entries); + if (!g_hash_table_remove (kargs->table, key)) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, @@ -254,10 +366,6 @@ ostree_kernel_args_delete_key_entry (OstreeKernelArgs *kargs, return FALSE; } - /* Then remove the key from order table */ - guint key_index; - g_assert (ot_ptr_array_find_with_equal_func (kargs->order, key, g_str_equal, &key_index)); - g_assert (g_ptr_array_remove_index (kargs->order, key_index)); return TRUE; } @@ -294,16 +402,17 @@ ostree_kernel_args_delete (OstreeKernelArgs *kargs, const char *key = arg_owned; const char *val = split_keyeq (arg_owned); - GPtrArray *values = g_hash_table_lookup (kargs->table, key); - if (!values) + GPtrArray *entries = g_hash_table_lookup (kargs->table, key); + if (!entries) return glnx_throw (error, "No key '%s' found", key); - g_assert_cmpuint (values->len, >, 0); + g_assert_cmpuint (entries->len, >, 0); /* special-case: we allow deleting by key only if there's only one val */ - if (values->len == 1) + if (entries->len == 1) { /* but if a specific val was passed, check that it's the same */ - if (val && !strcmp0_equal (val, values->pdata[0])) + OstreeKernelArgsEntry *e = entries->pdata[0]; + if (val && !strcmp0_equal (val, _ostree_kernel_args_entry_get_value (e))) return glnx_throw (error, "No karg '%s=%s' found", key, val); return ostree_kernel_args_delete_key_entry (kargs, key, error); } @@ -311,7 +420,7 @@ ostree_kernel_args_delete (OstreeKernelArgs *kargs, /* note val might be NULL here, in which case we're looking for `key`, not `key=` or * `key=val` */ guint i = 0; - if (!ot_ptr_array_find_with_equal_func (values, val, strcmp0_equal, &i)) + if (!ot_ptr_array_find_with_equal_func (entries, val, kernel_args_entry_value_equal, &i)) { if (!val) /* didn't find NULL -> only key= key=val1 key=val2 style things left, so the user @@ -320,7 +429,8 @@ ostree_kernel_args_delete (OstreeKernelArgs *kargs, return glnx_throw (error, "No karg '%s' found", arg); } - g_ptr_array_remove_index (values, i); + g_assert (g_ptr_array_remove (kargs->order, entries->pdata[i])); + g_assert (g_ptr_array_remove_index (entries, i)); return TRUE; } @@ -340,22 +450,38 @@ ostree_kernel_args_replace_take (OstreeKernelArgs *kargs, char *arg) { gboolean existed; - GPtrArray *values = g_ptr_array_new_with_free_func (g_free); + GPtrArray *entries = g_ptr_array_new_with_free_func (kernel_args_entry_free_from_table); const char *value = split_keyeq (arg); gpointer old_key; - g_ptr_array_add (values, g_strdup (value)); - existed = g_hash_table_lookup_extended (kargs->table, arg, &old_key, NULL); + OstreeKernelArgsEntry *entry = g_new0 (OstreeKernelArgsEntry, 1); + _ostree_kernel_args_entry_set_value (entry, g_strdup (value)); + g_ptr_array_add (entries, entry); + + gpointer old_entries_ptr; + existed = g_hash_table_lookup_extended (kargs->table, arg, &old_key, &old_entries_ptr); + GPtrArray *old_entries = old_entries_ptr; if (existed) { - g_hash_table_replace (kargs->table, old_key, values); - g_free (arg); + g_assert (old_entries); + g_assert_cmpuint (old_entries->len, >, 0); + + guint old_order_index = 0; + g_assert (ot_ptr_array_find_with_equal_func (kargs->order, old_key, kernel_args_entry_key_equal, &old_order_index)); + kernel_args_remove_entries_from_order (kargs->order, old_entries); + + g_assert_cmpstr (old_key, ==, arg); + _ostree_kernel_args_entry_set_key (entry, old_key); + g_ptr_array_insert (kargs->order, old_order_index, entry); + // `arg` is freed by the `g_hash_table_insert` call. + g_hash_table_insert (kargs->table, arg, entries); } else { - g_ptr_array_add (kargs->order, arg); - g_hash_table_replace (kargs->table, arg, values); + _ostree_kernel_args_entry_set_key (entry, arg); + g_hash_table_replace (kargs->table, arg, entries); + g_ptr_array_add (kargs->order, entry); } } @@ -393,28 +519,26 @@ ostree_kernel_args_append (OstreeKernelArgs *kargs, const char *arg) { gboolean existed = TRUE; - GPtrArray *values; + GPtrArray *entries = NULL; char *duped = g_strdup (arg); const char *val = split_keyeq (duped); - values = g_hash_table_lookup (kargs->table, duped); - if (!values) + entries = g_hash_table_lookup (kargs->table, duped); + if (!entries) { - values = g_ptr_array_new_with_free_func (g_free); + entries = g_ptr_array_new_with_free_func (kernel_args_entry_free_from_table); existed = FALSE; } - g_ptr_array_add (values, g_strdup (val)); + OstreeKernelArgsEntry *entry = _ostree_kernel_args_entry_new (); + _ostree_kernel_args_entry_set_key (entry, duped); + _ostree_kernel_args_entry_set_value (entry, g_strdup (val)); + + g_ptr_array_add (entries, entry); + g_ptr_array_add (kargs->order, entry); if (!existed) - { - g_hash_table_replace (kargs->table, duped, values); - g_ptr_array_add (kargs->order, duped); - } - else - { - g_free (duped); - } + g_hash_table_replace (kargs->table, duped, entries); } /** @@ -598,20 +722,13 @@ ostree_kernel_args_to_strv (OstreeKernelArgs *kargs) for (i = 0; i < kargs->order->len; i++) { - const char *key = kargs->order->pdata[i]; - GPtrArray *values = g_hash_table_lookup (kargs->table, key); - guint j; + const char *key = _ostree_kernel_args_get_key_index (kargs, i); + const char *value = _ostree_kernel_args_get_value_index (kargs, i); - g_assert (values != NULL); - - for (j = 0; j < values->len; j++) - { - const char *value = values->pdata[j]; - if (value == NULL) - g_ptr_array_add (strv, g_strconcat (key, NULL)); - else - g_ptr_array_add (strv, g_strconcat (key, "=", value, NULL)); - } + if (value == NULL) + g_ptr_array_add (strv, g_strconcat (key, NULL)); + else + g_ptr_array_add (strv, g_strconcat (key, "=", value, NULL)); } g_ptr_array_add (strv, NULL); @@ -644,27 +761,19 @@ ostree_kernel_args_to_string (OstreeKernelArgs *kargs) for (i = 0; i < kargs->order->len; i++) { - const char *key = kargs->order->pdata[i]; - GPtrArray *values = g_hash_table_lookup (kargs->table, key); - guint j; + const char *key = _ostree_kernel_args_get_key_index (kargs, i); + const char *value = _ostree_kernel_args_get_value_index (kargs, i); - g_assert (values != NULL); + if (first) + first = FALSE; + else + g_string_append_c (buf, ' '); - for (j = 0; j < values->len; j++) + g_string_append (buf, key); + if (value != NULL) { - const char *value = values->pdata[j]; - - if (first) - first = FALSE; - else - g_string_append_c (buf, ' '); - - g_string_append (buf, key); - if (value != NULL) - { - g_string_append_c (buf, '='); - g_string_append (buf, value); - } + g_string_append_c (buf, '='); + g_string_append (buf, value); } } @@ -688,11 +797,12 @@ ostree_kernel_args_to_string (OstreeKernelArgs *kargs) const char * ostree_kernel_args_get_last_value (OstreeKernelArgs *kargs, const char *key) { - GPtrArray *values = g_hash_table_lookup (kargs->table, key); + const GPtrArray *entries = g_hash_table_lookup (kargs->table, key); - if (!values) + if (!entries) return NULL; - g_assert (values->len > 0); - return (char*)values->pdata[values->len-1]; + g_assert (entries->len > 0); + const OstreeKernelArgsEntry *e = entries->pdata[entries->len-1]; + return _ostree_kernel_args_entry_get_value (e); } diff --git a/src/libostree/ostree-kernel-args.h b/src/libostree/ostree-kernel-args.h index 3975ae5c..5c8be0c0 100644 --- a/src/libostree/ostree-kernel-args.h +++ b/src/libostree/ostree-kernel-args.h @@ -27,11 +27,40 @@ G_BEGIN_DECLS typedef struct _OstreeKernelArgs OstreeKernelArgs; +typedef struct _OstreeKernelArgsEntry OstreeKernelArgsEntry; GHashTable *_ostree_kernel_arg_get_kargs_table (OstreeKernelArgs *kargs); GPtrArray *_ostree_kernel_arg_get_key_array (OstreeKernelArgs *kargs); +char * +_ostree_kernel_args_entry_get_key (const OstreeKernelArgsEntry *e); + +char * +_ostree_kernel_args_entry_get_value (const OstreeKernelArgsEntry *e); + +void +_ostree_kernel_args_entry_set_key (OstreeKernelArgsEntry *e, + char *key); + +void +_ostree_kernel_args_entry_set_value (OstreeKernelArgsEntry *e, + char *value); + +char * +_ostree_kernel_args_get_key_index (const OstreeKernelArgs *kargs, + int i); + +char * +_ostree_kernel_args_get_value_index (const OstreeKernelArgs *kargs, + int i); + +OstreeKernelArgsEntry * +_ostree_kernel_args_entry_new (void); + +void +_ostree_kernel_args_entry_value_free (OstreeKernelArgsEntry *e); + _OSTREE_PUBLIC void ostree_kernel_args_free (OstreeKernelArgs *kargs); diff --git a/tests/test-admin-deploy-karg.sh b/tests/test-admin-deploy-karg.sh index aade011c..ccf66b0e 100755 --- a/tests/test-admin-deploy-karg.sh +++ b/tests/test-admin-deploy-karg.sh @@ -66,4 +66,8 @@ assert_file_has_content sysroot/boot/loader/entries/ostree-2-testos.conf 'option assert_file_has_content sysroot/boot/loader/entries/ostree-2-testos.conf 'options.*TESTARG=TESTVALUE' assert_file_has_content sysroot/boot/loader/entries/ostree-2-testos.conf 'options.*APPENDARG=VALAPPEND .*APPENDARG=2NDAPPEND' +# Check correct ordering of different-valued args of the same key. +${CMD_PREFIX} ostree admin deploy --os=testos --karg-append=FOO=TESTORDERED --karg-append=APPENDARG=3RDAPPEND testos:testos/buildmaster/x86_64-runtime +assert_file_has_content sysroot/boot/loader/entries/ostree-2-testos.conf 'options.*APPENDARG=VALAPPEND .*APPENDARG=2NDAPPEND .*FOO=TESTORDERED .*APPENDARG=3RDAPPEND' + echo "ok deploy --karg-append" diff --git a/tests/test-kargs.c b/tests/test-kargs.c index 8d34f73c..d8370555 100644 --- a/tests/test-kargs.c +++ b/tests/test-kargs.c @@ -32,6 +32,22 @@ check_string_existance (OstreeKernelArgs *karg, return g_strv_contains ((const char* const*) string_list, string_to_find); } +static gboolean +kernel_args_entry_value_equal (gconstpointer data, + gconstpointer value) +{ + const OstreeKernelArgsEntry *e = data; + return g_strcmp0 (_ostree_kernel_args_entry_get_value (e), value) == 0; +} + +static gboolean +kernel_args_entry_key_equal (gconstpointer data, + gconstpointer key) +{ + const OstreeKernelArgsEntry *e = data; + return g_strcmp0 (_ostree_kernel_args_entry_get_key (e), key) == 0; +} + static void test_kargs_delete (void) { @@ -82,7 +98,7 @@ test_kargs_delete (void) g_assert (ret); /* verify the value array is properly updated */ GPtrArray *kargs_array = _ostree_kernel_arg_get_key_array (karg); - g_assert (!ot_ptr_array_find_with_equal_func (kargs_array, "single_key", g_str_equal, NULL)); + g_assert (!ot_ptr_array_find_with_equal_func (kargs_array, "single_key", kernel_args_entry_value_equal, NULL)); g_assert (!check_string_existance (karg, "single_key")); /* Delete specific key/value pair */ @@ -177,13 +193,6 @@ test_kargs_replace (void) g_assert (check_string_existance (karg, "test=newval")); } -static gboolean -strcmp0_equal (gconstpointer v1, - gconstpointer v2) -{ - return g_strcmp0 (v1, v2) == 0; -} - /* In this function, we want to verify that ostree_kernel_args_append * and ostree_kernel_args_to_string is correct. After that * we will use these two functions(append and tostring) in other tests: delete and replace @@ -208,22 +217,22 @@ test_kargs_append (void) { if (g_str_equal (key, "test")) { - g_assert (ot_ptr_array_find_with_equal_func (value_array, "valid", strcmp0_equal, NULL)); - g_assert (ot_ptr_array_find_with_equal_func (value_array, "secondvalid", strcmp0_equal, NULL)); - g_assert (ot_ptr_array_find_with_equal_func (value_array, "", strcmp0_equal, NULL)); - g_assert (ot_ptr_array_find_with_equal_func (value_array, NULL, strcmp0_equal, NULL)); + g_assert (ot_ptr_array_find_with_equal_func (value_array, "valid", kernel_args_entry_value_equal, NULL)); + g_assert (ot_ptr_array_find_with_equal_func (value_array, "secondvalid", kernel_args_entry_value_equal, NULL)); + g_assert (ot_ptr_array_find_with_equal_func (value_array, "", kernel_args_entry_value_equal, NULL)); + g_assert (ot_ptr_array_find_with_equal_func (value_array, NULL, kernel_args_entry_value_equal, NULL)); } else { g_assert_cmpstr (key, ==, "second_test"); - g_assert (ot_ptr_array_find_with_equal_func (value_array, NULL, strcmp0_equal, NULL)); + g_assert (ot_ptr_array_find_with_equal_func (value_array, NULL, kernel_args_entry_value_equal, NULL)); } } /* verify the value array is properly updated */ GPtrArray *kargs_array = _ostree_kernel_arg_get_key_array (append_arg); - g_assert (ot_ptr_array_find_with_equal_func (kargs_array, "test", g_str_equal, NULL)); - g_assert (ot_ptr_array_find_with_equal_func (kargs_array, "second_test", g_str_equal, NULL)); + g_assert (ot_ptr_array_find_with_equal_func (kargs_array, "test", kernel_args_entry_key_equal, NULL)); + g_assert (ot_ptr_array_find_with_equal_func (kargs_array, "second_test", kernel_args_entry_key_equal, NULL)); /* Up till this point, we verified that the above was all correct, we then * check ostree_kernel_args_to_string has the right result From f07838fa089c9039fcf8b699f6658de5365d3971 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Tue, 5 Nov 2019 22:26:23 +0000 Subject: [PATCH 09/17] build: create tests directory for split builds When `--disable-dependency-tracking` is in effect with separate build directory, the tests directory isn't created as a result of the dependency generation, which leads to a build race for the tests directory being created and failures: Making all in . make[2]: Entering directory 'TOPDIR/build/tmp/work/riscv64-yoe-linux-musl/ostree/2019.5-r0/build' (echo '[Test]' > tests/test-local-pull-depth.sh.test.tmp; \ echo 'Type=session' >> tests/test-local-pull-depth.sh.test.tmp; \ echo 'Exec=env G_TEST_SRCDIR=/usr/libexec/installed-tests/libostree G_TEST_BUILDDIR=/usr/libexec/installed-tests/libostree /usr/libexec/installed-tests/libostree/test-local-pull-depth.sh' >> tests/test-local-pull-depth.sh.test.tmp; \ mv tests/test-local-pull-depth.sh.test.tmp tests/test-local-pull-depth.sh.test) /bin/sh: tests/test-local-pull-depth.sh.test.tmp: No such file or directory /bin/sh: line 1: tests/test-local-pull-depth.sh.test.tmp: No such file or directory /bin/sh: line 2: tests/test-local-pull-depth.sh.test.tmp: No such file or directory mv: cannot stat 'tests/test-local-pull-depth.sh.test.tmp': No such file or directory make[2]: *** [Makefile:9282: tests/test-local-pull-depth.sh.test] Error 1 Signed-off-by: Alex Kiernan --- buildutil/glib-tap.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/buildutil/glib-tap.mk b/buildutil/glib-tap.mk index 34b8f21e..9a944a0a 100644 --- a/buildutil/glib-tap.mk +++ b/buildutil/glib-tap.mk @@ -116,6 +116,7 @@ installed_testcases = $(test_programs) $(installed_test_programs) \ installed_test_meta_DATA = $(installed_testcases:=.test) %.test: %$(EXEEXT) Makefile + @$(MKDIR_P) $(dir $@) $(AM_V_GEN) (echo '[Test]' > $@.tmp; \ echo 'Type=session' >> $@.tmp; \ echo 'Exec=env G_TEST_SRCDIR=$(installed_testdir) G_TEST_BUILDDIR=$(installed_testdir) $(installed_testdir)/$(notdir $<)' >> $@.tmp; \ From 1e0761849cc7278dbb83e1632a4cc12f4f06214c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 8 Nov 2019 13:36:23 +0000 Subject: [PATCH 10/17] Bump libglnx This has a few fixes, mainly I want to get this in as prep for fs-verity. Update submodule: libglnx ``` Alex Kiernan (1): macros: Add TEMP_FAILURE_RETRY for musl Alexander Larsson (1): Add glnx_open_anonymous_tmpfile_full() allowing you to specify the directory Colin Walters (8): Merge branch 'shutil-rm-rf-errprefix' into 'master' Merge branch 'us-temp-failure-retry' into 'master' Merge branch 'anonymous-tmpfile-dir' into 'master' Merge branch 'meson-older-compilers' into 'master' fdio: Add glnx_tmpfile_reopen_rdonly() Merge branch 'reopen-rdonly' into 'master' build-sys: Add libglnx-testlib.c to Automake Merge branch 'testlib-automake' into 'master' Jonathan Lebon (1): Merge branch 'uchar' into 'master' Simon McVittie (5): missing: Remove unused Run the fdio test in its own temporary directory meson: Define HAVE_DECL_FOO to 0 if foo isn't declared Make the Meson build work on older compilers CI: Target a Fedora stable release Will Thompson (3): Add meson.build files Document using this as a Meson subproject Add GitLab CI ``` --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index b1cb19b6..9d0711fe 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit b1cb19b6b2d712b492e6376248f3010d18e59daa +Subproject commit 9d0711fe1d6db58b4e24a6c92113930fecafd43e From 382ad59822ebbc7b3ca30058bb5bb450a6ff91be Mon Sep 17 00:00:00 2001 From: Philip Chimento Date: Tue, 5 Nov 2019 10:05:45 -0800 Subject: [PATCH 11/17] Bump version in symbols file There were no new symbols in 2019.5 and this version didn't get bumped when 2019.5 was released. --- src/libostree/libostree-devel.sym | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 0b876f3b..d9d02056 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,7 +18,7 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2019.5 { +LIBOSTREE_2019.6 { } LIBOSTREE_2019.4; /* Stub section for the stable release *after* this development one; don't From 54639c03ca7713d71088f1ac9d323063d4c116d7 Mon Sep 17 00:00:00 2001 From: Philip Chimento Date: Mon, 4 Nov 2019 13:21:36 -0800 Subject: [PATCH 12/17] libostree: Add ostree_async_progress_copy_state() This allows copying the state from one OstreeAsyncProgress object to another, atomically, without invoking the callback. This is needed in libflatpak, in order to chain OstreeAsyncProgress objects so that you can still receive progress updates when iterating a different GMainContext than the one that the OstreeAsyncProgress object was created under. See https://github.com/flatpak/flatpak/pull/3211 for the application of this API. --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-async-progress.c | 34 +++++++++++++++++++++++++++ src/libostree/ostree-async-progress.h | 4 ++++ 4 files changed, 40 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 252a563a..f99c4df5 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -3,6 +3,7 @@ OstreeAsyncProgress ostree_async_progress_new ostree_async_progress_new_and_connect +ostree_async_progress_copy_state ostree_async_progress_get_status ostree_async_progress_get ostree_async_progress_get_variant diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index d9d02056..646a4a21 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,6 +19,7 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2019.6 { + ostree_async_progress_copy_state; } LIBOSTREE_2019.4; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-async-progress.c b/src/libostree/ostree-async-progress.c index 64372c27..8d6fdfe5 100644 --- a/src/libostree/ostree-async-progress.c +++ b/src/libostree/ostree-async-progress.c @@ -424,6 +424,40 @@ ostree_async_progress_set_uint64 (OstreeAsyncProgress *self, ostree_async_progress_set_variant (self, key, g_variant_new_uint64 (value)); } +/** + * ostree_async_progress_copy_state: + * @self: An #OstreeAsyncProgress to copy from + * @dest: An #OstreeAsyncProgress to copy to + * + * Atomically copies all the state from @self to @dest, without invoking the + * callback. + * This is used for proxying progress objects across different #GMainContexts. + * + * Since: 2019.6 + */ +void +ostree_async_progress_copy_state (OstreeAsyncProgress *self, + OstreeAsyncProgress *dest) +{ + g_return_if_fail (OSTREE_IS_ASYNC_PROGRESS (self)); + g_return_if_fail (OSTREE_IS_ASYNC_PROGRESS (dest)); + + g_mutex_lock (&self->lock); + + if (self->dead) + goto out; + + GLNX_HASH_TABLE_FOREACH_KV (self->values, void *, key, GVariant *, value) + { + if (value) + g_variant_ref (value); + g_hash_table_replace (dest->values, key, value); + } + + out: + g_mutex_unlock (&self->lock); +} + /** * ostree_async_progress_new: * diff --git a/src/libostree/ostree-async-progress.h b/src/libostree/ostree-async-progress.h index 45a80cfb..475d7f62 100644 --- a/src/libostree/ostree-async-progress.h +++ b/src/libostree/ostree-async-progress.h @@ -92,4 +92,8 @@ void ostree_async_progress_set_variant (OstreeAsyncProgress *self, _OSTREE_PUBLIC void ostree_async_progress_finish (OstreeAsyncProgress *self); +_OSTREE_PUBLIC +void ostree_async_progress_copy_state (OstreeAsyncProgress *self, + OstreeAsyncProgress *dest); + G_END_DECLS From 16bb294384bb4ae4ba83af51fbb64fdd14d02838 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 21 Nov 2019 18:37:48 +0000 Subject: [PATCH 13/17] tests/repo-finder: Run realpath() on /tmp This fixes running this test case inside https://github.com/cgwalters/coretoolbox --- tests/test-repo-finder-mount.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test-repo-finder-mount.c b/tests/test-repo-finder-mount.c index 847f4bec..af2f5e08 100644 --- a/tests/test-repo-finder-mount.c +++ b/tests/test-repo-finder-mount.c @@ -62,7 +62,9 @@ setup (Fixture *fixture, g_clear_error (&error); g_assert_no_error (error); - fixture->working_dir = g_file_new_for_path (fixture->tmpdir.path); + /* Realpath since at least coretoolbox makes /tmp a symlink to /host/tmp */ + g_autofree char *tmpdir_real_path = realpath (fixture->tmpdir.path, NULL); + fixture->working_dir = g_file_new_for_path (tmpdir_real_path); fixture->parent_repo = ot_test_setup_repo (NULL, &error); g_assert_no_error (error); @@ -443,9 +445,8 @@ test_repo_finder_mount_mixed_mounts (Fixture *fixture, } else { - g_test_message ("Unknown result ‘%s’ with keyring ‘%s’.", - result->remote->name, result->remote->keyring); - g_assert_not_reached (); + g_error ("Unknown result ‘%s’ with keyring ‘%s’", + result->remote->name, result->remote->keyring); } } From 77b27c07df2bdff86945766a4728b6012062535c Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Tue, 26 Nov 2019 09:20:30 +0100 Subject: [PATCH 14/17] Avoid race condition when building outside of source tree When building outside of source tree it can happen that src/ostree/ does not exist (yet) when bison is called. This leads to an build error like so: bison: src/ostree/parse-datetime.c: cannot open: No such file or directory Make sure that src/ostree/ exists when parse-datetime.c is built. --- Makefile-ostree.am | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 76f39cad..f861afe4 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -123,6 +123,7 @@ ostree_SOURCES += src/ostree/ot-remote-builtin-add-cookie.c \ endif src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile + mkdir -p src/ostree/ $(AM_V_GEN) $(YACC) $< -o $@ EXTRA_DIST += src/ostree/parse-datetime.y From a4e2b97e651b4c90328fc2c326e37630af519585 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sun, 1 Dec 2019 20:43:14 +0100 Subject: [PATCH 15/17] README.md: Tweak text about BuildStream BuildStream no longer uses libostree internally (see https://gitlab.com/BuildStream/buildstream/issues/387). It still has first class support for interoperating with libostree repos. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index a4146f04..1e6348dd 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,7 @@ systems. which uses libostree. The [BuildStream](https://gitlab.com/BuildStream/buildstream) build and -integration tool uses libostree as a caching system to store and share -built artifacts. +integration tool supports importing and exporting from libostree repos. Fedora [coreos-assembler](https://github.com/coreos/coreos-assembler) is the build tool used to generate Fedora CoreOS derivatives. From 36039759e2fbcadfb7841a035e98879e478ff8fb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 26 Sep 2017 22:02:25 -0400 Subject: [PATCH 16/17] pull: Add support for basic auth This has been requested a few times for people delivering OSTree content and wanting to do access control. --- Makefile-tests.am | 1 + src/libostree/ostree-fetcher-curl.c | 12 +++++-- src/ostree/ostree-trivial-httpd.c | 18 ++++++++++ tests/test-pull-basicauth.sh | 55 +++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 3 deletions(-) create mode 100755 tests/test-pull-basicauth.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index f5a65278..fc2f2d91 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -84,6 +84,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-metalink.sh \ tests/test-pull-summary-sigs.sh \ tests/test-pull-resume.sh \ + tests/test-pull-basicauth.sh \ tests/test-pull-repeated.sh \ tests/test-pull-untrusted.sh \ tests/test-pull-override-url.sh \ diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index 294b2078..fdf8a2ef 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -261,9 +261,8 @@ destroy_and_unref_source (GSource *source) } static char * -request_get_uri (FetcherRequest *req, guint idx) +request_get_uri (FetcherRequest *req, SoupURI *baseuri) { - SoupURI *baseuri = req->mirrorlist->pdata[idx]; if (!req->filename) return soup_uri_to_string (baseuri, FALSE); { g_autofree char *uristr = soup_uri_to_string (baseuri, FALSE); @@ -715,7 +714,8 @@ initiate_next_curl_request (FetcherRequest *req, g_assert_cmpint (req->idx, <, req->mirrorlist->len); - { g_autofree char *uri = request_get_uri (req, req->idx); + SoupURI *baseuri = req->mirrorlist->pdata[req->idx]; + { g_autofree char *uri = request_get_uri (req, baseuri); curl_easy_setopt (req->easy, CURLOPT_URL, uri); } @@ -771,6 +771,12 @@ initiate_next_curl_request (FetcherRequest *req, if ((self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) > 0) curl_easy_setopt (req->easy, CURLOPT_ACCEPT_ENCODING, ""); + /* If we have e.g. basic auth in the URL string, let's honor that */ + const char *username = soup_uri_get_user (baseuri); + curl_easy_setopt (req->easy, CURLOPT_USERNAME, username); + const char *password = soup_uri_get_password (baseuri); + curl_easy_setopt (req->easy, CURLOPT_PASSWORD, password); + /* We should only speak HTTP; TODO: only enable file if specified */ curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE)); /* Picked the current version in F25 as of 20170127, since diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index 1b7aa2a2..5d3a004e 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -50,6 +50,7 @@ static int opt_random_408s_max = 100; static gint opt_port = 0; static gchar **opt_expected_cookies; static gchar **opt_expected_headers; +static gboolean opt_require_basic_auth; static guint emitted_random_500s_count = 0; static guint emitted_random_408s_count = 0; @@ -71,6 +72,7 @@ static GOptionEntry options[] = { { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", "PORT" }, { "port-file", 'p', 0, G_OPTION_ARG_FILENAME, &opt_port_file, "Write port number to PATH (- for standard output)", "PATH" }, { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL }, + { "require-basic-auth", 0, 0, G_OPTION_ARG_NONE, &opt_require_basic_auth, "Require username foouser, password barpw", NULL }, { "random-500s", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_percentage, "Generate random HTTP 500 errors approximately for PERCENTAGE requests", "PERCENTAGE" }, { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" }, { "random-408s", 0, 0, G_OPTION_ARG_INT, &opt_random_408s_percentage, "Generate random HTTP 408 errors approximately for PERCENTAGE requests", "PERCENTAGE" }, @@ -474,6 +476,13 @@ httpd_callback (SoupServer *server, SoupMessage *msg, soup_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED); } +static gboolean +basic_auth_callback (SoupAuthDomain *auth_domain, SoupMessage *msg, + const char *username, const char *password, gpointer data) +{ + return g_str_equal (username, "foouser") && g_str_equal (password, "barpw"); +} + static void on_dir_changed (GFileMonitor *mon, GFile *file, @@ -571,6 +580,15 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL); #endif + if (opt_require_basic_auth) + { + glnx_unref_object SoupAuthDomain *auth_domain = + soup_auth_domain_basic_new (SOUP_AUTH_DOMAIN_REALM, "auth-test", + SOUP_AUTH_DOMAIN_ADD_PATH, "/", + SOUP_AUTH_DOMAIN_BASIC_AUTH_CALLBACK, basic_auth_callback, + NULL); + soup_server_add_auth_domain (server, auth_domain); + } soup_server_add_handler (server, NULL, httpd_callback, app, NULL); if (opt_port_file) diff --git a/tests/test-pull-basicauth.sh b/tests/test-pull-basicauth.sh new file mode 100755 index 00000000..4a2622a7 --- /dev/null +++ b/tests/test-pull-basicauth.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Copyright (C) 2017 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive" "" "--require-basic-auth" + +echo '1..3' + +repopath=${test_tmpdir}/ostree-srv/gnomerepo +cp -a ${repopath} ${repopath}.orig + +cd ${test_tmpdir} +rm repo -rf +ostree_repo_init repo +unauthaddress=$(cat httpd-address) +badauthaddress=$(echo $unauthaddress | sed -e 's,http://,http://foo:bar@,') +goodauthaddress=$(echo $unauthaddress | sed -e 's,http://,http://foouser:barpw@,') +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin-unauth ${unauthaddress}/ostree/gnomerepo +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin-badauth ${badauthaddress}/ostree/gnomerepo +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin-goodauth ${goodauthaddress}/ostree/gnomerepo + +if ${CMD_PREFIX} ostree --repo=repo pull origin-unauth main 2>err.txt; then + fatal "Pulled via unauth" +fi +assert_file_has_content err.txt "401" +echo "ok unauth" +rm -f err.txt +if ${CMD_PREFIX} ostree --repo=repo pull origin-badauth main 2>err.txt; then + fatal "Pulled via badauth" +fi +assert_file_has_content err.txt "401" +rm -f err.txt +echo "ok badauth" + +${CMD_PREFIX} ostree --repo=repo pull origin-goodauth main +echo "ok basic auth" From 978bd19f977119e391a7c4ea706eeb4740970757 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 9 Dec 2019 16:10:50 +0000 Subject: [PATCH 17/17] Release 2019.6 Nothing really big here, but let's get a release out so some bigger things like ro-sysroot, signing, sizes can bake in master for a bit. --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 5 ++--- src/libostree/libostree-released.sym | 4 ++++ tests/test-symbols.sh | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 2f9579f5..46a900f5 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ m4_define([year_version], [2019]) m4_define([release_version], [6]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 646a4a21..a5d15e93 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,9 +18,8 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2019.6 { - ostree_async_progress_copy_state; -} LIBOSTREE_2019.4; +LIBOSTREE_2019.7 { +} LIBOSTREE_2019.6; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index f6a83b77..e81a6595 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -577,6 +577,10 @@ LIBOSTREE_2019.4 { /* No new symbols in 2019.5 */ +LIBOSTREE_2019.6 { + ostree_async_progress_copy_state; +} LIBOSTREE_2019.4; + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 2287710d..a041ddb1 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -54,7 +54,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt <