New upstream version 2020.2

tests/gpg-verify-data/README.md

@@ -1,5 +1,7 @@
 This is a GPG config directory for use with the OstreeGpgVerifyResult
-test cases. The test data (`lgplv2`) is signed with a variety of valid
-and invalid GPG keys in a detached signature file (`lgplv2.sig`).
+test cases. The test data (`lgpl2`) is signed with a variety of valid
+and invalid GPG keys in a detached signature file (`lgpl2.sig`). In
+addition, each detached signature is available in a separate file
+(`lgpgl2.sig<N>`).
 
 The passphrase for all the keys is `redhat`.

tests/gpghome/revocations/key1.rev

@@ -0,0 +1,12 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQE9BCABCgAnFiEEXmXedascUBhi1HY0f8oj2Ecs2voFAl0MvqgJHQBUZXN0aW5n
+AAoJEH/KI9hHLNr6/dQH/iPZjfJgFAc/TIR4xE4kB0yL4zdMqxgV1ef/atQDLEN4
+MBiqIltzb8WyG+cpNfNZgFmqXmCRN+1IAla9piixe76ZwOqcQ6S5MU/8nMcyMsD9
+edg+9sg0DH8SEzejVma3ZLfaJ/6ZpU7c6a4vCPNcRBC7PxAvAc0LnAN6KQYGU7GR
+gv2k/JsGYgvmUAajhVFy0jc4jGkhRBHMDksGGFdYY94RATFF4gWtlUXyRYMTXBCf
+eM3bxEeSMbU7lXCZg9zjoxP6XzJuNW1SLz3zL90GnO19uNh8Pf6pHmkCNTO/L1Ua
+Cc6fb3ubtdqgs6an84U/aod1VcK7BNASqZ2gYUsF2KE=
+=owvo
+-----END PGP PUBLIC KEY BLOCK-----

tests/gpghome/revocations/key2.rev

@@ -0,0 +1,12 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQE9BCABCgAnFiEEezsQINdEeWh/2yJz2CKM/sqVDUEFAl0Mvs4JHQBUZXN0aW5n
+AAoJENgijP7KlQ1BzH4IAMUoTrW6XraDYq/d/b0qa3sZ1NTBPUXLp4gFaedZwKk/
+AKSUCK05RWRQO3HrSvmhdSUF6/9tFLGpbu7P56ihjAnq2vpzRyeNTEGQ02IzfCpM
+SJup0R6iA7KmjiDutDoEgjhAzxCKbnU71SQ3PmjyaQT1KCBCDJVptcY6HDbo0dRN
+vcjTfjtFqkPgqbHyXwGv3rlm4uctSVfACrOS/fKF4Q9Fs4prsUXjQpGLaTHdxhiL
+pMRCTfZ4DBEMwAY7s9FpMpIh9rdOwE+zkv5CsE0uJVZq0WW5r0CBDCta6Sopt6uk
+pIA+QHL9GPOrG2E3SJxyIBC37Sl40MGAJQ1djmecIGk=
+=0KEe
+-----END PGP PUBLIC KEY BLOCK-----

tests/gpghome/revocations/key3.rev

@@ -0,0 +1,12 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQE9BCABCgAnFiEEfSnPBguCac32O/vdDRX6599ETWcFAl0MvuYJHQBUZXN0aW5n
+AAoJEA0V+uffRE1n/p8H/3mmSK2gtbxJ5sfu1z44ra82fLRAUupJzf53dAvvJCEK
+4RSJFtHYu+hoPVFd9bmToxo2YQWe67MMZW7cHtq9D/a755SYOrty//KpXsGS22W/
+ZGatBjl36zuE6BoR18Q6VAMgVBwovPSlSuCEW+Ro8JZYyA/LbA95AnMprNod6Jw9
+VSsGC39au5rUlhEOHLL1Iw3dl4blxa6tf/roljbXzaN+Qh2/ez7Cy532oocak2FL
+bbblBGrIdfYLAXpLqhnQk2vgEHZ+ZylvStBndpLWwEskXhmaHpW7+WapFhLCUOr+
+arzbc9XQ7ghhF9hSoKiToJqU5PRjaOex85BEDwE5gWY=
+=ykAF
+-----END PGP PUBLIC KEY BLOCK-----

tests/libtest-core.sh

@@ -41,7 +41,7 @@ assert_not_reached () {
 # If we can't find the locale command assume we have support for C.UTF-8
 # (e.g. musl based systems)
 if type -p locale >/dev/null; then
-    export LC_ALL=$(locale -a | grep -Ee '\.(UTF-8|utf8)' | grep -iEe '^(C|en_US)' | head -1 || true)
+    export LC_ALL=$(locale -a | grep -iEe '^(C|en_US)\.(UTF-8|utf8)$' | head -n1 || true)
     if [ -z "${LC_ALL}" ]; then fatal "Can't find suitable UTF-8 locale"; fi
 else
     export LC_ALL=C.UTF-8

tests/libtest.sh

@@ -78,16 +78,20 @@ unset TAR_OPTIONS
 # easily clean up.
 export OSTREE_SYSROOT_DEBUG=mutable-deployments
 
-export TEST_GPG_KEYID_1="472CDAFA"
-export TEST_GPG_KEYID_2="CA950D41"
-export TEST_GPG_KEYID_3="DF444D67"
+export TEST_GPG_KEYID_1="7FCA23D8472CDAFA"
+export TEST_GPG_KEYFPR_1="5E65DE75AB1C501862D476347FCA23D8472CDAFA"
+export TEST_GPG_KEYID_2="D8228CFECA950D41"
+export TEST_GPG_KEYFPR_2="7B3B1020D74479687FDB2273D8228CFECA950D41"
+export TEST_GPG_KEYID_3="0D15FAE7DF444D67"
+export TEST_GPG_KEYFPR_3="7D29CF060B8269CDF63BFBDD0D15FAE7DF444D67"
 
-# GPG when creating signatures demands a writable
+# GPG when creating signatures demands a private writable
 # homedir in order to create lockfiles.  Work around
 # this by copying locally.
 echo "Copying gpghome to ${test_tmpdir}"
 cp -a "${test_srcdir}/gpghome" ${test_tmpdir}
 chmod -R u+w "${test_tmpdir}"
+chmod 700 "${test_tmpdir}/gpghome"
 export TEST_GPG_KEYHOME=${test_tmpdir}/gpghome
 export OSTREE_GPG_HOME=${test_tmpdir}/gpghome/trusted
 
@@ -559,6 +563,18 @@ skip_without_user_xattrs () {
     fi
 }
 
+_have_systemd_and_libmount=''
+have_systemd_and_libmount() {
+    if test "${_have_systemd_and_libmount}" = ''; then
+        if [ $(ostree --version | grep -c -e '- systemd' -e '- libmount') -eq 2 ]; then
+            _have_systemd_and_libmount=yes
+        else
+            _have_systemd_and_libmount=no
+        fi
+    fi
+    test ${_have_systemd_and_libmount} = yes
+}
+
 # Skip unless SELinux is disabled, or we can relabel.
 # Default Docker has security.selinux xattrs, but returns
 # EOPNOTSUPP when trying to set them, even to the existing value.

tests/test-gpg-verify-result.c

@@ -36,6 +36,18 @@
     } \
   } G_STMT_END
 
+#define assert_str_contains(s1, s2) \
+  G_STMT_START { \
+    const char *__s1 = (s1), *__s2 = (s2); \
+    if (strstr (__s1, __s2) == NULL) { \
+      g_autoptr(GString) string = g_string_new ("assertion failed (" #s1 " contains " #s2 "): "); \
+      g_autofree char *__es1 = g_strescape (__s1, NULL); \
+      g_autofree char *__es2 = g_strescape (__s2, NULL); \
+      g_string_append_printf (string, "(\"%s\", \"%s\")", __es1, __es2); \
+      g_assertion_message (G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, string->str); \
+    } \
+  } G_STMT_END
+
 typedef struct {
   OstreeGpgVerifyResult *result;
 } TestFixture;
@@ -53,12 +65,12 @@ static void
 test_fixture_setup (TestFixture *fixture,
                     gconstpointer user_data)
 {
+  const char * const *sig_files = user_data;
   gpgme_error_t gpg_error;
   gpgme_data_t data_buffer;
   gpgme_data_t signature_buffer;
   OstreeGpgVerifyResult *result;
   g_autofree char *homedir = NULL;
-  g_autofree char *filename = NULL;
   GError *local_error = NULL;
 
   /* Mimic what OstreeGpgVerifier does to create OstreeGpgVerifyResult.
@@ -74,15 +86,47 @@ test_fixture_setup (TestFixture *fixture,
                            NULL, &local_error, NULL);
   g_assert_no_error (local_error);
 
-  filename = g_build_filename (homedir, "lgpl2", NULL);
-  gpg_error = gpgme_data_new_from_file (&data_buffer, filename, 1);
-  assert_no_gpg_error (gpg_error, filename);
+  g_autofree char *data_filename = g_build_filename (homedir, "lgpl2", NULL);
+  gpg_error = gpgme_data_new_from_file (&data_buffer, data_filename, 1);
+  assert_no_gpg_error (gpg_error, data_filename);
 
-  g_clear_pointer (&filename, g_free);
+  if (sig_files == NULL)
+    {
+      /* No signature files specified, use full lgpl2.sig file */
+      g_autofree char *filename = g_build_filename (homedir, "lgpl2.sig", NULL);
+      gpg_error = gpgme_data_new_from_file (&signature_buffer, filename, 1);
+      assert_no_gpg_error (gpg_error, filename);
+    }
+  else
+    {
+      /* Read all the specified files into the signature buffer */
+      gpg_error = gpgme_data_new (&signature_buffer);
+      assert_no_gpg_error (gpg_error, NULL);
 
-  filename = g_build_filename (homedir, "lgpl2.sig", NULL);
-  gpg_error = gpgme_data_new_from_file (&signature_buffer, filename, 1);
-  assert_no_gpg_error (gpg_error, filename);
+      for (const char * const *name = sig_files; *name != NULL; name++)
+        {
+          g_autofree char *path = g_build_filename (homedir, *name, NULL);
+          g_autoptr(GFile) sig_file = g_file_new_for_path (path);
+
+          g_autofree char *contents = NULL;
+          gsize len;
+          g_assert_true (g_file_load_contents (sig_file, NULL, &contents,
+                                               &len, NULL, &local_error));
+          g_assert_no_error (local_error);
+
+          char *cur = contents;
+          while (len > 0)
+            {
+              ssize_t written = gpgme_data_write (signature_buffer, cur, len);
+              if (written == -1)
+                assert_no_gpg_error (gpgme_error_from_syserror (), path);
+              cur += written;
+              len -= written;
+            }
+        }
+
+      gpgme_data_seek (signature_buffer, 0, SEEK_SET);
+    }
 
   gpg_error = gpgme_op_verify (result->context,
                                signature_buffer, data_buffer, NULL);
@@ -115,7 +159,7 @@ test_check_counts (TestFixture *fixture,
   count_valid = ostree_gpg_verify_result_count_valid (fixture->result);
 
   g_assert_cmpint (count_all, ==, 5);
-  g_assert_cmpint (count_valid, ==, 2);
+  g_assert_cmpint (count_valid, ==, 1);
 }
 
 static void
@@ -123,7 +167,7 @@ test_signature_lookup (TestFixture *fixture,
                        gconstpointer user_data)
 {
   /* Checking the signature with the revoked key for this case. */
-  guint expected_signature_index = GPOINTER_TO_UINT (user_data);
+  guint expected_signature_index = 2;
 
   /* Lowercase letters to ensure OstreeGpgVerifyResult handles it. */
   const char *fingerprint = "68dcc2db4bec5811c2573590bd9d2a44b7f541a6";
@@ -215,7 +259,7 @@ static void
 test_valid_signature (TestFixture *fixture,
                       gconstpointer user_data)
 {
-  guint signature_index = GPOINTER_TO_UINT (user_data);
+  guint signature_index = 0;
   g_autoptr(GVariant) tuple = NULL;
   gboolean valid;
   gboolean sig_expired;
@@ -249,7 +293,7 @@ static void
 test_expired_key (TestFixture *fixture,
                   gconstpointer user_data)
 {
-  guint signature_index = GPOINTER_TO_UINT (user_data);
+  guint signature_index = 1;
   g_autoptr(GVariant) tuple = NULL;
   gboolean valid;
   gboolean sig_expired;
@@ -283,7 +327,7 @@ static void
 test_revoked_key (TestFixture *fixture,
                   gconstpointer user_data)
 {
-  guint signature_index = GPOINTER_TO_UINT (user_data);
+  guint signature_index = 2;
   g_autoptr(GVariant) tuple = NULL;
   gboolean valid;
   gboolean sig_expired;
@@ -317,7 +361,7 @@ static void
 test_missing_key (TestFixture *fixture,
                   gconstpointer user_data)
 {
-  guint signature_index = GPOINTER_TO_UINT (user_data);
+  guint signature_index = 3;
   g_autoptr(GVariant) tuple = NULL;
   gboolean valid;
   gboolean sig_expired;
@@ -351,7 +395,7 @@ static void
 test_expired_signature (TestFixture *fixture,
                         gconstpointer user_data)
 {
-  guint signature_index = GPOINTER_TO_UINT (user_data);
+  guint signature_index = 4;
   g_autoptr(GVariant) tuple = NULL;
   gboolean valid;
   gboolean sig_expired;
@@ -373,7 +417,7 @@ test_expired_signature (TestFixture *fixture,
                  &key_missing,
                  &key_exp_timestamp);
 
-  g_assert_true (valid);
+  g_assert_false (valid);
   g_assert_true (sig_expired);
   g_assert_false (key_expired);
   g_assert_false (key_revoked);
@@ -381,6 +425,83 @@ test_expired_signature (TestFixture *fixture,
   g_assert_cmpint (key_exp_timestamp, ==, 0);
 }
 
+static void
+test_require_valid_signature (TestFixture *fixture,
+                              gconstpointer user_data)
+{
+  GError *error = NULL;
+  gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result,
+                                                                   &error);
+  g_assert_true (res);
+  g_assert_no_error (error);
+}
+
+static void
+test_require_valid_signature_expired_key (TestFixture *fixture,
+                                          gconstpointer user_data)
+{
+  GError *error = NULL;
+  gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result,
+                                                                   &error);
+  g_assert_false (res);
+  g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_EXPIRED_KEY);
+  assert_str_contains (error->message, "Key expired");
+}
+
+static void
+test_require_valid_signature_revoked_key (TestFixture *fixture,
+                                          gconstpointer user_data)
+{
+  GError *error = NULL;
+  gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result,
+                                                                   &error);
+  g_assert_false (res);
+  g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_REVOKED_KEY);
+  assert_str_contains (error->message, "Key revoked");
+}
+
+static void
+test_require_valid_signature_missing_key (TestFixture *fixture,
+                                          gconstpointer user_data)
+{
+  GError *error = NULL;
+  gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result,
+                                                                   &error);
+  g_assert_false (res);
+  g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_MISSING_KEY);
+  assert_str_contains (error->message, "public key not found");
+}
+
+static void
+test_require_valid_signature_expired_signature (TestFixture *fixture,
+                                                gconstpointer user_data)
+{
+  GError *error = NULL;
+  gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result,
+                                                                   &error);
+  g_assert_false (res);
+  g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_EXPIRED_SIGNATURE);
+  assert_str_contains (error->message, "Signature expired");
+}
+
+static void
+test_require_valid_signature_expired_missing_key (TestFixture *fixture,
+                                                  gconstpointer user_data)
+{
+  GError *error = NULL;
+  gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result,
+                                                                   &error);
+  g_assert_false (res);
+
+  /*
+   * The error will be for the last signature, which is for a missing key, but
+   * the message should show both issues.
+   */
+  g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_MISSING_KEY);
+  assert_str_contains (error->message, "Key expired");
+  assert_str_contains (error->message, "public key not found");
+}
+
 int
 main (int argc, char **argv)
 {
@@ -397,7 +518,7 @@ main (int argc, char **argv)
 
   g_test_add ("/gpg-verify-result/signature-lookup",
               TestFixture,
-              GINT_TO_POINTER (2),
+              NULL,
               test_fixture_setup,
               test_signature_lookup,
               test_fixture_teardown);
@@ -411,38 +532,85 @@ main (int argc, char **argv)
 
   g_test_add ("/gpg-verify-result/valid-signature",
               TestFixture,
-              GINT_TO_POINTER (0),  /* signature index */
+              NULL,
               test_fixture_setup,
               test_valid_signature,
               test_fixture_teardown);
 
   g_test_add ("/gpg-verify-result/expired-key",
               TestFixture,
-              GINT_TO_POINTER (1),  /* signature index */
+              NULL,
               test_fixture_setup,
               test_expired_key,
               test_fixture_teardown);
 
   g_test_add ("/gpg-verify-result/revoked-key",
               TestFixture,
-              GINT_TO_POINTER (2),  /* signature index */
+              NULL,
               test_fixture_setup,
               test_revoked_key,
               test_fixture_teardown);
 
   g_test_add ("/gpg-verify-result/missing-key",
               TestFixture,
-              GINT_TO_POINTER (3),  /* signature index */
+              NULL,
               test_fixture_setup,
               test_missing_key,
               test_fixture_teardown);
 
   g_test_add ("/gpg-verify-result/expired-signature",
               TestFixture,
-              GINT_TO_POINTER (4),  /* signature index */
+              NULL,
               test_fixture_setup,
               test_expired_signature,
               test_fixture_teardown);
 
+  g_test_add ("/gpg-verify-result/require-valid-signature",
+              TestFixture,
+              NULL,
+              test_fixture_setup,
+              test_require_valid_signature,
+              test_fixture_teardown);
+
+  const char *expired_key_files[] = { "lgpl2.sig1", NULL };
+  g_test_add ("/gpg-verify-result/require-valid-signature-expired-key",
+              TestFixture,
+              expired_key_files,
+              test_fixture_setup,
+              test_require_valid_signature_expired_key,
+              test_fixture_teardown);
+
+  const char *revoked_key_files[] = { "lgpl2.sig2", NULL };
+  g_test_add ("/gpg-verify-result/require-valid-signature-revoked-key",
+              TestFixture,
+              revoked_key_files,
+              test_fixture_setup,
+              test_require_valid_signature_revoked_key,
+              test_fixture_teardown);
+
+  const char *missing_key_files[] = { "lgpl2.sig3", NULL };
+  g_test_add ("/gpg-verify-result/require-valid-signature-missing-key",
+              TestFixture,
+              missing_key_files,
+              test_fixture_setup,
+              test_require_valid_signature_missing_key,
+              test_fixture_teardown);
+
+  const char *expired_signature_files[] = { "lgpl2.sig4", NULL };
+  g_test_add ("/gpg-verify-result/require-valid-signature-expired-signature",
+              TestFixture,
+              expired_signature_files,
+              test_fixture_setup,
+              test_require_valid_signature_expired_signature,
+              test_fixture_teardown);
+
+  const char *expired_missing_key_files[] = { "lgpl2.sig1", "lgpl2.sig3", NULL };
+  g_test_add ("/gpg-verify-result/require-valid-signature-expired-missing-key",
+              TestFixture,
+              expired_missing_key_files,
+              test_fixture_setup,
+              test_require_valid_signature_expired_missing_key,
+              test_fixture_teardown);
+
   return g_test_run ();
 }

tests/test-libarchive-import.c

@@ -544,6 +544,12 @@ test_libarchive_selinux (gconstpointer data)
 
   if (skip_if_no_xattr (td))
     goto out;
+  if (getenv ("container"))
+    {
+      // FIXME dedup this with libtest.sh have_selinux_relabel
+      g_test_skip ("skip in containers for now");
+      goto out;
+    }
 
   if (td->skip_all != NULL)
     {

tests/test-libarchive.sh

@@ -28,7 +28,7 @@ fi
 
 . $(dirname $0)/libtest.sh
 
-echo "1..17"
+echo "1..18"
 
 setup_test_repository "bare"
 
@@ -234,3 +234,17 @@ for filter in '^usr/bin/,usr/sbin/' '/bin/,/sbin/'; do
     assert_file_has_content usr/lib/libfoo.so 'a library'
     echo "ok tar pathname filter modification: ${filter}"
 done
+
+# Test sizes metadata. This needs an archive repo, so a separate repo is used.
+cd ${test_tmpdir}
+rm -rf repo2
+ostree_repo_init repo2 --mode=archive
+${CMD_PREFIX} ostree --repo=repo2 commit \
+  -s "from tar" -b test-tar \
+  --generate-sizes \
+  --tree=tar=foo.tar.gz
+${CMD_PREFIX} ostree --repo=repo2 show --print-sizes test-tar > sizes.txt
+assert_file_has_content sizes.txt 'Compressed size (needed/total): 0[  ]bytes/1.1[  ]kB'
+assert_file_has_content sizes.txt 'Unpacked size (needed/total): 0[  ]bytes/900[  ]bytes'
+assert_file_has_content sizes.txt 'Number of objects (needed/total): 0/12'
+echo "ok tar sizes metadata"

tests/test-pull-sizes.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Copyright (C) 2019 Endless Mobile, Inc.
+#
+# SPDX-License-Identifier: LGPL-2.0+
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive" "--generate-sizes"
+
+echo '1..3'
+
+cd ${test_tmpdir}
+mkdir repo
+ostree_repo_init repo
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+
+# Pull commit metadata only. All size and objects will be needed.
+${CMD_PREFIX} ostree --repo=repo pull --commit-metadata-only origin main
+${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt
+assert_file_has_content show.txt 'Compressed size (needed/total): 637[  ]bytes/637[  ]bytes'
+assert_file_has_content show.txt 'Unpacked size (needed/total): 457[  ]bytes/457[  ]bytes'
+assert_file_has_content show.txt 'Number of objects (needed/total): 10/10'
+echo "ok sizes commit metadata only"
+
+# Pull the parent commit so we get most of the objects
+parent=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main^)
+${CMD_PREFIX} ostree --repo=repo pull origin ${parent}
+${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt
+assert_file_has_content show.txt 'Compressed size (needed/total): 501[  ]bytes/637[  ]bytes'
+assert_file_has_content show.txt 'Unpacked size (needed/total): 429[  ]bytes/457[  ]bytes'
+assert_file_has_content show.txt 'Number of objects (needed/total): 6/10'
+echo "ok sizes commit partial"
+
+# Finish pulling the commit and check that no objects needed
+${CMD_PREFIX} ostree --repo=repo pull origin main
+${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt
+assert_file_has_content show.txt 'Compressed size (needed/total): 0[  ]bytes/637[  ]bytes'
+assert_file_has_content show.txt 'Unpacked size (needed/total): 0[  ]bytes/457[  ]bytes'
+assert_file_has_content show.txt 'Number of objects (needed/total): 0/10'
+echo "ok sizes commit full"

tests/test-pull-summary-sigs.sh

@@ -189,7 +189,7 @@ cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.2,}
 if ${OSTREE} --repo=repo pull origin main 2>err.txt; then
     assert_not_reached "Successful pull with old summary"
 fi
-assert_file_has_content err.txt "none are in trusted keyring"
+assert_file_has_content err.txt "BAD signature"
 assert_has_file repo/tmp/cache/summaries/origin
 assert_has_file repo/tmp/cache/summaries/origin.sig
 cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2

tests/test-remote-gpg-import.sh

@@ -28,7 +28,12 @@ unset OSTREE_GPG_HOME
 
 setup_fake_remote_repo1 "archive"
 
-echo "1..4"
+# Some tests require an appropriate gpg
+num_non_gpg_tests=4
+num_gpg_tests=2
+num_tests=$((num_non_gpg_tests + num_gpg_tests))
+
+echo "1..${num_tests}"
 
 cd ${test_tmpdir}
 mkdir repo
@@ -163,7 +168,7 @@ ${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/key1.asc,${test_tmp
 if ${OSTREE} pull R8:main 2>err.txt; then
     assert_not_reached "Unexpectedly succeeded at pulling with different key"
 fi
-assert_file_has_content err.txt "GPG signatures found, but none are in trusted keyring"
+assert_file_has_content err.txt "public key not found"
 
 # Test gpgkeypath success with directory containing a valid key
 ${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/ R9 $(cat httpd-address)/ostree/gnomerepo
@@ -243,7 +248,7 @@ ${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/key2.asc R6 $(cat h
 if ${OSTREE} pull R6:main 2>err.txt; then
     assert_not_reached "Unexpectedly succeeded at pulling with different key"
 fi
-assert_file_has_content err.txt "GPG signatures found, but none are in trusted keyring"
+assert_file_has_content err.txt "public key not found"
 
 echo "ok"
 
@@ -269,7 +274,7 @@ newrev=$(${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo rev-par
 if ${OSTREE} pull --require-static-deltas R1:main 2>err.txt; then
     assert_not_reached "Unexpectedly succeeded at pulling commit signed with untrusted key"
 fi
-assert_file_has_content err.txt "GPG signatures found, but none are in trusted keyring"
+assert_file_has_content err.txt "public key not found"
 
 echo "ok gpg untrusted signed commit for delta upgrades"
 
@@ -280,3 +285,57 @@ ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u --gpg
 ${OSTREE} pull --require-static-deltas R1:main
 
 echo "ok gpg trusted signed commit for delta upgrades"
+
+# Run some more tests if an appropriate gpg is available
+GPG=$(which_gpg)
+if [ -z "${GPG}" ]; then
+  # Print a skip message per skipped test
+  for (( i = 0; i < num_gpg_tests; i++ )); do
+    echo "ok # SKIP this test requires gpg"
+  done
+else
+  # Create a commit signed with keyid 1
+  echo $(date) > workdir/testfile-for-key-mangling
+  ${CMD_PREFIX} ostree  --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main --gpg-sign ${TEST_GPG_KEYID_1} --gpg-homedir ${test_tmpdir}/gpghome workdir
+
+  # Re-add the remote
+  ${OSTREE} remote delete R1
+  ${OSTREE} remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc R1 $(cat httpd-address)/ostree/gnomerepo | grep -o 'Imported [[:digit:]] GPG key' > result
+  assert_file_has_content result 'Imported 1 GPG key'
+
+  # Expire key 1, wait for it to be expired and import the expired key. Only
+  # new keys are reported.
+  ${GPG} --homedir=${test_tmpdir}/gpghome --quick-set-expire ${TEST_GPG_KEYFPR_1} seconds=1
+  sleep 2
+  ${GPG} --homedir=${test_tmpdir}/gpghome --armor --export ${TEST_GPG_KEYID_1} > ${test_tmpdir}/key1expired.asc
+  ${OSTREE} remote gpg-import --keyring ${test_tmpdir}/key1expired.asc R1 | grep -o 'Imported [[:digit:]] GPG key' > result
+  assert_file_has_content result 'Imported 0 GPG key'
+
+  # Pulling should fail since the key is expired
+  rm repo/refs/remotes/* -rf
+  ${OSTREE} prune --refs-only
+  if ${OSTREE} pull R1:main 2>err.txt; then
+    assert_not_reached "Unexpectedly succeeded at pulling commit signed with expired key"
+  fi
+  assert_file_has_content err.txt "Key expired"
+
+  echo "ok imported expired key"
+
+  # Unexpire keyid 1 and revoke it. Revoking is done by importing the
+  # pre-generated revocation certificate.
+  ${GPG} --homedir=${test_tmpdir}/gpghome --quick-set-expire ${TEST_GPG_KEYFPR_1} seconds=0
+  ${GPG} --homedir=${TEST_GPG_KEYHOME} --import ${TEST_GPG_KEYHOME}/revocations/key1.rev
+  ${GPG} --homedir=${test_tmpdir}/gpghome --armor --export ${TEST_GPG_KEYID_1} > ${test_tmpdir}/key1revoked.asc
+  ${OSTREE} remote gpg-import --keyring ${test_tmpdir}/key1revoked.asc R1 | grep -o 'Imported [[:digit:]] GPG key' > result
+  assert_file_has_content result 'Imported 0 GPG key'
+
+  # Pulling should fail since the key is revoked
+  rm repo/refs/remotes/* -rf
+  ${OSTREE} prune --refs-only
+  if ${OSTREE} pull R1:main 2>err.txt; then
+    assert_not_reached "Unexpectedly succeeded at pulling commit signed with revoked key"
+  fi
+  assert_file_has_content err.txt "Key revoked"
+
+  echo "ok imported revoked key"
+fi

tests/test-sizes.js

@@ -28,11 +28,110 @@ function assertEquals(a, b) {
 	throw new Error("assertion failed " + JSON.stringify(a) + " == " + JSON.stringify(b));
 }
 
-print('1..1')
+function assertGreater(a, b) {
+    if (a <= b)
+	throw new Error("assertion failed " + JSON.stringify(a) + " > " + JSON.stringify(b));
+}
+
+function assertGreaterEquals(a, b) {
+    if (a < b)
+	throw new Error("assertion failed " + JSON.stringify(a) + " >= " + JSON.stringify(b));
+}
+
+// Adapted from _ostree_read_varuint64()
+function readVarint(buffer) {
+    let result = 0;
+    let count = 0;
+    let len = buffer.length;
+    let cur;
+
+    do {
+        assertGreater(len, 0);
+        cur = buffer[count];
+        result = result | ((cur & 0x7F) << (7 * count));
+        count++;
+        len--;
+    } while (cur & 0x80);
+
+    return [result, count];
+}
+
+// There have been various bugs with byte array unpacking in GJS, so
+// just do it manually.
+function unpackByteArray(variant) {
+    let array = [];
+    let nBytes = variant.n_children();
+    for (let i = 0; i < nBytes; i++) {
+        array.push(variant.get_child_value(i).get_byte());
+    }
+    return array;
+}
+
+function validateSizes(repo, commit, expectedObjects) {
+    let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit);
+    let metadata = commitVariant.get_child_value(0);
+    let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay'));
+    let nObjects = sizes.n_children();
+    let expectedNObjects = Object.keys(expectedObjects).length
+    assertEquals(nObjects, expectedNObjects);
+
+    for (let i = 0; i < nObjects; i++) {
+        let sizeEntry = sizes.get_child_value(i);
+        assertGreaterEquals(sizeEntry.n_children(), 34);
+        let entryBytes = unpackByteArray(sizeEntry);
+        let checksumBytes = entryBytes.slice(0, 32);
+        let checksumString = OSTree.checksum_from_bytes(checksumBytes);
+        print("checksum = " + checksumString);
+
+        // Read the sizes from the next 2 varints
+        let remainingBytes = entryBytes.slice(32);
+        assertGreaterEquals(remainingBytes.length, 2);
+        let varintRead;
+        let compressedSize;
+        let uncompressedSize;
+        [compressedSize, varintRead] = readVarint(remainingBytes);
+        remainingBytes = remainingBytes.slice(varintRead);
+        assertGreaterEquals(remainingBytes.length, 1);
+        [uncompressedSize, varintRead] = readVarint(remainingBytes);
+        remainingBytes = remainingBytes.slice(varintRead);
+        assertEquals(remainingBytes.length, 1);
+        let objectType = remainingBytes[0];
+        let objectTypeString = OSTree.object_type_to_string(objectType);
+        print("compressed = " + compressedSize);
+        print("uncompressed = " + uncompressedSize);
+        print("objtype = " + objectTypeString + " (" + objectType + ")");
+        let objectName = OSTree.object_to_string(checksumString, objectType);
+        print("object = " + objectName);
+
+        if (!(objectName in expectedObjects)) {
+            throw new Error("Object " + objectName + " not in " +
+                            JSON.stringify(expectedObjects));
+        }
+        let expectedSizes = expectedObjects[objectName];
+        let expectedCompressedSize = expectedSizes[0];
+        let expectedUncompressedSize = expectedSizes[1];
+        if (compressedSize != expectedCompressedSize) {
+            throw new Error("Compressed size " + compressedSize +
+                            " for checksum " + checksumString +
+                            " does not match expected " + expectedCompressedSize);
+        }
+        if (uncompressedSize != expectedUncompressedSize) {
+            throw new Error("Uncompressed size " + uncompressedSize +
+                            " for checksum " + checksumString +
+                            " does not match expected " + expectedUncompressedSize);
+        }
+    }
+}
+
+print('1..3')
 
 let testDataDir = Gio.File.new_for_path('test-data');
 testDataDir.make_directory(null);
 testDataDir.get_child('some-file').replace_contents("hello world!", null, false, 0, null);
+testDataDir.get_child('some-file').copy(testDataDir.get_child('duplicate-file'),
+                                        Gio.FileCopyFlags.OVERWRITE,
+                                        null, null);
+testDataDir.get_child('link-file').make_symbolic_link('some-file', null);
 testDataDir.get_child('another-file').replace_contents("hello world again!", null, false, 0, null);
 
 let repoPath = Gio.File.new_for_path('repo');
@@ -41,7 +140,10 @@ repo.create(OSTree.RepoMode.ARCHIVE_Z2, null);
 
 repo.open(null);
 
-let commitModifier = OSTree.RepoCommitModifier.new(OSTree.RepoCommitModifierFlags.GENERATE_SIZES, null);
+let commitModifierFlags = (OSTree.RepoCommitModifierFlags.GENERATE_SIZES |
+                           OSTree.RepoCommitModifierFlags.SKIP_XATTRS |
+                           OSTree.RepoCommitModifierFlags.CANONICAL_PERMISSIONS);
+let commitModifier = OSTree.RepoCommitModifier.new(commitModifierFlags, null);
 
 assertEquals(repo.get_mode(), OSTree.RepoMode.ARCHIVE_Z2);
 
@@ -55,32 +157,61 @@ print("commit => " + commit);
 
 repo.commit_transaction(null);
 
-// Test the sizes metadata
-let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit);
-let metadata = commitVariant.get_child_value(0);
-let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay'));
-let nSizes = sizes.n_children();
-assertEquals(nSizes, 2);
-let expectedUncompressedSizes = [12, 18];
-let foundExpectedUncompressedSizes = 0;
-for (let i = 0; i < nSizes; i++) {
-    let sizeEntry = sizes.get_child_value(i);
-    assertEquals(sizeEntry.n_children(), 34);
-    let compressedSize = sizeEntry.get_child_value(32).get_byte();
-    let uncompressedSize = sizeEntry.get_child_value(33).get_byte();
-    print("compressed = " + compressedSize);
-    print("uncompressed = " + uncompressedSize);
-    for (let j = 0; j < expectedUncompressedSizes.length; j++) {
-	let expected = expectedUncompressedSizes[j];
-	if (expected == uncompressedSize) {
-	    print("Matched expected uncompressed size " + expected);
-	    expectedUncompressedSizes.splice(j, 1);
-	    break;
-	}
-    }
-}
-if (expectedUncompressedSizes.length > 0) {
-    throw new Error("Failed to match expectedUncompressedSizes: " + JSON.stringify(expectedUncompressedSizes));
-}
+// Test the sizes metadata. The key is the object and the value is an
+// array of compressed size and uncompressed size.
+let expectedObjects = {
+    'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file': [
+        54, 18
+    ],
+    'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b.file': [
+        48, 12
+    ],
+    '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0.file': [
+        43, 9
+    ],
+    '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree': [
+        185, 185
+    ],
+    '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488.dirmeta': [
+        12, 12
+    ],
+};
+validateSizes(repo, commit, expectedObjects);
 
 print("ok test-sizes");
+
+// Remove a file to make sure that metadata is not reused from the
+// previous commit. Remove that file from the expected metadata and
+// replace the dirtree object.
+testDataDir.get_child('another-file').delete(null);
+delete expectedObjects['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file'];
+delete expectedObjects['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree'];
+expectedObjects['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539.dirtree'] = [
+    138, 138
+];
+
+repo.prepare_transaction(null);
+mtree = OSTree.MutableTree.new();
+repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null);
+[,dirTree] = repo.write_mtree(mtree, null);
+[,commit] = repo.write_commit(null, 'Some subject', 'Some body', null, dirTree, null);
+print("commit => " + commit);
+repo.commit_transaction(null);
+
+validateSizes(repo, commit, expectedObjects);
+
+print("ok test-sizes file deleted");
+
+// Repeat the commit now that all the objects are cached and ensure the
+// metadata is still correct
+repo.prepare_transaction(null);
+mtree = OSTree.MutableTree.new();
+repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null);
+[,dirTree] = repo.write_mtree(mtree, null);
+[,commit] = repo.write_commit(null, 'Another subject', 'Another body', null, dirTree, null);
+print("commit => " + commit);
+repo.commit_transaction(null);
+
+validateSizes(repo, commit, expectedObjects);
+
+print("ok test-sizes repeated");

tests/test-switchroot.sh

@@ -2,6 +2,16 @@
 
 this_script="${BASH_SOURCE:-$(readlink -f "$0")}"
 
+OSTREE_PREPARE_ROOT=$(dirname "${this_script}")/../ostree-prepare-root
+if [ ! -x "${OSTREE_PREPARE_ROOT}" ]; then
+	# ostree-prepare-root is in $libdir by default, assume we can find it
+	# based on our test directory, if not we'll have to skip this test.
+	OSTREE_PREPARE_ROOT=$(dirname "${this_script}")/../../../lib/ostree/ostree-prepare-root
+	if [ ! -x "${OSTREE_PREPARE_ROOT}" ]; then
+		OSTREE_PREPARE_ROOT=""
+	fi
+fi
+
 setup_bootfs() {
 	mkdir -p "$1/proc" "$1/bin"
 
@@ -13,7 +23,7 @@ setup_bootfs() {
 	mount --bind "$1/override_cmdline" "$1/proc/cmdline"
 
 	touch "$1/this_is_bootfs"
-	cp "$(dirname "$this_script")/../ostree-prepare-root" "$1/bin"
+	cp "${OSTREE_PREPARE_ROOT}" "$1/bin"
 }
 
 setup_rootfs() {
@@ -56,7 +66,7 @@ find_in_env() {
 		"$1" "$tmpdir"
 		enter_fs "$tmpdir"
 		ostree-prepare-root /sysroot
-		find /
+		find / \( -path /proc -o -path /sysroot/proc \) -prune -o -print
 		touch /usr/usr_writable 2>/null \
 			&& echo "/usr is writable" \
 			|| echo "/usr is not writable"
@@ -82,7 +92,9 @@ test_that_prepare_root_sets_sysroot_up_correctly_with_initrd() {
 	grep -qx "/this_is_bootfs" files
 	grep -qx "/sysroot/this_is_ostree_root" files
 	grep -qx "/sysroot/sysroot/this_is_real_root" files
-	grep -qx "/sysroot/var/this_is_ostree_var" files
+	if ! have_systemd_and_libmount; then
+		grep -qx "/sysroot/var/this_is_ostree_var" files
+	fi
 	grep -qx "/sysroot/usr/this_is_ostree_usr" files
 
 	grep -qx "/sysroot/usr is not writable" files
@@ -101,7 +113,9 @@ test_that_prepare_root_sets_root_up_correctly_with_no_initrd() {
 	grep -qx "/this_is_ostree_root" files
 	grep -qx "/sysroot/this_is_bootfs" files
 	grep -qx "/sysroot/this_is_real_root" files
-	grep -qx "/var/this_is_ostree_var" files
+	if ! have_systemd_and_libmount; then
+		grep -qx "/var/this_is_ostree_var" files
+	fi
 	grep -qx "/usr/this_is_ostree_usr" files
 
 	grep -qx "/usr is not writable" files
@@ -130,6 +144,9 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 	[ -f /bin/busybox ] || \
 	    skip "this test needs busybox"
 
+	[ -n "${OSTREE_PREPARE_ROOT}" ] || \
+	    skip "this test needs ostree-prepare-root"
+
 	echo "1..3"
 	test_that_prepare_root_sets_sysroot_up_correctly_with_initrd
 	test_that_prepare_root_sets_root_up_correctly_with_no_initrd

tests/test-symbols.sh

@@ -54,7 +54,7 @@ echo 'ok documented symbols'
 
 # ONLY update this checksum in release commits!
 cat > released-sha256.txt <<EOF
-cba152efd13b9e80c35edfccde26ecd7f9555968239c5a275c9e411ad27e5183  ${released_syms}
+dd16acf1370f9b4ecd77d4eb7c4b45548b249f33f629d3147ddc9d264bd32e41  ${released_syms}
 EOF
 sha256sum -c released-sha256.txt