New upstream version 2019.1

src/libostree/libostree-devel.sym

@@ -18,6 +18,8 @@
 ***/
 
 /* Add new symbols here.  Release commits should copy this section into -released.sym. */
+LIBOSTREE_2018.10 {
+} LIBOSTREE_2018.9;
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the last number.  This is just a copy/paste

src/libostree/ostree-bootloader-grub2.c

@@ -191,6 +191,7 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot                 *sysroot
       const char *options;
       const char *kernel;
       const char *initrd;
+      const char *devicetree;
       char *quoted_title = NULL;
       char *uuid = NULL;
       char *quoted_uuid = NULL;
@@ -246,6 +247,15 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot                 *sysroot
           g_string_append_c (output, '\n');
         }
 
+      devicetree = ostree_bootconfig_parser_get (config, "devicetree");
+      if (devicetree)
+        {
+          g_string_append (output, "devicetree");
+          g_string_append_c (output, ' ');
+          g_string_append (output, devicetree);
+          g_string_append_c (output, '\n');
+        }
+
       g_string_append (output, "}\n");
     }
 

src/libostree/ostree-core.c

@@ -626,9 +626,9 @@ ostree_content_stream_parse (gboolean                compressed,
                                 cancellable, error))
     return FALSE;
   g_autoptr(GVariant) file_header =
-    g_variant_new_from_data (compressed ? _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT : _OSTREE_FILE_HEADER_GVARIANT_FORMAT,
-                             buf, archive_header_size, trusted,
-                             g_free, buf);
+    g_variant_ref_sink(g_variant_new_from_data (compressed ? _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT : _OSTREE_FILE_HEADER_GVARIANT_FORMAT,
+                                                buf, archive_header_size, trusted,
+                                                g_free, buf));
   buf = NULL;
   g_autoptr(GFileInfo) ret_file_info = NULL;
   g_autoptr(GVariant) ret_xattrs = NULL;

src/libostree/ostree-fetcher-curl.c

@@ -777,7 +777,6 @@ initiate_next_curl_request (FetcherRequest *req,
    * there are numerous HTTP/2 fixes since the original version in
    * libcurl 7.43.0.
    */
-#ifdef BUILDOPT_HTTP2
   if (!(self->config_flags & OSTREE_FETCHER_FLAGS_DISABLE_HTTP2))
     {
 #if CURL_AT_LEAST_VERSION(7, 51, 0)
@@ -789,7 +788,7 @@ initiate_next_curl_request (FetcherRequest *req,
       curl_easy_setopt (req->easy, CURLOPT_PIPEWAIT, 1L);
 #endif
     }
-#endif
+
   curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb);
   if (g_getenv ("OSTREE_DEBUG_HTTP"))
     curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L);

src/libostree/ostree-gpg-verifier.c

@@ -304,12 +304,82 @@ _ostree_gpg_verifier_add_key_ascii_file (OstreeGpgVerifier *self,
   g_ptr_array_add (self->key_ascii_files, g_strdup (path));
 }
 
+gboolean
+_ostree_gpg_verifier_add_keyfile_path (OstreeGpgVerifier   *self,
+                                       const char          *path,
+                                       GCancellable        *cancellable,
+                                       GError             **error)
+{
+  g_autoptr(GError) temp_error = NULL;
+  if (!_ostree_gpg_verifier_add_keyfile_dir_at (self, AT_FDCWD, path,
+                                                cancellable, &temp_error))
+    {
+      g_assert (temp_error);
+
+      /* If failed due to not being a directory, add the file as an ascii key. */
+      if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_DIRECTORY))
+        {
+          g_clear_error (&temp_error);
+
+          _ostree_gpg_verifier_add_key_ascii_file (self, path);
+        }
+      else
+        {
+          g_propagate_error (error, g_steal_pointer (&temp_error));
+
+          return FALSE;
+        }
+    }
+  return TRUE;
+}
+
+/* Add files that exist one level below the directory at @path as ascii
+ * key files. If @path cannot be opened as a directory,
+ * an error is returned.
+ */
+gboolean
+_ostree_gpg_verifier_add_keyfile_dir_at (OstreeGpgVerifier   *self,
+                                         int                  dfd,
+                                         const char          *path,
+                                         GCancellable        *cancellable,
+                                         GError             **error)
+{
+  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+
+  if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE,
+                                    &dfd_iter, error))
+    return FALSE;
+
+  g_debug ("Adding GPG keyfile dir %s to verifier", path);
+
+  while (TRUE)
+    {
+      struct dirent *dent;
+
+      if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent,
+                                                       cancellable, error))
+        return FALSE;
+      if (dent == NULL)
+        break;
+
+      if (dent->d_type != DT_REG)
+        continue;
+
+      /* TODO: Potentially open the files here and have the GPG verifier iterate
+      over the fds. See https://github.com/ostreedev/ostree/pull/1773#discussion_r235421900. */
+      g_autofree char *iter_path = g_build_filename (path, dent->d_name, NULL);
+
+      _ostree_gpg_verifier_add_key_ascii_file (self, iter_path);
+    }
+
+  return TRUE;
+}
+
 gboolean
 _ostree_gpg_verifier_add_keyring_dir (OstreeGpgVerifier   *self,
                                       GFile               *path,
                                       GCancellable        *cancellable,
                                       GError             **error)
-
 {
   return _ostree_gpg_verifier_add_keyring_dir_at (self, AT_FDCWD,
                                                   gs_file_get_path_cached (path),
@@ -322,7 +392,6 @@ _ostree_gpg_verifier_add_keyring_dir_at (OstreeGpgVerifier   *self,
                                          const char          *path,
                                          GCancellable        *cancellable,
                                          GError             **error)
-
 {
   g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
   if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE,

src/libostree/ostree-gpg-verifier.h

@@ -75,4 +75,17 @@ void _ostree_gpg_verifier_add_keyring_file (OstreeGpgVerifier *self,
 void _ostree_gpg_verifier_add_key_ascii_file (OstreeGpgVerifier *self,
                                               const char        *path);
 
+gboolean
+_ostree_gpg_verifier_add_keyfile_path (OstreeGpgVerifier   *self,
+                                       const char          *path,
+                                       GCancellable        *cancellable,
+                                       GError             **error);
+
+gboolean
+_ostree_gpg_verifier_add_keyfile_dir_at (OstreeGpgVerifier   *self,
+                                         int                  dfd,
+                                         const char          *path,
+                                         GCancellable        *cancellable,
+                                         GError             **error);
+
 G_END_DECLS

src/libostree/ostree-kernel-args.c

@@ -33,23 +33,14 @@ static char *
 split_keyeq (char *arg)
 {
   char *eq;
-      
+
   eq = strchr (arg, '=');
-  if (eq)
-    {
-      /* Note key/val are in one malloc block,
-       * so we don't free val...
-       */
-      *eq = '\0';
-      return eq+1;
-    }
-  else
-    {
-      /* ...and this allows us to insert a constant
-       * string.
-       */
-      return "";
-    }
+  if (eq == NULL)
+    return NULL;
+
+  // Note: key/val are in a single allocation block, so we don't free val.
+  *eq = '\0';
+  return eq+1;
 }
 
 static gboolean
@@ -264,8 +255,10 @@ _ostree_kernel_args_to_strv (OstreeKernelArgs *kargs)
       for (j = 0; j < values->len; j++)
         {
           const char *value = values->pdata[j];
-
-          g_ptr_array_add (strv, g_strconcat (key, "=", value, NULL));
+          if (value == NULL)
+            g_ptr_array_add (strv, g_strconcat (key, NULL));
+          else
+            g_ptr_array_add (strv, g_strconcat (key, "=", value, NULL));
         }
     }
   g_ptr_array_add (strv, NULL);
@@ -297,14 +290,12 @@ _ostree_kernel_args_to_string (OstreeKernelArgs *kargs)
           else
             g_string_append_c (buf, ' ');
 
-          if (value && *value)
+          g_string_append (buf, key);
+          if (value != NULL)
             {
-              g_string_append (buf, key);
               g_string_append_c (buf, '=');
               g_string_append (buf, value);
             }
-          else
-            g_string_append (buf, key);
         }
     }
 

src/libostree/ostree-repo-checkout.c

@@ -35,6 +35,7 @@
 #include "ostree-repo-private.h"
 
 #define WHITEOUT_PREFIX ".wh."
+#define OPAQUE_WHITEOUT_NAME ".wh..wh..opq"
 
 /* Per-checkout call state/caching */
 typedef struct {
@@ -879,6 +880,7 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
                           GError                           **error)
 {
   gboolean did_exist = FALSE;
+  gboolean is_opaque_whiteout = FALSE;
   const gboolean sepolicy_enabled = options->sepolicy && !self->disable_xattrs;
   g_autoptr(GVariant) dirtree = NULL;
   g_autoptr(GVariant) dirmeta = NULL;
@@ -912,6 +914,22 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
         return TRUE; /* Note early return */
     }
 
+  if (options->process_whiteouts)
+    {
+      g_autoptr(GVariant) dir_file_contents = g_variant_get_child_value (dirtree, 0);
+      GVariantIter viter;
+      const char *fname;
+      g_autoptr(GVariant) contents_csum_v = NULL;
+      g_variant_iter_init (&viter, dir_file_contents);
+      while (g_variant_iter_loop (&viter, "(&s@ay)", &fname, &contents_csum_v))
+        {
+          is_opaque_whiteout = (g_str_equal (fname, OPAQUE_WHITEOUT_NAME));
+          if (is_opaque_whiteout)
+            break;
+        }
+      contents_csum_v = NULL; /* iter_loop freed it */
+    }
+
   /* First, make the directory.  Push a new scope in case we end up using
    * setfscreatecon().
    */
@@ -931,6 +949,13 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
           return FALSE;
       }
 
+    /* If it is an opaque whiteout, ensure the destination is empty first. */
+    if (is_opaque_whiteout)
+      {
+        if (!glnx_shutil_rm_rf_at (destination_parent_fd, destination_name, cancellable, error))
+          return FALSE;
+      }
+
     /* Create initially with mode 0700, then chown/chmod only when we're
      * done.  This avoids anyone else being able to operate on partially
      * constructed dirs.

src/libostree/ostree-repo-commit.c

@@ -1624,9 +1624,18 @@ ostree_repo_prepare_transaction (OstreeRepo     *self,
     self->txn.max_blocks = bfree - self->reserved_blocks;
   else
     {
-      self->cleanup_stagedir = TRUE;
-      g_mutex_unlock (&self->txn_lock);
-      return throw_min_free_space_error (self, 0, error);
+      self->txn.max_blocks = 0;
+      /* Don't throw_min_free_space_error here; reason being that
+       * this transaction could be just committing metadata objects
+       * which are relatively small in size and we do not really
+       * want to block them via min-free-space-* value. Metadata
+       * objects helps in housekeeping and hence should be kept
+       * out of the strict min-free-space values.
+       *
+       * The main drivers for writing content objects will always honor
+       * the min-free-space value and throw_min_free_space_error in
+       * case of overstepping the number of reserved blocks.
+       */
     }
   g_mutex_unlock (&self->txn_lock);
 
@@ -3338,6 +3347,9 @@ write_content_to_mtree_internal (OstreeRepo                  *self,
               if (!glnx_shutil_rm_rf_at (dfd_iter->fd, name, cancellable, error))
                 return FALSE;
             }
+          g_mutex_lock (&self->txn_lock);
+          self->txn.stats.devino_cache_hits++;
+          g_mutex_unlock (&self->txn_lock);
           return TRUE; /* Early return */
         }
     }
@@ -3469,6 +3481,10 @@ write_content_to_mtree_internal (OstreeRepo                  *self,
       if (!ostree_mutable_tree_replace_file (mtree, name, loose_checksum,
                                              error))
         return FALSE;
+
+      g_mutex_lock (&self->txn_lock);
+      self->txn.stats.devino_cache_hits++;
+      g_mutex_unlock (&self->txn_lock);
     }
   /* Next fast path - we can "adopt" the file */
   else if (can_adopt)
@@ -4094,12 +4110,18 @@ import_is_bareuser_only_conversion (OstreeRepo *src_repo,
     && objtype == OSTREE_OBJECT_TYPE_FILE;
 }
 
-/* Returns TRUE if we can potentially just call link() to copy an object. */
+/* Returns TRUE if we can potentially just call link() to copy an object;
+ * if untrusted the repos must be owned by the same uid.
+ */
 static gboolean
 import_via_reflink_is_possible (OstreeRepo *src_repo,
                                 OstreeRepo *dest_repo,
-                                OstreeObjectType objtype)
+                                OstreeObjectType objtype,
+                                gboolean    trusted)
 {
+  /* Untrusted pulls require matching ownership */
+  if (!trusted && (src_repo->owner_uid != dest_repo->owner_uid))
+    return FALSE;
   /* Equal modes are always compatible, and metadata
    * is identical between all modes.
    */
@@ -4160,13 +4182,6 @@ import_one_object_direct (OstreeRepo    *dest_repo,
   char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
   _ostree_loose_path (loose_path_buf, checksum, objtype, dest_repo->mode);
 
-  if (!import_via_reflink_is_possible (src_repo, dest_repo, objtype))
-    {
-      /* If we can't reflink, nothing to do here */
-      *out_was_supported = FALSE;
-      return TRUE;
-    }
-
   /* hardlinks require the owner to match and to be on the same device */
   const gboolean can_hardlink =
     src_repo->owner_uid == dest_repo->owner_uid &&
@@ -4254,15 +4269,29 @@ import_one_object_direct (OstreeRepo    *dest_repo,
         G_IN_SET (src_repo->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER);
       if (src_is_bare_or_bare_user && !OSTREE_OBJECT_TYPE_IS_META(objtype))
         {
-          g_autoptr(GVariant) xattrs = NULL;
+          if (src_repo == OSTREE_REPO_MODE_BARE)
+            {
+              g_autoptr(GVariant) xattrs = NULL;
+              if (!glnx_fd_get_all_xattrs (src_fd, &xattrs,
+                                           cancellable, error))
+                return FALSE;
+              if (!glnx_fd_set_all_xattrs (tmp_dest.fd, xattrs,
+                                           cancellable, error))
+                return FALSE;
+            }
+          else
+            {
+              /* bare-user; we just want ostree.usermeta */
+              g_autoptr(GBytes) bytes =
+                glnx_fgetxattr_bytes (src_fd, "user.ostreemeta", error);
+              if (bytes == NULL)
+                return FALSE;
 
-          if (!glnx_fd_get_all_xattrs (src_fd, &xattrs,
-                                       cancellable, error))
-            return FALSE;
-
-          if (!glnx_fd_set_all_xattrs (tmp_dest.fd, xattrs,
-                                       cancellable, error))
-            return FALSE;
+              if (TEMP_FAILURE_RETRY (fsetxattr (src_fd, "user.ostreemeta",
+                                                 (char*)g_bytes_get_data (bytes, NULL),
+                                                 g_bytes_get_size (bytes), 0)) != 0)
+                return glnx_throw_errno_prefix (error, "fsetxattr");
+            }
         }
 
       if (fchmod (tmp_dest.fd, stbuf.st_mode & ~S_IFMT) != 0)
@@ -4318,7 +4347,7 @@ _ostree_repo_import_object (OstreeRepo           *self,
    */
   const gboolean is_bareuseronly_conversion =
     import_is_bareuser_only_conversion (source, self, objtype);
-  gboolean try_direct = trusted;
+  gboolean try_direct = TRUE;
 
   /* If we need to do bareuseronly verification, or we're potentially doing a
    * bareuseronly conversion, let's verify those first so we don't complicate
@@ -4358,11 +4387,20 @@ _ostree_repo_import_object (OstreeRepo           *self,
         }
     }
 
-   /* We try to import via reflink/hardlink. If the remote is explicitly not trusted
-   * (i.e.) their checksums may be incorrect, we skip that.
-   */
-  if (try_direct)
+  /* First, let's see if we can import via reflink/hardlink. */
+  if (try_direct && import_via_reflink_is_possible (source, self, objtype, trusted))
     {
+      /* For local repositories, if the untrusted flag is set, we verify the
+       * checksum first. This assumes then that the files are immutable - the
+       * above check verified that the owner uids match.
+       */
+      if (!trusted)
+        {
+          if (!ostree_repo_fsck_object (source, objtype, checksum,
+                                        cancellable, error))
+            return FALSE;
+        }
+
       gboolean direct_was_supported = FALSE;
       if (!import_one_object_direct (self, source, checksum, objtype,
                                      &direct_was_supported,

src/libostree/ostree-repo-pull.c

@@ -2948,9 +2948,14 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
   if (gzip)
     fetcher_flags |= OSTREE_FETCHER_FLAGS_TRANSFER_GZIP;
 
-  { gboolean http2 = TRUE;
+  { gboolean http2_default = TRUE;
+#ifndef BUILDOPT_HTTP2
+    http2_default = FALSE;
+#endif
+    gboolean http2;
+
     if (!ostree_repo_get_remote_boolean_option (self, remote_name,
-                                                "http2", TRUE,
+                                                "http2", http2_default,
                                                 &http2, error))
       goto out;
     if (!http2)
@@ -4643,6 +4648,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_clear_object (&pull_data->cancellable);
   g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref);
   g_clear_object (&pull_data->remote_repo_local);
+  g_free (pull_data->remote_refspec_name);
   g_free (pull_data->remote_name);
   g_free (pull_data->append_user_agent);
   g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);

src/libostree/ostree-repo-refs.c

@@ -1002,7 +1002,7 @@ ostree_repo_remote_list_collection_refs (OstreeRepo    *self,
     summary_collection_map = NULL;
 
   while (summary_collection_map != NULL &&
-         g_variant_iter_loop (summary_collection_map, "{s@a(s(taya{sv}))}", &summary_collection_id, &summary_refs))
+         g_variant_iter_loop (summary_collection_map, "{&s@a(s(taya{sv}))}", &summary_collection_id, &summary_refs))
     {
       if (!remote_list_collection_refs_process_refs (self, remote_name,
                                                      summary_collection_id, summary_refs,

src/libostree/ostree-repo.c

@@ -4214,9 +4214,9 @@ ostree_repo_import_object_from (OstreeRepo           *self,
  * @error: Error
  *
  * Copy object named by @objtype and @checksum into @self from the
- * source repository @source.  If both repositories are of the same
- * type and on the same filesystem, this will simply be a fast Unix
- * hard link operation.
+ * source repository @source. If @trusted is %TRUE and both
+ * repositories are of the same type and on the same filesystem,
+ * this will simply be a fast Unix hard link operation.
  *
  * Otherwise, a copy will be performed.
  */
@@ -5081,7 +5081,6 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo    *self,
     }
   else if (remote_name != NULL)
     {
-      g_autofree char *gpgkeypath = NULL;
       /* Add the remote's keyring file if it exists. */
 
       g_autoptr(OstreeRemote) remote = NULL;
@@ -5100,12 +5099,19 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo    *self,
           add_global_keyring_dir = FALSE;
         }
 
-      if (!ot_keyfile_get_value_with_default (remote->options, remote->group, "gpgkeypath", NULL,
-                                              &gpgkeypath, error))
+      g_auto(GStrv) gpgkeypath_list = NULL;
+
+      if (!ot_keyfile_get_string_as_list (remote->options, remote->group, "gpgkeypath",
+                                          ";,", &gpgkeypath_list, error))
         return NULL;
 
-      if (gpgkeypath)
-        _ostree_gpg_verifier_add_key_ascii_file (verifier, gpgkeypath);
+      if (gpgkeypath_list)
+        {
+          for (char **iter = gpgkeypath_list; *iter != NULL; ++iter)
+            if (!_ostree_gpg_verifier_add_keyfile_path (verifier, *iter,
+                                                        cancellable, error))
+              return NULL;
+        }
     }
 
   if (add_global_keyring_dir)

src/libostree/ostree-repo.h

@@ -291,8 +291,9 @@ struct _OstreeRepoTransactionStats {
   guint content_objects_total;
   guint content_objects_written;
   guint64 content_bytes_written;
+  guint devino_cache_hits;
 
-  guint64 padding1;
+  guint   padding1;
   guint64 padding2;
   guint64 padding3;
   guint64 padding4;

src/libostree/ostree-version.h

@@ -34,7 +34,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_YEAR_VERSION (2018)
+#define OSTREE_YEAR_VERSION (2019)
 
 /**
  * OSTREE_RELEASE_VERSION:
@@ -43,7 +43,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_RELEASE_VERSION (9)
+#define OSTREE_RELEASE_VERSION (1)
 
 /**
  * OSTREE_VERSION
@@ -52,7 +52,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION (2018.9)
+#define OSTREE_VERSION (2019.1)
 
 /**
  * OSTREE_VERSION_S:
@@ -62,7 +62,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION_S "2018.9"
+#define OSTREE_VERSION_S "2019.1"
 
 #define OSTREE_ENCODE_VERSION(year,release) \
         ((year) << 16 | (release))