lib/checkout: add filter API to skip over files

This is analogous to the filtering support for the commit API: we allow
library users to skip over checking out specific files. This is useful
in some tricky situations where we *know* that the files to be checked
out will conflict with existing files in subtle ways.

One such example is in rpm-ostree support for multilib. There, we want
to allow checking out a package onto an existing tree, but skipping over
files that are not coloured to our preferred value (e.g. not overwriting
an i686 version of `ldconfig` if we already have the `x86_64` version).
See https://github.com/projectatomic/rpm-ostree/pull/1227 for details.

Closes: #1441
Approved by: cgwalters

src/libostree/ostree-core-private.h

@@ -84,6 +84,7 @@ _ostree_make_temporary_symlink_at (int             tmp_dirfd,
                                    GError        **error);
 
 GFileInfo * _ostree_stbuf_to_gfileinfo (const struct stat *stbuf);
+void _ostree_gfileinfo_to_stbuf (GFileInfo    *file_info, struct stat  *out_stbuf);
 gboolean _ostree_gfileinfo_equal (GFileInfo *a, GFileInfo *b);
 gboolean _ostree_stbuf_equal (struct stat *stbuf_a, struct stat *stbuf_b);
 GFileInfo * _ostree_mode_uidgid_to_gfileinfo (mode_t mode, uid_t uid, gid_t gid);

src/libostree/ostree-core.c

@@ -1686,6 +1686,26 @@ _ostree_stbuf_to_gfileinfo (const struct stat *stbuf)
   return ret;
 }
 
+/**
+ * _ostree_gfileinfo_to_stbuf:
+ * @file_info: File info
+ * @out_stbuf: (out): stat buffer
+ *
+ * Map GFileInfo data from @file_info onto @out_stbuf.
+ */
+void
+_ostree_gfileinfo_to_stbuf (GFileInfo    *file_info,
+                            struct stat  *out_stbuf)
+{
+  struct stat stbuf = {0,};
+  stbuf.st_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+  stbuf.st_uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
+  stbuf.st_gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
+  if (S_ISREG (stbuf.st_mode))
+    stbuf.st_size = g_file_info_get_attribute_uint64 (file_info, "standard::size");
+  *out_stbuf = stbuf;
+}
+
 /**
  * _ostree_gfileinfo_equal:
  * @a: First file info

src/libostree/ostree-repo-checkout.c

@@ -38,13 +38,17 @@
 
 /* Per-checkout call state/caching */
 typedef struct {
-  GString *selabel_path_buf;
+  GString *path_buf; /* buffer for real path if filtering enabled */
+  GString *selabel_path_buf; /* buffer for selinux path if labeling enabled; this may be
+                                the same buffer as path_buf */
 } CheckoutState;
 
 static void
 checkout_state_clear (CheckoutState *state)
 {
-  if (state->selabel_path_buf)
+  if (state->path_buf)
+    g_string_free (state->path_buf, TRUE);
+  if (state->selabel_path_buf && (state->selabel_path_buf != state->path_buf))
     g_string_free (state->selabel_path_buf, TRUE);
 }
 G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(CheckoutState, checkout_state_clear)
@@ -529,7 +533,7 @@ checkout_file_hardlink (OstreeRepo                          *self,
 
 static gboolean
 checkout_one_file_at (OstreeRepo                        *repo,
-                      OstreeRepoCheckoutAtOptions         *options,
+                      OstreeRepoCheckoutAtOptions       *options,
                       CheckoutState                     *state,
                       const char                        *checksum,
                       int                                destination_dfd,
@@ -545,12 +549,24 @@ checkout_one_file_at (OstreeRepo                        *repo,
   gboolean is_bare_user_symlink = FALSE;
   char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
 
+
   /* FIXME - avoid the GFileInfo here */
   g_autoptr(GFileInfo) source_info = NULL;
   if (!ostree_repo_load_file (repo, checksum, NULL, &source_info, NULL,
                               cancellable, error))
     return FALSE;
 
+  if (options->filter)
+    {
+      /* use struct stat for when we can get rid of GFileInfo; though for now, we end up
+       * packing and unpacking in the non-archive case; blehh */
+      struct stat stbuf = {0,};
+      _ostree_gfileinfo_to_stbuf (source_info, &stbuf);
+      if (options->filter (repo, state->path_buf->str, &stbuf, options->filter_user_data) ==
+          OSTREE_REPO_CHECKOUT_FILTER_SKIP)
+        return TRUE; /* Note early return */
+    }
+
   const gboolean is_symlink = (g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK);
   const gboolean is_whiteout = (!is_symlink && options->process_whiteouts &&
                                 g_str_has_prefix (destination_name, WHITEOUT_PREFIX));
@@ -750,6 +766,41 @@ checkout_one_file_at (OstreeRepo                        *repo,
   return TRUE;
 }
 
+static inline void
+push_path_element_once (GString    *buf,
+                        const char *name,
+                        gboolean    is_dir)
+{
+  g_string_append (buf, name);
+  if (is_dir)
+    g_string_append_c (buf, '/');
+}
+
+static inline void
+push_path_element (OstreeRepoCheckoutAtOptions *options,
+                   CheckoutState               *state,
+                   const char                  *name,
+                   gboolean                     is_dir)
+{
+  if (state->path_buf)
+    push_path_element_once (state->path_buf, name, is_dir);
+  if (state->selabel_path_buf && (state->selabel_path_buf != state->path_buf))
+    push_path_element_once (state->selabel_path_buf, name, is_dir);
+}
+
+static inline void
+pop_path_element (OstreeRepoCheckoutAtOptions *options,
+                  CheckoutState               *state,
+                  const char                  *name,
+                  gboolean                     is_dir)
+{
+  const size_t n = strlen (name) + (is_dir ? 1 : 0);
+  if (state->path_buf)
+    g_string_truncate (state->path_buf, state->path_buf->len - n);
+  if (state->selabel_path_buf && (state->selabel_path_buf != state->path_buf))
+    g_string_truncate (state->selabel_path_buf, state->selabel_path_buf->len - n);
+}
+
 /*
  * checkout_tree_at:
  * @self: Repo
@@ -800,6 +851,17 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
   gid = GUINT32_FROM_BE (gid);
   mode = GUINT32_FROM_BE (mode);
 
+  if (options->filter)
+    {
+      struct stat stbuf = { 0, };
+      stbuf.st_mode = mode;
+      stbuf.st_uid = uid;
+      stbuf.st_gid = gid;
+      if (options->filter (self, state->path_buf->str, &stbuf, options->filter_user_data)
+          == OSTREE_REPO_CHECKOUT_FILTER_SKIP)
+        return TRUE; /* Note early return */
+    }
+
   /* First, make the directory.  Push a new scope in case we end up using
    * setfscreatecon().
    */
@@ -865,7 +927,6 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
         return FALSE;
     }
 
-  GString *selabel_path_buf = state->selabel_path_buf;
   /* Process files in this subdir */
   { g_autoptr(GVariant) dir_file_contents = g_variant_get_child_value (dirtree, 0);
     GVariantIter viter;
@@ -874,9 +935,7 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
     g_autoptr(GVariant) contents_csum_v = NULL;
     while (g_variant_iter_loop (&viter, "(&s@ay)", &fname, &contents_csum_v))
       {
-        const size_t origlen = selabel_path_buf ? selabel_path_buf->len : 0;
+        push_path_element (options, state, fname, FALSE);
-        if (selabel_path_buf)
-          g_string_append (selabel_path_buf, fname);
 
         char tmp_checksum[OSTREE_SHA256_STRING_LEN+1];
         _ostree_checksum_inplace_from_bytes_v (contents_csum_v, tmp_checksum);
@@ -887,8 +946,7 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
                                    cancellable, error))
           return FALSE;
 
-        if (selabel_path_buf)
+        pop_path_element (options, state, fname, FALSE);
-          g_string_truncate (selabel_path_buf, origlen);
       }
     contents_csum_v = NULL; /* iter_loop freed it */
   }
@@ -912,12 +970,7 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
         if (!ot_util_filename_validate (dname, error))
           return FALSE;
 
-        const size_t origlen = selabel_path_buf ? selabel_path_buf->len : 0;
+        push_path_element (options, state, dname, TRUE);
-        if (selabel_path_buf)
-          {
-            g_string_append (selabel_path_buf, dname);
-            g_string_append_c (selabel_path_buf, '/');
-          }
 
         char subdirtree_checksum[OSTREE_SHA256_STRING_LEN+1];
         _ostree_checksum_inplace_from_bytes_v (subdirtree_csum_v, subdirtree_checksum);
@@ -929,8 +982,7 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
                                        cancellable, error))
           return FALSE;
 
-        if (selabel_path_buf)
+        pop_path_element (options, state, dname, TRUE);
-          g_string_truncate (selabel_path_buf, origlen);
       }
   }
 
@@ -992,18 +1044,31 @@ checkout_tree_at (OstreeRepo                        *self,
                   GError                           **error)
 {
   g_auto(CheckoutState) state = { 0, };
-  // If SELinux labeling is enabled, we need to keep track of the full path string
+
+  if (options->filter)
+    state.path_buf = g_string_new ("/");
+
+  /* If SELinux labeling is enabled, we need to keep track of the full path string */
   if (options->sepolicy)
     {
-      GString *buf = g_string_new (options->sepolicy_prefix ?: options->subpath);
-      g_assert_cmpint (buf->len, >, 0);
-      // Ensure it ends with /
-      if (buf->str[buf->len-1] != '/')
-        g_string_append_c (buf, '/');
-      state.selabel_path_buf = buf;
-
       /* Otherwise it'd just be corrupting things, and there's no use case */
       g_assert (options->force_copy);
+
+      const char *prefix = options->sepolicy_prefix ?: options->subpath;
+      if (g_str_equal (prefix, "/") && state.path_buf)
+        {
+          /* just use the same scratchpad if we can */
+          state.selabel_path_buf = state.path_buf;
+        }
+      else
+        {
+          GString *buf = g_string_new (prefix);
+          g_assert_cmpint (buf->len, >, 0);
+          /* Ensure it ends with / */
+          if (buf->str[buf->len-1] != '/')
+            g_string_append_c (buf, '/');
+          state.selabel_path_buf = buf;
+        }
     }
 
   /* Special case handling for subpath of a non-directory */
@@ -1017,7 +1082,7 @@ checkout_tree_at (OstreeRepo                        *self,
        */
       int destination_dfd = destination_parent_fd;
       glnx_autofd int destination_dfd_owned = -1;
-      if (strcmp (destination_name, ".") != 0)
+      if (!g_str_equal (destination_name, "."))
         {
           if (mkdirat (destination_parent_fd, destination_name, 0700) < 0
               && errno != EEXIST)
@@ -1027,6 +1092,9 @@ checkout_tree_at (OstreeRepo                        *self,
             return FALSE;
           destination_dfd = destination_dfd_owned;
         }
+      /* let's just ignore filter here; I can't think of a useful case for filtering when
+       * only checking out one path */
+      options->filter = NULL;
       return checkout_one_file_at (self, options, &state,
                                    ostree_repo_file_get_checksum (source),
                                    destination_dfd,

src/libostree/ostree-repo.h

@@ -942,6 +942,34 @@ ostree_repo_checkout_tree (OstreeRepo               *self,
                            GCancellable             *cancellable,
                            GError                  **error);
 
+/**
+ * OstreeRepoCheckoutFilterResult:
+ * @OSTREE_REPO_CHECKOUT_FILTER_ALLOW: Do checkout this object
+ * @OSTREE_REPO_CHECKOUT_FILTER_SKIP: Ignore this object
+ *
+ * Since: 2018.2
+ */
+typedef enum {
+  OSTREE_REPO_CHECKOUT_FILTER_ALLOW,
+  OSTREE_REPO_CHECKOUT_FILTER_SKIP
+} OstreeRepoCheckoutFilterResult;
+
+/**
+ * OstreeRepoCheckoutFilter:
+ * @repo: Repo
+ * @path: Path to file
+ * @stbuf: File information
+ * @user_data: User data
+ *
+ * Returns: #OstreeRepoCheckoutFilterResult saying whether or not to checkout this file
+ *
+ * Since: 2018.2
+ */
+typedef OstreeRepoCheckoutFilterResult (*OstreeRepoCheckoutFilter) (OstreeRepo    *repo,
+                                                                    const char    *path,
+                                                                    struct stat   *stbuf,
+                                                                    gpointer       user_data);
+
 /**
  * OstreeRepoCheckoutAtOptions:
  *
@@ -969,7 +997,9 @@ typedef struct {
   OstreeRepoDevInoCache *devino_to_csum_cache;
 
   int unused_ints[6];
-  gpointer unused_ptrs[5];
+  gpointer unused_ptrs[3];
+  OstreeRepoCheckoutFilter filter; /* Since: 2018.2 */
+  gpointer filter_user_data; /* Since: 2018.2 */
   OstreeSePolicy *sepolicy; /* Since: 2017.6 */
   const char *sepolicy_prefix;
 } OstreeRepoCheckoutAtOptions;

src/ostree/ot-builtin-checkout.c

@@ -46,6 +46,7 @@ static gboolean opt_disable_fsync;
 static gboolean opt_require_hardlinks;
 static gboolean opt_force_copy;
 static gboolean opt_bareuseronly_dirs;
+static char *opt_skiplist_file;
 static char *opt_selinux_policy;
 static char *opt_selinux_prefix;
 
@@ -85,11 +86,34 @@ static GOptionEntry options[] = {
   { "require-hardlinks", 'H', 0, G_OPTION_ARG_NONE, &opt_require_hardlinks, "Do not fall back to full copies if hardlinking fails", NULL },
   { "force-copy", 'C', 0, G_OPTION_ARG_NONE, &opt_force_copy, "Never hardlink (but may reflink if available)", NULL },
   { "bareuseronly-dirs", 'M', 0, G_OPTION_ARG_NONE, &opt_bareuseronly_dirs, "Suppress mode bits outside of 0775 for directories (suid, world writable, etc.)", NULL },
+  { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" },
   { "selinux-policy", 0, 0, G_OPTION_ARG_FILENAME, &opt_selinux_policy, "Set SELinux labels based on policy in root filesystem PATH (may be /); implies --force-copy", "PATH" },
   { "selinux-prefix", 0, 0, G_OPTION_ARG_STRING, &opt_selinux_prefix, "When setting SELinux labels, prefix all paths by PREFIX", "PREFIX" },
   { NULL }
 };
 
+static gboolean
+handle_skiplist_line (const char  *line,
+                      void        *data,
+                      GError     **error)
+{
+  GHashTable *files = data;
+  g_hash_table_add (files, g_strdup (line));
+  return TRUE;
+}
+
+static OstreeRepoCheckoutFilterResult
+checkout_filter (OstreeRepo         *self,
+                 const char         *path,
+                 struct stat        *st_buf,
+                 gpointer            user_data)
+{
+  GHashTable *skiplist = user_data;
+  if (g_hash_table_contains (skiplist, path))
+    return OSTREE_REPO_CHECKOUT_FILTER_SKIP;
+  return OSTREE_REPO_CHECKOUT_FILTER_ALLOW;
+}
+
 static gboolean
 process_one_checkout (OstreeRepo           *repo,
                       const char           *resolved_commit,
@@ -107,7 +131,7 @@ process_one_checkout (OstreeRepo           *repo,
    */
   if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks ||
       opt_union_add || opt_force_copy || opt_bareuseronly_dirs || opt_union_identical ||
-      opt_selinux_policy || opt_selinux_prefix)
+      opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix)
     {
       OstreeRepoCheckoutAtOptions options = { 0, };
 
@@ -181,6 +205,17 @@ process_one_checkout (OstreeRepo           *repo,
           options.sepolicy_prefix = opt_selinux_prefix;
         }
 
+      g_autoptr(GHashTable) skip_list =
+        g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+      if (opt_skiplist_file)
+        {
+          if (!ot_parse_file_by_line (opt_skiplist_file, handle_skiplist_line, skip_list,
+                                      cancellable, error))
+            goto out;
+          options.filter = checkout_filter;
+          options.filter_user_data = skip_list;
+        }
+
       options.no_copy_fallback = opt_require_hardlinks;
       options.force_copy = opt_force_copy;
       options.bareuseronly_dirs = opt_bareuseronly_dirs;

tests/basic-test.sh

@@ -21,7 +21,7 @@
 
 set -euo pipefail
 
-echo "1..$((79 + ${extra_basic_tests:-0}))"
+echo "1..$((81 + ${extra_basic_tests:-0}))"
 
 CHECKOUT_U_ARG=""
 CHECKOUT_H_ARGS="-H"
@@ -518,6 +518,35 @@ assert_file_has_content saucer alien
 rm t -rf
 echo "ok checkout subpath"
 
+cd ${test_tmpdir}
+rm -rf checkout-test2-skiplist
+cat > test-skiplist.txt <<EOF
+/baz/saucer
+/yet/another/tree
+EOF
+$OSTREE checkout --skip-list test-skiplist.txt test2 checkout-test2-skiplist
+cd checkout-test2-skiplist
+! test -f baz/saucer
+! test -d yet/another/tree
+test -f baz/cow
+test -d baz/deeper
+echo "ok checkout skip-list"
+
+cd ${test_tmpdir}
+rm -rf checkout-test2-skiplist
+cat > test-skiplist.txt <<EOF
+/saucer
+/deeper
+EOF
+$OSTREE checkout --skip-list test-skiplist.txt --subpath /baz \
+  test2 checkout-test2-skiplist
+cd checkout-test2-skiplist
+! test -f saucer
+! test -d deeper
+test -f cow
+test -d another
+echo "ok checkout skip-list with subpath"
+
 cd ${test_tmpdir}
 $OSTREE checkout  --union test2 checkout-test2-union
 find checkout-test2-union | wc -l > union-files-count

tests/installed/itest-label-selinux.sh

@@ -69,7 +69,19 @@ ostree checkout testbranch --selinux-policy / \
   --subpath subdir --selinux-prefix / co
 newcon=$(getfattr --only-values -m security.selinux co/usr/bin/bash)
 assert_streq "${oldcon}" "${newcon}"
-
-ostree refs --delete testbranch
 rm co -rf
 echo "ok checkout with sepolicy and selinux-prefix"
+
+# Now check that combining --selinux-policy with --skip-list doesn't blow up
+echo > skip-list.txt << EOF
+/usr/bin/true
+EOF
+ostree checkout testbranch --selinux-policy / --skip-list skip-list.txt \
+  --subpath subdir --selinux-prefix / co
+! test -f co/usr/bin/true
+test -f co/usr/bin/bash
+newcon=$(getfattr --only-values -m security.selinux co/usr/bin/bash)
+assert_streq "${oldcon}" "${newcon}"
+rm co -rf
+ostree refs --delete testbranch
+echo "ok checkout selinux and skip-list"