libarchive: Add support for translating paths during commit

For rpm-ostree, I want to move RPM files in `/boot` to `/usr/lib/ostree-boot`.
This is currently impossible without forking the libarchive code.  Supporting
this is pretty straightforward; we already had pathname translation in
the libarchive code, we just need to expose it as an option.

On the command line side, I chose to wrap this as a regexp. That should be good
enough for a lot of use cases; sophisticated users should as always be making
use of the API. Note that this required some new `#ifdef LIBARCHIVE` bits to use
the new API. Following previous patterns here, we use the new API only if a
relevant option is enabled, ensuring unit test coverage of both paths.

For the test cases, I ended up changing the accounting to avoid having to
multiply the test count.

Closes: #1105
Approved by: jlebon

src/libostree/ostree-libarchive-private.h

@@ -38,6 +38,28 @@ typedef struct archive OtAutoArchiveWrite;
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtAutoArchiveWrite, archive_write_free)
 typedef struct archive OtAutoArchiveRead;
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtAutoArchiveRead, archive_read_free)
+
+static inline OtAutoArchiveRead *
+ot_open_archive_read (const char *path, GError **error)
+{
+  g_autoptr(OtAutoArchiveRead) a = archive_read_new ();
+
+#ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL
+  archive_read_support_filter_all (a);
+#else
+  archive_read_support_compression_all (a);
+#endif
+  archive_read_support_format_all (a);
+  if (archive_read_open_filename (a, path, 8192) != ARCHIVE_OK)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "%s", archive_error_string (a));
+      return NULL;
+    }
+
+  return g_steal_pointer (&a);
+}
+
 #endif
 
 G_END_DECLS

src/libostree/ostree-repo-libarchive.c

@@ -123,24 +123,30 @@ squash_trailing_slashes (char *path)
     *endp = '\0';
 }
 
+/* Like archive_entry_stat(), but since some archives only store the permission
+ * mode bits in hardlink entries, so let's just make it into a regular file.
+ * Yes, this hack will work even if it's a hardlink to a symlink.
+ */
+static void
+read_archive_entry_stat (struct archive_entry *entry,
+                         struct stat          *stbuf)
+{
+  const struct stat *st = archive_entry_stat (entry);
+
+  *stbuf = *st;
+  if (archive_entry_hardlink (entry))
+    stbuf->st_mode |= S_IFREG;
+}
+
+/* Create a GFileInfo from archive_entry_stat() */
 static GFileInfo *
 file_info_from_archive_entry (struct archive_entry *entry)
 {
-  const struct stat *st = archive_entry_stat (entry);
+  struct stat stbuf;
-  struct stat st_copy;
+  read_archive_entry_stat (entry, &stbuf);
 
-  /* Some archives only store the permission mode bits in hardlink entries, so
+  g_autoptr(GFileInfo) info = _ostree_stbuf_to_gfileinfo (&stbuf);
-   * let's just make it into a regular file. Yes, this hack will work even if
+  if (S_ISLNK (stbuf.st_mode))
-   * it's a hardlink to a symlink. */
-  if (archive_entry_hardlink (entry))
-    {
-      st_copy = *st;
-      st_copy.st_mode |= S_IFREG;
-      st = &st_copy;
-    }
-
-  g_autoptr(GFileInfo) info = _ostree_stbuf_to_gfileinfo (st);
-  if (S_ISLNK (st->st_mode))
     g_file_info_set_attribute_byte_string (info, "standard::symlink-target",
                                            archive_entry_symlink (entry));
 
@@ -247,7 +253,18 @@ aic_get_final_path (OstreeRepoArchiveImportContext *ctx,
                     const char  *path,
                     GError     **error)
 {
-  if (ctx->opts->use_ostree_convention)
+  if (ctx->opts->translate_pathname)
+    {
+      struct stat stbuf;
+      path = path_relative (path, error);
+      read_archive_entry_stat (ctx->entry, &stbuf);
+      char *ret = ctx->opts->translate_pathname (ctx->repo, &stbuf, path,
+                                                 ctx->opts->translate_pathname_user_data);
+      if (ret)
+        return ret;
+      /* Fall through */
+    }
+  else if (ctx->opts->use_ostree_convention)
     return path_relative_ostree (path, error);
   return g_strdup (path_relative (path, error));
 }
@@ -258,7 +275,6 @@ aic_get_final_entry_pathname (OstreeRepoArchiveImportContext *ctx,
 {
   const char *pathname = archive_entry_pathname (ctx->entry);
   g_autofree char *final = aic_get_final_path (ctx, pathname, error);
-
   if (final == NULL)
     return NULL;
 
@@ -642,17 +658,17 @@ aic_import_entry (OstreeRepoArchiveImportContext *ctx,
                   GCancellable  *cancellable,
                   GError       **error)
 {
-  g_autoptr(GFileInfo) fi = NULL;
-  g_autoptr(OstreeMutableTree) parent = NULL;
   g_autofree char *path = aic_get_final_entry_pathname (ctx, error);
 
   if (path == NULL)
     return FALSE;
 
+  g_autoptr(GFileInfo) fi = NULL;
   if (aic_apply_modifier_filter (ctx, path, &fi)
         == OSTREE_REPO_COMMIT_FILTER_SKIP)
     return TRUE;
 
+  g_autoptr(OstreeMutableTree) parent = NULL;
   if (!aic_get_parent_dir (ctx, path, &parent, cancellable, error))
     return FALSE;
 
@@ -907,18 +923,9 @@ ostree_repo_write_archive_to_mtree (OstreeRepo                *self,
   g_autoptr(OtAutoArchiveRead) a = archive_read_new ();
   OstreeRepoImportArchiveOptions opts = { 0, };
 
-#ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL
+  a = ot_open_archive_read (gs_file_get_path_cached (archive), error);
-  archive_read_support_filter_all (a);
+  if (!a)
-#else
-  archive_read_support_compression_all (a);
-#endif
-  archive_read_support_format_all (a);
-  if (archive_read_open_filename (a, gs_file_get_path_cached (archive), 8192) != ARCHIVE_OK)
-    {
-      propagate_libarchive_error (error, a);
     goto out;
-    }
-
   opts.autocreate_parents = !!autocreate_parents;
 
   if (!ostree_repo_import_archive_to_mtree (self, &opts, a, mtree, modifier, cancellable, error))

src/libostree/ostree-repo.h

@@ -22,6 +22,8 @@
 
 #pragma once
 
+#include <sys/stat.h>
+
 #include "ostree-core.h"
 #include "ostree-types.h"
 #include "ostree-async-progress.h"
@@ -688,6 +690,31 @@ gboolean      ostree_repo_write_archive_to_mtree (OstreeRepo                   *
                                                   GCancellable                 *cancellable,
                                                   GError                      **error);
 
+/**
+ * OstreeRepoImportArchiveTranslatePathname:
+ * @repo: Repo
+ * @stbuf: Stat buffer
+ * @src_path: Path in the archive
+ * @user_data: User data
+ *
+ * Possibly change a pathname while importing an archive. If %NULL is returned,
+ * then @src_path will be used unchanged.  Otherwise, return a new pathname which
+ * will be freed via `g_free()`.
+ *
+ * This pathname translation will be performed *before* any processing from an
+ * active `OstreeRepoCommitModifier`. Will be invoked for all directory and file
+ * types, first with outer directories, then their sub-files and directories.
+ *
+ * Note that enabling pathname translation will always override the setting for
+ * `use_ostree_convention`.
+ *
+ * Since: 2017.11
+ */
+typedef char *(*OstreeRepoImportArchiveTranslatePathname) (OstreeRepo     *repo,
+                                                           const struct stat *stbuf,
+                                                           const char     *src_path,
+                                                           gpointer        user_data);
+
 /**
  * OstreeRepoImportArchiveOptions: (skip)
  *
@@ -703,7 +730,9 @@ typedef struct {
   guint reserved : 28;
 
   guint unused_uint[8];
-  gpointer unused_ptrs[8];
+  OstreeRepoImportArchiveTranslatePathname translate_pathname;
+  gpointer translate_pathname_user_data;
+  gpointer unused_ptrs[6];
 } OstreeRepoImportArchiveOptions;
 
 _OSTREE_PUBLIC

src/ostree/ot-builtin-commit.c

@@ -30,6 +30,7 @@
 #include "ot-tool-util.h"
 #include "parse-datetime.h"
 #include "ostree-repo-private.h"
+#include "ostree-libarchive-private.h"
 
 static char *opt_subject;
 static char *opt_body;
@@ -46,6 +47,7 @@ static char **opt_detached_metadata_strings;
 static gboolean opt_link_checkout_speedup;
 static gboolean opt_skip_if_unchanged;
 static gboolean opt_tar_autocreate_parents;
+static char *opt_tar_pathname_filter;
 static gboolean opt_no_xattrs;
 static char *opt_selinux_policy;
 static gboolean opt_canonical_permissions;
@@ -97,6 +99,7 @@ static GOptionEntry options[] = {
   { "selinux-policy", 0, 0, G_OPTION_ARG_FILENAME, &opt_selinux_policy, "Set SELinux labels based on policy in root filesystem PATH (may be /)", "PATH" },
   { "link-checkout-speedup", 0, 0, G_OPTION_ARG_NONE, &opt_link_checkout_speedup, "Optimize for commits of trees composed of hardlinks into the repository", NULL },
   { "tar-autocreate-parents", 0, 0, G_OPTION_ARG_NONE, &opt_tar_autocreate_parents, "When loading tar archives, automatically create parent directories as needed", NULL },
+  { "tar-pathname-filter", 0, 0, G_OPTION_ARG_STRING, &opt_tar_pathname_filter, "When loading tar archives, use REGEX,REPLACEMENT against path names", "REGEX,REPLACEMENT" },
   { "skip-if-unchanged", 0, 0, G_OPTION_ARG_NONE, &opt_skip_if_unchanged, "If the contents are unchanged from previous commit, do nothing", NULL },
   { "statoverride", 0, 0, G_OPTION_ARG_FILENAME, &opt_statoverride_file, "File containing list of modifications to make to permissions", "PATH" },
   { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" },
@@ -221,6 +224,28 @@ commit_filter (OstreeRepo         *self,
   return OSTREE_REPO_COMMIT_FILTER_ALLOW;
 }
 
+typedef struct {
+  GRegex *regex;
+  const char *replacement;
+} TranslatePathnameData;
+
+/* Implement --tar-pathname-filter */
+static char *
+handle_translate_pathname (OstreeRepo *repo,
+                           const struct stat *stbuf,
+                           const char *path,
+                           gpointer user_data)
+{
+  TranslatePathnameData *tpdata = user_data;
+  g_autoptr(GError) tmp_error = NULL;
+  char *ret =
+    g_regex_replace (tpdata->regex, path, -1, 0,
+                     tpdata->replacement, 0, &tmp_error);
+  g_assert_no_error (tmp_error);
+  g_assert (ret);
+  return ret;
+}
+
 static gboolean
 commit_editor (OstreeRepo     *repo,
                const char     *branch,
@@ -567,6 +592,8 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError
                 goto out;
             }
           else if (strcmp (tree_type, "tar") == 0)
+            {
+              if (!opt_tar_pathname_filter)
                 {
                   object_to_commit = g_file_new_for_path (tree);
                   if (!ostree_repo_write_archive_to_mtree (repo, object_to_commit, mtree, modifier,
@@ -574,6 +601,43 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError
                                                            cancellable, error))
                     goto out;
                 }
+              else
+                {
+#ifdef HAVE_LIBARCHIVE
+                  const char *comma = strchr (opt_tar_pathname_filter, ',');
+                  if (!comma)
+                    {
+                      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                                           "Missing ',' in --tar-pathname-filter");
+                      goto out;
+                    }
+                  const char *replacement = comma + 1;
+                  g_autofree char *regexp_text = g_strndup (opt_tar_pathname_filter, comma - opt_tar_pathname_filter);
+                  /* Use new API if we have a pathname filter */
+                  OstreeRepoImportArchiveOptions opts = { 0, };
+                  opts.autocreate_parents = opt_tar_autocreate_parents;
+                  opts.translate_pathname = handle_translate_pathname;
+                  g_autoptr(GRegex) regexp = g_regex_new (regexp_text, 0, 0, error);
+                  TranslatePathnameData tpdata = { regexp, replacement };
+                  if (!regexp)
+                    {
+                      g_prefix_error (error, "--tar-pathname-filter: ");
+                      goto out;
+                    }
+                  opts.translate_pathname_user_data = &tpdata;
+                  g_autoptr(OtAutoArchiveRead) archive = ot_open_archive_read (tree, error);
+                  if (!archive)
+                    goto out;
+                  if (!ostree_repo_import_archive_to_mtree (repo, &opts, archive, mtree,
+                                                            modifier, cancellable, error))
+                    goto out;
+                }
+#else
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+                           "This version of ostree is not compiled with libarchive support");
+              return FALSE;
+#endif
+            }
           else if (strcmp (tree_type, "ref") == 0)
             {
               if (!ostree_repo_read_commit (repo, tree, &object_to_commit, NULL, cancellable, error))

tests/test-libarchive.sh

@@ -26,14 +26,14 @@ fi
 
 . $(dirname $0)/libtest.sh
 
-echo "1..21"
+echo "1..13"
 
 setup_test_repository "bare"
 
 cd ${test_tmpdir}
 mkdir foo
 cd foo
-mkdir -p usr/bin
+mkdir -p usr/bin usr/lib
 echo contents > usr/bin/foo
 touch usr/bin/foo0
 ln usr/bin/foo usr/bin/bar
@@ -45,8 +45,12 @@ ln usr/bin/foo0 usr/local/bin/baz0
 ln usr/bin/sl usr/local/bin/slhl
 touch usr/bin/setuidme
 touch usr/bin/skipme
+echo "a library" > usr/lib/libfoo.so
+echo "another library" > usr/lib/libbar.so
 
+# Create a tar archive
 tar -c -z -f ../foo.tar.gz .
+# Create a cpio archive
 find . | cpio -o -H newc > ../foo.cpio
 
 cd ..
@@ -71,10 +75,17 @@ $OSTREE commit -s "from cpio" -b test-cpio \
 echo "ok cpio commit"
 
 assert_valid_checkout () {
-  cd ${test_tmpdir}
+  ref=$1
-  $OSTREE checkout test-$1 test-$1-checkout
+  rm test-${ref}-checkout -rf
-  cd test-$1-checkout
+  $OSTREE checkout test-${ref} test-${ref}-checkout
 
+  assert_valid_content test-${ref}-checkout
+  rm -rf test-${ref}-checkout
+}
+
+assert_valid_content () {
+  dn=$1
+  cd ${dn}
   # basic content check
   assert_file_has_content usr/bin/foo contents
   assert_file_has_content usr/bin/bar contents
@@ -82,39 +93,35 @@ assert_valid_checkout () {
   assert_file_empty usr/bin/foo0
   assert_file_empty usr/bin/bar0
   assert_file_empty usr/local/bin/baz0
-  echo "ok $1 contents"
+  assert_file_has_content usr/lib/libfoo.so 'a library'
+  assert_file_has_content usr/lib/libbar.so 'another library'
 
   # hardlinks
   assert_files_hardlinked usr/bin/foo usr/bin/bar
   assert_files_hardlinked usr/bin/foo usr/local/bin/baz
-  echo "ok $1 hardlink"
   assert_files_hardlinked usr/bin/foo0 usr/bin/bar0
   assert_files_hardlinked usr/bin/foo0 usr/local/bin/baz0
-  echo "ok $1 hardlink to empty files"
 
   # symlinks
   assert_symlink_has_content usr/bin/sl foo
   assert_file_has_content usr/bin/sl contents
-  echo "ok $1 symlink"
   # ostree checkout doesn't care if two symlinks are actually hardlinked
   # together (which is fine). checking that it's also a symlink is good enough.
   assert_symlink_has_content usr/local/bin/slhl foo
-  echo "ok $1 hardlink to symlink"
 
   # stat override
   test -u usr/bin/setuidme
-  echo "ok $1 setuid"
 
   # skip list
   test ! -f usr/bin/skipme
-  echo "ok $1 file skip"
 
   cd ${test_tmpdir}
-  rm -rf test-$1-checkout
 }
 
 assert_valid_checkout tar
+echo "ok tar contents"
 assert_valid_checkout cpio
+echo "ok cpio contents"
 
 cd ${test_tmpdir}
 mkdir multicommit-files
@@ -155,12 +162,59 @@ cd partial-checkout
 assert_file_has_content subdir/original "original"
 echo "ok tar partial commit contents"
 
-cd ${test_tmpdir}
-tar -cf empty.tar.gz -T /dev/null
 uid=$(id -u)
 gid=$(id -g)
-$OSTREE commit -b tar-empty --tar-autocreate-parents \
+autocreate_args="--tar-autocreate-parents --owner-uid=${uid} --owner-gid=${gid}"
-        --owner-uid=${uid} --owner-gid=${gid} --tree=tar=empty.tar.gz
+
+cd ${test_tmpdir}
+tar -cf empty.tar.gz -T /dev/null
+$OSTREE commit -b tar-empty ${autocreate_args} --tree=tar=empty.tar.gz
 $OSTREE ls tar-empty > ls.txt
 assert_file_has_content ls.txt "d00755 ${uid} ${gid}      0 /"
 echo "ok tar autocreate with owner uid/gid"
+
+# noop pathname filter
+cd ${test_tmpdir}
+$OSTREE commit -b test-tar ${autocreate_args} \
+        --tar-pathname-filter='^nosuchfile/,nootherfile/' \
+        --statoverride=statoverride.txt \
+        --skip-list=skiplist.txt \
+        --tree=tar=foo.tar.gz
+rm test-tar-co -rf
+$OSTREE checkout test-tar test-tar-co
+assert_valid_content ${test_tmpdir}/test-tar-co
+echo "ok tar pathname filter prefix (noop)"
+
+# Add a prefix
+cd ${test_tmpdir}
+# Update the metadata overrides matching our pathname filter
+for f in statoverride.txt skiplist.txt; do
+    sed -i -e 's,/usr/,/foo/usr/,' $f
+done
+$OSTREE commit -b test-tar ${autocreate_args} \
+        --tar-pathname-filter='^,foo/' \
+        --statoverride=statoverride.txt \
+        --skip-list=skiplist.txt \
+        --tree=tar=foo.tar.gz
+rm test-tar-co -rf
+$OSTREE checkout test-tar test-tar-co
+assert_has_dir test-tar-co/foo
+assert_valid_content ${test_tmpdir}/test-tar-co/foo
+echo "ok tar pathname filter prefix"
+
+# Test anchored and not-anchored
+for filter in '^usr/bin/,usr/sbin/' '/bin/,/sbin/'; do
+    cd ${test_tmpdir}
+    $OSTREE commit -b test-tar ${autocreate_args} \
+            --tar-pathname-filter=$filter \
+            --tree=tar=foo.tar.gz
+    rm test-tar-co -rf
+    $OSTREE checkout test-tar test-tar-co
+    cd test-tar-co
+    # Check that we just had usr/bin → usr/sbin
+    assert_not_has_file usr/bin/foo
+    assert_file_has_content usr/sbin/foo contents
+    assert_not_has_file usr/sbin/libfoo.so
+    assert_file_has_content usr/lib/libfoo.so 'a library'
+    echo "ok tar pathname filter modification: ${filter}"
+done