From 3a555114bc22f599d72d95d71c2579fd2345cf6f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 18 Feb 2016 16:49:30 -0500 Subject: [PATCH] repo: Add ostree_repo_import_archive_to_mtree This is a more flexible version of the previous ostree_repo_write_archive_to_mtree() which took a file reference. This has an extensible options structure, and in particular now supports `ignore_unsupported_content`. I plan to use this for importing Docker images which contain device nodes. (There's no reason for container images to have those, so we'll just ignore them). Also here, just like the export variant, the caller is responsible for setting up libarchive. --- Makefile-tests.am | 8 + src/libostree/ostree-repo-libarchive.c | 143 +++++++++++++----- src/libostree/ostree-repo.h | 23 +++ tests/test-libarchive-import.c | 200 +++++++++++++++++++++++++ 4 files changed, 334 insertions(+), 40 deletions(-) create mode 100644 tests/test-libarchive-import.c diff --git a/Makefile-tests.am b/Makefile-tests.am index 292699ef..475519d7 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -140,6 +140,10 @@ TESTS = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test- tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \ tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum +if USE_LIBARCHIVE +TESTS += tests/test-libarchive-import +endif + check_PROGRAMS = $(TESTS) TESTS_ENVIRONMENT = \ G_TEST_SRCDIR=$(abs_srcdir)/tests \ @@ -172,6 +176,10 @@ tests_test_checksum_SOURCES = src/libostree/ostree-core.c tests/test-checksum.c tests_test_checksum_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) tests_test_checksum_LDADD = $(TESTS_LDADD) +tests_test_libarchive_import_SOURCES = tests/test-libarchive-import.c +tests_test_libarchive_import_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) +tests_test_libarchive_import_LDADD = $(TESTS_LDADD) + tests_test_keyfile_utils_CFLAGS = $(TESTS_CFLAGS) tests_test_keyfile_utils_LDADD = $(TESTS_LDADD) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 19fae757..64410cb2 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -78,6 +78,7 @@ file_info_from_archive_entry_and_modifier (OstreeRepo *repo, static gboolean import_libarchive_entry_file (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, struct archive *a, struct archive_entry *entry, GFileInfo *file_info, @@ -93,8 +94,27 @@ import_libarchive_entry_file (OstreeRepo *self, if (g_cancellable_set_error_if_cancelled (cancellable, error)) return FALSE; - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) - archive_stream = _ostree_libarchive_input_stream_new (a); + switch (g_file_info_get_file_type (file_info)) + { + case G_FILE_TYPE_REGULAR: + archive_stream = _ostree_libarchive_input_stream_new (a); + break; + case G_FILE_TYPE_SYMBOLIC_LINK: + break; + default: + if (opts->ignore_unsupported_content) + { + ret = TRUE; + goto out; + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unable to import non-regular/non-symlink file '%s'", + archive_entry_pathname (entry)); + goto out; + } + } if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL, &file_object_input, &length, cancellable, error)) @@ -111,6 +131,7 @@ import_libarchive_entry_file (OstreeRepo *self, static gboolean write_libarchive_entry_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, OstreeMutableTree *root, struct archive *a, struct archive_entry *entry, @@ -249,16 +270,19 @@ write_libarchive_entry_to_mtree (OstreeRepo *self, goto out; } - if (!import_libarchive_entry_file (self, a, entry, file_info, &tmp_csum, + if (!import_libarchive_entry_file (self, opts, a, entry, file_info, &tmp_csum, cancellable, error)) goto out; - - g_free (tmp_checksum); - tmp_checksum = ostree_checksum_from_bytes (tmp_csum); - if (!ostree_mutable_tree_replace_file (parent, basename, - tmp_checksum, - error)) - goto out; + + if (tmp_csum) + { + g_free (tmp_checksum); + tmp_checksum = ostree_checksum_from_bytes (tmp_csum); + if (!ostree_mutable_tree_replace_file (parent, basename, + tmp_checksum, + error)) + goto out; + } } } @@ -267,6 +291,71 @@ write_libarchive_entry_to_mtree (OstreeRepo *self, return ret; } #endif + +/** + * ostree_repo_import_archive_to_mtree: + * @self: An #OstreeRepo + * @opts: Options structure, ensure this is zeroed, then set specific variables + * @archive: Really this is "struct archive*" + * @mtree: The #OstreeMutableTree to write to + * @modifier: (allow-none): Optional commit modifier + * @cancellable: Cancellable + * @error: Error + * + * Import an archive file @archive into the repository, and write its + * file structure to @mtree. + */ +gboolean +ostree_repo_import_archive_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, + void *archive, + OstreeMutableTree *mtree, + OstreeRepoCommitModifier *modifier, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + struct archive *a = archive; + struct archive_entry *entry; + g_autofree guchar *tmp_csum = NULL; + int r; + + while (TRUE) + { + r = archive_read_next_header (a, &entry); + if (r == ARCHIVE_EOF) + break; + else if (r != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + /* TODO - refactor this to only create the metadata on demand + * (i.e. if there is a missing parent dir) + */ + if (opts->autocreate_parents && !tmp_csum) + { + g_autoptr(GFileInfo) tmp_dir_info = g_file_info_new (); + + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry)); + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry)); + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); + + if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error)) + goto out; + } + + if (!write_libarchive_entry_to_mtree (self, opts, mtree, a, + entry, modifier, tmp_csum, + cancellable, error)) + goto out; + } + + ret = TRUE; + out: + return ret; +} /** * ostree_repo_write_archive_to_mtree: @@ -293,10 +382,8 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, #ifdef HAVE_LIBARCHIVE gboolean ret = FALSE; struct archive *a = NULL; - struct archive_entry *entry; - int r; g_autoptr(GFileInfo) tmp_dir_info = NULL; - g_autofree guchar *tmp_csum = NULL; + OstreeRepoImportArchiveOptions opts = { 0, }; a = archive_read_new (); #ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL @@ -311,35 +398,11 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, goto out; } - while (TRUE) - { - r = archive_read_next_header (a, &entry); - if (r == ARCHIVE_EOF) - break; - else if (r != ARCHIVE_OK) - { - propagate_libarchive_error (error, a); - goto out; - } + opts.autocreate_parents = !!autocreate_parents; - if (autocreate_parents && !tmp_csum) - { - tmp_dir_info = g_file_info_new (); - - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry)); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry)); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); - - if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error)) - goto out; - } + if (!ostree_repo_import_archive_to_mtree (self, &opts, a, mtree, modifier, cancellable, error)) + goto out; - if (!write_libarchive_entry_to_mtree (self, mtree, a, - entry, modifier, - autocreate_parents ? tmp_csum : NULL, - cancellable, error)) - goto out; - } if (archive_read_close (a) != ARCHIVE_OK) { propagate_libarchive_error (error, a); diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 3b1040ab..4629ea55 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -450,6 +450,29 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo * GCancellable *cancellable, GError **error); +/** + * OstreeRepoImportArchiveOptions: + * + * An extensible options structure controlling archive import. Ensure that + * you have entirely zeroed the structure, then set just the desired + * options. This is used by ostree_repo_import_archive_to_mtree(). + */ +typedef struct { + guint ignore_unsupported_content : 1; + guint autocreate_parents : 1; + guint reserved : 30; + + guint unused_uint[8]; + gpointer unused_ptrs[8]; +} OstreeRepoImportArchiveOptions; + +gboolean ostree_repo_import_archive_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, + void *archive, /* Really struct archive * */ + OstreeMutableTree *mtree, + OstreeRepoCommitModifier *modifier, + GCancellable *cancellable, + GError **error); /** * OstreeRepoExportArchiveOptions: * diff --git a/tests/test-libarchive-import.c b/tests/test-libarchive-import.c new file mode 100644 index 00000000..1dd0c68c --- /dev/null +++ b/tests/test-libarchive-import.c @@ -0,0 +1,200 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" +#include "libglnx.h" +#include +#include +#include +#include + +#include +#include +#include + +typedef struct { + OstreeRepo *repo; + int fd; + char *tmpd; +} TestData; + +static void +test_data_init (TestData *td) +{ + GError *error = NULL; + struct archive *a = archive_write_new (); + struct archive_entry *ae; + + td->tmpd = g_mkdtemp (g_strdup ("/var/tmp/test-libarchive-import-XXXXXX")); + g_assert_cmpint (0, ==, chdir (td->tmpd)); + + td->fd = openat (AT_FDCWD, "foo.tar.gz", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0644); + (void) unlink ("foo.tar.gz"); + + g_assert_no_error (error); + g_assert (td->fd >= 0); + + g_assert_cmpint (0, ==, archive_write_set_format_pax (a)); + g_assert_cmpint (0, ==, archive_write_add_filter_gzip (a)); + g_assert_cmpint (0, ==, archive_write_open_fd (a, td->fd)); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/"); + archive_entry_set_mode (ae, S_IFDIR | 0755); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/file"); + archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_size (ae, 4); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + g_assert_cmpint (4, ==, archive_write_data (a, "foo\n", 4)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/devnull"); + archive_entry_set_mode (ae, S_IFCHR | 0777); + archive_entry_set_devmajor (ae, 1); + archive_entry_set_devminor (ae, 3); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/anotherfile"); + archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_size (ae, 4); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + g_assert_cmpint (4, ==, archive_write_data (a, "bar\n", 4)); + archive_entry_free (ae); + + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_close (a)); + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_free (a)); + + { g_autoptr(GFile) repopath = g_file_new_for_path ("repo"); + td->repo = ostree_repo_new (repopath); + + g_assert_cmpint (0, ==, mkdir ("repo", 0755)); + + ostree_repo_create (td->repo, OSTREE_REPO_MODE_BARE_USER, NULL, &error); + g_assert_no_error (error); + } +} + +static gboolean +spawn_cmdline (const char *cmd, GError **error) +{ + int estatus; + if (!g_spawn_command_line_sync (cmd, NULL, NULL, &estatus, error)) + return FALSE; + if (!g_spawn_check_exit_status (estatus, error)) + return FALSE; + return TRUE; +} + +static void +test_libarchive_error_device_file (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + + g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192)); + + (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); + g_assert (error != NULL); +} + +static void +test_libarchive_ignore_device_file (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + GCancellable *cancellable = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + glnx_unref_object GFile *root = NULL; + g_autofree char *commit_checksum = NULL; + + g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192)); + + opts.ignore_unsupported_content = TRUE; + + if (!ostree_repo_prepare_transaction (td->repo, NULL, cancellable, &error)) + goto out; + + if (!ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error)) + goto out; + + if (!ostree_repo_write_mtree (td->repo, mtree, &root, cancellable, &error)) + goto out; + + if (!ostree_repo_write_commit (td->repo, NULL, "", "", NULL, + OSTREE_REPO_FILE (root), + &commit_checksum, cancellable, &error)) + goto out; + + ostree_repo_transaction_set_ref (td->repo, NULL, "foo", commit_checksum); + + if (!ostree_repo_commit_transaction (td->repo, NULL, cancellable, &error)) + goto out; + + if (!spawn_cmdline ("ostree --repo=repo ls foo file", &error)) + goto out; + + if (!spawn_cmdline ("ostree --repo=repo ls foo anotherfile", &error)) + goto out; + + if (spawn_cmdline ("ostree --repo=repo ls foo devnull", &error)) + g_assert_not_reached (); + g_assert (error != NULL); + g_clear_error (&error); + + out: + g_assert_no_error (error); +} + +int main (int argc, char **argv) +{ + TestData td = {NULL,}; + int r; + + test_data_init (&td); + + g_test_init (&argc, &argv, NULL); + + g_test_add_data_func ("/libarchive/error-device-file", &td, test_libarchive_error_device_file); + g_test_add_data_func ("/libarchive/ignore-device-file", &td, test_libarchive_ignore_device_file); + + r = g_test_run(); + + if (td.tmpd) + (void) glnx_shutil_rm_rf_at (AT_FDCWD, td.tmpd, NULL, NULL); + return r; +}