From 569e43c280d6b09b5a683c5b4ea41a448f1f7ce2 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 23 May 2016 13:28:04 +0200 Subject: [PATCH] core: Add a function creating an archive-z2 content stream It is quite similar to the already existing ostree_raw_file_to_content_stream function, so I factored the common part to a separate function. The difference is that we cannot report the size of the resulting stream. Can be useful for serving a "bare" repository as a faked "archive-z2" repository. Closes: #308 Approved by: cgwalters --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 2 +- src/libostree/ostree-core.c | 134 +++++++++++++++++++++++++--------- src/libostree/ostree-core.h | 9 +++ tests/test-basic-c.c | 138 ++++++++++++++++++++++++++++++++++++ 5 files changed, 250 insertions(+), 34 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index b9b292c7..105783f4 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -114,6 +114,7 @@ ostree_object_from_string ostree_content_stream_parse ostree_content_file_parse ostree_content_file_parse_at +ostree_raw_file_to_archive_z2_stream ostree_raw_file_to_content_stream ostree_checksum_file_from_input ostree_checksum_file diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 2aa72512..71f4bc9b 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -341,5 +341,5 @@ global: LIBOSTREE_2016.6 { global: ostree_repo_remote_fetch_summary_with_options; - + ostree_raw_file_to_archive_z2_stream; } LIBOSTREE_2016.5; diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 00f767b3..d393c496 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -400,6 +400,98 @@ write_file_header_update_checksum (GOutputStream *out, return ret; } +/* + * header_and_input_to_stream: + * @file_header: A file header + * @input: File raw content stream + * @out_input: (out): Serialized object stream + * @out_header_size: (out): Length of the header + * @cancellable: Cancellable + * @error: Error + * + * Combines @file_header and @input into a single stream. + */ +static gboolean +header_and_input_to_stream (GVariant *file_header, + GInputStream *input, + GInputStream **out_input, + guint64 *out_header_size, + GCancellable *cancellable, + GError **error) +{ + gpointer header_data; + gsize header_size; + g_autoptr(GInputStream) ret_input = NULL; + g_autoptr(GPtrArray) streams = NULL; + g_autoptr(GOutputStream) header_out_stream = NULL; + g_autoptr(GInputStream) header_in_stream = NULL; + + header_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free); + + if (!_ostree_write_variant_with_size (header_out_stream, file_header, 0, NULL, NULL, + cancellable, error)) + return FALSE; + + if (!g_output_stream_close (header_out_stream, cancellable, error)) + return FALSE; + + header_size = g_memory_output_stream_get_data_size ((GMemoryOutputStream*) header_out_stream); + header_data = g_memory_output_stream_steal_data ((GMemoryOutputStream*) header_out_stream); + header_in_stream = g_memory_input_stream_new_from_data (header_data, header_size, g_free); + + streams = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); + + g_ptr_array_add (streams, g_object_ref (header_in_stream)); + if (input) + g_ptr_array_add (streams, g_object_ref (input)); + + ret_input = (GInputStream*)ostree_chain_input_stream_new (streams); + ot_transfer_out_value (out_input, &ret_input); + if (out_header_size) + *out_header_size = header_size; + + return TRUE; +} + +/** + * ostree_raw_file_to_archive_z2_stream: + * @input: File raw content stream + * @file_info: A file info + * @xattrs: (allow-none): Optional extended attributes + * @out_input: (out): Serialized object stream + * @cancellable: Cancellable + * @error: Error + * + * Convert from a "bare" file representation into an + * OSTREE_OBJECT_TYPE_FILE stream suitable for ostree pull. + */ +gboolean +ostree_raw_file_to_archive_z2_stream (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + GInputStream **out_input, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariant) file_header = NULL; + g_autoptr(GInputStream) zlib_input = NULL; + + file_header = _ostree_zlib_file_header_new (file_info, xattrs); + if (input != NULL) + { + g_autoptr(GConverter) zlib_compressor = NULL; + + zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9)); + zlib_input = g_converter_input_stream_new (input, zlib_compressor); + } + return header_and_input_to_stream (file_header, + zlib_input, + out_input, + NULL, + cancellable, + error); +} + /** * ostree_raw_file_to_content_stream: * @input: File raw content stream @@ -423,44 +515,20 @@ ostree_raw_file_to_content_stream (GInputStream *input, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - gpointer header_data; - gsize header_size; - g_autoptr(GInputStream) ret_input = NULL; g_autoptr(GVariant) file_header = NULL; - g_autoptr(GPtrArray) streams = NULL; - g_autoptr(GOutputStream) header_out_stream = NULL; - g_autoptr(GInputStream) header_in_stream = NULL; + guint64 header_size; file_header = file_header_new (file_info, xattrs); - - header_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free); - - if (!_ostree_write_variant_with_size (header_out_stream, file_header, 0, NULL, NULL, - cancellable, error)) - goto out; - - if (!g_output_stream_close (header_out_stream, cancellable, error)) - goto out; - - header_size = g_memory_output_stream_get_data_size ((GMemoryOutputStream*) header_out_stream); - header_data = g_memory_output_stream_steal_data ((GMemoryOutputStream*) header_out_stream); - header_in_stream = g_memory_input_stream_new_from_data (header_data, header_size, g_free); - - streams = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - - g_ptr_array_add (streams, g_object_ref (header_in_stream)); - if (input) - g_ptr_array_add (streams, g_object_ref (input)); - - ret_input = (GInputStream*)ostree_chain_input_stream_new (streams); - - ret = TRUE; - ot_transfer_out_value (out_input, &ret_input); + if (!header_and_input_to_stream (file_header, + input, + out_input, + &header_size, + cancellable, + error)) + return FALSE; if (out_length) *out_length = header_size + g_file_info_get_size (file_info); - out: - return ret; + return TRUE; } /** diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 29b5a1c0..4d788b20 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -274,6 +274,15 @@ gboolean ostree_content_file_parse_at (gboolean compressed, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean +ostree_raw_file_to_archive_z2_stream (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + GInputStream **out_input, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_raw_file_to_content_stream (GInputStream *input, GFileInfo *file_info, diff --git a/tests/test-basic-c.c b/tests/test-basic-c.c index d5dcc811..447c46ea 100644 --- a/tests/test-basic-c.c +++ b/tests/test-basic-c.c @@ -34,6 +34,143 @@ test_repo_is_not_system (gconstpointer data) g_assert (!ostree_repo_is_system (repo)); } +static GBytes * +input_stream_to_bytes (GInputStream *input) +{ + g_autoptr(GOutputStream) mem_out_stream = NULL; + g_autoptr(GError) error = NULL; + + if (input == NULL) + return g_bytes_new (NULL, 0); + + mem_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free); + g_output_stream_splice (mem_out_stream, + input, + G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, + NULL, + &error); + g_assert_no_error (error); + + return g_memory_output_stream_steal_as_bytes (G_MEMORY_OUTPUT_STREAM (mem_out_stream)); +} + +static void +test_raw_file_to_archive_z2_stream (gconstpointer data) +{ + OstreeRepo *repo = OSTREE_REPO (data); + g_autofree gchar *commit_checksum = NULL; + g_autoptr(GHashTable) reachable = NULL; + g_autoptr(GError) error = NULL; + /* branch name of the test repository, see setup_test_repository in libtest.sh */ + const gchar *rev = "test2"; + GHashTableIter iter; + GVariant *serialized_object; + guint checks = 0; + + ostree_repo_resolve_rev (repo, + rev, + FALSE, + &commit_checksum, + &error); + g_assert_no_error (error); + reachable = ostree_repo_traverse_new_reachable (); + ostree_repo_traverse_commit (repo, + commit_checksum, + -1, + &reachable, + NULL, + &error); + g_assert_no_error (error); + g_hash_table_iter_init (&iter, reachable); + while (g_hash_table_iter_next (&iter, (gpointer*)&serialized_object, NULL)) + { + const gchar *object_checksum; + OstreeObjectType object_type; + g_autoptr(GInputStream) input = NULL; + g_autoptr(GFileInfo) info = NULL; + g_autoptr(GVariant) xattrs = NULL; + g_autoptr(GBytes) input_bytes = NULL; + g_autoptr(GInputStream) mem_input = NULL; + g_autoptr(GInputStream) zlib_stream = NULL; + g_autoptr(GBytes) zlib_bytes = NULL; + g_autoptr(GInputStream) mem_zlib = NULL; + g_autoptr(GInputStream) input2 = NULL; + g_autoptr(GFileInfo) info2 = NULL; + g_autoptr(GVariant) xattrs2 = NULL; + g_autoptr(GBytes) input2_bytes = NULL; + + ostree_object_name_deserialize (serialized_object, &object_checksum, &object_type); + if (object_type != OSTREE_OBJECT_TYPE_FILE) + continue; + + ostree_repo_load_file (repo, + object_checksum, + &input, + &info, + &xattrs, + NULL, + &error); + g_assert_no_error (error); + + input_bytes = input_stream_to_bytes (input); + /* This is to simulate NULL input received from + * ostree_repo_load_file. Instead of creating the mem_input + * variable, I could also rewind the input stream and pass it to + * the function below, but this would assume that the input + * stream implements either the GSeekable or + * GFileDescriptorBased interface. */ + if (input != NULL) + mem_input = g_memory_input_stream_new_from_bytes (input_bytes); + ostree_raw_file_to_archive_z2_stream (mem_input, + info, + xattrs, + &zlib_stream, + NULL, + &error); + g_assert_no_error (error); + + zlib_bytes = input_stream_to_bytes (zlib_stream); + mem_zlib = g_memory_input_stream_new_from_bytes (zlib_bytes); + ostree_content_stream_parse (FALSE, + mem_zlib, + g_bytes_get_size (zlib_bytes), + FALSE, + &input2, + &info2, + &xattrs2, + NULL, + &error); + g_assert_error (error, G_IO_ERROR, G_IO_ERROR_FAILED); + g_clear_error (&error); + + g_seekable_seek (G_SEEKABLE (mem_zlib), + 0, + G_SEEK_SET, + NULL, + &error); + g_assert_no_error (error); + + ostree_content_stream_parse (TRUE, + mem_zlib, + g_bytes_get_size (zlib_bytes), + FALSE, + &input2, + &info2, + &xattrs2, + NULL, + &error); + g_assert_no_error (error); + + input2_bytes = input_stream_to_bytes (input2); + g_assert_true (g_bytes_equal (input_bytes, input2_bytes)); + g_assert_true (g_variant_equal (xattrs, xattrs2)); + /* TODO: Not sure how to compare fileinfos */ + ++checks; + } + /* to make sure we really tested the function */ + g_assert_cmpint (checks, >, 0); +} + int main (int argc, char **argv) { g_autoptr(GError) error = NULL; @@ -46,6 +183,7 @@ int main (int argc, char **argv) goto out; g_test_add_data_func ("/repo-not-system", repo, test_repo_is_not_system); + g_test_add_data_func ("/raw-file-to-archive-z2-stream", repo, test_raw_file_to_archive_z2_stream); return g_test_run(); out: