From 3d38f03e4fd4cc20f754bb787feb0d109387f4f8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 7 Feb 2017 08:59:32 -0500 Subject: [PATCH] repo: Add archive/zlib-level option, drop default compression to 6 The gzip default is 6. When I was writing this code, I chose 9 under the assumption that for long-term archival, the extra compression was worth it. Turns out level 9 is really, really not worth it. Here's run at level 9 compressing the current Fedora Atomic Host into archive: ``` ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host real 2m38.115s user 2m31.210s sys 0m3.114s 617M repo ``` And here's the new default level of 6: ``` ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host real 0m53.712s user 0m43.727s sys 0m3.601s 619M repo 619M total ``` As you can see, we run almost *three times* faster, and we take up *less than one percent* more space. Conclusion: Using level 9 is dumb. And here's a run at compression level 1: ``` ostree --repo=repo pull-local repo-build fedora-atomic/25/x86_64/docker-host real 0m24.073s user 0m17.574s sys 0m2.636s 643M repo 643M total ``` I would argue actually many people would prefer even this for "devel" repos. For production repos, you want static deltas anyways. (However, perhaps we should support a model where generating a delta involves re-compressing fallback objects with a bit stronger compression level). Anyways, let's make everyone's life better and switch the default to 6. Closes: #671 Approved by: jlebon --- src/libostree/ostree-core-private.h | 15 +++++++++ src/libostree/ostree-core.c | 48 +++++++++++++++++++---------- src/libostree/ostree-repo-commit.c | 2 +- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 13 ++++++++ 5 files changed, 61 insertions(+), 18 deletions(-) diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 0c5fb0eb..cfd8a998 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -24,6 +24,9 @@ G_BEGIN_DECLS +/* It's what gzip does, 9 is too slow */ +#define OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL (6) + /* This file contains private implementation data format definitions * read by multiple implementation .c files. */ @@ -143,4 +146,16 @@ _ostree_detached_metadata_append_gpg_sig (GVariant *existing_metadata, GFile * _ostree_get_default_sysroot_path (void); +_OSTREE_PUBLIC +gboolean +_ostree_raw_file_to_archive_stream (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + guint compression_level, + GInputStream **out_input, + GCancellable *cancellable, + GError **error); + + + G_END_DECLS diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index e3f0a771..af36d98b 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -453,6 +453,34 @@ header_and_input_to_stream (GVariant *file_header, return TRUE; } +gboolean +_ostree_raw_file_to_archive_stream (GInputStream *input, + GFileInfo *file_info, + GVariant *xattrs, + guint compression_level, + GInputStream **out_input, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariant) file_header = NULL; + g_autoptr(GInputStream) zlib_input = NULL; + + file_header = _ostree_zlib_file_header_new (file_info, xattrs); + if (input != NULL) + { + g_autoptr(GConverter) zlib_compressor = NULL; + + zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, compression_level)); + zlib_input = g_converter_input_stream_new (input, zlib_compressor); + } + return header_and_input_to_stream (file_header, + zlib_input, + out_input, + NULL, + cancellable, + error); +} + /** * ostree_raw_file_to_archive_z2_stream: * @input: File raw content stream @@ -473,23 +501,9 @@ ostree_raw_file_to_archive_z2_stream (GInputStream *input, GCancellable *cancellable, GError **error) { - g_autoptr(GVariant) file_header = NULL; - g_autoptr(GInputStream) zlib_input = NULL; - - file_header = _ostree_zlib_file_header_new (file_info, xattrs); - if (input != NULL) - { - g_autoptr(GConverter) zlib_compressor = NULL; - - zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9)); - zlib_input = g_converter_input_stream_new (input, zlib_compressor); - } - return header_and_input_to_stream (file_header, - zlib_input, - out_input, - NULL, - cancellable, - error); + return _ostree_raw_file_to_archive_stream (input, file_info, xattrs, + OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL, + out_input, cancellable, error); } /** diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index eb9733e8..07940f48 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -719,7 +719,7 @@ write_object (OstreeRepo *self, if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { - zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9); + zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, self->zlib_compression_level); compressed_out_stream = g_converter_output_stream_new (temp_out, zlib_compressor); /* Don't close the base; we'll do that later */ g_filter_output_stream_set_close_base_stream ((GFilterOutputStream*)compressed_out_stream, FALSE); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index a4e59e44..cfc178f3 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -89,6 +89,7 @@ struct OstreeRepo { GError *writable_error; gboolean in_transaction; gboolean disable_fsync; + guint zlib_compression_level; GHashTable *loose_object_devino_hash; GHashTable *updated_uncompressed_dirs; GHashTable *object_sizes; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index fd49f0fc..4ac39d11 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2083,6 +2083,19 @@ reload_core_config (OstreeRepo *self, self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10); } + { g_autofree char *compression_level_str = NULL; + + /* gzip defaults to 6 */ + (void)ot_keyfile_get_value_with_default (self->config, "archive", "zlib-level", NULL, + &compression_level_str, NULL); + + if (compression_level_str) + /* Ensure level is in [1,9] */ + self->zlib_compression_level = MAX (1, MIN (9, g_ascii_strtoull (compression_level_str, NULL, 10))); + else + self->zlib_compression_level = OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL; + } + if (!ot_keyfile_get_value_with_default (self->config, "core", "parent", NULL, &parent_repo_path, error)) return FALSE;