From 2a0601efc790a0c8f783043f2db682eec9ceffaa Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 23 Sep 2012 15:16:33 -0400 Subject: [PATCH] core: Drop packfiles as they are now They're not a large efficiency win at the moment, because we don't do any delta compression. At the moment, they simply served to compress data, but we will change the archive mode to do that by default. --- Makefile-ostree.am | 2 - src/libostree/ostree-core.c | 332 ------- src/libostree/ostree-core.h | 79 -- src/libostree/ostree-repo.c | 1289 +--------------------------- src/libostree/ostree-repo.h | 88 -- src/ostree/main.c | 2 - src/ostree/ostree-pull.c | 508 +---------- src/ostree/ot-builtin-fsck.c | 126 --- src/ostree/ot-builtin-init.c | 5 +- src/ostree/ot-builtin-pack.c | 967 --------------------- src/ostree/ot-builtin-pull-local.c | 4 +- src/ostree/ot-builtin-unpack.c | 266 ------ src/ostree/ot-builtins.h | 2 - tests/t0001-archive.sh | 41 +- tests/t0010-pull.sh | 20 +- 15 files changed, 37 insertions(+), 3694 deletions(-) delete mode 100644 src/ostree/ot-builtin-pack.c delete mode 100644 src/ostree/ot-builtin-unpack.c diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 59b4775e..ce4e40c6 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -37,8 +37,6 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-ls.c \ src/ostree/ot-builtin-prune.c \ src/ostree/ot-builtin-remote.c \ - src/ostree/ot-builtin-pack.c \ - src/ostree/ot-builtin-unpack.c \ src/ostree/ot-builtin-rev-parse.c \ src/ostree/ot-builtin-show.c \ src/ostree/ot-builtin-write-refs.c \ diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 39a6531d..03b8b4e4 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -944,58 +944,6 @@ ostree_get_relative_archive_content_path (const char *checksum) return g_string_free (path, FALSE); } -static char * -get_pack_name (gboolean is_meta, - gboolean is_index, - const char *prefix, - const char *checksum) -{ - GString *path; - - g_assert (strlen (checksum) == 64); - - path = g_string_new (prefix); - if (is_meta) - g_string_append (path, "ostmetapack-"); - else - g_string_append (path, "ostdatapack-"); - g_string_append (path, checksum); - if (is_index) - g_string_append (path, ".index"); - else - g_string_append (path, ".data"); - - return g_string_free (path, FALSE); -} - -char * -ostree_get_pack_index_name (gboolean is_meta, - const char *checksum) -{ - return get_pack_name (is_meta, TRUE, "", checksum); -} - -char * -ostree_get_pack_data_name (gboolean is_meta, - const char *checksum) -{ - return get_pack_name (is_meta, FALSE, "", checksum); -} - -char * -ostree_get_relative_pack_index_path (gboolean is_meta, - const char *checksum) -{ - return get_pack_name (is_meta, TRUE, "objects/pack/", checksum); -} - -char * -ostree_get_relative_pack_data_path (gboolean is_meta, - const char *checksum) -{ - return get_pack_name (is_meta, FALSE, "objects/pack/", checksum); -} - gboolean ostree_file_header_parse (GVariant *metadata, GFileInfo **out_file_info, @@ -1343,194 +1291,6 @@ ostree_create_temp_dir (GFile *dir, return ret; } -gboolean -ostree_read_pack_entry_raw (guchar *pack_data, - guint64 pack_len, - guint64 offset, - gboolean trusted, - gboolean is_meta, - GVariant **out_entry, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint64 entry_start; - guint64 entry_end; - guint32 entry_len; - ot_lvariant GVariant *ret_entry = NULL; - - if (G_UNLIKELY (!(offset <= pack_len))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted pack index; out of range offset %" G_GUINT64_FORMAT, - offset); - goto out; - } - if (G_UNLIKELY (!((offset & 0x3) == 0))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT, - offset); - goto out; - } - - entry_start = ALIGN_VALUE (offset + 4, 8); - if (G_UNLIKELY (!(entry_start <= pack_len))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT, - entry_start); - goto out; - } - - g_assert ((((guint64)pack_data+offset) & 0x3) == 0); - entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset))); - - entry_end = entry_start + entry_len; - if (G_UNLIKELY (!(entry_end <= pack_len))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted pack index; out of range entry length %u", - entry_len); - goto out; - } - - ret_entry = g_variant_new_from_data (is_meta ? OSTREE_PACK_META_FILE_VARIANT_FORMAT : - OSTREE_PACK_DATA_FILE_VARIANT_FORMAT, - pack_data+entry_start, entry_len, - trusted, NULL, NULL); - g_variant_ref_sink (ret_entry); - ret = TRUE; - ot_transfer_out_value (out_entry, &ret_entry); - out: - return ret; -} - -gboolean -ostree_parse_file_pack_entry (GVariant *pack_entry, - GInputStream **out_input, - GFileInfo **out_info, - GVariant **out_xattrs, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guchar entry_flags; - ot_lvariant GVariant *file_header = NULL; - ot_lvariant GVariant *pack_data = NULL; - ot_lobj GInputStream *memory_input = NULL; - ot_lobj GInputStream *ret_input = NULL; - ot_lobj GFileInfo *ret_info = NULL; - ot_lvariant GVariant *ret_xattrs = NULL; - - g_variant_get_child (pack_entry, 1, "y", &entry_flags); - g_variant_get_child (pack_entry, 2, "@(uuuusa(ayay))", &file_header); - g_variant_get_child (pack_entry, 3, "@ay", &pack_data); - - if (!ostree_file_header_parse (file_header, &ret_info, &ret_xattrs, - error)) - goto out; - g_file_info_set_size (ret_info, g_variant_get_size (pack_data)); - - if (g_file_info_get_file_type (ret_info) == G_FILE_TYPE_REGULAR) - { - memory_input = ot_variant_read (pack_data); - - if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP) - { - ot_lobj GConverter *decompressor = NULL; - - decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP); - ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM, - "converter", decompressor, - "base-stream", memory_input, - "close-base-stream", TRUE, - NULL); - } - else - { - ret_input = g_object_ref (memory_input); - } - } - - ret = TRUE; - ot_transfer_out_value (out_input, &ret_input); - ot_transfer_out_value (out_info, &ret_info); - ot_transfer_out_value (out_xattrs, &ret_xattrs); - out: - return ret; -} - -gboolean -ostree_pack_index_search (GVariant *index, - GVariant *csum_v, - OstreeObjectType objtype, - guint64 *out_offset) -{ - gboolean ret = FALSE; - gsize imax, imin; - gsize n; - guint32 target_objtype; - const guchar *csum; - ot_lvariant GVariant *index_contents = NULL; - - csum = ostree_checksum_bytes_peek (csum_v); - - index_contents = g_variant_get_child_value (index, 2); - - target_objtype = (guint32) objtype; - - n = g_variant_n_children (index_contents); - - if (n == 0) - goto out; - - imax = n - 1; - imin = 0; - while (imax >= imin) - { - GVariant *cur_csum_bytes; - guchar cur_objtype; - guint64 cur_offset; - gsize imid; - int c; - - imid = (imin + imax) / 2; - - g_variant_get_child (index_contents, imid, "(y@ayt)", &cur_objtype, - &cur_csum_bytes, &cur_offset); - - c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum); - if (c == 0) - { - if (cur_objtype < target_objtype) - c = -1; - else if (cur_objtype > target_objtype) - c = 1; - } - g_variant_unref (cur_csum_bytes); - - if (c < 0) - imin = imid + 1; - else if (c > 0) - { - if (imid == 0) - goto out; - imax = imid - 1; - } - else - { - if (out_offset) - *out_offset = GUINT64_FROM_BE (cur_offset); - ret = TRUE; - goto out; - } - } - - out: - return ret; -} - gboolean ostree_validate_structureof_objtype (guchar objtype, GError **error) @@ -1766,95 +1526,3 @@ ostree_validate_structureof_dirmeta (GVariant *dirmeta, out: return ret; } - -gboolean -ostree_validate_structureof_pack_index (GVariant *index, - GError **error) -{ - gboolean ret = FALSE; - const char *header; - guchar objtype_u8; - guint64 offset; - ot_lvariant GVariant *csum_v = NULL; - GVariantIter *content_iter = NULL; - - if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error)) - goto out; - - g_variant_get_child (index, 0, "&s", &header); - - if (strcmp (header, "OSTv0PACKINDEX") != 0) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid pack index; doesn't match header"); - goto out; - } - - g_variant_get_child (index, 2, "a(yayt)", &content_iter); - - while (g_variant_iter_loop (content_iter, "(y@ayt)", - &objtype_u8, &csum_v, &offset)) - { - if (!ostree_validate_structureof_objtype (objtype_u8, error)) - goto out; - if (!ostree_validate_structureof_csum_v (csum_v, error)) - goto out; - } - csum_v = NULL; - - ret = TRUE; - out: - if (content_iter) - g_variant_iter_free (content_iter); - return ret; -} - -gboolean -ostree_validate_structureof_pack_superindex (GVariant *superindex, - GError **error) -{ - gboolean ret = FALSE; - const char *header; - ot_lvariant GVariant *csum_v = NULL; - ot_lvariant GVariant *bloom = NULL; - GVariantIter *content_iter = NULL; - - if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error)) - goto out; - - g_variant_get_child (superindex, 0, "&s", &header); - - if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid pack superindex; doesn't match header"); - goto out; - } - - g_variant_get_child (superindex, 2, "a(ayay)", &content_iter); - while (g_variant_iter_loop (content_iter, "(@ay@ay)", - &csum_v, &bloom)) - { - if (!ostree_validate_structureof_csum_v (csum_v, error)) - goto out; - } - csum_v = NULL; - bloom = NULL; - - g_variant_iter_free (content_iter); - g_variant_get_child (superindex, 3, "a(ayay)", &content_iter); - while (g_variant_iter_loop (content_iter, "(@ay@ay)", - &csum_v, &bloom)) - { - if (!ostree_validate_structureof_csum_v (csum_v, error)) - goto out; - } - csum_v = NULL; - bloom = NULL; - - ret = TRUE; - out: - if (content_iter) - g_variant_iter_free (content_iter); - return ret; -} diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 3617b01e..5b038180 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -87,48 +87,6 @@ typedef enum { */ #define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE ("(a{sv}aya(say)sstayay)") -/* Pack super index - * s - OSTv0SUPERPACKINDEX - * a{sv} - Metadata - * a(ayay) - metadata packs (pack file checksum, bloom filter) - * a(ayay) - data packs (pack file checksum, bloom filter) - */ -#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay)a(ayay))") - -/* Pack index - * s - OSTv0PACKINDEX - * a{sv} - Metadata - * a(yayt) - (objtype, checksum, offset into packfile) - */ -#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(yayt))") - -typedef enum { - OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0, - OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0) -} OstreePackFileEntryFlag; - -/* Data Pack files - * s - OSTv0PACKDATAFILE - * a{sv} - Metadata - * t - number of entries - * - * Repeating pair of: - * - * ( ayy(uuuusa(ayay))ay) ) - checksum, flags, file meta, data - */ -#define OSTREE_PACK_DATA_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(ayy(uuuusa(ayay))ay)") - -/* Meta Pack files - * s - OSTv0PACKMETAFILE - * a{sv} - Metadata - * t - number of entries - * - * Repeating pair of: - * - * ( yayv ) - objtype, checksum, data - */ -#define OSTREE_PACK_META_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yayv)") - const GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype); gboolean ostree_validate_checksum_string (const char *sha256, @@ -173,16 +131,6 @@ char *ostree_get_relative_object_path (const char *checksum, char *ostree_get_relative_archive_content_path (const char *checksum); -char *ostree_get_pack_index_name (gboolean is_meta, - const char *checksum); -char *ostree_get_pack_data_name (gboolean is_meta, - const char *checksum); - -char *ostree_get_relative_pack_index_path (gboolean is_meta, - const char *checksum); -char *ostree_get_relative_pack_data_path (gboolean is_meta, - const char *checksum); - gboolean ostree_get_xattrs_for_file (GFile *f, GVariant **out_xattrs, GCancellable *cancellable, @@ -304,27 +252,6 @@ gboolean ostree_create_temp_dir (GFile *dir, GCancellable *cancellable, GError **error); -gboolean ostree_read_pack_entry_raw (guchar *pack_data, - guint64 pack_len, - guint64 object_offset, - gboolean trusted, - gboolean is_meta, - GVariant **out_entry, - GCancellable *cancellable, - GError **error); - -gboolean ostree_parse_file_pack_entry (GVariant *pack_entry, - GInputStream **out_input, - GFileInfo **out_info, - GVariant **out_xattrs, - GCancellable *cancellable, - GError **error); - -gboolean ostree_pack_index_search (GVariant *index, - GVariant *csum_bytes, - OstreeObjectType objtype, - guint64 *out_offset); - /** VALIDATION **/ gboolean ostree_validate_structureof_objtype (guchar objtype, @@ -348,10 +275,4 @@ gboolean ostree_validate_structureof_dirtree (GVariant *index, gboolean ostree_validate_structureof_dirmeta (GVariant *index, GError **error); -gboolean ostree_validate_structureof_pack_index (GVariant *index, - GError **error); - -gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex, - GError **error); - #endif /* _OSTREE_REPO */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 9bd44f17..05041b45 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -50,7 +50,6 @@ struct OstreeRepo { GFile *local_heads_dir; GFile *remote_heads_dir; GFile *objects_dir; - GFile *pack_dir; GFile *remote_cache_dir; GFile *config_file; @@ -61,8 +60,6 @@ struct OstreeRepo { #endif GPtrArray *cached_meta_indexes; GPtrArray *cached_content_indexes; - GHashTable *cached_pack_index_mappings; - GHashTable *cached_pack_data_mappings; gboolean inited; gboolean in_transaction; @@ -82,10 +79,7 @@ static gboolean repo_find_object (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, - gboolean lookup_all, GFile **out_stored_path, - char **out_pack_checksum, - guint64 *out_pack_offset, GCancellable *cancellable, GError **error); @@ -110,7 +104,6 @@ ostree_repo_finalize (GObject *object) g_clear_object (&self->local_heads_dir); g_clear_object (&self->remote_heads_dir); g_clear_object (&self->objects_dir); - g_clear_object (&self->pack_dir); g_clear_object (&self->remote_cache_dir); g_clear_object (&self->config_file); if (self->loose_object_devino_hash) @@ -119,8 +112,6 @@ ostree_repo_finalize (GObject *object) g_key_file_free (self->config); g_clear_pointer (&self->cached_meta_indexes, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&self->cached_content_indexes, (GDestroyNotify) g_ptr_array_unref); - g_hash_table_destroy (self->cached_pack_index_mappings); - g_hash_table_destroy (self->cached_pack_data_mappings); g_mutex_clear (&self->cache_lock); G_OBJECT_CLASS (ostree_repo_parent_class)->finalize (object); @@ -186,7 +177,6 @@ ostree_repo_constructor (GType gtype, self->remote_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/remotes"); self->objects_dir = g_file_get_child (self->repodir, "objects"); - self->pack_dir = g_file_get_child (self->objects_dir, "pack"); self->remote_cache_dir = g_file_get_child (self->repodir, "remote-cache"); self->config_file = g_file_get_child (self->repodir, "config"); @@ -216,12 +206,6 @@ static void ostree_repo_init (OstreeRepo *self) { g_mutex_init (&self->cache_lock); - self->cached_pack_index_mappings = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, - (GDestroyNotify)g_variant_unref); - self->cached_pack_data_mappings = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, - (GDestroyNotify)g_mapped_file_unref); } OstreeRepo* @@ -841,8 +825,7 @@ commit_loose_object_trusted (OstreeRepo *self, } typedef enum { - OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED = (1<<0), - OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<1) + OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<0) } OstreeRepoStageFlags; static gboolean @@ -863,9 +846,9 @@ stage_object_internal (OstreeRepo *self, ot_lobj GFile *temp_file = NULL; ot_lobj GFile *raw_temp_file = NULL; ot_lobj GFile *stored_path = NULL; - ot_lfree char *pack_checksum = NULL; ot_lfree guchar *ret_csum = NULL; ot_lobj OstreeChecksumInputStream *checksum_input = NULL; + gboolean have_obj; GChecksum *checksum = NULL; gboolean staged_raw_file = FALSE; gboolean staged_archive_file = FALSE; @@ -984,18 +967,11 @@ stage_object_internal (OstreeRepo *self, } } - if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED)) - { - gboolean have_obj; + if (!ostree_repo_has_object (self, objtype, actual_checksum, &have_obj, + cancellable, error)) + goto out; - if (!ostree_repo_has_object (self, objtype, actual_checksum, &have_obj, - cancellable, error)) - goto out; - - do_commit = !have_obj; - } - else - do_commit = TRUE; + do_commit = !have_obj; if (do_commit) { @@ -1074,9 +1050,7 @@ stage_object (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - guint64 pack_offset; ot_lobj GFile *stored_path = NULL; - ot_lfree char *pack_checksum = NULL; ot_lfree guchar *ret_csum = NULL; g_return_val_if_fail (self->in_transaction, FALSE); @@ -1088,23 +1062,12 @@ stage_object (OstreeRepo *self, if (expected_checksum) { - if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED)) - { - if (!repo_find_object (self, objtype, expected_checksum, FALSE, - &stored_path, &pack_checksum, &pack_offset, - cancellable, error)) - goto out; - } - else - { - if (!repo_find_object (self, objtype, expected_checksum, FALSE, - &stored_path, NULL, NULL, - cancellable, error)) - goto out; - } + if (!repo_find_object (self, objtype, expected_checksum, &stored_path, + cancellable, error)) + goto out; } - if (stored_path == NULL && pack_checksum == NULL) + if (stored_path == NULL) { if (!stage_object_internal (self, flags, objtype, input, file_object_length, expected_checksum, @@ -1425,14 +1388,11 @@ gboolean ostree_repo_stage_object_trusted (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, - gboolean store_if_packed, GInputStream *object_input, GCancellable *cancellable, GError **error) { int flags = 0; - if (store_if_packed) - flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED; return stage_object (self, flags, objtype, object_input, 0, checksum, NULL, cancellable, error); @@ -1462,15 +1422,12 @@ ostree_repo_stage_object (OstreeRepo *self, gboolean ostree_repo_stage_file_object_trusted (OstreeRepo *self, const char *checksum, - gboolean store_if_packed, GInputStream *object_input, guint64 length, GCancellable *cancellable, GError **error) { int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID; - if (store_if_packed) - flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED; return stage_object (self, flags, OSTREE_OBJECT_TYPE_FILE, object_input, length, checksum, NULL, cancellable, error); @@ -1715,836 +1672,6 @@ ostree_repo_stage_commit (OstreeRepo *self, return ret; } -static gboolean -list_files_in_dir_matching (GFile *dir, - const char *prefix, - const char *suffix, - GPtrArray **out_files, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - GError *temp_error = NULL; - ot_lobj GFileEnumerator *enumerator = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lptrarray GPtrArray *ret_files = NULL; - - g_return_val_if_fail (prefix != NULL || suffix != NULL, FALSE); - - ret_files = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - - enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); - if (!enumerator) - goto out; - - while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL) - { - const char *name; - guint32 type; - - name = g_file_info_get_attribute_byte_string (file_info, "standard::name"); - type = g_file_info_get_attribute_uint32 (file_info, "standard::type"); - - if (type != G_FILE_TYPE_REGULAR) - goto loop_next; - - if (prefix) - { - if (!g_str_has_prefix (name, prefix)) - goto loop_next; - } - if (suffix) - { - if (!g_str_has_suffix (name, suffix)) - goto loop_next; - } - - g_ptr_array_add (ret_files, g_file_get_child (dir, name)); - - loop_next: - g_clear_object (&file_info); - } - if (temp_error != NULL) - { - g_propagate_error (error, temp_error); - goto out; - } - if (!g_file_enumerator_close (enumerator, cancellable, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_files, &ret_files); - out: - return ret; -} - -static gboolean -map_variant_file_check_header_string (GFile *path, - const GVariantType *variant_type, - const char *expected_header, - gboolean trusted, - GVariant **out_variant, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - const char *header; - ot_lvariant GVariant *ret_variant = NULL; - - if (!ot_util_variant_map (path, variant_type, trusted, &ret_variant, error)) - goto out; - - g_variant_get_child (ret_variant, 0, "&s", &header); - - if (strcmp (header, expected_header) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid variant file '%s', expected header '%s'", - ot_gfile_get_path_cached (path), - expected_header); - goto out; - } - - ret = TRUE; - ot_transfer_out_value (out_variant, &ret_variant); - out: - return ret; -} - - -static char * -get_checksum_from_pack_name (const char *name) -{ - const char *dash; - const char *dot; - - dash = strchr (name, '-'); - g_assert (dash); - dot = strrchr (name, '.'); - g_assert (dot); - - g_assert_cmpint (dot - (dash + 1), ==, 64); - - return g_strndup (dash + 1, 64); -} - -static gboolean -list_pack_indexes_from_dir (OstreeRepo *self, - gboolean is_meta, - GPtrArray **out_indexes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lptrarray GPtrArray *index_files = NULL; - ot_lptrarray GPtrArray *ret_indexes = NULL; - - if (!list_files_in_dir_matching (self->pack_dir, - is_meta ? "ostmetapack-" : "ostdatapack-", ".index", - &index_files, - cancellable, error)) - goto out; - - ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - for (i = 0; i < index_files->len; i++) - { - GFile *index_path = index_files->pdata[i]; - const char *basename = ot_gfile_get_basename_cached (index_path); - g_ptr_array_add (ret_indexes, get_checksum_from_pack_name (basename)); - } - - ret = TRUE; - ot_transfer_out_value (out_indexes, &ret_indexes); - out: - return ret; -} - -static gboolean -list_pack_checksums_from_superindex_file (GFile *superindex_path, - GPtrArray **out_meta_indexes, - GPtrArray **out_data_indexes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - const char *magic; - ot_lptrarray GPtrArray *ret_meta_indexes = NULL; - ot_lptrarray GPtrArray *ret_data_indexes = NULL; - ot_lvariant GVariant *superindex_variant = NULL; - ot_lvariant GVariant *checksum = NULL; - ot_lvariant GVariant *bloom = NULL; - GVariantIter *meta_variant_iter = NULL; - GVariantIter *data_variant_iter = NULL; - - if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, - TRUE, &superindex_variant, error)) - goto out; - - g_variant_get (superindex_variant, "(&s@a{sv}a(ayay)a(ayay))", - &magic, NULL, &meta_variant_iter, &data_variant_iter); - - if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid header in super pack index"); - goto out; - } - - ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - while (g_variant_iter_loop (meta_variant_iter, "(@ay@ay)", - &checksum, &bloom)) - g_ptr_array_add (ret_meta_indexes, ostree_checksum_from_bytes_v (checksum)); - checksum = NULL; - bloom = NULL; - - ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - while (g_variant_iter_loop (data_variant_iter, "(@ay@ay)", - &checksum, &bloom)) - g_ptr_array_add (ret_data_indexes, ostree_checksum_from_bytes_v (checksum)); - checksum = NULL; - bloom = NULL; - - ret = TRUE; - ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes); - ot_transfer_out_value (out_data_indexes, &ret_data_indexes); - out: - if (meta_variant_iter) - g_variant_iter_free (meta_variant_iter); - if (data_variant_iter) - g_variant_iter_free (data_variant_iter); - return ret; -} - -gboolean -ostree_repo_list_pack_indexes (OstreeRepo *self, - GPtrArray **out_meta_indexes, - GPtrArray **out_data_indexes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *superindex_path = NULL; - ot_lptrarray GPtrArray *ret_meta_indexes = NULL; - ot_lptrarray GPtrArray *ret_data_indexes = NULL; - - g_mutex_lock (&self->cache_lock); - if (self->cached_meta_indexes) - { - ret_meta_indexes = g_ptr_array_ref (self->cached_meta_indexes); - ret_data_indexes = g_ptr_array_ref (self->cached_content_indexes); - } - else - { - superindex_path = g_file_get_child (self->pack_dir, "index"); - - if (g_file_query_exists (superindex_path, cancellable)) - { - if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_meta_indexes, - &ret_data_indexes, - cancellable, error)) - goto out; - } - else - { - ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - } - - self->cached_meta_indexes = g_ptr_array_ref (ret_meta_indexes); - self->cached_content_indexes = g_ptr_array_ref (ret_data_indexes); - } - - ret = TRUE; - ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes); - ot_transfer_out_value (out_data_indexes, &ret_data_indexes); - out: - g_mutex_unlock (&self->cache_lock); - return ret; -} - -static gboolean -create_index_bloom (OstreeRepo *self, - const char *pack_checksum, - GVariant **out_bloom, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lvariant GVariant *ret_bloom; - - /* TODO - define and compute bloom filter */ - - ret_bloom = ot_gvariant_new_bytearray (NULL, 0); - g_variant_ref_sink (ret_bloom); - - ret = TRUE; - ot_transfer_out_value (out_bloom, &ret_bloom); - /* out: */ - return ret; -} - -static gboolean -append_index_builder (OstreeRepo *self, - GPtrArray *indexes, - GVariantBuilder *builder, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - - for (i = 0; i < indexes->len; i++) - { - const char *pack_checksum = indexes->pdata[i]; - ot_lvariant GVariant *bloom = NULL; - - if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error)) - goto out; - - g_variant_builder_add (builder, - "(@ay@ay)", - ostree_checksum_to_bytes_v (pack_checksum), - bloom); - } - - ret = TRUE; - out: - return ret; -} - -/** - * Regenerate the pack superindex file based on the set of pack - * indexes currently in the filesystem. - */ -gboolean -ostree_repo_regenerate_pack_index (OstreeRepo *self, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *superindex_path = NULL; - ot_lptrarray GPtrArray *pack_indexes = NULL; - ot_lvariant GVariant *superindex_variant = NULL; - GVariantBuilder *meta_index_content_builder = NULL; - GVariantBuilder *data_index_content_builder = NULL; - - g_clear_pointer (&self->cached_meta_indexes, (GDestroyNotify) g_ptr_array_unref); - g_clear_pointer (&self->cached_content_indexes, (GDestroyNotify) g_ptr_array_unref); - - superindex_path = g_file_get_child (self->pack_dir, "index"); - - g_clear_pointer (&pack_indexes, (GDestroyNotify) g_ptr_array_unref); - if (!list_pack_indexes_from_dir (self, TRUE, &pack_indexes, - cancellable, error)) - goto out; - meta_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); - if (!append_index_builder (self, pack_indexes, meta_index_content_builder, - cancellable, error)) - goto out; - - g_clear_pointer (&pack_indexes, (GDestroyNotify) g_ptr_array_unref); - if (!list_pack_indexes_from_dir (self, FALSE, &pack_indexes, - cancellable, error)) - goto out; - data_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); - if (!append_index_builder (self, pack_indexes, data_index_content_builder, - cancellable, error)) - goto out; - - superindex_variant = g_variant_new ("(s@a{sv}@a(ayay)@a(ayay))", - "OSTv0SUPERPACKINDEX", - g_variant_new_array (G_VARIANT_TYPE ("{sv}"), - NULL, 0), - g_variant_builder_end (meta_index_content_builder), - g_variant_builder_end (data_index_content_builder)); - g_variant_ref_sink (superindex_variant); - - if (!ot_util_variant_save (superindex_path, superindex_variant, - cancellable, error)) - goto out; - - ret = TRUE; - out: - if (meta_index_content_builder) - g_variant_builder_unref (meta_index_content_builder); - if (data_index_content_builder) - g_variant_builder_unref (data_index_content_builder); - return ret; -} - -static GFile * -get_pack_index_path (GFile *parent, - gboolean is_meta, - const char *checksum) -{ - char *path = ostree_get_pack_index_name (is_meta, checksum); - GFile *ret = g_file_resolve_relative_path (parent, path); - g_free (path); - return ret; -} - -static GFile * -get_pack_data_path (GFile *parent, - gboolean is_meta, - const char *checksum) -{ - char *path = ostree_get_pack_data_name (is_meta, checksum); - GFile *ret = g_file_resolve_relative_path (parent, path); - g_free (path); - return ret; -} - -gboolean -ostree_repo_add_pack_file (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - GFile *index_path, - GFile *data_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *pack_index_path = NULL; - ot_lobj GFile *pack_data_path = NULL; - - if (!ot_gfile_ensure_directory (self->pack_dir, FALSE, error)) - goto out; - - pack_data_path = get_pack_data_path (self->pack_dir, is_meta, pack_checksum); - if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error)) - goto out; - - pack_index_path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum); - if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; -} - -static gboolean -ensure_remote_cache_dir (OstreeRepo *self, - const char *remote_name, - GFile **out_cache_dir, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *ret_cache_dir = NULL; - - ret_cache_dir = g_file_get_child (self->remote_cache_dir, remote_name); - - if (!ot_gfile_ensure_directory (ret_cache_dir, FALSE, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_cache_dir, &ret_cache_dir); - out: - return ret; -} - -static gboolean -delete_no_longer_referenced (OstreeRepo *self, - GFile *cache_path, - const char *prefix, - const char *suffix, - GHashTable *new_files, - GPtrArray *inout_cached, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lptrarray GPtrArray *current_files = NULL; - ot_lfree char *pack_checksum = NULL; - - if (!list_files_in_dir_matching (cache_path, - prefix, suffix, - ¤t_files, - cancellable, error)) - goto out; - for (i = 0; i < current_files->len; i++) - { - GFile *file = current_files->pdata[i]; - - g_free (pack_checksum); - pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (file)); - - if (!g_hash_table_lookup (new_files, pack_checksum)) - { - if (!ot_gfile_unlink (file, cancellable, error)) - goto out; - } - - if (inout_cached) - { - g_ptr_array_add (inout_cached, pack_checksum); - pack_checksum = NULL; /* transfer ownership */ - } - } - ret = TRUE; - out: - return ret; -} - -static void -gather_uncached (GHashTable *new_files, - GPtrArray *cached, - GPtrArray *inout_uncached) -{ - guint i; - GHashTableIter hash_iter; - gpointer key, value; - - g_hash_table_iter_init (&hash_iter, new_files); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *cur_pack_checksum = key; - gboolean found = FALSE; - - for (i = 0; i < cached->len; i++) - { - const char *checksum = cached->pdata[i]; - if (strcmp (cur_pack_checksum, checksum) == 0) - { - found = TRUE; - break; - } - } - - if (!found) - g_ptr_array_add (inout_uncached, g_strdup (cur_pack_checksum)); - } -} - -/** - * Take a pack superindex file @superindex_path, and clean up any - * no-longer-referenced pack files in the lookaside cache for - * @remote_name. The updated index file will also be saved into the - * cache. - * - * Upon successful return, @out_cached_indexes will hold checksum - * strings for indexes which are already in the cache, and - * @out_uncached_indexes will hold strings for those which are not. - */ -gboolean -ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, - const char *remote_name, - GFile *superindex_path, - GPtrArray **out_cached_meta_indexes, - GPtrArray **out_cached_data_indexes, - GPtrArray **out_uncached_meta_indexes, - GPtrArray **out_uncached_data_indexes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lvariant GVariant *superindex_variant = NULL; - ot_lobj GFile *cache_path = NULL; - ot_lobj GFile *superindex_cache_path = NULL; - ot_lptrarray GPtrArray *meta_index_files = NULL; - ot_lptrarray GPtrArray *data_index_files = NULL; - ot_lptrarray GPtrArray *meta_data_files = NULL; - ot_lptrarray GPtrArray *data_data_files = NULL; - ot_lhash GHashTable *new_pack_meta_indexes = NULL; - ot_lhash GHashTable *new_pack_data_indexes = NULL; - ot_lptrarray GPtrArray *ret_cached_meta_indexes = NULL; - ot_lptrarray GPtrArray *ret_cached_data_indexes = NULL; - ot_lptrarray GPtrArray *ret_uncached_meta_indexes = NULL; - ot_lptrarray GPtrArray *ret_uncached_data_indexes = NULL; - ot_lvariant GVariant *csum_bytes = NULL; - ot_lvariant GVariant *bloom = NULL; - ot_lfree char *pack_checksum = NULL; - GVariantIter *superindex_contents_iter = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error)) - goto out; - - ret_cached_meta_indexes = g_ptr_array_new_with_free_func (g_free); - ret_cached_data_indexes = g_ptr_array_new_with_free_func (g_free); - ret_uncached_meta_indexes = g_ptr_array_new_with_free_func (g_free); - ret_uncached_data_indexes = g_ptr_array_new_with_free_func (g_free); - - if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, - FALSE, &superindex_variant, error)) - goto out; - - if (!ostree_validate_structureof_pack_superindex (superindex_variant, error)) - goto out; - - new_pack_meta_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - new_pack_data_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - - g_variant_get_child (superindex_variant, 2, "a(ayay)", - &superindex_contents_iter); - while (g_variant_iter_loop (superindex_contents_iter, - "(@ay@ay)", &csum_bytes, &bloom)) - { - pack_checksum = ostree_checksum_from_bytes_v (csum_bytes); - g_hash_table_insert (new_pack_meta_indexes, pack_checksum, pack_checksum); - pack_checksum = NULL; /* transfer ownership */ - } - g_variant_iter_free (superindex_contents_iter); - - g_variant_get_child (superindex_variant, 3, "a(ayay)", - &superindex_contents_iter); - while (g_variant_iter_loop (superindex_contents_iter, - "(@ay@ay)", &csum_bytes, &bloom)) - { - pack_checksum = ostree_checksum_from_bytes_v (csum_bytes); - g_hash_table_insert (new_pack_data_indexes, pack_checksum, pack_checksum); - pack_checksum = NULL; /* transfer ownership */ - } - - if (!delete_no_longer_referenced (self, cache_path, - "ostmetapack-", ".index", - new_pack_meta_indexes, - ret_cached_meta_indexes, - cancellable, error)) - goto out; - - if (!delete_no_longer_referenced (self, cache_path, - "ostdatapack-", ".index", - new_pack_data_indexes, - ret_cached_data_indexes, - cancellable, error)) - goto out; - - gather_uncached (new_pack_meta_indexes, ret_cached_meta_indexes, ret_uncached_meta_indexes); - gather_uncached (new_pack_data_indexes, ret_cached_data_indexes, ret_uncached_data_indexes); - - superindex_cache_path = g_file_get_child (cache_path, "index"); - if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error)) - goto out; - - /* Now also delete stale pack files */ - - if (!delete_no_longer_referenced (self, cache_path, - "ostmetapack-", ".data", - new_pack_meta_indexes, NULL, - cancellable, error)) - goto out; - if (!delete_no_longer_referenced (self, cache_path, - "ostdatapack-", ".data", - new_pack_data_indexes, NULL, - cancellable, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_cached_meta_indexes, &ret_cached_meta_indexes); - ot_transfer_out_value (out_cached_data_indexes, &ret_cached_data_indexes); - ot_transfer_out_value (out_uncached_meta_indexes, &ret_uncached_meta_indexes); - ot_transfer_out_value (out_uncached_data_indexes, &ret_uncached_data_indexes); - out: - if (superindex_contents_iter) - g_variant_iter_free (superindex_contents_iter); - return ret; -} - -gboolean -ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lobj GFile *cache_path = NULL; - ot_lptrarray GPtrArray *data_files = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error)) - goto out; - - if (!list_files_in_dir_matching (cache_path, - "ostmetapack-", ".data", - &data_files, - cancellable, error)) - goto out; - for (i = 0; i < data_files->len; i++) - { - GFile *data_file = data_files->pdata[i]; - - if (!ot_gfile_unlink (data_file, cancellable, error)) - goto out; - } - - g_clear_pointer (&data_files, (GDestroyNotify) g_ptr_array_unref); - if (!list_files_in_dir_matching (cache_path, - "ostdatapack-", ".data", - &data_files, - cancellable, error)) - goto out; - for (i = 0; i < data_files->len; i++) - { - GFile *data_file = data_files->pdata[i]; - - if (!ot_gfile_unlink (data_file, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - return ret; -} - -/** - * Load the index for pack @pack_checksum from cache directory for - * @remote_name. - */ -gboolean -ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GVariant **out_variant, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lvariant GVariant *ret_variant = NULL; - ot_lobj GFile *cache_dir = NULL; - ot_lobj GFile *cached_pack_path = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cache_dir, - cancellable, error)) - goto out; - - cached_pack_path = get_pack_index_path (cache_dir, is_meta, pack_checksum); - if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT, - FALSE, &ret_variant, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_variant, &ret_variant); - out: - return ret; -} - -/** - * The variable @cached_path should refer to a file containing a pack - * index. It will be validated and added to the cache directory for - * @remote_name. - */ -gboolean -ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile *cached_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *cachedir = NULL; - ot_lobj GFile *target_path = NULL; - ot_lvariant GVariant *input_index_variant = NULL; - ot_lvariant GVariant *output_index_variant = NULL; - - if (!map_variant_file_check_header_string (cached_path, - OSTREE_PACK_INDEX_VARIANT_FORMAT, - "OSTv0PACKINDEX", - FALSE, &input_index_variant, - cancellable, error)) - goto out; - - if (!ostree_validate_structureof_pack_index (input_index_variant, error)) - goto out; - - output_index_variant = g_variant_get_normal_form (input_index_variant); - - if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) - goto out; - - target_path = get_pack_index_path (cachedir, is_meta, pack_checksum); - if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; -} - -/** - * Check for availability of the pack index pointing to @pack_checksum - * in the lookaside cache for @remote_name. If not found, then the - * output parameter @out_cached_path will be %NULL. - */ -gboolean -ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile **out_cached_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *cache_dir = NULL; - ot_lobj GFile *cached_pack_path = NULL; - ot_lobj GFile *ret_cached_path = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cache_dir, - cancellable, error)) - goto out; - - cached_pack_path = get_pack_data_path (cache_dir, is_meta, pack_checksum); - if (g_file_query_exists (cached_pack_path, cancellable)) - { - ret_cached_path = cached_pack_path; - cached_pack_path = NULL; - } - - ret = TRUE; - ot_transfer_out_value (out_cached_path, &ret_cached_path); - out: - return ret; -} - -/** - * Add file @cached_path into the cache for given @remote_name. If - * @cached_path is %NULL, delete the cached pack data (if any). - * - * - * This unlinks @cached_path. - * - */ -gboolean -ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile *cached_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *cachedir = NULL; - ot_lobj GFile *target_path = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) - goto out; - - target_path = get_pack_data_path (cachedir, is_meta, pack_checksum); - if (cached_path) - { - if (!ot_gfile_rename (cached_path, target_path, cancellable, error)) - goto out; - } - else - { - (void) ot_gfile_unlink (target_path, cancellable, NULL); - } - - ret = TRUE; - out: - return ret; -} - static GVariant * create_tree_variant_from_hashes (GHashTable *file_checksums, GHashTable *dir_contents_checksums, @@ -3387,95 +2514,6 @@ list_loose_objects (OstreeRepo *self, return ret; } -gboolean -ostree_repo_load_pack_index (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - GVariant **out_variant, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lvariant GVariant *ret_variant = NULL; - ot_lobj GFile *path = NULL; - - g_mutex_lock (&self->cache_lock); - - ret_variant = g_hash_table_lookup (self->cached_pack_index_mappings, pack_checksum); - if (ret_variant) - { - g_variant_ref (ret_variant); - } - else - { - path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum); - if (!map_variant_file_check_header_string (path, - OSTREE_PACK_INDEX_VARIANT_FORMAT, - "OSTv0PACKINDEX", TRUE, - &ret_variant, - cancellable, error)) - goto out; - g_hash_table_insert (self->cached_pack_index_mappings, g_strdup (pack_checksum), - g_variant_ref (ret_variant)); - } - - ret = TRUE; - ot_transfer_out_value (out_variant, &ret_variant); - out: - g_mutex_unlock (&self->cache_lock); - return ret; -} - -/** - * @sha256: Checksum of pack file - * @out_data: (out): Pointer to pack file data - * - * Ensure that the given pack file is mapped into - * memory. - */ -gboolean -ostree_repo_map_pack_file (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - guchar **out_data, - guint64 *out_len, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gpointer ret_data; - guint64 ret_len; - GMappedFile *map = NULL; - ot_lobj GFile *path = NULL; - - g_mutex_lock (&self->cache_lock); - - map = g_hash_table_lookup (self->cached_pack_data_mappings, pack_checksum); - if (map == NULL) - { - path = get_pack_data_path (self->pack_dir, is_meta, pack_checksum); - - map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error); - if (!map) - goto out; - - g_hash_table_insert (self->cached_pack_data_mappings, g_strdup (pack_checksum), map); - ret_data = g_mapped_file_get_contents (map); - } - - ret_data = g_mapped_file_get_contents (map); - ret_len = (guint64)g_mapped_file_get_length (map); - - ret = TRUE; - if (out_data) - *out_data = ret_data; - if (out_len) - *out_len = ret_len; - out: - g_mutex_unlock (&self->cache_lock); - return ret; -} - gboolean ostree_repo_load_file (OstreeRepo *self, const char *checksum, @@ -3486,21 +2524,14 @@ ostree_repo_load_file (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - guchar *pack_data; - guint64 pack_len; - guint64 pack_offset; - ot_lvariant GVariant *packed_object = NULL; ot_lvariant GVariant *file_data = NULL; ot_lobj GFile *loose_path = NULL; ot_lobj GFileInfo *content_loose_info = NULL; - ot_lfree char *pack_checksum = NULL; ot_lobj GInputStream *ret_input = NULL; ot_lobj GFileInfo *ret_file_info = NULL; ot_lvariant GVariant *ret_xattrs = NULL; - if (!repo_find_object (self, OSTREE_OBJECT_TYPE_FILE, - checksum, FALSE, &loose_path, - &pack_checksum, &pack_offset, + if (!repo_find_object (self, OSTREE_OBJECT_TYPE_FILE, checksum, &loose_path, cancellable, error)) goto out; @@ -3565,25 +2596,6 @@ ostree_repo_load_file (OstreeRepo *self, } } } - else if (pack_checksum) - { - if (!ostree_repo_map_pack_file (self, pack_checksum, FALSE, - &pack_data, &pack_len, - cancellable, error)) - goto out; - - if (!ostree_read_pack_entry_raw (pack_data, pack_len, - pack_offset, TRUE, FALSE, - &packed_object, cancellable, error)) - goto out; - - if (!ostree_parse_file_pack_entry (packed_object, - out_input ? &ret_input : NULL, - out_file_info ? &ret_file_info : NULL, - out_xattrs ? &ret_xattrs : NULL, - cancellable, error)) - goto out; - } else if (self->parent_repo) { if (!ostree_repo_load_file (self->parent_repo, checksum, @@ -3608,258 +2620,34 @@ ostree_repo_load_file (OstreeRepo *self, return ret; } -static gboolean -list_objects_in_index (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - GHashTable *inout_objects, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint8 objtype_u8; - guint64 offset; - ot_lobj GFile *index_path = NULL; - ot_lvariant GVariant *index_variant = NULL; - ot_lvariant GVariant *contents = NULL; - ot_lvariant GVariant *csum_bytes = NULL; - ot_lfree char *checksum = NULL; - GVariantIter content_iter; - - index_path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum); - - if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, - &index_variant, cancellable, error)) - goto out; - - contents = g_variant_get_child_value (index_variant, 2); - g_variant_iter_init (&content_iter, contents); - - while (g_variant_iter_loop (&content_iter, "(y@ayt)", &objtype_u8, &csum_bytes, &offset)) - { - GVariant *obj_key; - GVariant *objdata; - OstreeObjectType objtype; - GVariantBuilder pack_contents_builder; - gboolean is_loose; - - objtype = (OstreeObjectType) objtype_u8; - offset = GUINT64_FROM_BE (offset); - - g_variant_builder_init (&pack_contents_builder, - G_VARIANT_TYPE_STRING_ARRAY); - - g_free (checksum); - checksum = ostree_checksum_from_bytes_v (csum_bytes); - obj_key = ostree_object_name_serialize (checksum, objtype); - ot_util_variant_take_ref (obj_key); - - objdata = g_hash_table_lookup (inout_objects, obj_key); - if (!objdata) - { - is_loose = FALSE; - } - else - { - GVariantIter *current_packs_iter; - const char *current_pack_checksum; - - g_variant_get (objdata, "(bas)", &is_loose, ¤t_packs_iter); - - while (g_variant_iter_loop (current_packs_iter, "&s", ¤t_pack_checksum)) - { - g_variant_builder_add (&pack_contents_builder, "s", current_pack_checksum); - } - g_variant_iter_free (current_packs_iter); - } - g_variant_builder_add (&pack_contents_builder, "s", pack_checksum); - objdata = g_variant_new ("(b@as)", is_loose, - g_variant_builder_end (&pack_contents_builder)); - g_variant_ref_sink (objdata); - g_hash_table_replace (inout_objects, obj_key, objdata); - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -list_packed_objects (OstreeRepo *self, - GHashTable *inout_objects, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lptrarray GPtrArray *meta_index_checksums = NULL; - ot_lptrarray GPtrArray *data_index_checksums = NULL; - - if (!ostree_repo_list_pack_indexes (self, &meta_index_checksums, &data_index_checksums, - cancellable, error)) - goto out; - - for (i = 0; i < meta_index_checksums->len; i++) - { - const char *checksum = meta_index_checksums->pdata[i]; - if (!list_objects_in_index (self, checksum, TRUE, inout_objects, cancellable, error)) - goto out; - } - - for (i = 0; i < data_index_checksums->len; i++) - { - const char *checksum = data_index_checksums->pdata[i]; - if (!list_objects_in_index (self, checksum, FALSE, inout_objects, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -find_object_in_packs (OstreeRepo *self, - const char *checksum, - OstreeObjectType objtype, - char **out_pack_checksum, - guint64 *out_pack_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - guint64 ret_pack_offset = 0; - gboolean is_meta; - ot_lptrarray GPtrArray *index_checksums = NULL; - ot_lfree char *ret_pack_checksum = NULL; - ot_lvariant GVariant *csum_bytes = NULL; - ot_lvariant GVariant *index_variant = NULL; - - csum_bytes = ostree_checksum_to_bytes_v (checksum); - - is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); - - if (is_meta) - { - if (!ostree_repo_list_pack_indexes (self, &index_checksums, NULL, - cancellable, error)) - goto out; - } - else - { - if (!ostree_repo_list_pack_indexes (self, NULL, &index_checksums, - cancellable, error)) - goto out; - } - - for (i = 0; i < index_checksums->len; i++) - { - const char *pack_checksum = index_checksums->pdata[i]; - guint64 offset; - - g_clear_pointer (&index_variant, (GDestroyNotify) g_variant_unref); - if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, &index_variant, - cancellable, error)) - goto out; - - if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset)) - continue; - - ret_pack_checksum = g_strdup (pack_checksum); - ret_pack_offset = offset; - break; - } - - ret = TRUE; - ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); - if (out_pack_offset) - *out_pack_offset = ret_pack_offset; - out: - return ret; -} - static gboolean repo_find_object (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, - gboolean lookup_all, GFile **out_stored_path, - char **out_pack_checksum, - guint64 *out_pack_offset, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - guint64 ret_pack_offset = 0; struct stat stbuf; ot_lobj GFile *object_path = NULL; ot_lobj GFile *ret_stored_path = NULL; - ot_lfree char *ret_pack_checksum = NULL; - /* Look up metadata in packs first, but content loose first. We - * want to find loose content since that's preferable for - * hardlinking scenarios. - * - * Metadata is much more efficient packed. - */ - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) + object_path = ostree_repo_get_object_path (self, checksum, objtype); + + if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) { - if (out_pack_checksum) - { - if (!find_object_in_packs (self, checksum, objtype, - &ret_pack_checksum, &ret_pack_offset, - cancellable, error)) - goto out; - } - if (!ret_pack_checksum || lookup_all) - { - object_path = ostree_repo_get_object_path (self, checksum, objtype); - - if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) - { - ret_stored_path = object_path; - object_path = NULL; - } - else - { - g_clear_object (&object_path); - } - } + ret_stored_path = object_path; + object_path = NULL; /* Transfer ownership */ } - else + else if (errno != ENOENT) { - if (out_stored_path) - { - object_path = ostree_repo_get_object_path (self, checksum, objtype); - - if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) - { - ret_stored_path = object_path; - object_path = NULL; - } - else - { - g_clear_object (&object_path); - } - } - if (!ret_stored_path || lookup_all) - { - if (out_pack_checksum) - { - if (!find_object_in_packs (self, checksum, objtype, - &ret_pack_checksum, &ret_pack_offset, - cancellable, error)) - goto out; - } - } + ot_util_set_error_from_errno (error, errno); + goto out; } - + ret = TRUE; ot_transfer_out_value (out_stored_path, &ret_stored_path); - ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); - if (out_pack_offset) - *out_pack_offset = ret_pack_offset; out: return ret; } @@ -3875,15 +2663,12 @@ ostree_repo_has_object (OstreeRepo *self, gboolean ret = FALSE; gboolean ret_have_object; ot_lobj GFile *loose_path = NULL; - ot_lfree char *pack_checksum = NULL; - if (!repo_find_object (self, objtype, checksum, FALSE, - &loose_path, - &pack_checksum, NULL, + if (!repo_find_object (self, objtype, checksum, &loose_path, cancellable, error)) goto out; - ret_have_object = (loose_path != NULL) || (pack_checksum != NULL); + ret_have_object = (loose_path != NULL); if (!ret_have_object && self->parent_repo) { @@ -3928,35 +2713,17 @@ load_variant_internal (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - guchar *pack_data; - guint64 pack_len; - guint64 object_offset; GCancellable *cancellable = NULL; ot_lobj GFile *object_path = NULL; - ot_lvariant GVariant *packed_object = NULL; ot_lvariant GVariant *ret_variant = NULL; - ot_lfree char *pack_checksum = NULL; g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE); - if (!repo_find_object (self, objtype, sha256, FALSE, - &object_path, &pack_checksum, &object_offset, + if (!repo_find_object (self, objtype, sha256, &object_path, cancellable, error)) goto out; - if (pack_checksum != NULL) - { - if (!ostree_repo_map_pack_file (self, pack_checksum, TRUE, &pack_data, &pack_len, - cancellable, error)) - goto out; - - if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset, - TRUE, TRUE, &packed_object, cancellable, error)) - goto out; - - g_variant_get_child (packed_object, 2, "v", &ret_variant); - } - else if (object_path != NULL) + if (object_path != NULL) { if (!ot_util_variant_map (object_path, ostree_metadata_variant_type (objtype), TRUE, &ret_variant, error)) @@ -4065,13 +2832,7 @@ ostree_repo_list_objects (OstreeRepo *self, if (flags & OSTREE_REPO_LIST_OBJECTS_PACKED) { - if (!list_packed_objects (self, ret_objects, cancellable, error)) - goto out; - if (self->parent_repo) - { - if (!list_packed_objects (self->parent_repo, ret_objects, cancellable, error)) - goto out; - } + /* Nothing for now... */ } ret = TRUE; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 5f5e0dfa..151ea3af 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -111,14 +111,12 @@ gboolean ostree_repo_stage_file_object (OstreeRepo *self, gboolean ostree_repo_stage_object_trusted (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, - gboolean store_if_packed, GInputStream *content, GCancellable *cancellable, GError **error); gboolean ostree_repo_stage_file_object_trusted (OstreeRepo *self, const char *checksum, - gboolean store_if_packed, GInputStream *content, guint64 content_length, GCancellable *cancellable, @@ -159,27 +157,6 @@ gboolean ostree_repo_load_variant_if_exists (OstreeRepo *self, GVariant **out_variant, GError **error); -gboolean ostree_repo_load_pack_index (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - GVariant **out_variant, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_load_pack_data (OstreeRepo *self, - const char *pack_checksum, - guchar **out_data, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_map_pack_file (OstreeRepo *self, - const char *sha256, - gboolean is_meta, - guchar **out_data, - guint64 *out_len, - GCancellable *cancellable, - GError **error); - gboolean ostree_repo_load_file (OstreeRepo *self, const char *entry_sha256, GInputStream **out_input, @@ -248,65 +225,6 @@ gboolean ostree_repo_stage_commit (OstreeRepo *self, GCancellable *cancellable, GError **error); -gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_add_pack_file (OstreeRepo *self, - const char *checksum, - gboolean is_meta, - GFile *pack_index_path, - GFile *pack_data_path, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, - const char *remote_name, - GFile *superindex_path, - GPtrArray **out_cached_meta_indexes, - GPtrArray **out_cached_data_indexes, - GPtrArray **out_uncached_meta_indexes, - GPtrArray **out_uncached_data_indexes, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GVariant **out_variant, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile *cached_path, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile **out_cached_path, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile *cached_path, - GCancellable *cancellable, - GError **error); - typedef enum { OSTREE_REPO_CHECKOUT_MODE_NONE = 0, OSTREE_REPO_CHECKOUT_MODE_USER = 1 @@ -359,12 +277,6 @@ gboolean ostree_repo_list_objects (OstreeRepo *self, GCancellable *cancellable, GError **error); -gboolean ostree_repo_list_pack_indexes (OstreeRepo *self, - GPtrArray **out_meta_indexes, - GPtrArray **out_data_indexes, - GCancellable *cancellable, - GError **error); - G_END_DECLS #endif /* _OSTREE_REPO */ diff --git a/src/ostree/main.c b/src/ostree/main.c index 994a8373..8e322411 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -42,14 +42,12 @@ static OstreeCommand commands[] = { { "init", ostree_builtin_init, 0 }, { "log", ostree_builtin_log, 0 }, { "ls", ostree_builtin_ls, 0 }, - { "pack", ostree_builtin_pack, 0 }, { "prune", ostree_builtin_prune, 0 }, { "pull", NULL, 0 }, { "pull-local", ostree_builtin_pull_local, 0 }, { "remote", ostree_builtin_remote, 0 }, { "rev-parse", ostree_builtin_rev_parse, 0 }, { "show", ostree_builtin_show, 0 }, - { "unpack", ostree_builtin_unpack, 0 }, { "write-refs", ostree_builtin_write_refs, 0 }, { NULL } }; diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c index f1804dab..d1934b47 100644 --- a/src/ostree/ostree-pull.c +++ b/src/ostree/ostree-pull.c @@ -67,13 +67,11 @@ #include "ostree-fetcher.h" gboolean verbose; -gint opt_packfile_threshold = 66; gboolean opt_related; gint opt_depth; static GOptionEntry options[] = { { "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose, "Show more information", NULL }, - { "packfile-threshold", 't', 0, G_OPTION_ARG_INT, &opt_packfile_threshold, "Only download packfiles if more than PERCENT objects are needed (default: 66)", "PERCENT" }, { "related", 0, 0, G_OPTION_ARG_NONE, &opt_related, "Download related commits", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Download parent commits up to this depth (default: 0)", NULL }, { NULL }, @@ -85,10 +83,6 @@ typedef struct { OstreeFetcher *fetcher; SoupURI *base_uri; - gboolean fetched_packs; - GPtrArray *cached_meta_pack_indexes; - GPtrArray *cached_data_pack_indexes; - GHashTable *file_checksums_to_fetch; GMainLoop *loop; @@ -325,234 +319,6 @@ fetch_uri_contents_utf8 (OtPullData *pull_data, return ret; } -static gboolean -fetch_one_pack_file (OtPullData *pull_data, - const char *pack_checksum, - gboolean is_meta, - GFile **out_cached_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *ret_cached_path = NULL; - ot_lobj GFile *tmp_path = NULL; - ot_lfree char *pack_name = NULL; - SoupURI *pack_uri = NULL; - - if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, is_meta, &ret_cached_path, - cancellable, error)) - goto out; - - if (ret_cached_path == NULL) - { - pack_name = ostree_get_pack_data_name (is_meta, pack_checksum); - pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL); - - if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error)) - goto out; - - if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, is_meta, tmp_path, - cancellable, error)) - goto out; - } - - g_clear_object (&ret_cached_path); - if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, is_meta, &ret_cached_path, - cancellable, error)) - goto out; - - g_assert (ret_cached_path != NULL); - - ret = TRUE; - ot_transfer_out_value (out_cached_path, &ret_cached_path); - out: - if (pack_uri) - soup_uri_free (pack_uri); - return ret; -} - -static gboolean -find_object_in_one_remote_pack (OtPullData *pull_data, - GVariant *csum_bytes_v, - OstreeObjectType objtype, - const char *pack_checksum, - gboolean *out_exists, - guint64 *out_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint64 ret_offset; - gboolean ret_exists; - ot_lvariant GVariant *mapped_pack = NULL; - - if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype), - &mapped_pack, - cancellable, error)) - goto out; - - ret_exists = ostree_pack_index_search (mapped_pack, csum_bytes_v, objtype, &ret_offset); - - ret = TRUE; - if (out_exists) - *out_exists = ret_exists; - if (out_offset) - *out_offset = ret_offset; - out: - return ret; -} - -static gboolean -find_object_in_remote_packs (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - char **out_pack_checksum, - guint64 *out_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint64 ret_offset = 0; - guint i; - GPtrArray *iter; - ot_lvariant GVariant *csum_bytes_v = NULL; - ot_lfree char *ret_pack_checksum = NULL; - - csum_bytes_v = ostree_checksum_to_bytes_v (checksum); - - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - iter = pull_data->cached_meta_pack_indexes; - else - iter = pull_data->cached_data_pack_indexes; - for (i = 0; i < iter->len; i++) - { - const char *pack_checksum = iter->pdata[i]; - gboolean exists; - - if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, objtype, - pack_checksum, &exists, &ret_offset, - cancellable, error)) - goto out; - - if (exists) - { - ret_pack_checksum = g_strdup (pack_checksum); - break; - } - } - - ret = TRUE; - ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); - if (out_offset) - *out_offset = ret_offset; - out: - return ret; -} - -static gboolean -fetch_one_cache_index (OtPullData *pull_data, - const char *pack_checksum, - gboolean is_meta, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *tmp_path = NULL; - ot_lfree char *pack_index_name = NULL; - SoupURI *index_uri = NULL; - - pack_index_name = ostree_get_pack_index_name (is_meta, pack_checksum); - index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL); - - if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path, - cancellable, error)) - goto out; - - if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, is_meta, tmp_path, - cancellable, error)) - goto out; - - if (!ot_gfile_unlink (tmp_path, cancellable, error)) - goto out; - - g_clear_object (&tmp_path); - - ret = TRUE; - out: - if (tmp_path != NULL) - (void) ot_gfile_unlink (tmp_path, NULL, NULL); - if (index_uri) - soup_uri_free (index_uri); - return ret; -} - -static gboolean -fetch_and_cache_pack_indexes (OtPullData *pull_data, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lobj GFile *superindex_tmppath = NULL; - ot_lptrarray GPtrArray *cached_meta_indexes = NULL; - ot_lptrarray GPtrArray *cached_data_indexes = NULL; - ot_lptrarray GPtrArray *uncached_meta_indexes = NULL; - ot_lptrarray GPtrArray *uncached_data_indexes = NULL; - ot_lvariant GVariant *superindex_variant = NULL; - GVariantIter *contents_iter = NULL; - SoupURI *superindex_uri = NULL; - - superindex_uri = suburi_new (pull_data->base_uri, "objects", "pack", "index", NULL); - - if (!fetch_uri (pull_data, superindex_uri, "index-", - &superindex_tmppath, cancellable, error)) - goto out; - - if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name, - superindex_tmppath, - &cached_meta_indexes, - &cached_data_indexes, - &uncached_meta_indexes, - &uncached_data_indexes, - cancellable, error)) - goto out; - - for (i = 0; i < cached_meta_indexes->len; i++) - g_ptr_array_add (pull_data->cached_meta_pack_indexes, - g_strdup (cached_meta_indexes->pdata[i])); - for (i = 0; i < cached_data_indexes->len; i++) - g_ptr_array_add (pull_data->cached_data_pack_indexes, - g_strdup (cached_data_indexes->pdata[i])); - - for (i = 0; i < uncached_meta_indexes->len; i++) - { - const char *pack_checksum = uncached_meta_indexes->pdata[i]; - if (!fetch_one_cache_index (pull_data, pack_checksum, TRUE, cancellable, error)) - goto out; - g_ptr_array_add (pull_data->cached_meta_pack_indexes, g_strdup (pack_checksum)); - } - for (i = 0; i < uncached_data_indexes->len; i++) - { - const char *pack_checksum = uncached_data_indexes->pdata[i]; - if (!fetch_one_cache_index (pull_data, pack_checksum, FALSE, cancellable, error)) - goto out; - g_ptr_array_add (pull_data->cached_data_pack_indexes, g_strdup (pack_checksum)); - } - - ret = TRUE; - out: - if (superindex_uri) - soup_uri_free (superindex_uri); - if (contents_iter) - g_variant_iter_free (contents_iter); - return ret; -} - static gboolean fetch_loose_object (OtPullData *pull_data, const char *checksum, @@ -581,89 +347,6 @@ fetch_loose_object (OtPullData *pull_data, return ret; } -static gboolean -find_object_ensure_indexes (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - gboolean *out_is_stored, - char **out_remote_pack_checksum, - guint64 *out_remote_pack_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean ret_is_stored; - ot_lfree char *ret_remote_pack_checksum = NULL; - - if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &ret_is_stored, - cancellable, error)) - goto out; - - if (!ret_is_stored) - { - if (!pull_data->fetched_packs) - { - pull_data->fetched_packs = TRUE; - pull_data->cached_meta_pack_indexes = g_ptr_array_new_with_free_func (g_free); - pull_data->cached_data_pack_indexes = g_ptr_array_new_with_free_func (g_free); - - if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error)) - goto out; - } - - if (!find_object_in_remote_packs (pull_data, checksum, objtype, - &ret_remote_pack_checksum, out_remote_pack_offset, - cancellable, error)) - goto out; - } - - ret = TRUE; - if (out_is_stored) - *out_is_stored = ret_is_stored; - ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum); - out: - return ret; -} - -static gboolean -find_object_ensure_pack_data (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - gboolean *out_is_stored, - GFile **out_remote_pack_path, - guint64 *out_remote_pack_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean ret_is_stored; - ot_lfree char *remote_pack_checksum = NULL; - ot_lobj GFile *ret_remote_pack_path = NULL; - - if (!find_object_ensure_indexes (pull_data, checksum, objtype, &ret_is_stored, - &remote_pack_checksum, out_remote_pack_offset, - cancellable, error)) - goto out; - - if (!ret_is_stored) - { - if (remote_pack_checksum) - { - if (!fetch_one_pack_file (pull_data, remote_pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype), - &ret_remote_pack_path, cancellable, error)) - goto out; - } - } - - ret = TRUE; - if (out_is_stored) - *out_is_stored = ret_is_stored; - ot_transfer_out_value (out_remote_pack_path, &ret_remote_pack_path); - /* offset set above */ - out: - return ret; -} - static gboolean fetch_and_store_metadata (OtPullData *pull_data, const char *checksum, @@ -673,42 +356,19 @@ fetch_and_store_metadata (OtPullData *pull_data, GError **error) { gboolean ret = FALSE; - guint64 pack_offset = 0; gboolean is_stored; ot_lvariant GVariant *ret_variant = NULL; - ot_lobj GFile *remote_pack_path = NULL; ot_lobj GFile *temp_path = NULL; ot_lobj GInputStream *input = NULL; - ot_lvariant GVariant *pack_entry = NULL; ot_lvariant GVariant *metadata = NULL; - GMappedFile *pack_map = NULL; g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype)); - if (!find_object_ensure_pack_data (pull_data, checksum, objtype, - &is_stored, &remote_pack_path, &pack_offset, - cancellable, error)) + if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored, + cancellable, error)) goto out; - if (remote_pack_path != NULL) - { - g_assert (!is_stored); - - pack_map = g_mapped_file_new (ot_gfile_get_path_cached (remote_pack_path), FALSE, error); - if (!pack_map) - goto out; - - if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), - g_mapped_file_get_length (pack_map), - pack_offset, FALSE, TRUE, &pack_entry, - cancellable, error)) - goto out; - - g_variant_get_child (pack_entry, 2, "v", &metadata); - - input = ot_variant_read (metadata); - } - else if (!is_stored) + if (!is_stored) { if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error)) goto out; @@ -716,11 +376,7 @@ fetch_and_store_metadata (OtPullData *pull_data, input = (GInputStream*)g_file_read (temp_path, cancellable, error); if (!input) goto out; - } - if (input) - { - g_assert (remote_pack_path != NULL || !is_stored); if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, input, cancellable, error)) goto out; @@ -735,8 +391,6 @@ fetch_and_store_metadata (OtPullData *pull_data, out: if (temp_path) (void) ot_gfile_unlink (temp_path, NULL, NULL); - if (pack_map) - g_mapped_file_unref (pack_map); return ret; } @@ -753,7 +407,6 @@ fetch_and_store_tree_metadata_recurse (OtPullData *pull_data, ot_lvariant GVariant *files_variant = NULL; ot_lvariant GVariant *dirs_variant = NULL; ot_lobj GFile *stored_path = NULL; - ot_lfree char *pack_checksum = NULL; if (depth > OSTREE_MAX_RECURSION) { @@ -936,66 +589,6 @@ fetch_ref_contents (OtPullData *pull_data, return ret; } -static gboolean -store_file_from_pack (OtPullData *pull_data, - const char *checksum, - const char *pack_checksum, - GFile *pack_file, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean exists; - guint64 pack_offset; - ot_lobj GFile *remote_pack_path = NULL; - ot_lobj GFile *temp_path = NULL; - ot_lvariant GVariant *pack_entry = NULL; - ot_lobj GInputStream *input = NULL; - ot_lobj GInputStream *file_object_input = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lvariant GVariant *xattrs = NULL; - ot_lvariant GVariant *csum_bytes_v = NULL; - GMappedFile *pack_map = NULL; - - csum_bytes_v = ostree_checksum_to_bytes_v (checksum); - - if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, OSTREE_OBJECT_TYPE_FILE, - pack_checksum, &exists, &pack_offset, - cancellable, error)) - goto out; - - g_assert (exists); - - pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_file), FALSE, error); - if (!pack_map) - goto out; - - if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), - g_mapped_file_get_length (pack_map), - pack_offset, FALSE, FALSE, &pack_entry, - cancellable, error)) - goto out; - - if (!ostree_parse_file_pack_entry (pack_entry, &input, &file_info, &xattrs, - cancellable, error)) - goto out; - - if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, - &file_object_input, NULL, cancellable, error)) - goto out; - - if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, - file_object_input, - cancellable, error)) - goto out; - - ret = TRUE; - out: - if (pack_map) - g_mapped_file_unref (pack_map); - return ret; -} - typedef struct { OtPullData *pull_data; @@ -1072,7 +665,7 @@ content_fetch_on_checksum_complete (GObject *object, } if (!ostree_repo_stage_file_object_trusted (data->pull_data->repo, checksum, - FALSE, file_object_input, length, + file_object_input, length, cancellable, error)) goto out; @@ -1231,42 +824,23 @@ fetch_content (OtPullData *pull_data, gpointer key, value; ot_lobj GFile *temp_path = NULL; ot_lobj GFile *content_temp_path = NULL; - ot_lhash GHashTable *data_packs_to_fetch = NULL; ot_lhash GHashTable *loose_files = NULL; SoupURI *content_uri = NULL; guint n_objects_to_fetch = 0; - data_packs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, - (GDestroyNotify) g_ptr_array_unref); loose_files = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); g_hash_table_iter_init (&hash_iter, pull_data->file_checksums_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *checksum = key; - GPtrArray *files_to_fetch; gboolean is_stored; - ot_lfree char *remote_pack_checksum = NULL; - if (!find_object_ensure_indexes (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, - &is_stored, &remote_pack_checksum, NULL, - cancellable, error)) + if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, &is_stored, + cancellable, error)) goto out; - if (remote_pack_checksum) - { - files_to_fetch = g_hash_table_lookup (data_packs_to_fetch, remote_pack_checksum); - if (files_to_fetch == NULL) - { - files_to_fetch = g_ptr_array_new_with_free_func (g_free); - g_hash_table_insert (data_packs_to_fetch, remote_pack_checksum, files_to_fetch); - /* transfer ownership */ - remote_pack_checksum = NULL; - } - g_ptr_array_add (files_to_fetch, g_strdup (checksum)); - n_objects_to_fetch++; - } - else if (!is_stored) + if (!is_stored) { char *key = g_strdup (checksum); g_hash_table_insert (loose_files, key, key); @@ -1274,71 +848,9 @@ fetch_content (OtPullData *pull_data, } } - g_hash_table_iter_init (&hash_iter, data_packs_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *pack_checksum = key; - GPtrArray *files = value; - ot_lvariant GVariant *mapped_pack = NULL; - ot_lvariant GVariant *content_list = NULL; - gboolean fetch; - - if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, FALSE, - &mapped_pack, - cancellable, error)) - goto out; - - content_list = g_variant_get_child_value (mapped_pack, 2); - - fetch = (((double)files->len) / g_variant_n_children (content_list)) > ((double)opt_packfile_threshold / 100); - if (!fetch) - { - guint i; - for (i = 0; i < files->len; i++) - { - g_hash_table_insert (loose_files, files->pdata[i], files->pdata[i]); - files->pdata[i] = NULL; /* steal data */ - } - g_hash_table_iter_remove (&hash_iter); - } - } - if (n_objects_to_fetch > 0) g_print ("%u content objects to fetch\n", n_objects_to_fetch); - if (g_hash_table_size (data_packs_to_fetch) > 0) - g_print ("Fetching %u content packs\n", - g_hash_table_size (data_packs_to_fetch)); - - g_hash_table_iter_init (&hash_iter, data_packs_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *pack_checksum = key; - GPtrArray *file_checksums = value; - guint i; - ot_lobj GFile *pack_path = NULL; - - if (!fetch_one_pack_file (pull_data, pack_checksum, FALSE, - &pack_path, cancellable, error)) - goto out; - - g_print ("Storing %u objects from content pack %s\n", file_checksums->len, - pack_checksum); - for (i = 0; i < file_checksums->len; i++) - { - const char *checksum = file_checksums->pdata[i]; - if (!store_file_from_pack (pull_data, checksum, pack_checksum, pack_path, - cancellable, error)) - goto out; - } - - if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, FALSE, NULL, - cancellable, error)) - goto out; - } - if (g_hash_table_size (loose_files) > 0) g_print ("Fetching %u loose objects\n", g_hash_table_size (loose_files)); @@ -1729,10 +1241,6 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) g_print ("remote %s is now %s\n", remote_ref, checksum); } - if (!ostree_repo_clean_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - cancellable, error)) - goto out; - bytes_transferred = ostree_fetcher_bytes_transferred (pull_data->fetcher); if (bytes_transferred > 0) { @@ -1751,8 +1259,6 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) if (pull_data->base_uri) soup_uri_free (pull_data->base_uri); g_clear_pointer (&pull_data->file_checksums_to_fetch, (GDestroyNotify) g_hash_table_unref); - g_clear_pointer (&pull_data->cached_meta_pack_indexes, (GDestroyNotify) g_ptr_array_unref); - g_clear_pointer (&pull_data->cached_data_pack_indexes, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref); if (summary_uri) soup_uri_free (summary_uri); diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 1a003623..9aa7b66b 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -39,129 +39,8 @@ static GOptionEntry options[] = { typedef struct { OstreeRepo *repo; - guint n_pack_files; } OtFsckData; -static gboolean -fsck_one_pack_file (OtFsckData *data, - const char *pack_checksum, - gboolean is_meta, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guchar objtype_u8; - guint64 offset; - guint64 pack_size; - ot_lfree char *path = NULL; - ot_lobj GFileInfo *pack_info = NULL; - ot_lobj GInputStream *input = NULL; - ot_lvariant GVariant *index_variant = NULL; - ot_lobj GFile *pack_index_path = NULL; - ot_lobj GFile *pack_data_path = NULL; - ot_lfree guchar *pack_content_csum = NULL; - ot_lfree char *tmp_checksum = NULL; - GVariantIter *index_content_iter = NULL; - - g_free (path); - path = ostree_get_relative_pack_index_path (is_meta, pack_checksum); - pack_index_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path); - - if (!ot_util_variant_map (pack_index_path, - OSTREE_PACK_INDEX_VARIANT_FORMAT, FALSE, - &index_variant, error)) - goto out; - - if (!ostree_validate_structureof_pack_index (index_variant, error)) - goto out; - - g_free (path); - path = ostree_get_relative_pack_data_path (is_meta, pack_checksum); - pack_data_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path); - - input = (GInputStream*)g_file_read (pack_data_path, cancellable, error); - if (!input) - goto out; - - pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO, - cancellable, error); - if (!pack_info) - goto out; - pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size"); - - if (!ot_gio_checksum_stream (input, &pack_content_csum, cancellable, error)) - goto out; - - tmp_checksum = ostree_checksum_from_bytes (pack_content_csum); - if (strcmp (tmp_checksum, pack_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "corrupted pack '%s', actual checksum is %s", - pack_checksum, tmp_checksum); - goto out; - } - - g_variant_get_child (index_variant, 2, "a(yayt)", &index_content_iter); - - while (g_variant_iter_loop (index_content_iter, "(y@ayt)", - &objtype_u8, NULL, &offset)) - { - offset = GUINT64_FROM_BE (offset); - if (offset > pack_size) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT, - pack_checksum, - offset, pack_size); - goto out; - } - } - - ret = TRUE; - out: - if (index_content_iter) - g_variant_iter_free (index_content_iter); - return ret; -} - -static gboolean -fsck_pack_files (OtFsckData *data, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lptrarray GPtrArray *meta_pack_indexes = NULL; - ot_lptrarray GPtrArray *data_pack_indexes = NULL; - - if (!ostree_repo_list_pack_indexes (data->repo, &meta_pack_indexes, &data_pack_indexes, - cancellable, error)) - goto out; - - for (i = 0; i < meta_pack_indexes->len; i++) - { - const char *pack_checksum = meta_pack_indexes->pdata[i]; - - if (!fsck_one_pack_file (data, pack_checksum, TRUE, cancellable, error)) - goto out; - - data->n_pack_files++; - } - for (i = 0; i < data_pack_indexes->len; i++) - { - const char *pack_checksum = data_pack_indexes->pdata[i]; - - if (!fsck_one_pack_file (data, pack_checksum, FALSE, cancellable, error)) - goto out; - - data->n_pack_files++; - } - - ret = TRUE; - out: - return ret; -} - static gboolean fsck_reachable_objects_from_commits (OtFsckData *data, GHashTable *commits, @@ -355,11 +234,6 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error) if (!fsck_reachable_objects_from_commits (&data, commits, cancellable, error)) goto out; - g_print ("Verifying structure of pack files...\n"); - - if (!fsck_pack_files (&data, cancellable, error)) - goto out; - ret = TRUE; out: if (context) diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c index 6c8c052c..0707242e 100644 --- a/src/ostree/ot-builtin-init.c +++ b/src/ostree/ot-builtin-init.c @@ -43,7 +43,7 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error) { GOptionContext *context = NULL; gboolean ret = FALSE; - GCancellable *cancellable = NULL; + __attribute__ ((unused)) GCancellable *cancellable = NULL; ot_lobj GFile *child = NULL; ot_lobj GFile *grandchild = NULL; ot_lobj OstreeRepo *repo = NULL; @@ -110,9 +110,6 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error) if (!ostree_repo_check (repo, error)) goto out; - if (!ostree_repo_regenerate_pack_index (repo, cancellable, error)) - goto out; - ret = TRUE; out: if (context) diff --git a/src/ostree/ot-builtin-pack.c b/src/ostree/ot-builtin-pack.c deleted file mode 100644 index 74564355..00000000 --- a/src/ostree/ot-builtin-pack.c +++ /dev/null @@ -1,967 +0,0 @@ -/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- - * - * Copyright (C) 2012 Colin Walters - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 02111-1307, USA. - * - * Author: Colin Walters - */ - -#include "config.h" - -#include "ot-builtins.h" -#include "ostree.h" - -#include -#include - -#include -#include - -#define OT_GZIP_COMPRESSION_LEVEL (8) - -static gboolean opt_analyze_only; -static gboolean opt_metadata_only; -static gboolean opt_content_only; -static gboolean opt_reindex_only; -static gboolean opt_delete_all_loose; -static gboolean opt_keep_all_loose; -static char* opt_pack_size = "50m"; -static char* opt_int_compression; -static char* opt_ext_compression; - -typedef enum { - OT_COMPRESSION_NONE, - OT_COMPRESSION_GZIP, - OT_COMPRESSION_XZ -} OtCompressionType; - -static GOptionEntry options[] = { - { "pack-size", 0, 0, G_OPTION_ARG_STRING, &opt_pack_size, "Maximum uncompressed size of packfiles in bytes; may be suffixed with k, m, or g (default: 50m)", "BYTES" }, - { "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION", "COMPRESSION" }, - { "external-compression", 0, 0, G_OPTION_ARG_STRING, &opt_ext_compression, "Compress entire packfiles using COMPRESSION", "COMPRESSION" }, - { "metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_metadata_only, "Only pack metadata objects", NULL }, - { "content-only", 0, 0, G_OPTION_ARG_NONE, &opt_content_only, "Only pack content objects", NULL }, - { "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL }, - { "reindex-only", 0, 0, G_OPTION_ARG_NONE, &opt_reindex_only, "Regenerate pack index", NULL }, - { "delete-all-loose", 0, 0, G_OPTION_ARG_NONE, &opt_delete_all_loose, "Delete all loose objects (default: delete unreferenced loose)", NULL }, - { "keep-all-loose", 0, 0, G_OPTION_ARG_NONE, &opt_keep_all_loose, "Don't delete any loose objects (default: delete unreferenced loose)", NULL }, - { NULL } -}; - -typedef struct { - OstreeRepo *repo; - - guint64 pack_size; - OtCompressionType int_compression; - OtCompressionType ext_compression; - - gboolean had_error; - GError **error; -} OtRepackData; - -typedef struct { - GOutputStream *out; - GPtrArray *compressor_argv; - GPid compress_child_pid; -} OtBuildRepackFile; - -static gint -compare_object_data_by_size (gconstpointer ap, - gconstpointer bp) -{ - GVariant *a = *(void **)ap; - GVariant *b = *(void **)bp; - guint64 a_size; - guint64 b_size; - - g_variant_get_child (a, 2, "t", &a_size); - g_variant_get_child (b, 2, "t", &b_size); - if (a == b) - return 0; - else if (a > b) - return 1; - else - return -1; -} - -static gboolean -write_bytes_update_checksum (GOutputStream *output, - gconstpointer bytes, - gsize len, - GChecksum *checksum, - guint64 *inout_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gsize bytes_written; - - if (len > 0) - { - g_checksum_update (checksum, (guchar*) bytes, len); - if (!g_output_stream_write_all (output, bytes, len, &bytes_written, - cancellable, error)) - goto out; - g_assert_cmpint (bytes_written, ==, len); - *inout_offset += bytes_written; - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -write_padding (GOutputStream *output, - guint alignment, - GChecksum *checksum, - guint64 *inout_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint bits; - guint padding_len; - guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0}; - - if (alignment == 8) - bits = ((*inout_offset) & 7); - else - bits = ((*inout_offset) & 3); - - if (bits > 0) - { - padding_len = alignment - bits; - if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding_len, - checksum, inout_offset, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - return ret; -} - -static gint -compare_index_content (gconstpointer ap, - gconstpointer bp) -{ - gpointer a = *((gpointer*)ap); - gpointer b = *((gpointer*)bp); - GVariant *a_v = a; - GVariant *b_v = b; - guchar a_objtype; - guchar b_objtype; - guint64 a_offset; - guint64 b_offset; - int c; - ot_lvariant GVariant *a_csum_bytes = NULL; - ot_lvariant GVariant *b_csum_bytes = NULL; - - g_variant_get (a_v, "(y@ayt)", &a_objtype, &a_csum_bytes, &a_offset); - g_variant_get (b_v, "(y@ayt)", &b_objtype, &b_csum_bytes, &b_offset); - c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (a_csum_bytes), - ostree_checksum_bytes_peek (b_csum_bytes)); - if (c == 0) - { - if (a_objtype < b_objtype) - c = -1; - else if (a_objtype > b_objtype) - c = 1; - } - return c; -} - -static gboolean -delete_loose_object (OtRepackData *data, - const char *checksum, - OstreeObjectType objtype, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean do_delete = FALSE; - GError *temp_error = NULL; - ot_lobj GFile *object_path = NULL; - ot_lobj GFile *file_content_object_path = NULL; - - object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); - - if (objtype == OSTREE_OBJECT_TYPE_FILE) - { - ot_lobj GFileInfo *file_info = NULL; - - if (ostree_repo_get_mode (data->repo) == OSTREE_REPO_MODE_BARE) - { - file_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, &temp_error); - } - else - { - ot_lobj GFile *content_object_path = NULL; - - content_object_path = ostree_repo_get_archive_content_path (data->repo, checksum); - - file_info = g_file_query_info (content_object_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, &temp_error); - } - - if (!file_info) - { - if (!g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_propagate_error (error, temp_error); - goto out; - } - else - { - g_clear_error (&temp_error); - } - } - - do_delete = opt_delete_all_loose - || (file_info && g_file_info_get_attribute_uint32 (file_info, "unix::nlink") <= 1); - } - else - do_delete = TRUE; - - if (do_delete) - { - if (!ot_gfile_unlink (object_path, cancellable, error)) - { - g_prefix_error (error, "Failed to delete loose object '%s'", - ot_gfile_get_path_cached (object_path)); - goto out; - } - - if (objtype == OSTREE_OBJECT_TYPE_FILE - && ostree_repo_get_mode (data->repo) == OSTREE_REPO_MODE_ARCHIVE) - { - ot_lobj GFile *content_object_path = NULL; - - content_object_path = ostree_repo_get_archive_content_path (data->repo, checksum); - - /* Ignoring errors for now; later should only be trying to - * delete files with content. - */ - (void) ot_gfile_unlink (content_object_path, NULL, NULL); - } - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -pack_one_meta_object (OtRepackData *data, - const char *checksum, - OstreeObjectType objtype, - GVariant **out_packed_object, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *object_path = NULL; - ot_lvariant GVariant *metadata_v = NULL; - ot_lvariant GVariant *ret_packed_object = NULL; - - object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); - - if (!ot_util_variant_map (object_path, ostree_metadata_variant_type (objtype), - TRUE, &metadata_v, error)) - goto out; - - ret_packed_object = g_variant_new ("(y@ayv)", (guchar) objtype, - ostree_checksum_to_bytes_v (checksum), - metadata_v); - - ret = TRUE; - ot_transfer_out_value (out_packed_object, &ret_packed_object); - out: - return ret; -} - -static gboolean -pack_one_data_object (OtRepackData *data, - const char *checksum, - OstreeObjectType objtype, - guint64 expected_objsize, - GVariant **out_packed_object, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guchar entry_flags = 0; - GInputStream *read_object_in; /* nofree */ - ot_lobj GInputStream *input = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lvariant GVariant *xattrs = NULL; - ot_lobj GMemoryOutputStream *object_data_stream = NULL; - ot_lobj GConverter *compressor = NULL; - ot_lobj GConverterInputStream *compressed_object_input = NULL; - ot_lvariant GVariant *file_header = NULL; - ot_lvariant GVariant *ret_packed_object = NULL; - - switch (data->int_compression) - { - case OT_COMPRESSION_GZIP: - { - entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP; - break; - } - default: - { - g_assert_not_reached (); - } - } - - if (!ostree_repo_load_file (data->repo, checksum, &input, &file_info, &xattrs, - cancellable, error)) - goto out; - - file_header = ostree_file_header_new (file_info, xattrs); - - object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free); - if (input != NULL) - { - if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP) - { - compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL); - compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM, - "converter", compressor, - "base-stream", input, - "close-base-stream", TRUE, - NULL); - read_object_in = (GInputStream*)compressed_object_input; - } - else - { - read_object_in = (GInputStream*)input; - } - - if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in, - G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, - cancellable, error)) - goto out; - } - - { - guchar *data = g_memory_output_stream_get_data (object_data_stream); - gsize data_len = g_memory_output_stream_get_data_size (object_data_stream); - ret_packed_object = g_variant_new ("(@ayy@(uuuusa(ayay))@ay)", - ostree_checksum_to_bytes_v (checksum), - entry_flags, - file_header, - ot_gvariant_new_bytearray (data, data_len)); - } - - ret = TRUE; - ot_transfer_out_value (out_packed_object, &ret_packed_object); - out: - return ret; -} - -static gboolean -create_pack_file (OtRepackData *data, - gboolean is_meta, - GPtrArray *objects, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - guint64 offset; - gsize bytes_written; - ot_lobj GFile *pack_dir = NULL; - ot_lobj GFile *index_temppath = NULL; - ot_lobj GOutputStream *index_out = NULL; - ot_lobj GFile *pack_temppath = NULL; - ot_lobj GOutputStream *pack_out = NULL; - ot_lptrarray GPtrArray *index_content_list = NULL; - ot_lvariant GVariant *pack_header = NULL; - ot_lvariant GVariant *index_content = NULL; - ot_lfree char *pack_name = NULL; - ot_lobj GFile *pack_file_path = NULL; - ot_lobj GFile *pack_index_path = NULL; - GVariantBuilder index_content_builder; - GChecksum *pack_checksum = NULL; - - if (g_cancellable_set_error_if_cancelled (cancellable, error)) - return FALSE; - - if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo), - "pack-index", NULL, - &index_temppath, - &index_out, - cancellable, error)) - goto out; - - if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo), - "pack-content", NULL, - &pack_temppath, - &pack_out, - cancellable, error)) - goto out; - - index_content_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - - offset = 0; - pack_checksum = g_checksum_new (G_CHECKSUM_SHA256); - - pack_header = g_variant_new ("(s@a{sv}t)", - is_meta ? "OSTv0PACKMETAFILE" : "OSTv0PACKDATAFILE", - g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), - (guint64)objects->len); - - if (!ostree_write_variant_with_size (pack_out, pack_header, offset, &bytes_written, pack_checksum, - cancellable, error)) - goto out; - offset += bytes_written; - - for (i = 0; i < objects->len; i++) - { - GVariant *object_data = objects->pdata[i]; - const char *checksum; - guint32 objtype_u32; - OstreeObjectType objtype; - guint64 expected_objsize; - ot_lvariant GVariant *packed_object = NULL; - ot_lvariant GVariant *index_entry = NULL; - - g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize); - - objtype = (OstreeObjectType) objtype_u32; - - if (is_meta) - { - if (!pack_one_meta_object (data, checksum, objtype, &packed_object, - cancellable, error)) - goto out; - } - else - { - if (!pack_one_data_object (data, checksum, objtype, expected_objsize, - &packed_object, cancellable, error)) - goto out; - } - - if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error)) - goto out; - - /* offset points to aligned header size */ - index_entry = g_variant_new ("(y@ayt)", - (guchar)objtype, - ostree_checksum_to_bytes_v (checksum), - GUINT64_TO_BE (offset)); - g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry)); - index_entry = NULL; - - bytes_written = 0; - if (!ostree_write_variant_with_size (pack_out, packed_object, offset, &bytes_written, - pack_checksum, cancellable, error)) - goto out; - offset += bytes_written; - } - - if (!g_output_stream_close (pack_out, cancellable, error)) - goto out; - - g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(yayt)")); - g_ptr_array_sort (index_content_list, compare_index_content); - for (i = 0; i < index_content_list->len; i++) - { - GVariant *index_item = index_content_list->pdata[i]; - g_variant_builder_add_value (&index_content_builder, index_item); - } - index_content = g_variant_new ("(s@a{sv}@a(yayt))", - "OSTv0PACKINDEX", - g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), - g_variant_builder_end (&index_content_builder)); - - if (!g_output_stream_write_all (index_out, - g_variant_get_data (index_content), - g_variant_get_size (index_content), - &bytes_written, - cancellable, - error)) - goto out; - - if (!g_output_stream_close (index_out, cancellable, error)) - goto out; - - if (!ostree_repo_add_pack_file (data->repo, - g_checksum_get_string (pack_checksum), - is_meta, - index_temppath, - pack_temppath, - cancellable, - error)) - goto out; - - if (!ostree_repo_regenerate_pack_index (data->repo, cancellable, error)) - goto out; - - g_print ("Created %s pack file '%s' with %u objects\n", is_meta ? "metadata" : "content", - g_checksum_get_string (pack_checksum), objects->len); - - if (!opt_keep_all_loose) - { - for (i = 0; i < objects->len; i++) - { - GVariant *object_data = objects->pdata[i]; - const char *checksum; - guint32 objtype_u32; - OstreeObjectType objtype; - guint64 expected_objsize; - - g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize); - - objtype = (OstreeObjectType) objtype_u32; - - if (!delete_loose_object (data, checksum, objtype, cancellable, error)) - goto out; - } - } - - ret = TRUE; - out: - if (index_temppath) - (void) unlink (ot_gfile_get_path_cached (index_temppath)); - if (pack_temppath) - (void) unlink (ot_gfile_get_path_cached (pack_temppath)); - if (pack_checksum) - g_checksum_free (pack_checksum); - return ret; -} - -static void -cluster_one_object_chain (OtRepackData *data, - GPtrArray *object_list, - GPtrArray *inout_clusters) -{ - guint i; - guint64 current_size; - guint current_offset; - - current_size = 0; - current_offset = 0; - for (i = 0; i < object_list->len; i++) - { - GVariant *objdata = object_list->pdata[i]; - guint64 objsize; - - g_variant_get_child (objdata, 2, "t", &objsize); - - if (current_size + objsize > data->pack_size || i == (object_list->len - 1)) - { - guint j; - GPtrArray *current; - - if (current_offset < i) - { - current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - for (j = current_offset; j <= i; j++) - { - g_ptr_array_add (current, g_variant_ref (object_list->pdata[j])); - } - g_ptr_array_add (inout_clusters, current); - current_size = objsize; - current_offset = i+1; - } - } - else if (objsize > data->pack_size) - { - break; - } - else - { - current_size += objsize; - } - } -} - -/** - * cluster_objects_stupidly: - * @objects: Map from serialized object name to objdata - * @out_meta_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). - * @out_data_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). - * - * Just sorts by size currently. Also filters out non-regular object - * content. - */ -static gboolean -cluster_objects_stupidly (OtRepackData *data, - GHashTable *objects, - GPtrArray **out_meta_clusters, - GPtrArray **out_data_clusters, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - GHashTableIter hash_iter; - gpointer key, value; - ot_lptrarray GPtrArray *ret_meta_clusters = NULL; - ot_lptrarray GPtrArray *ret_data_clusters = NULL; - ot_lptrarray GPtrArray *meta_object_list = NULL; - ot_lptrarray GPtrArray *data_object_list = NULL; - - meta_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - data_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - - g_hash_table_iter_init (&hash_iter, objects); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - GVariant *serialized_key = key; - const char *checksum; - OstreeObjectType objtype; - guint64 size; - GVariant *v; - ot_lobj GFile *object_path = NULL; - ot_lobj GFileInfo *object_info = NULL; - - ostree_object_name_deserialize (serialized_key, &checksum, &objtype); - - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - { - object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); - - object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); - if (!object_info) - goto out; - } - else - { - if (!ostree_repo_load_file (data->repo, checksum, NULL, &object_info, NULL, - cancellable, error)) - goto out; - } - - size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE); - - v = g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size)); - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - g_ptr_array_add (meta_object_list, v); - else - g_ptr_array_add (data_object_list, v); - } - - g_ptr_array_sort (meta_object_list, compare_object_data_by_size); - g_ptr_array_sort (data_object_list, compare_object_data_by_size); - - ret_meta_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref); - ret_data_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref); - - cluster_one_object_chain (data, meta_object_list, ret_meta_clusters); - cluster_one_object_chain (data, data_object_list, ret_data_clusters); - - ret = TRUE; - ot_transfer_out_value (out_meta_clusters, &ret_meta_clusters); - ot_transfer_out_value (out_data_clusters, &ret_data_clusters); - out: - return ret; -} - -static gboolean -parse_size_spec_with_suffix (const char *spec, - guint64 *out_size, - GError **error) -{ - gboolean ret = FALSE; - char *endptr = NULL; - guint64 ret_size; - - ret_size = g_ascii_strtoull (spec, &endptr, 10); - - if (endptr && *endptr) - { - char suffix = *endptr; - - switch (suffix) - { - case 'k': - case 'K': - { - ret_size *= 1024; - break; - } - case 'm': - case 'M': - { - ret_size *= (1024 * 1024); - break; - } - case 'g': - case 'G': - { - ret_size *= (1024 * 1024 * 1024); - break; - } - default: - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid size suffix '%c'", suffix); - goto out; - } - } - - ret = TRUE; - *out_size = ret_size; - out: - return ret; -} - -static gboolean -parse_compression_string (const char *compstr, - OtCompressionType *out_comptype, - GError **error) -{ - gboolean ret = FALSE; - OtCompressionType ret_comptype; - - if (compstr == NULL) - ret_comptype = OT_COMPRESSION_NONE; - else if (strcmp (compstr, "gzip") == 0) - ret_comptype = OT_COMPRESSION_GZIP; - else if (strcmp (compstr, "xz") == 0) - ret_comptype = OT_COMPRESSION_XZ; - else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid compression '%s'", compstr); - goto out; - } - - ret = TRUE; - *out_comptype = ret_comptype; - out: - return ret; -} - -static gboolean -do_stats_gather_loose (OtRepackData *data, - GHashTable *objects, - GHashTable **out_loose, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint n_loose = 0; - guint n_loose_and_packed = 0; - guint n_packed = 0; - guint n_dup_packed = 0; - guint n_commits = 0; - guint n_dirmeta = 0; - guint n_dirtree = 0; - guint n_files = 0; - GHashTableIter hash_iter; - gpointer key, value; - ot_lhash GHashTable *ret_loose = NULL; - - ret_loose = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, - (GDestroyNotify) g_variant_unref, - NULL); - - g_hash_table_iter_init (&hash_iter, objects); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - GVariant *serialized_key = key; - GVariant *objdata = value; - const char *checksum; - OstreeObjectType objtype; - gboolean is_loose; - gboolean is_packed; - ot_lvariant GVariant *pack_array = NULL; - - ostree_object_name_deserialize (serialized_key, &checksum, &objtype); - - g_variant_get (objdata, "(b@as)", &is_loose, &pack_array); - - is_packed = g_variant_n_children (pack_array) > 0; - - if (is_loose && is_packed) - { - n_loose_and_packed++; - } - else if (is_loose) - { - if (!(opt_metadata_only && !OSTREE_OBJECT_TYPE_IS_META(objtype)) - && !(opt_content_only && OSTREE_OBJECT_TYPE_IS_META(objtype))) - { - GVariant *copy = g_variant_ref (serialized_key); - g_hash_table_replace (ret_loose, copy, copy); - } - n_loose++; - } - else if (g_variant_n_children (pack_array) > 1) - { - n_dup_packed++; - } - else - { - n_packed++; - } - - switch (objtype) - { - case OSTREE_OBJECT_TYPE_COMMIT: - n_commits++; - break; - case OSTREE_OBJECT_TYPE_DIR_TREE: - n_dirtree++; - break; - case OSTREE_OBJECT_TYPE_DIR_META: - n_dirmeta++; - break; - case OSTREE_OBJECT_TYPE_FILE: - n_files++; - break; - default: - g_assert_not_reached (); - } - } - - g_print ("Commits: %u\n", n_commits); - g_print ("Tree contents: %u\n", n_dirtree); - g_print ("Tree meta: %u\n", n_dirmeta); - g_print ("Files: %u\n", n_files); - g_print ("\n"); - g_print ("Loose+packed objects: %u\n", n_loose_and_packed); - g_print ("Loose-only objects: %u\n", n_loose); - g_print ("Duplicate packed objects: %u\n", n_dup_packed); - g_print ("Packed-only objects: %u\n", n_packed); - - ret = TRUE; - ot_transfer_out_value (out_loose, &ret_loose); - /* out: */ - return ret; -} - -static gboolean -do_incremental_pack (OtRepackData *data, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lhash GHashTable *objects = NULL; - ot_lptrarray GPtrArray *meta_clusters = NULL; - ot_lptrarray GPtrArray *data_clusters = NULL; - ot_lhash GHashTable *loose_objects = NULL; - - if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, - cancellable, error)) - goto out; - - if (!do_stats_gather_loose (data, objects, &loose_objects, cancellable, error)) - goto out; - - g_print ("\n"); - g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size); - - if (!cluster_objects_stupidly (data, loose_objects, &meta_clusters, &data_clusters, - cancellable, error)) - goto out; - - if (meta_clusters->len > 0 || data_clusters->len > 0) - g_print ("Going to create %u meta packfiles, %u data packfiles\n", - meta_clusters->len, data_clusters->len); - else - g_print ("Nothing to do\n"); - - if (!opt_analyze_only) - { - for (i = 0; i < meta_clusters->len; i++) - { - GPtrArray *cluster = meta_clusters->pdata[i]; - - if (!create_pack_file (data, TRUE, cluster, cancellable, error)) - goto out; - } - for (i = 0; i < data_clusters->len; i++) - { - GPtrArray *cluster = data_clusters->pdata[i]; - - if (!create_pack_file (data, FALSE, cluster, cancellable, error)) - goto out; - } - } - - ret = TRUE; - out: - return ret; -} - -gboolean -ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error) -{ - gboolean ret = FALSE; - GOptionContext *context; - GCancellable *cancellable = NULL; - OtRepackData data; - ot_lobj OstreeRepo *repo = NULL; - - memset (&data, 0, sizeof (data)); - - context = g_option_context_new ("- Recompress objects"); - g_option_context_add_main_entries (context, options, NULL); - - if (!g_option_context_parse (context, &argc, &argv, error)) - goto out; - - if (opt_metadata_only && opt_content_only) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "--content-only cannot be specified with --metadata-only"); - goto out; - } - - repo = ostree_repo_new (repo_path); - if (!ostree_repo_check (repo, error)) - goto out; - - data.repo = repo; - data.error = error; - - if (!parse_size_spec_with_suffix (opt_pack_size, &data.pack_size, error)) - goto out; - /* Default internal compression to gzip */ - if (!parse_compression_string (opt_int_compression ? opt_int_compression : "gzip", &data.int_compression, error)) - goto out; - if (!parse_compression_string (opt_ext_compression, &data.ext_compression, error)) - goto out; - - if (opt_reindex_only) - { - if (!ostree_repo_regenerate_pack_index (repo, cancellable, error)) - goto out; - } - else - { - if (!do_incremental_pack (&data, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - if (context) - g_option_context_free (context); - return ret; -} diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index 2dec7627..7d6e05d0 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -67,7 +67,7 @@ import_one_object (OtLocalCloneData *data, cancellable, error)) goto out; - if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum, FALSE, + if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum, file_object, length, cancellable, error)) goto out; @@ -81,7 +81,7 @@ import_one_object (OtLocalCloneData *data, input = ot_variant_read (metadata); if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype, - checksum, FALSE, input, + checksum, input, cancellable, error)) goto out; } diff --git a/src/ostree/ot-builtin-unpack.c b/src/ostree/ot-builtin-unpack.c deleted file mode 100644 index 53e66e6c..00000000 --- a/src/ostree/ot-builtin-unpack.c +++ /dev/null @@ -1,266 +0,0 @@ -/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- - * - * Copyright (C) 2012 Colin Walters - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 02111-1307, USA. - * - * Author: Colin Walters - */ - -#include "config.h" - -#include "ot-builtins.h" -#include "ostree.h" - -#include -#include - -#include -#include - -static gboolean opt_keep_packs; - -static GOptionEntry options[] = { - { "keep-packs", 0, 0, G_OPTION_ARG_NONE, &opt_keep_packs, "Don't delete pack files", NULL }, - { NULL } -}; - -typedef struct { - OstreeRepo *repo; -} OtUnpackData; - -static gboolean -unpack_one_object (OstreeRepo *repo, - const char *checksum, - OstreeObjectType objtype, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GInputStream *input = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lvariant GVariant *xattrs = NULL; - ot_lvariant GVariant *meta = NULL; - - if (objtype == OSTREE_OBJECT_TYPE_FILE) - { - ot_lobj GInputStream *file_object = NULL; - guint64 length; - - if (!ostree_repo_load_file (repo, checksum, - &input, &file_info, &xattrs, - cancellable, error)) - goto out; - - if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, &length, - cancellable, error)) - goto out; - - if (!ostree_repo_stage_file_object_trusted (repo, checksum, TRUE, file_object, length, - cancellable, error)) - goto out; - } - else - { - if (!ostree_repo_load_variant (repo, objtype, checksum, &meta, error)) - goto out; - - input = ot_variant_read (meta); - - if (!ostree_repo_stage_object_trusted (repo, objtype, checksum, TRUE, - input, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -delete_one_packfile (OstreeRepo *repo, - const char *pack_checksum, - gboolean is_meta, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lfree char *data_name = NULL; - ot_lobj GFile *data_path = NULL; - ot_lfree char *index_name = NULL; - ot_lobj GFile *index_path = NULL; - - index_name = ostree_get_relative_pack_index_path (is_meta, pack_checksum); - index_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), index_name); - data_name = ostree_get_relative_pack_data_path (is_meta, pack_checksum); - data_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), data_name); - - if (!ot_gfile_unlink (index_path, cancellable, error)) - { - g_prefix_error (error, "Failed to delete pack index '%s': ", ot_gfile_get_path_cached (index_path)); - goto out; - } - if (!ot_gfile_unlink (data_path, cancellable, error)) - { - g_prefix_error (error, "Failed to delete pack data '%s': ", ot_gfile_get_path_cached (data_path)); - goto out; - } - - ret = TRUE; - out: - return ret; -} - -gboolean -ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error) -{ - gboolean ret = FALSE; - GOptionContext *context; - GCancellable *cancellable = NULL; - gboolean in_transaction = FALSE; - OtUnpackData data; - gpointer key, value; - guint64 unpacked_object_count = 0; - GHashTableIter hash_iter; - ot_lobj OstreeRepo *repo = NULL; - ot_lhash GHashTable *objects = NULL; - ot_lptrarray GPtrArray *clusters = NULL; - ot_lhash GHashTable *meta_packfiles_to_delete = NULL; - ot_lhash GHashTable *data_packfiles_to_delete = NULL; - ot_lobj GFile *objpath = NULL; - - memset (&data, 0, sizeof (data)); - - context = g_option_context_new ("- Uncompress objects"); - g_option_context_add_main_entries (context, options, NULL); - - if (!g_option_context_parse (context, &argc, &argv, error)) - goto out; - - repo = ostree_repo_new (repo_path); - if (!ostree_repo_check (repo, error)) - goto out; - - if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "Can't unpack bare repositories yet"); - goto out; - } - - data.repo = repo; - - if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error)) - goto out; - - if (!ostree_repo_prepare_transaction (repo, cancellable, error)) - goto out; - - in_transaction = TRUE; - - meta_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - data_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - - g_hash_table_iter_init (&hash_iter, objects); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - GVariant *objkey = key; - GVariant *objdata; - const char *checksum; - const char *pack_checksum; - OstreeObjectType objtype; - gboolean is_loose; - gboolean is_packed = FALSE; - GVariantIter *pack_array_iter; - GHashTable *target_hash; - - ostree_object_name_deserialize (objkey, &checksum, &objtype); - - objdata = g_hash_table_lookup (objects, objkey); - g_assert (objdata); - - g_variant_get (objdata, "(bas)", &is_loose, &pack_array_iter); - - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - target_hash = meta_packfiles_to_delete; - else - target_hash = data_packfiles_to_delete; - - while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum)) - { - is_packed = TRUE; - if (!g_hash_table_lookup (target_hash, pack_checksum)) - { - gchar *duped_checksum = g_strdup (pack_checksum); - g_hash_table_replace (target_hash, duped_checksum, duped_checksum); - } - } - g_variant_iter_free (pack_array_iter); - - if (is_packed) - { - if (!unpack_one_object (repo, checksum, objtype, cancellable, error)) - goto out; - - unpacked_object_count++; - } - } - - if (!ostree_repo_commit_transaction (repo, cancellable, error)) - goto out; - - if (!opt_keep_packs) - { - if (g_hash_table_size (meta_packfiles_to_delete) == 0 - && g_hash_table_size (data_packfiles_to_delete) == 0) - g_print ("No pack files; nothing to do\n"); - - g_hash_table_iter_init (&hash_iter, meta_packfiles_to_delete); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *pack_checksum = key; - - if (!delete_one_packfile (repo, pack_checksum, TRUE, cancellable, error)) - goto out; - - g_print ("Deleted packfile '%s'\n", pack_checksum); - } - - g_hash_table_iter_init (&hash_iter, data_packfiles_to_delete); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *pack_checksum = key; - - if (!delete_one_packfile (repo, pack_checksum, FALSE, cancellable, error)) - goto out; - - g_print ("Deleted packfile '%s'\n", pack_checksum); - } - - if (!ostree_repo_regenerate_pack_index (repo, cancellable, error)) - goto out; - } - - g_print ("Unpacked %" G_GUINT64_FORMAT " objects\n", unpacked_object_count); - - ret = TRUE; - out: - if (in_transaction) - (void) ostree_repo_abort_transaction (repo, cancellable, NULL); - if (context) - g_option_context_free (context); - return ret; -} diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index aa7a645d..b3d446ea 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -41,10 +41,8 @@ gboolean ostree_builtin_ls (int argc, char **argv, GFile *repo_path, GError **er gboolean ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_show (int argc, char **argv, GFile *repo_path, GError **error); -gboolean ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_rev_parse (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_remote (int argc, char **argv, GFile *repo_path, GError **error); -gboolean ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_write_refs (int argc, char **argv, GFile *repo_path, GError **error); G_END_DECLS diff --git a/tests/t0001-archive.sh b/tests/t0001-archive.sh index 3a3e706d..0e9f88cf 100755 --- a/tests/t0001-archive.sh +++ b/tests/t0001-archive.sh @@ -21,7 +21,7 @@ set -e . libtest.sh -echo '1..22' +echo '1..11' setup_test_repository "archive" echo "ok setup" @@ -68,45 +68,6 @@ $OSTREE cat test2 /baz/cow > cow-contents assert_file_has_content cow-contents "moo" echo "ok cat-file" -cd ${test_tmpdir} -$OSTREE pack -echo "ok pack" - cd ${test_tmpdir} $OSTREE fsck echo "ok fsck" - -$OSTREE checkout test2 checkout-test2-from-packed -echo "ok checkout union 1" - -cd ${test_tmpdir} -$OSTREE fsck -echo "ok fsck" - -$OSTREE pack --analyze-only -echo "ok pack analyze" - -$OSTREE unpack -echo "ok unpack" - -cd ${test_tmpdir} -$OSTREE fsck -echo "ok fsck" - -cd ${test_tmpdir} -$OSTREE checkout test2 checkout-test2-from-unpacked -echo "ok checkout union 2" - -$OSTREE pack --metadata-only -echo "ok pack metadata" - -$OSTREE fsck -echo "ok fsck" - -cd ${test_tmpdir} -rm -rf checkout-test2 -$OSTREE checkout test2 checkout-test2 -echo "ok checkout metadata-packed" - -$OSTREE unpack -echo "ok unpack" diff --git a/tests/t0010-pull.sh b/tests/t0010-pull.sh index 57416c60..1fe002ee 100755 --- a/tests/t0010-pull.sh +++ b/tests/t0010-pull.sh @@ -21,7 +21,7 @@ set -e . libtest.sh -echo '1..4' +echo '1..2' setup_fake_remote_repo1 cd ${test_tmpdir} @@ -38,21 +38,3 @@ cd checkout-origin-main assert_file_has_content firstfile '^first$' assert_file_has_content baz/cow '^moo$' echo "ok pull contents" - -cd ${test_tmpdir} -ostree --repo=$(pwd)/ostree-srv/gnomerepo pack -rm -rf repo -mkdir repo -${CMD_PREFIX} ostree --repo=repo init -${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo -${CMD_PREFIX} ostree-pull --repo=repo origin main -${CMD_PREFIX} ostree --repo=repo fsck -echo "ok pull packed" - -cd ${test_tmpdir} -rm -rf checkout-origin-main -$OSTREE checkout origin/main checkout-origin-main -cd checkout-origin-main -assert_file_has_content firstfile '^first$' -assert_file_has_content baz/cow '^moo$' -echo "ok pull contents packed"