diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 59b4775e..ce4e40c6 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -37,8 +37,6 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-ls.c \ src/ostree/ot-builtin-prune.c \ src/ostree/ot-builtin-remote.c \ - src/ostree/ot-builtin-pack.c \ - src/ostree/ot-builtin-unpack.c \ src/ostree/ot-builtin-rev-parse.c \ src/ostree/ot-builtin-show.c \ src/ostree/ot-builtin-write-refs.c \ diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 39a6531d..03b8b4e4 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -944,58 +944,6 @@ ostree_get_relative_archive_content_path (const char *checksum) return g_string_free (path, FALSE); } -static char * -get_pack_name (gboolean is_meta, - gboolean is_index, - const char *prefix, - const char *checksum) -{ - GString *path; - - g_assert (strlen (checksum) == 64); - - path = g_string_new (prefix); - if (is_meta) - g_string_append (path, "ostmetapack-"); - else - g_string_append (path, "ostdatapack-"); - g_string_append (path, checksum); - if (is_index) - g_string_append (path, ".index"); - else - g_string_append (path, ".data"); - - return g_string_free (path, FALSE); -} - -char * -ostree_get_pack_index_name (gboolean is_meta, - const char *checksum) -{ - return get_pack_name (is_meta, TRUE, "", checksum); -} - -char * -ostree_get_pack_data_name (gboolean is_meta, - const char *checksum) -{ - return get_pack_name (is_meta, FALSE, "", checksum); -} - -char * -ostree_get_relative_pack_index_path (gboolean is_meta, - const char *checksum) -{ - return get_pack_name (is_meta, TRUE, "objects/pack/", checksum); -} - -char * -ostree_get_relative_pack_data_path (gboolean is_meta, - const char *checksum) -{ - return get_pack_name (is_meta, FALSE, "objects/pack/", checksum); -} - gboolean ostree_file_header_parse (GVariant *metadata, GFileInfo **out_file_info, @@ -1343,194 +1291,6 @@ ostree_create_temp_dir (GFile *dir, return ret; } -gboolean -ostree_read_pack_entry_raw (guchar *pack_data, - guint64 pack_len, - guint64 offset, - gboolean trusted, - gboolean is_meta, - GVariant **out_entry, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint64 entry_start; - guint64 entry_end; - guint32 entry_len; - ot_lvariant GVariant *ret_entry = NULL; - - if (G_UNLIKELY (!(offset <= pack_len))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted pack index; out of range offset %" G_GUINT64_FORMAT, - offset); - goto out; - } - if (G_UNLIKELY (!((offset & 0x3) == 0))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT, - offset); - goto out; - } - - entry_start = ALIGN_VALUE (offset + 4, 8); - if (G_UNLIKELY (!(entry_start <= pack_len))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT, - entry_start); - goto out; - } - - g_assert ((((guint64)pack_data+offset) & 0x3) == 0); - entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset))); - - entry_end = entry_start + entry_len; - if (G_UNLIKELY (!(entry_end <= pack_len))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted pack index; out of range entry length %u", - entry_len); - goto out; - } - - ret_entry = g_variant_new_from_data (is_meta ? OSTREE_PACK_META_FILE_VARIANT_FORMAT : - OSTREE_PACK_DATA_FILE_VARIANT_FORMAT, - pack_data+entry_start, entry_len, - trusted, NULL, NULL); - g_variant_ref_sink (ret_entry); - ret = TRUE; - ot_transfer_out_value (out_entry, &ret_entry); - out: - return ret; -} - -gboolean -ostree_parse_file_pack_entry (GVariant *pack_entry, - GInputStream **out_input, - GFileInfo **out_info, - GVariant **out_xattrs, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guchar entry_flags; - ot_lvariant GVariant *file_header = NULL; - ot_lvariant GVariant *pack_data = NULL; - ot_lobj GInputStream *memory_input = NULL; - ot_lobj GInputStream *ret_input = NULL; - ot_lobj GFileInfo *ret_info = NULL; - ot_lvariant GVariant *ret_xattrs = NULL; - - g_variant_get_child (pack_entry, 1, "y", &entry_flags); - g_variant_get_child (pack_entry, 2, "@(uuuusa(ayay))", &file_header); - g_variant_get_child (pack_entry, 3, "@ay", &pack_data); - - if (!ostree_file_header_parse (file_header, &ret_info, &ret_xattrs, - error)) - goto out; - g_file_info_set_size (ret_info, g_variant_get_size (pack_data)); - - if (g_file_info_get_file_type (ret_info) == G_FILE_TYPE_REGULAR) - { - memory_input = ot_variant_read (pack_data); - - if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP) - { - ot_lobj GConverter *decompressor = NULL; - - decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP); - ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM, - "converter", decompressor, - "base-stream", memory_input, - "close-base-stream", TRUE, - NULL); - } - else - { - ret_input = g_object_ref (memory_input); - } - } - - ret = TRUE; - ot_transfer_out_value (out_input, &ret_input); - ot_transfer_out_value (out_info, &ret_info); - ot_transfer_out_value (out_xattrs, &ret_xattrs); - out: - return ret; -} - -gboolean -ostree_pack_index_search (GVariant *index, - GVariant *csum_v, - OstreeObjectType objtype, - guint64 *out_offset) -{ - gboolean ret = FALSE; - gsize imax, imin; - gsize n; - guint32 target_objtype; - const guchar *csum; - ot_lvariant GVariant *index_contents = NULL; - - csum = ostree_checksum_bytes_peek (csum_v); - - index_contents = g_variant_get_child_value (index, 2); - - target_objtype = (guint32) objtype; - - n = g_variant_n_children (index_contents); - - if (n == 0) - goto out; - - imax = n - 1; - imin = 0; - while (imax >= imin) - { - GVariant *cur_csum_bytes; - guchar cur_objtype; - guint64 cur_offset; - gsize imid; - int c; - - imid = (imin + imax) / 2; - - g_variant_get_child (index_contents, imid, "(y@ayt)", &cur_objtype, - &cur_csum_bytes, &cur_offset); - - c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum); - if (c == 0) - { - if (cur_objtype < target_objtype) - c = -1; - else if (cur_objtype > target_objtype) - c = 1; - } - g_variant_unref (cur_csum_bytes); - - if (c < 0) - imin = imid + 1; - else if (c > 0) - { - if (imid == 0) - goto out; - imax = imid - 1; - } - else - { - if (out_offset) - *out_offset = GUINT64_FROM_BE (cur_offset); - ret = TRUE; - goto out; - } - } - - out: - return ret; -} - gboolean ostree_validate_structureof_objtype (guchar objtype, GError **error) @@ -1766,95 +1526,3 @@ ostree_validate_structureof_dirmeta (GVariant *dirmeta, out: return ret; } - -gboolean -ostree_validate_structureof_pack_index (GVariant *index, - GError **error) -{ - gboolean ret = FALSE; - const char *header; - guchar objtype_u8; - guint64 offset; - ot_lvariant GVariant *csum_v = NULL; - GVariantIter *content_iter = NULL; - - if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error)) - goto out; - - g_variant_get_child (index, 0, "&s", &header); - - if (strcmp (header, "OSTv0PACKINDEX") != 0) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid pack index; doesn't match header"); - goto out; - } - - g_variant_get_child (index, 2, "a(yayt)", &content_iter); - - while (g_variant_iter_loop (content_iter, "(y@ayt)", - &objtype_u8, &csum_v, &offset)) - { - if (!ostree_validate_structureof_objtype (objtype_u8, error)) - goto out; - if (!ostree_validate_structureof_csum_v (csum_v, error)) - goto out; - } - csum_v = NULL; - - ret = TRUE; - out: - if (content_iter) - g_variant_iter_free (content_iter); - return ret; -} - -gboolean -ostree_validate_structureof_pack_superindex (GVariant *superindex, - GError **error) -{ - gboolean ret = FALSE; - const char *header; - ot_lvariant GVariant *csum_v = NULL; - ot_lvariant GVariant *bloom = NULL; - GVariantIter *content_iter = NULL; - - if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error)) - goto out; - - g_variant_get_child (superindex, 0, "&s", &header); - - if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid pack superindex; doesn't match header"); - goto out; - } - - g_variant_get_child (superindex, 2, "a(ayay)", &content_iter); - while (g_variant_iter_loop (content_iter, "(@ay@ay)", - &csum_v, &bloom)) - { - if (!ostree_validate_structureof_csum_v (csum_v, error)) - goto out; - } - csum_v = NULL; - bloom = NULL; - - g_variant_iter_free (content_iter); - g_variant_get_child (superindex, 3, "a(ayay)", &content_iter); - while (g_variant_iter_loop (content_iter, "(@ay@ay)", - &csum_v, &bloom)) - { - if (!ostree_validate_structureof_csum_v (csum_v, error)) - goto out; - } - csum_v = NULL; - bloom = NULL; - - ret = TRUE; - out: - if (content_iter) - g_variant_iter_free (content_iter); - return ret; -} diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 3617b01e..5b038180 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -87,48 +87,6 @@ typedef enum { */ #define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE ("(a{sv}aya(say)sstayay)") -/* Pack super index - * s - OSTv0SUPERPACKINDEX - * a{sv} - Metadata - * a(ayay) - metadata packs (pack file checksum, bloom filter) - * a(ayay) - data packs (pack file checksum, bloom filter) - */ -#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay)a(ayay))") - -/* Pack index - * s - OSTv0PACKINDEX - * a{sv} - Metadata - * a(yayt) - (objtype, checksum, offset into packfile) - */ -#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(yayt))") - -typedef enum { - OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0, - OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0) -} OstreePackFileEntryFlag; - -/* Data Pack files - * s - OSTv0PACKDATAFILE - * a{sv} - Metadata - * t - number of entries - * - * Repeating pair of: - * - * ( ayy(uuuusa(ayay))ay) ) - checksum, flags, file meta, data - */ -#define OSTREE_PACK_DATA_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(ayy(uuuusa(ayay))ay)") - -/* Meta Pack files - * s - OSTv0PACKMETAFILE - * a{sv} - Metadata - * t - number of entries - * - * Repeating pair of: - * - * ( yayv ) - objtype, checksum, data - */ -#define OSTREE_PACK_META_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yayv)") - const GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype); gboolean ostree_validate_checksum_string (const char *sha256, @@ -173,16 +131,6 @@ char *ostree_get_relative_object_path (const char *checksum, char *ostree_get_relative_archive_content_path (const char *checksum); -char *ostree_get_pack_index_name (gboolean is_meta, - const char *checksum); -char *ostree_get_pack_data_name (gboolean is_meta, - const char *checksum); - -char *ostree_get_relative_pack_index_path (gboolean is_meta, - const char *checksum); -char *ostree_get_relative_pack_data_path (gboolean is_meta, - const char *checksum); - gboolean ostree_get_xattrs_for_file (GFile *f, GVariant **out_xattrs, GCancellable *cancellable, @@ -304,27 +252,6 @@ gboolean ostree_create_temp_dir (GFile *dir, GCancellable *cancellable, GError **error); -gboolean ostree_read_pack_entry_raw (guchar *pack_data, - guint64 pack_len, - guint64 object_offset, - gboolean trusted, - gboolean is_meta, - GVariant **out_entry, - GCancellable *cancellable, - GError **error); - -gboolean ostree_parse_file_pack_entry (GVariant *pack_entry, - GInputStream **out_input, - GFileInfo **out_info, - GVariant **out_xattrs, - GCancellable *cancellable, - GError **error); - -gboolean ostree_pack_index_search (GVariant *index, - GVariant *csum_bytes, - OstreeObjectType objtype, - guint64 *out_offset); - /** VALIDATION **/ gboolean ostree_validate_structureof_objtype (guchar objtype, @@ -348,10 +275,4 @@ gboolean ostree_validate_structureof_dirtree (GVariant *index, gboolean ostree_validate_structureof_dirmeta (GVariant *index, GError **error); -gboolean ostree_validate_structureof_pack_index (GVariant *index, - GError **error); - -gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex, - GError **error); - #endif /* _OSTREE_REPO */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 9bd44f17..05041b45 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -50,7 +50,6 @@ struct OstreeRepo { GFile *local_heads_dir; GFile *remote_heads_dir; GFile *objects_dir; - GFile *pack_dir; GFile *remote_cache_dir; GFile *config_file; @@ -61,8 +60,6 @@ struct OstreeRepo { #endif GPtrArray *cached_meta_indexes; GPtrArray *cached_content_indexes; - GHashTable *cached_pack_index_mappings; - GHashTable *cached_pack_data_mappings; gboolean inited; gboolean in_transaction; @@ -82,10 +79,7 @@ static gboolean repo_find_object (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, - gboolean lookup_all, GFile **out_stored_path, - char **out_pack_checksum, - guint64 *out_pack_offset, GCancellable *cancellable, GError **error); @@ -110,7 +104,6 @@ ostree_repo_finalize (GObject *object) g_clear_object (&self->local_heads_dir); g_clear_object (&self->remote_heads_dir); g_clear_object (&self->objects_dir); - g_clear_object (&self->pack_dir); g_clear_object (&self->remote_cache_dir); g_clear_object (&self->config_file); if (self->loose_object_devino_hash) @@ -119,8 +112,6 @@ ostree_repo_finalize (GObject *object) g_key_file_free (self->config); g_clear_pointer (&self->cached_meta_indexes, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&self->cached_content_indexes, (GDestroyNotify) g_ptr_array_unref); - g_hash_table_destroy (self->cached_pack_index_mappings); - g_hash_table_destroy (self->cached_pack_data_mappings); g_mutex_clear (&self->cache_lock); G_OBJECT_CLASS (ostree_repo_parent_class)->finalize (object); @@ -186,7 +177,6 @@ ostree_repo_constructor (GType gtype, self->remote_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/remotes"); self->objects_dir = g_file_get_child (self->repodir, "objects"); - self->pack_dir = g_file_get_child (self->objects_dir, "pack"); self->remote_cache_dir = g_file_get_child (self->repodir, "remote-cache"); self->config_file = g_file_get_child (self->repodir, "config"); @@ -216,12 +206,6 @@ static void ostree_repo_init (OstreeRepo *self) { g_mutex_init (&self->cache_lock); - self->cached_pack_index_mappings = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, - (GDestroyNotify)g_variant_unref); - self->cached_pack_data_mappings = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, - (GDestroyNotify)g_mapped_file_unref); } OstreeRepo* @@ -841,8 +825,7 @@ commit_loose_object_trusted (OstreeRepo *self, } typedef enum { - OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED = (1<<0), - OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<1) + OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<0) } OstreeRepoStageFlags; static gboolean @@ -863,9 +846,9 @@ stage_object_internal (OstreeRepo *self, ot_lobj GFile *temp_file = NULL; ot_lobj GFile *raw_temp_file = NULL; ot_lobj GFile *stored_path = NULL; - ot_lfree char *pack_checksum = NULL; ot_lfree guchar *ret_csum = NULL; ot_lobj OstreeChecksumInputStream *checksum_input = NULL; + gboolean have_obj; GChecksum *checksum = NULL; gboolean staged_raw_file = FALSE; gboolean staged_archive_file = FALSE; @@ -984,18 +967,11 @@ stage_object_internal (OstreeRepo *self, } } - if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED)) - { - gboolean have_obj; + if (!ostree_repo_has_object (self, objtype, actual_checksum, &have_obj, + cancellable, error)) + goto out; - if (!ostree_repo_has_object (self, objtype, actual_checksum, &have_obj, - cancellable, error)) - goto out; - - do_commit = !have_obj; - } - else - do_commit = TRUE; + do_commit = !have_obj; if (do_commit) { @@ -1074,9 +1050,7 @@ stage_object (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - guint64 pack_offset; ot_lobj GFile *stored_path = NULL; - ot_lfree char *pack_checksum = NULL; ot_lfree guchar *ret_csum = NULL; g_return_val_if_fail (self->in_transaction, FALSE); @@ -1088,23 +1062,12 @@ stage_object (OstreeRepo *self, if (expected_checksum) { - if (!(flags & OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED)) - { - if (!repo_find_object (self, objtype, expected_checksum, FALSE, - &stored_path, &pack_checksum, &pack_offset, - cancellable, error)) - goto out; - } - else - { - if (!repo_find_object (self, objtype, expected_checksum, FALSE, - &stored_path, NULL, NULL, - cancellable, error)) - goto out; - } + if (!repo_find_object (self, objtype, expected_checksum, &stored_path, + cancellable, error)) + goto out; } - if (stored_path == NULL && pack_checksum == NULL) + if (stored_path == NULL) { if (!stage_object_internal (self, flags, objtype, input, file_object_length, expected_checksum, @@ -1425,14 +1388,11 @@ gboolean ostree_repo_stage_object_trusted (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, - gboolean store_if_packed, GInputStream *object_input, GCancellable *cancellable, GError **error) { int flags = 0; - if (store_if_packed) - flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED; return stage_object (self, flags, objtype, object_input, 0, checksum, NULL, cancellable, error); @@ -1462,15 +1422,12 @@ ostree_repo_stage_object (OstreeRepo *self, gboolean ostree_repo_stage_file_object_trusted (OstreeRepo *self, const char *checksum, - gboolean store_if_packed, GInputStream *object_input, guint64 length, GCancellable *cancellable, GError **error) { int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID; - if (store_if_packed) - flags |= OSTREE_REPO_STAGE_FLAGS_STORE_IF_PACKED; return stage_object (self, flags, OSTREE_OBJECT_TYPE_FILE, object_input, length, checksum, NULL, cancellable, error); @@ -1715,836 +1672,6 @@ ostree_repo_stage_commit (OstreeRepo *self, return ret; } -static gboolean -list_files_in_dir_matching (GFile *dir, - const char *prefix, - const char *suffix, - GPtrArray **out_files, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - GError *temp_error = NULL; - ot_lobj GFileEnumerator *enumerator = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lptrarray GPtrArray *ret_files = NULL; - - g_return_val_if_fail (prefix != NULL || suffix != NULL, FALSE); - - ret_files = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - - enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); - if (!enumerator) - goto out; - - while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL) - { - const char *name; - guint32 type; - - name = g_file_info_get_attribute_byte_string (file_info, "standard::name"); - type = g_file_info_get_attribute_uint32 (file_info, "standard::type"); - - if (type != G_FILE_TYPE_REGULAR) - goto loop_next; - - if (prefix) - { - if (!g_str_has_prefix (name, prefix)) - goto loop_next; - } - if (suffix) - { - if (!g_str_has_suffix (name, suffix)) - goto loop_next; - } - - g_ptr_array_add (ret_files, g_file_get_child (dir, name)); - - loop_next: - g_clear_object (&file_info); - } - if (temp_error != NULL) - { - g_propagate_error (error, temp_error); - goto out; - } - if (!g_file_enumerator_close (enumerator, cancellable, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_files, &ret_files); - out: - return ret; -} - -static gboolean -map_variant_file_check_header_string (GFile *path, - const GVariantType *variant_type, - const char *expected_header, - gboolean trusted, - GVariant **out_variant, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - const char *header; - ot_lvariant GVariant *ret_variant = NULL; - - if (!ot_util_variant_map (path, variant_type, trusted, &ret_variant, error)) - goto out; - - g_variant_get_child (ret_variant, 0, "&s", &header); - - if (strcmp (header, expected_header) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid variant file '%s', expected header '%s'", - ot_gfile_get_path_cached (path), - expected_header); - goto out; - } - - ret = TRUE; - ot_transfer_out_value (out_variant, &ret_variant); - out: - return ret; -} - - -static char * -get_checksum_from_pack_name (const char *name) -{ - const char *dash; - const char *dot; - - dash = strchr (name, '-'); - g_assert (dash); - dot = strrchr (name, '.'); - g_assert (dot); - - g_assert_cmpint (dot - (dash + 1), ==, 64); - - return g_strndup (dash + 1, 64); -} - -static gboolean -list_pack_indexes_from_dir (OstreeRepo *self, - gboolean is_meta, - GPtrArray **out_indexes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lptrarray GPtrArray *index_files = NULL; - ot_lptrarray GPtrArray *ret_indexes = NULL; - - if (!list_files_in_dir_matching (self->pack_dir, - is_meta ? "ostmetapack-" : "ostdatapack-", ".index", - &index_files, - cancellable, error)) - goto out; - - ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - for (i = 0; i < index_files->len; i++) - { - GFile *index_path = index_files->pdata[i]; - const char *basename = ot_gfile_get_basename_cached (index_path); - g_ptr_array_add (ret_indexes, get_checksum_from_pack_name (basename)); - } - - ret = TRUE; - ot_transfer_out_value (out_indexes, &ret_indexes); - out: - return ret; -} - -static gboolean -list_pack_checksums_from_superindex_file (GFile *superindex_path, - GPtrArray **out_meta_indexes, - GPtrArray **out_data_indexes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - const char *magic; - ot_lptrarray GPtrArray *ret_meta_indexes = NULL; - ot_lptrarray GPtrArray *ret_data_indexes = NULL; - ot_lvariant GVariant *superindex_variant = NULL; - ot_lvariant GVariant *checksum = NULL; - ot_lvariant GVariant *bloom = NULL; - GVariantIter *meta_variant_iter = NULL; - GVariantIter *data_variant_iter = NULL; - - if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, - TRUE, &superindex_variant, error)) - goto out; - - g_variant_get (superindex_variant, "(&s@a{sv}a(ayay)a(ayay))", - &magic, NULL, &meta_variant_iter, &data_variant_iter); - - if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid header in super pack index"); - goto out; - } - - ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - while (g_variant_iter_loop (meta_variant_iter, "(@ay@ay)", - &checksum, &bloom)) - g_ptr_array_add (ret_meta_indexes, ostree_checksum_from_bytes_v (checksum)); - checksum = NULL; - bloom = NULL; - - ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - while (g_variant_iter_loop (data_variant_iter, "(@ay@ay)", - &checksum, &bloom)) - g_ptr_array_add (ret_data_indexes, ostree_checksum_from_bytes_v (checksum)); - checksum = NULL; - bloom = NULL; - - ret = TRUE; - ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes); - ot_transfer_out_value (out_data_indexes, &ret_data_indexes); - out: - if (meta_variant_iter) - g_variant_iter_free (meta_variant_iter); - if (data_variant_iter) - g_variant_iter_free (data_variant_iter); - return ret; -} - -gboolean -ostree_repo_list_pack_indexes (OstreeRepo *self, - GPtrArray **out_meta_indexes, - GPtrArray **out_data_indexes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *superindex_path = NULL; - ot_lptrarray GPtrArray *ret_meta_indexes = NULL; - ot_lptrarray GPtrArray *ret_data_indexes = NULL; - - g_mutex_lock (&self->cache_lock); - if (self->cached_meta_indexes) - { - ret_meta_indexes = g_ptr_array_ref (self->cached_meta_indexes); - ret_data_indexes = g_ptr_array_ref (self->cached_content_indexes); - } - else - { - superindex_path = g_file_get_child (self->pack_dir, "index"); - - if (g_file_query_exists (superindex_path, cancellable)) - { - if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_meta_indexes, - &ret_data_indexes, - cancellable, error)) - goto out; - } - else - { - ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - } - - self->cached_meta_indexes = g_ptr_array_ref (ret_meta_indexes); - self->cached_content_indexes = g_ptr_array_ref (ret_data_indexes); - } - - ret = TRUE; - ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes); - ot_transfer_out_value (out_data_indexes, &ret_data_indexes); - out: - g_mutex_unlock (&self->cache_lock); - return ret; -} - -static gboolean -create_index_bloom (OstreeRepo *self, - const char *pack_checksum, - GVariant **out_bloom, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lvariant GVariant *ret_bloom; - - /* TODO - define and compute bloom filter */ - - ret_bloom = ot_gvariant_new_bytearray (NULL, 0); - g_variant_ref_sink (ret_bloom); - - ret = TRUE; - ot_transfer_out_value (out_bloom, &ret_bloom); - /* out: */ - return ret; -} - -static gboolean -append_index_builder (OstreeRepo *self, - GPtrArray *indexes, - GVariantBuilder *builder, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - - for (i = 0; i < indexes->len; i++) - { - const char *pack_checksum = indexes->pdata[i]; - ot_lvariant GVariant *bloom = NULL; - - if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error)) - goto out; - - g_variant_builder_add (builder, - "(@ay@ay)", - ostree_checksum_to_bytes_v (pack_checksum), - bloom); - } - - ret = TRUE; - out: - return ret; -} - -/** - * Regenerate the pack superindex file based on the set of pack - * indexes currently in the filesystem. - */ -gboolean -ostree_repo_regenerate_pack_index (OstreeRepo *self, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *superindex_path = NULL; - ot_lptrarray GPtrArray *pack_indexes = NULL; - ot_lvariant GVariant *superindex_variant = NULL; - GVariantBuilder *meta_index_content_builder = NULL; - GVariantBuilder *data_index_content_builder = NULL; - - g_clear_pointer (&self->cached_meta_indexes, (GDestroyNotify) g_ptr_array_unref); - g_clear_pointer (&self->cached_content_indexes, (GDestroyNotify) g_ptr_array_unref); - - superindex_path = g_file_get_child (self->pack_dir, "index"); - - g_clear_pointer (&pack_indexes, (GDestroyNotify) g_ptr_array_unref); - if (!list_pack_indexes_from_dir (self, TRUE, &pack_indexes, - cancellable, error)) - goto out; - meta_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); - if (!append_index_builder (self, pack_indexes, meta_index_content_builder, - cancellable, error)) - goto out; - - g_clear_pointer (&pack_indexes, (GDestroyNotify) g_ptr_array_unref); - if (!list_pack_indexes_from_dir (self, FALSE, &pack_indexes, - cancellable, error)) - goto out; - data_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); - if (!append_index_builder (self, pack_indexes, data_index_content_builder, - cancellable, error)) - goto out; - - superindex_variant = g_variant_new ("(s@a{sv}@a(ayay)@a(ayay))", - "OSTv0SUPERPACKINDEX", - g_variant_new_array (G_VARIANT_TYPE ("{sv}"), - NULL, 0), - g_variant_builder_end (meta_index_content_builder), - g_variant_builder_end (data_index_content_builder)); - g_variant_ref_sink (superindex_variant); - - if (!ot_util_variant_save (superindex_path, superindex_variant, - cancellable, error)) - goto out; - - ret = TRUE; - out: - if (meta_index_content_builder) - g_variant_builder_unref (meta_index_content_builder); - if (data_index_content_builder) - g_variant_builder_unref (data_index_content_builder); - return ret; -} - -static GFile * -get_pack_index_path (GFile *parent, - gboolean is_meta, - const char *checksum) -{ - char *path = ostree_get_pack_index_name (is_meta, checksum); - GFile *ret = g_file_resolve_relative_path (parent, path); - g_free (path); - return ret; -} - -static GFile * -get_pack_data_path (GFile *parent, - gboolean is_meta, - const char *checksum) -{ - char *path = ostree_get_pack_data_name (is_meta, checksum); - GFile *ret = g_file_resolve_relative_path (parent, path); - g_free (path); - return ret; -} - -gboolean -ostree_repo_add_pack_file (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - GFile *index_path, - GFile *data_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *pack_index_path = NULL; - ot_lobj GFile *pack_data_path = NULL; - - if (!ot_gfile_ensure_directory (self->pack_dir, FALSE, error)) - goto out; - - pack_data_path = get_pack_data_path (self->pack_dir, is_meta, pack_checksum); - if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error)) - goto out; - - pack_index_path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum); - if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; -} - -static gboolean -ensure_remote_cache_dir (OstreeRepo *self, - const char *remote_name, - GFile **out_cache_dir, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *ret_cache_dir = NULL; - - ret_cache_dir = g_file_get_child (self->remote_cache_dir, remote_name); - - if (!ot_gfile_ensure_directory (ret_cache_dir, FALSE, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_cache_dir, &ret_cache_dir); - out: - return ret; -} - -static gboolean -delete_no_longer_referenced (OstreeRepo *self, - GFile *cache_path, - const char *prefix, - const char *suffix, - GHashTable *new_files, - GPtrArray *inout_cached, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lptrarray GPtrArray *current_files = NULL; - ot_lfree char *pack_checksum = NULL; - - if (!list_files_in_dir_matching (cache_path, - prefix, suffix, - ¤t_files, - cancellable, error)) - goto out; - for (i = 0; i < current_files->len; i++) - { - GFile *file = current_files->pdata[i]; - - g_free (pack_checksum); - pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (file)); - - if (!g_hash_table_lookup (new_files, pack_checksum)) - { - if (!ot_gfile_unlink (file, cancellable, error)) - goto out; - } - - if (inout_cached) - { - g_ptr_array_add (inout_cached, pack_checksum); - pack_checksum = NULL; /* transfer ownership */ - } - } - ret = TRUE; - out: - return ret; -} - -static void -gather_uncached (GHashTable *new_files, - GPtrArray *cached, - GPtrArray *inout_uncached) -{ - guint i; - GHashTableIter hash_iter; - gpointer key, value; - - g_hash_table_iter_init (&hash_iter, new_files); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *cur_pack_checksum = key; - gboolean found = FALSE; - - for (i = 0; i < cached->len; i++) - { - const char *checksum = cached->pdata[i]; - if (strcmp (cur_pack_checksum, checksum) == 0) - { - found = TRUE; - break; - } - } - - if (!found) - g_ptr_array_add (inout_uncached, g_strdup (cur_pack_checksum)); - } -} - -/** - * Take a pack superindex file @superindex_path, and clean up any - * no-longer-referenced pack files in the lookaside cache for - * @remote_name. The updated index file will also be saved into the - * cache. - * - * Upon successful return, @out_cached_indexes will hold checksum - * strings for indexes which are already in the cache, and - * @out_uncached_indexes will hold strings for those which are not. - */ -gboolean -ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, - const char *remote_name, - GFile *superindex_path, - GPtrArray **out_cached_meta_indexes, - GPtrArray **out_cached_data_indexes, - GPtrArray **out_uncached_meta_indexes, - GPtrArray **out_uncached_data_indexes, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lvariant GVariant *superindex_variant = NULL; - ot_lobj GFile *cache_path = NULL; - ot_lobj GFile *superindex_cache_path = NULL; - ot_lptrarray GPtrArray *meta_index_files = NULL; - ot_lptrarray GPtrArray *data_index_files = NULL; - ot_lptrarray GPtrArray *meta_data_files = NULL; - ot_lptrarray GPtrArray *data_data_files = NULL; - ot_lhash GHashTable *new_pack_meta_indexes = NULL; - ot_lhash GHashTable *new_pack_data_indexes = NULL; - ot_lptrarray GPtrArray *ret_cached_meta_indexes = NULL; - ot_lptrarray GPtrArray *ret_cached_data_indexes = NULL; - ot_lptrarray GPtrArray *ret_uncached_meta_indexes = NULL; - ot_lptrarray GPtrArray *ret_uncached_data_indexes = NULL; - ot_lvariant GVariant *csum_bytes = NULL; - ot_lvariant GVariant *bloom = NULL; - ot_lfree char *pack_checksum = NULL; - GVariantIter *superindex_contents_iter = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error)) - goto out; - - ret_cached_meta_indexes = g_ptr_array_new_with_free_func (g_free); - ret_cached_data_indexes = g_ptr_array_new_with_free_func (g_free); - ret_uncached_meta_indexes = g_ptr_array_new_with_free_func (g_free); - ret_uncached_data_indexes = g_ptr_array_new_with_free_func (g_free); - - if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, - FALSE, &superindex_variant, error)) - goto out; - - if (!ostree_validate_structureof_pack_superindex (superindex_variant, error)) - goto out; - - new_pack_meta_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - new_pack_data_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - - g_variant_get_child (superindex_variant, 2, "a(ayay)", - &superindex_contents_iter); - while (g_variant_iter_loop (superindex_contents_iter, - "(@ay@ay)", &csum_bytes, &bloom)) - { - pack_checksum = ostree_checksum_from_bytes_v (csum_bytes); - g_hash_table_insert (new_pack_meta_indexes, pack_checksum, pack_checksum); - pack_checksum = NULL; /* transfer ownership */ - } - g_variant_iter_free (superindex_contents_iter); - - g_variant_get_child (superindex_variant, 3, "a(ayay)", - &superindex_contents_iter); - while (g_variant_iter_loop (superindex_contents_iter, - "(@ay@ay)", &csum_bytes, &bloom)) - { - pack_checksum = ostree_checksum_from_bytes_v (csum_bytes); - g_hash_table_insert (new_pack_data_indexes, pack_checksum, pack_checksum); - pack_checksum = NULL; /* transfer ownership */ - } - - if (!delete_no_longer_referenced (self, cache_path, - "ostmetapack-", ".index", - new_pack_meta_indexes, - ret_cached_meta_indexes, - cancellable, error)) - goto out; - - if (!delete_no_longer_referenced (self, cache_path, - "ostdatapack-", ".index", - new_pack_data_indexes, - ret_cached_data_indexes, - cancellable, error)) - goto out; - - gather_uncached (new_pack_meta_indexes, ret_cached_meta_indexes, ret_uncached_meta_indexes); - gather_uncached (new_pack_data_indexes, ret_cached_data_indexes, ret_uncached_data_indexes); - - superindex_cache_path = g_file_get_child (cache_path, "index"); - if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error)) - goto out; - - /* Now also delete stale pack files */ - - if (!delete_no_longer_referenced (self, cache_path, - "ostmetapack-", ".data", - new_pack_meta_indexes, NULL, - cancellable, error)) - goto out; - if (!delete_no_longer_referenced (self, cache_path, - "ostdatapack-", ".data", - new_pack_data_indexes, NULL, - cancellable, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_cached_meta_indexes, &ret_cached_meta_indexes); - ot_transfer_out_value (out_cached_data_indexes, &ret_cached_data_indexes); - ot_transfer_out_value (out_uncached_meta_indexes, &ret_uncached_meta_indexes); - ot_transfer_out_value (out_uncached_data_indexes, &ret_uncached_data_indexes); - out: - if (superindex_contents_iter) - g_variant_iter_free (superindex_contents_iter); - return ret; -} - -gboolean -ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lobj GFile *cache_path = NULL; - ot_lptrarray GPtrArray *data_files = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error)) - goto out; - - if (!list_files_in_dir_matching (cache_path, - "ostmetapack-", ".data", - &data_files, - cancellable, error)) - goto out; - for (i = 0; i < data_files->len; i++) - { - GFile *data_file = data_files->pdata[i]; - - if (!ot_gfile_unlink (data_file, cancellable, error)) - goto out; - } - - g_clear_pointer (&data_files, (GDestroyNotify) g_ptr_array_unref); - if (!list_files_in_dir_matching (cache_path, - "ostdatapack-", ".data", - &data_files, - cancellable, error)) - goto out; - for (i = 0; i < data_files->len; i++) - { - GFile *data_file = data_files->pdata[i]; - - if (!ot_gfile_unlink (data_file, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - return ret; -} - -/** - * Load the index for pack @pack_checksum from cache directory for - * @remote_name. - */ -gboolean -ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GVariant **out_variant, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lvariant GVariant *ret_variant = NULL; - ot_lobj GFile *cache_dir = NULL; - ot_lobj GFile *cached_pack_path = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cache_dir, - cancellable, error)) - goto out; - - cached_pack_path = get_pack_index_path (cache_dir, is_meta, pack_checksum); - if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT, - FALSE, &ret_variant, error)) - goto out; - - ret = TRUE; - ot_transfer_out_value (out_variant, &ret_variant); - out: - return ret; -} - -/** - * The variable @cached_path should refer to a file containing a pack - * index. It will be validated and added to the cache directory for - * @remote_name. - */ -gboolean -ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile *cached_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *cachedir = NULL; - ot_lobj GFile *target_path = NULL; - ot_lvariant GVariant *input_index_variant = NULL; - ot_lvariant GVariant *output_index_variant = NULL; - - if (!map_variant_file_check_header_string (cached_path, - OSTREE_PACK_INDEX_VARIANT_FORMAT, - "OSTv0PACKINDEX", - FALSE, &input_index_variant, - cancellable, error)) - goto out; - - if (!ostree_validate_structureof_pack_index (input_index_variant, error)) - goto out; - - output_index_variant = g_variant_get_normal_form (input_index_variant); - - if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) - goto out; - - target_path = get_pack_index_path (cachedir, is_meta, pack_checksum); - if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; -} - -/** - * Check for availability of the pack index pointing to @pack_checksum - * in the lookaside cache for @remote_name. If not found, then the - * output parameter @out_cached_path will be %NULL. - */ -gboolean -ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile **out_cached_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *cache_dir = NULL; - ot_lobj GFile *cached_pack_path = NULL; - ot_lobj GFile *ret_cached_path = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cache_dir, - cancellable, error)) - goto out; - - cached_pack_path = get_pack_data_path (cache_dir, is_meta, pack_checksum); - if (g_file_query_exists (cached_pack_path, cancellable)) - { - ret_cached_path = cached_pack_path; - cached_pack_path = NULL; - } - - ret = TRUE; - ot_transfer_out_value (out_cached_path, &ret_cached_path); - out: - return ret; -} - -/** - * Add file @cached_path into the cache for given @remote_name. If - * @cached_path is %NULL, delete the cached pack data (if any). - * - * - * This unlinks @cached_path. - * - */ -gboolean -ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile *cached_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *cachedir = NULL; - ot_lobj GFile *target_path = NULL; - - if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) - goto out; - - target_path = get_pack_data_path (cachedir, is_meta, pack_checksum); - if (cached_path) - { - if (!ot_gfile_rename (cached_path, target_path, cancellable, error)) - goto out; - } - else - { - (void) ot_gfile_unlink (target_path, cancellable, NULL); - } - - ret = TRUE; - out: - return ret; -} - static GVariant * create_tree_variant_from_hashes (GHashTable *file_checksums, GHashTable *dir_contents_checksums, @@ -3387,95 +2514,6 @@ list_loose_objects (OstreeRepo *self, return ret; } -gboolean -ostree_repo_load_pack_index (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - GVariant **out_variant, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lvariant GVariant *ret_variant = NULL; - ot_lobj GFile *path = NULL; - - g_mutex_lock (&self->cache_lock); - - ret_variant = g_hash_table_lookup (self->cached_pack_index_mappings, pack_checksum); - if (ret_variant) - { - g_variant_ref (ret_variant); - } - else - { - path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum); - if (!map_variant_file_check_header_string (path, - OSTREE_PACK_INDEX_VARIANT_FORMAT, - "OSTv0PACKINDEX", TRUE, - &ret_variant, - cancellable, error)) - goto out; - g_hash_table_insert (self->cached_pack_index_mappings, g_strdup (pack_checksum), - g_variant_ref (ret_variant)); - } - - ret = TRUE; - ot_transfer_out_value (out_variant, &ret_variant); - out: - g_mutex_unlock (&self->cache_lock); - return ret; -} - -/** - * @sha256: Checksum of pack file - * @out_data: (out): Pointer to pack file data - * - * Ensure that the given pack file is mapped into - * memory. - */ -gboolean -ostree_repo_map_pack_file (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - guchar **out_data, - guint64 *out_len, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gpointer ret_data; - guint64 ret_len; - GMappedFile *map = NULL; - ot_lobj GFile *path = NULL; - - g_mutex_lock (&self->cache_lock); - - map = g_hash_table_lookup (self->cached_pack_data_mappings, pack_checksum); - if (map == NULL) - { - path = get_pack_data_path (self->pack_dir, is_meta, pack_checksum); - - map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error); - if (!map) - goto out; - - g_hash_table_insert (self->cached_pack_data_mappings, g_strdup (pack_checksum), map); - ret_data = g_mapped_file_get_contents (map); - } - - ret_data = g_mapped_file_get_contents (map); - ret_len = (guint64)g_mapped_file_get_length (map); - - ret = TRUE; - if (out_data) - *out_data = ret_data; - if (out_len) - *out_len = ret_len; - out: - g_mutex_unlock (&self->cache_lock); - return ret; -} - gboolean ostree_repo_load_file (OstreeRepo *self, const char *checksum, @@ -3486,21 +2524,14 @@ ostree_repo_load_file (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - guchar *pack_data; - guint64 pack_len; - guint64 pack_offset; - ot_lvariant GVariant *packed_object = NULL; ot_lvariant GVariant *file_data = NULL; ot_lobj GFile *loose_path = NULL; ot_lobj GFileInfo *content_loose_info = NULL; - ot_lfree char *pack_checksum = NULL; ot_lobj GInputStream *ret_input = NULL; ot_lobj GFileInfo *ret_file_info = NULL; ot_lvariant GVariant *ret_xattrs = NULL; - if (!repo_find_object (self, OSTREE_OBJECT_TYPE_FILE, - checksum, FALSE, &loose_path, - &pack_checksum, &pack_offset, + if (!repo_find_object (self, OSTREE_OBJECT_TYPE_FILE, checksum, &loose_path, cancellable, error)) goto out; @@ -3565,25 +2596,6 @@ ostree_repo_load_file (OstreeRepo *self, } } } - else if (pack_checksum) - { - if (!ostree_repo_map_pack_file (self, pack_checksum, FALSE, - &pack_data, &pack_len, - cancellable, error)) - goto out; - - if (!ostree_read_pack_entry_raw (pack_data, pack_len, - pack_offset, TRUE, FALSE, - &packed_object, cancellable, error)) - goto out; - - if (!ostree_parse_file_pack_entry (packed_object, - out_input ? &ret_input : NULL, - out_file_info ? &ret_file_info : NULL, - out_xattrs ? &ret_xattrs : NULL, - cancellable, error)) - goto out; - } else if (self->parent_repo) { if (!ostree_repo_load_file (self->parent_repo, checksum, @@ -3608,258 +2620,34 @@ ostree_repo_load_file (OstreeRepo *self, return ret; } -static gboolean -list_objects_in_index (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - GHashTable *inout_objects, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint8 objtype_u8; - guint64 offset; - ot_lobj GFile *index_path = NULL; - ot_lvariant GVariant *index_variant = NULL; - ot_lvariant GVariant *contents = NULL; - ot_lvariant GVariant *csum_bytes = NULL; - ot_lfree char *checksum = NULL; - GVariantIter content_iter; - - index_path = get_pack_index_path (self->pack_dir, is_meta, pack_checksum); - - if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, - &index_variant, cancellable, error)) - goto out; - - contents = g_variant_get_child_value (index_variant, 2); - g_variant_iter_init (&content_iter, contents); - - while (g_variant_iter_loop (&content_iter, "(y@ayt)", &objtype_u8, &csum_bytes, &offset)) - { - GVariant *obj_key; - GVariant *objdata; - OstreeObjectType objtype; - GVariantBuilder pack_contents_builder; - gboolean is_loose; - - objtype = (OstreeObjectType) objtype_u8; - offset = GUINT64_FROM_BE (offset); - - g_variant_builder_init (&pack_contents_builder, - G_VARIANT_TYPE_STRING_ARRAY); - - g_free (checksum); - checksum = ostree_checksum_from_bytes_v (csum_bytes); - obj_key = ostree_object_name_serialize (checksum, objtype); - ot_util_variant_take_ref (obj_key); - - objdata = g_hash_table_lookup (inout_objects, obj_key); - if (!objdata) - { - is_loose = FALSE; - } - else - { - GVariantIter *current_packs_iter; - const char *current_pack_checksum; - - g_variant_get (objdata, "(bas)", &is_loose, ¤t_packs_iter); - - while (g_variant_iter_loop (current_packs_iter, "&s", ¤t_pack_checksum)) - { - g_variant_builder_add (&pack_contents_builder, "s", current_pack_checksum); - } - g_variant_iter_free (current_packs_iter); - } - g_variant_builder_add (&pack_contents_builder, "s", pack_checksum); - objdata = g_variant_new ("(b@as)", is_loose, - g_variant_builder_end (&pack_contents_builder)); - g_variant_ref_sink (objdata); - g_hash_table_replace (inout_objects, obj_key, objdata); - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -list_packed_objects (OstreeRepo *self, - GHashTable *inout_objects, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lptrarray GPtrArray *meta_index_checksums = NULL; - ot_lptrarray GPtrArray *data_index_checksums = NULL; - - if (!ostree_repo_list_pack_indexes (self, &meta_index_checksums, &data_index_checksums, - cancellable, error)) - goto out; - - for (i = 0; i < meta_index_checksums->len; i++) - { - const char *checksum = meta_index_checksums->pdata[i]; - if (!list_objects_in_index (self, checksum, TRUE, inout_objects, cancellable, error)) - goto out; - } - - for (i = 0; i < data_index_checksums->len; i++) - { - const char *checksum = data_index_checksums->pdata[i]; - if (!list_objects_in_index (self, checksum, FALSE, inout_objects, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -find_object_in_packs (OstreeRepo *self, - const char *checksum, - OstreeObjectType objtype, - char **out_pack_checksum, - guint64 *out_pack_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - guint64 ret_pack_offset = 0; - gboolean is_meta; - ot_lptrarray GPtrArray *index_checksums = NULL; - ot_lfree char *ret_pack_checksum = NULL; - ot_lvariant GVariant *csum_bytes = NULL; - ot_lvariant GVariant *index_variant = NULL; - - csum_bytes = ostree_checksum_to_bytes_v (checksum); - - is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); - - if (is_meta) - { - if (!ostree_repo_list_pack_indexes (self, &index_checksums, NULL, - cancellable, error)) - goto out; - } - else - { - if (!ostree_repo_list_pack_indexes (self, NULL, &index_checksums, - cancellable, error)) - goto out; - } - - for (i = 0; i < index_checksums->len; i++) - { - const char *pack_checksum = index_checksums->pdata[i]; - guint64 offset; - - g_clear_pointer (&index_variant, (GDestroyNotify) g_variant_unref); - if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, &index_variant, - cancellable, error)) - goto out; - - if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset)) - continue; - - ret_pack_checksum = g_strdup (pack_checksum); - ret_pack_offset = offset; - break; - } - - ret = TRUE; - ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); - if (out_pack_offset) - *out_pack_offset = ret_pack_offset; - out: - return ret; -} - static gboolean repo_find_object (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, - gboolean lookup_all, GFile **out_stored_path, - char **out_pack_checksum, - guint64 *out_pack_offset, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - guint64 ret_pack_offset = 0; struct stat stbuf; ot_lobj GFile *object_path = NULL; ot_lobj GFile *ret_stored_path = NULL; - ot_lfree char *ret_pack_checksum = NULL; - /* Look up metadata in packs first, but content loose first. We - * want to find loose content since that's preferable for - * hardlinking scenarios. - * - * Metadata is much more efficient packed. - */ - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) + object_path = ostree_repo_get_object_path (self, checksum, objtype); + + if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) { - if (out_pack_checksum) - { - if (!find_object_in_packs (self, checksum, objtype, - &ret_pack_checksum, &ret_pack_offset, - cancellable, error)) - goto out; - } - if (!ret_pack_checksum || lookup_all) - { - object_path = ostree_repo_get_object_path (self, checksum, objtype); - - if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) - { - ret_stored_path = object_path; - object_path = NULL; - } - else - { - g_clear_object (&object_path); - } - } + ret_stored_path = object_path; + object_path = NULL; /* Transfer ownership */ } - else + else if (errno != ENOENT) { - if (out_stored_path) - { - object_path = ostree_repo_get_object_path (self, checksum, objtype); - - if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) - { - ret_stored_path = object_path; - object_path = NULL; - } - else - { - g_clear_object (&object_path); - } - } - if (!ret_stored_path || lookup_all) - { - if (out_pack_checksum) - { - if (!find_object_in_packs (self, checksum, objtype, - &ret_pack_checksum, &ret_pack_offset, - cancellable, error)) - goto out; - } - } + ot_util_set_error_from_errno (error, errno); + goto out; } - + ret = TRUE; ot_transfer_out_value (out_stored_path, &ret_stored_path); - ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); - if (out_pack_offset) - *out_pack_offset = ret_pack_offset; out: return ret; } @@ -3875,15 +2663,12 @@ ostree_repo_has_object (OstreeRepo *self, gboolean ret = FALSE; gboolean ret_have_object; ot_lobj GFile *loose_path = NULL; - ot_lfree char *pack_checksum = NULL; - if (!repo_find_object (self, objtype, checksum, FALSE, - &loose_path, - &pack_checksum, NULL, + if (!repo_find_object (self, objtype, checksum, &loose_path, cancellable, error)) goto out; - ret_have_object = (loose_path != NULL) || (pack_checksum != NULL); + ret_have_object = (loose_path != NULL); if (!ret_have_object && self->parent_repo) { @@ -3928,35 +2713,17 @@ load_variant_internal (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - guchar *pack_data; - guint64 pack_len; - guint64 object_offset; GCancellable *cancellable = NULL; ot_lobj GFile *object_path = NULL; - ot_lvariant GVariant *packed_object = NULL; ot_lvariant GVariant *ret_variant = NULL; - ot_lfree char *pack_checksum = NULL; g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE); - if (!repo_find_object (self, objtype, sha256, FALSE, - &object_path, &pack_checksum, &object_offset, + if (!repo_find_object (self, objtype, sha256, &object_path, cancellable, error)) goto out; - if (pack_checksum != NULL) - { - if (!ostree_repo_map_pack_file (self, pack_checksum, TRUE, &pack_data, &pack_len, - cancellable, error)) - goto out; - - if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset, - TRUE, TRUE, &packed_object, cancellable, error)) - goto out; - - g_variant_get_child (packed_object, 2, "v", &ret_variant); - } - else if (object_path != NULL) + if (object_path != NULL) { if (!ot_util_variant_map (object_path, ostree_metadata_variant_type (objtype), TRUE, &ret_variant, error)) @@ -4065,13 +2832,7 @@ ostree_repo_list_objects (OstreeRepo *self, if (flags & OSTREE_REPO_LIST_OBJECTS_PACKED) { - if (!list_packed_objects (self, ret_objects, cancellable, error)) - goto out; - if (self->parent_repo) - { - if (!list_packed_objects (self->parent_repo, ret_objects, cancellable, error)) - goto out; - } + /* Nothing for now... */ } ret = TRUE; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 5f5e0dfa..151ea3af 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -111,14 +111,12 @@ gboolean ostree_repo_stage_file_object (OstreeRepo *self, gboolean ostree_repo_stage_object_trusted (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, - gboolean store_if_packed, GInputStream *content, GCancellable *cancellable, GError **error); gboolean ostree_repo_stage_file_object_trusted (OstreeRepo *self, const char *checksum, - gboolean store_if_packed, GInputStream *content, guint64 content_length, GCancellable *cancellable, @@ -159,27 +157,6 @@ gboolean ostree_repo_load_variant_if_exists (OstreeRepo *self, GVariant **out_variant, GError **error); -gboolean ostree_repo_load_pack_index (OstreeRepo *self, - const char *pack_checksum, - gboolean is_meta, - GVariant **out_variant, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_load_pack_data (OstreeRepo *self, - const char *pack_checksum, - guchar **out_data, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_map_pack_file (OstreeRepo *self, - const char *sha256, - gboolean is_meta, - guchar **out_data, - guint64 *out_len, - GCancellable *cancellable, - GError **error); - gboolean ostree_repo_load_file (OstreeRepo *self, const char *entry_sha256, GInputStream **out_input, @@ -248,65 +225,6 @@ gboolean ostree_repo_stage_commit (OstreeRepo *self, GCancellable *cancellable, GError **error); -gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_add_pack_file (OstreeRepo *self, - const char *checksum, - gboolean is_meta, - GFile *pack_index_path, - GFile *pack_data_path, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, - const char *remote_name, - GFile *superindex_path, - GPtrArray **out_cached_meta_indexes, - GPtrArray **out_cached_data_indexes, - GPtrArray **out_uncached_meta_indexes, - GPtrArray **out_uncached_data_indexes, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GVariant **out_variant, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile *cached_path, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile **out_cached_path, - GCancellable *cancellable, - GError **error); - -gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, - const char *remote_name, - const char *pack_checksum, - gboolean is_meta, - GFile *cached_path, - GCancellable *cancellable, - GError **error); - typedef enum { OSTREE_REPO_CHECKOUT_MODE_NONE = 0, OSTREE_REPO_CHECKOUT_MODE_USER = 1 @@ -359,12 +277,6 @@ gboolean ostree_repo_list_objects (OstreeRepo *self, GCancellable *cancellable, GError **error); -gboolean ostree_repo_list_pack_indexes (OstreeRepo *self, - GPtrArray **out_meta_indexes, - GPtrArray **out_data_indexes, - GCancellable *cancellable, - GError **error); - G_END_DECLS #endif /* _OSTREE_REPO */ diff --git a/src/ostree/main.c b/src/ostree/main.c index 994a8373..8e322411 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -42,14 +42,12 @@ static OstreeCommand commands[] = { { "init", ostree_builtin_init, 0 }, { "log", ostree_builtin_log, 0 }, { "ls", ostree_builtin_ls, 0 }, - { "pack", ostree_builtin_pack, 0 }, { "prune", ostree_builtin_prune, 0 }, { "pull", NULL, 0 }, { "pull-local", ostree_builtin_pull_local, 0 }, { "remote", ostree_builtin_remote, 0 }, { "rev-parse", ostree_builtin_rev_parse, 0 }, { "show", ostree_builtin_show, 0 }, - { "unpack", ostree_builtin_unpack, 0 }, { "write-refs", ostree_builtin_write_refs, 0 }, { NULL } }; diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c index f1804dab..d1934b47 100644 --- a/src/ostree/ostree-pull.c +++ b/src/ostree/ostree-pull.c @@ -67,13 +67,11 @@ #include "ostree-fetcher.h" gboolean verbose; -gint opt_packfile_threshold = 66; gboolean opt_related; gint opt_depth; static GOptionEntry options[] = { { "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose, "Show more information", NULL }, - { "packfile-threshold", 't', 0, G_OPTION_ARG_INT, &opt_packfile_threshold, "Only download packfiles if more than PERCENT objects are needed (default: 66)", "PERCENT" }, { "related", 0, 0, G_OPTION_ARG_NONE, &opt_related, "Download related commits", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Download parent commits up to this depth (default: 0)", NULL }, { NULL }, @@ -85,10 +83,6 @@ typedef struct { OstreeFetcher *fetcher; SoupURI *base_uri; - gboolean fetched_packs; - GPtrArray *cached_meta_pack_indexes; - GPtrArray *cached_data_pack_indexes; - GHashTable *file_checksums_to_fetch; GMainLoop *loop; @@ -325,234 +319,6 @@ fetch_uri_contents_utf8 (OtPullData *pull_data, return ret; } -static gboolean -fetch_one_pack_file (OtPullData *pull_data, - const char *pack_checksum, - gboolean is_meta, - GFile **out_cached_path, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *ret_cached_path = NULL; - ot_lobj GFile *tmp_path = NULL; - ot_lfree char *pack_name = NULL; - SoupURI *pack_uri = NULL; - - if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, is_meta, &ret_cached_path, - cancellable, error)) - goto out; - - if (ret_cached_path == NULL) - { - pack_name = ostree_get_pack_data_name (is_meta, pack_checksum); - pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL); - - if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error)) - goto out; - - if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, is_meta, tmp_path, - cancellable, error)) - goto out; - } - - g_clear_object (&ret_cached_path); - if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, is_meta, &ret_cached_path, - cancellable, error)) - goto out; - - g_assert (ret_cached_path != NULL); - - ret = TRUE; - ot_transfer_out_value (out_cached_path, &ret_cached_path); - out: - if (pack_uri) - soup_uri_free (pack_uri); - return ret; -} - -static gboolean -find_object_in_one_remote_pack (OtPullData *pull_data, - GVariant *csum_bytes_v, - OstreeObjectType objtype, - const char *pack_checksum, - gboolean *out_exists, - guint64 *out_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint64 ret_offset; - gboolean ret_exists; - ot_lvariant GVariant *mapped_pack = NULL; - - if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype), - &mapped_pack, - cancellable, error)) - goto out; - - ret_exists = ostree_pack_index_search (mapped_pack, csum_bytes_v, objtype, &ret_offset); - - ret = TRUE; - if (out_exists) - *out_exists = ret_exists; - if (out_offset) - *out_offset = ret_offset; - out: - return ret; -} - -static gboolean -find_object_in_remote_packs (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - char **out_pack_checksum, - guint64 *out_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint64 ret_offset = 0; - guint i; - GPtrArray *iter; - ot_lvariant GVariant *csum_bytes_v = NULL; - ot_lfree char *ret_pack_checksum = NULL; - - csum_bytes_v = ostree_checksum_to_bytes_v (checksum); - - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - iter = pull_data->cached_meta_pack_indexes; - else - iter = pull_data->cached_data_pack_indexes; - for (i = 0; i < iter->len; i++) - { - const char *pack_checksum = iter->pdata[i]; - gboolean exists; - - if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, objtype, - pack_checksum, &exists, &ret_offset, - cancellable, error)) - goto out; - - if (exists) - { - ret_pack_checksum = g_strdup (pack_checksum); - break; - } - } - - ret = TRUE; - ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); - if (out_offset) - *out_offset = ret_offset; - out: - return ret; -} - -static gboolean -fetch_one_cache_index (OtPullData *pull_data, - const char *pack_checksum, - gboolean is_meta, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *tmp_path = NULL; - ot_lfree char *pack_index_name = NULL; - SoupURI *index_uri = NULL; - - pack_index_name = ostree_get_pack_index_name (is_meta, pack_checksum); - index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL); - - if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path, - cancellable, error)) - goto out; - - if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, is_meta, tmp_path, - cancellable, error)) - goto out; - - if (!ot_gfile_unlink (tmp_path, cancellable, error)) - goto out; - - g_clear_object (&tmp_path); - - ret = TRUE; - out: - if (tmp_path != NULL) - (void) ot_gfile_unlink (tmp_path, NULL, NULL); - if (index_uri) - soup_uri_free (index_uri); - return ret; -} - -static gboolean -fetch_and_cache_pack_indexes (OtPullData *pull_data, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lobj GFile *superindex_tmppath = NULL; - ot_lptrarray GPtrArray *cached_meta_indexes = NULL; - ot_lptrarray GPtrArray *cached_data_indexes = NULL; - ot_lptrarray GPtrArray *uncached_meta_indexes = NULL; - ot_lptrarray GPtrArray *uncached_data_indexes = NULL; - ot_lvariant GVariant *superindex_variant = NULL; - GVariantIter *contents_iter = NULL; - SoupURI *superindex_uri = NULL; - - superindex_uri = suburi_new (pull_data->base_uri, "objects", "pack", "index", NULL); - - if (!fetch_uri (pull_data, superindex_uri, "index-", - &superindex_tmppath, cancellable, error)) - goto out; - - if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name, - superindex_tmppath, - &cached_meta_indexes, - &cached_data_indexes, - &uncached_meta_indexes, - &uncached_data_indexes, - cancellable, error)) - goto out; - - for (i = 0; i < cached_meta_indexes->len; i++) - g_ptr_array_add (pull_data->cached_meta_pack_indexes, - g_strdup (cached_meta_indexes->pdata[i])); - for (i = 0; i < cached_data_indexes->len; i++) - g_ptr_array_add (pull_data->cached_data_pack_indexes, - g_strdup (cached_data_indexes->pdata[i])); - - for (i = 0; i < uncached_meta_indexes->len; i++) - { - const char *pack_checksum = uncached_meta_indexes->pdata[i]; - if (!fetch_one_cache_index (pull_data, pack_checksum, TRUE, cancellable, error)) - goto out; - g_ptr_array_add (pull_data->cached_meta_pack_indexes, g_strdup (pack_checksum)); - } - for (i = 0; i < uncached_data_indexes->len; i++) - { - const char *pack_checksum = uncached_data_indexes->pdata[i]; - if (!fetch_one_cache_index (pull_data, pack_checksum, FALSE, cancellable, error)) - goto out; - g_ptr_array_add (pull_data->cached_data_pack_indexes, g_strdup (pack_checksum)); - } - - ret = TRUE; - out: - if (superindex_uri) - soup_uri_free (superindex_uri); - if (contents_iter) - g_variant_iter_free (contents_iter); - return ret; -} - static gboolean fetch_loose_object (OtPullData *pull_data, const char *checksum, @@ -581,89 +347,6 @@ fetch_loose_object (OtPullData *pull_data, return ret; } -static gboolean -find_object_ensure_indexes (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - gboolean *out_is_stored, - char **out_remote_pack_checksum, - guint64 *out_remote_pack_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean ret_is_stored; - ot_lfree char *ret_remote_pack_checksum = NULL; - - if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &ret_is_stored, - cancellable, error)) - goto out; - - if (!ret_is_stored) - { - if (!pull_data->fetched_packs) - { - pull_data->fetched_packs = TRUE; - pull_data->cached_meta_pack_indexes = g_ptr_array_new_with_free_func (g_free); - pull_data->cached_data_pack_indexes = g_ptr_array_new_with_free_func (g_free); - - if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error)) - goto out; - } - - if (!find_object_in_remote_packs (pull_data, checksum, objtype, - &ret_remote_pack_checksum, out_remote_pack_offset, - cancellable, error)) - goto out; - } - - ret = TRUE; - if (out_is_stored) - *out_is_stored = ret_is_stored; - ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum); - out: - return ret; -} - -static gboolean -find_object_ensure_pack_data (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - gboolean *out_is_stored, - GFile **out_remote_pack_path, - guint64 *out_remote_pack_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean ret_is_stored; - ot_lfree char *remote_pack_checksum = NULL; - ot_lobj GFile *ret_remote_pack_path = NULL; - - if (!find_object_ensure_indexes (pull_data, checksum, objtype, &ret_is_stored, - &remote_pack_checksum, out_remote_pack_offset, - cancellable, error)) - goto out; - - if (!ret_is_stored) - { - if (remote_pack_checksum) - { - if (!fetch_one_pack_file (pull_data, remote_pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype), - &ret_remote_pack_path, cancellable, error)) - goto out; - } - } - - ret = TRUE; - if (out_is_stored) - *out_is_stored = ret_is_stored; - ot_transfer_out_value (out_remote_pack_path, &ret_remote_pack_path); - /* offset set above */ - out: - return ret; -} - static gboolean fetch_and_store_metadata (OtPullData *pull_data, const char *checksum, @@ -673,42 +356,19 @@ fetch_and_store_metadata (OtPullData *pull_data, GError **error) { gboolean ret = FALSE; - guint64 pack_offset = 0; gboolean is_stored; ot_lvariant GVariant *ret_variant = NULL; - ot_lobj GFile *remote_pack_path = NULL; ot_lobj GFile *temp_path = NULL; ot_lobj GInputStream *input = NULL; - ot_lvariant GVariant *pack_entry = NULL; ot_lvariant GVariant *metadata = NULL; - GMappedFile *pack_map = NULL; g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype)); - if (!find_object_ensure_pack_data (pull_data, checksum, objtype, - &is_stored, &remote_pack_path, &pack_offset, - cancellable, error)) + if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored, + cancellable, error)) goto out; - if (remote_pack_path != NULL) - { - g_assert (!is_stored); - - pack_map = g_mapped_file_new (ot_gfile_get_path_cached (remote_pack_path), FALSE, error); - if (!pack_map) - goto out; - - if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), - g_mapped_file_get_length (pack_map), - pack_offset, FALSE, TRUE, &pack_entry, - cancellable, error)) - goto out; - - g_variant_get_child (pack_entry, 2, "v", &metadata); - - input = ot_variant_read (metadata); - } - else if (!is_stored) + if (!is_stored) { if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error)) goto out; @@ -716,11 +376,7 @@ fetch_and_store_metadata (OtPullData *pull_data, input = (GInputStream*)g_file_read (temp_path, cancellable, error); if (!input) goto out; - } - if (input) - { - g_assert (remote_pack_path != NULL || !is_stored); if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, input, cancellable, error)) goto out; @@ -735,8 +391,6 @@ fetch_and_store_metadata (OtPullData *pull_data, out: if (temp_path) (void) ot_gfile_unlink (temp_path, NULL, NULL); - if (pack_map) - g_mapped_file_unref (pack_map); return ret; } @@ -753,7 +407,6 @@ fetch_and_store_tree_metadata_recurse (OtPullData *pull_data, ot_lvariant GVariant *files_variant = NULL; ot_lvariant GVariant *dirs_variant = NULL; ot_lobj GFile *stored_path = NULL; - ot_lfree char *pack_checksum = NULL; if (depth > OSTREE_MAX_RECURSION) { @@ -936,66 +589,6 @@ fetch_ref_contents (OtPullData *pull_data, return ret; } -static gboolean -store_file_from_pack (OtPullData *pull_data, - const char *checksum, - const char *pack_checksum, - GFile *pack_file, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean exists; - guint64 pack_offset; - ot_lobj GFile *remote_pack_path = NULL; - ot_lobj GFile *temp_path = NULL; - ot_lvariant GVariant *pack_entry = NULL; - ot_lobj GInputStream *input = NULL; - ot_lobj GInputStream *file_object_input = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lvariant GVariant *xattrs = NULL; - ot_lvariant GVariant *csum_bytes_v = NULL; - GMappedFile *pack_map = NULL; - - csum_bytes_v = ostree_checksum_to_bytes_v (checksum); - - if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, OSTREE_OBJECT_TYPE_FILE, - pack_checksum, &exists, &pack_offset, - cancellable, error)) - goto out; - - g_assert (exists); - - pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_file), FALSE, error); - if (!pack_map) - goto out; - - if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), - g_mapped_file_get_length (pack_map), - pack_offset, FALSE, FALSE, &pack_entry, - cancellable, error)) - goto out; - - if (!ostree_parse_file_pack_entry (pack_entry, &input, &file_info, &xattrs, - cancellable, error)) - goto out; - - if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, - &file_object_input, NULL, cancellable, error)) - goto out; - - if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, - file_object_input, - cancellable, error)) - goto out; - - ret = TRUE; - out: - if (pack_map) - g_mapped_file_unref (pack_map); - return ret; -} - typedef struct { OtPullData *pull_data; @@ -1072,7 +665,7 @@ content_fetch_on_checksum_complete (GObject *object, } if (!ostree_repo_stage_file_object_trusted (data->pull_data->repo, checksum, - FALSE, file_object_input, length, + file_object_input, length, cancellable, error)) goto out; @@ -1231,42 +824,23 @@ fetch_content (OtPullData *pull_data, gpointer key, value; ot_lobj GFile *temp_path = NULL; ot_lobj GFile *content_temp_path = NULL; - ot_lhash GHashTable *data_packs_to_fetch = NULL; ot_lhash GHashTable *loose_files = NULL; SoupURI *content_uri = NULL; guint n_objects_to_fetch = 0; - data_packs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, - (GDestroyNotify) g_ptr_array_unref); loose_files = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); g_hash_table_iter_init (&hash_iter, pull_data->file_checksums_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *checksum = key; - GPtrArray *files_to_fetch; gboolean is_stored; - ot_lfree char *remote_pack_checksum = NULL; - if (!find_object_ensure_indexes (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, - &is_stored, &remote_pack_checksum, NULL, - cancellable, error)) + if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, &is_stored, + cancellable, error)) goto out; - if (remote_pack_checksum) - { - files_to_fetch = g_hash_table_lookup (data_packs_to_fetch, remote_pack_checksum); - if (files_to_fetch == NULL) - { - files_to_fetch = g_ptr_array_new_with_free_func (g_free); - g_hash_table_insert (data_packs_to_fetch, remote_pack_checksum, files_to_fetch); - /* transfer ownership */ - remote_pack_checksum = NULL; - } - g_ptr_array_add (files_to_fetch, g_strdup (checksum)); - n_objects_to_fetch++; - } - else if (!is_stored) + if (!is_stored) { char *key = g_strdup (checksum); g_hash_table_insert (loose_files, key, key); @@ -1274,71 +848,9 @@ fetch_content (OtPullData *pull_data, } } - g_hash_table_iter_init (&hash_iter, data_packs_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *pack_checksum = key; - GPtrArray *files = value; - ot_lvariant GVariant *mapped_pack = NULL; - ot_lvariant GVariant *content_list = NULL; - gboolean fetch; - - if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, FALSE, - &mapped_pack, - cancellable, error)) - goto out; - - content_list = g_variant_get_child_value (mapped_pack, 2); - - fetch = (((double)files->len) / g_variant_n_children (content_list)) > ((double)opt_packfile_threshold / 100); - if (!fetch) - { - guint i; - for (i = 0; i < files->len; i++) - { - g_hash_table_insert (loose_files, files->pdata[i], files->pdata[i]); - files->pdata[i] = NULL; /* steal data */ - } - g_hash_table_iter_remove (&hash_iter); - } - } - if (n_objects_to_fetch > 0) g_print ("%u content objects to fetch\n", n_objects_to_fetch); - if (g_hash_table_size (data_packs_to_fetch) > 0) - g_print ("Fetching %u content packs\n", - g_hash_table_size (data_packs_to_fetch)); - - g_hash_table_iter_init (&hash_iter, data_packs_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *pack_checksum = key; - GPtrArray *file_checksums = value; - guint i; - ot_lobj GFile *pack_path = NULL; - - if (!fetch_one_pack_file (pull_data, pack_checksum, FALSE, - &pack_path, cancellable, error)) - goto out; - - g_print ("Storing %u objects from content pack %s\n", file_checksums->len, - pack_checksum); - for (i = 0; i < file_checksums->len; i++) - { - const char *checksum = file_checksums->pdata[i]; - if (!store_file_from_pack (pull_data, checksum, pack_checksum, pack_path, - cancellable, error)) - goto out; - } - - if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, FALSE, NULL, - cancellable, error)) - goto out; - } - if (g_hash_table_size (loose_files) > 0) g_print ("Fetching %u loose objects\n", g_hash_table_size (loose_files)); @@ -1729,10 +1241,6 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) g_print ("remote %s is now %s\n", remote_ref, checksum); } - if (!ostree_repo_clean_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - cancellable, error)) - goto out; - bytes_transferred = ostree_fetcher_bytes_transferred (pull_data->fetcher); if (bytes_transferred > 0) { @@ -1751,8 +1259,6 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) if (pull_data->base_uri) soup_uri_free (pull_data->base_uri); g_clear_pointer (&pull_data->file_checksums_to_fetch, (GDestroyNotify) g_hash_table_unref); - g_clear_pointer (&pull_data->cached_meta_pack_indexes, (GDestroyNotify) g_ptr_array_unref); - g_clear_pointer (&pull_data->cached_data_pack_indexes, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref); if (summary_uri) soup_uri_free (summary_uri); diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 1a003623..9aa7b66b 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -39,129 +39,8 @@ static GOptionEntry options[] = { typedef struct { OstreeRepo *repo; - guint n_pack_files; } OtFsckData; -static gboolean -fsck_one_pack_file (OtFsckData *data, - const char *pack_checksum, - gboolean is_meta, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guchar objtype_u8; - guint64 offset; - guint64 pack_size; - ot_lfree char *path = NULL; - ot_lobj GFileInfo *pack_info = NULL; - ot_lobj GInputStream *input = NULL; - ot_lvariant GVariant *index_variant = NULL; - ot_lobj GFile *pack_index_path = NULL; - ot_lobj GFile *pack_data_path = NULL; - ot_lfree guchar *pack_content_csum = NULL; - ot_lfree char *tmp_checksum = NULL; - GVariantIter *index_content_iter = NULL; - - g_free (path); - path = ostree_get_relative_pack_index_path (is_meta, pack_checksum); - pack_index_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path); - - if (!ot_util_variant_map (pack_index_path, - OSTREE_PACK_INDEX_VARIANT_FORMAT, FALSE, - &index_variant, error)) - goto out; - - if (!ostree_validate_structureof_pack_index (index_variant, error)) - goto out; - - g_free (path); - path = ostree_get_relative_pack_data_path (is_meta, pack_checksum); - pack_data_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path); - - input = (GInputStream*)g_file_read (pack_data_path, cancellable, error); - if (!input) - goto out; - - pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO, - cancellable, error); - if (!pack_info) - goto out; - pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size"); - - if (!ot_gio_checksum_stream (input, &pack_content_csum, cancellable, error)) - goto out; - - tmp_checksum = ostree_checksum_from_bytes (pack_content_csum); - if (strcmp (tmp_checksum, pack_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "corrupted pack '%s', actual checksum is %s", - pack_checksum, tmp_checksum); - goto out; - } - - g_variant_get_child (index_variant, 2, "a(yayt)", &index_content_iter); - - while (g_variant_iter_loop (index_content_iter, "(y@ayt)", - &objtype_u8, NULL, &offset)) - { - offset = GUINT64_FROM_BE (offset); - if (offset > pack_size) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT, - pack_checksum, - offset, pack_size); - goto out; - } - } - - ret = TRUE; - out: - if (index_content_iter) - g_variant_iter_free (index_content_iter); - return ret; -} - -static gboolean -fsck_pack_files (OtFsckData *data, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lptrarray GPtrArray *meta_pack_indexes = NULL; - ot_lptrarray GPtrArray *data_pack_indexes = NULL; - - if (!ostree_repo_list_pack_indexes (data->repo, &meta_pack_indexes, &data_pack_indexes, - cancellable, error)) - goto out; - - for (i = 0; i < meta_pack_indexes->len; i++) - { - const char *pack_checksum = meta_pack_indexes->pdata[i]; - - if (!fsck_one_pack_file (data, pack_checksum, TRUE, cancellable, error)) - goto out; - - data->n_pack_files++; - } - for (i = 0; i < data_pack_indexes->len; i++) - { - const char *pack_checksum = data_pack_indexes->pdata[i]; - - if (!fsck_one_pack_file (data, pack_checksum, FALSE, cancellable, error)) - goto out; - - data->n_pack_files++; - } - - ret = TRUE; - out: - return ret; -} - static gboolean fsck_reachable_objects_from_commits (OtFsckData *data, GHashTable *commits, @@ -355,11 +234,6 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error) if (!fsck_reachable_objects_from_commits (&data, commits, cancellable, error)) goto out; - g_print ("Verifying structure of pack files...\n"); - - if (!fsck_pack_files (&data, cancellable, error)) - goto out; - ret = TRUE; out: if (context) diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c index 6c8c052c..0707242e 100644 --- a/src/ostree/ot-builtin-init.c +++ b/src/ostree/ot-builtin-init.c @@ -43,7 +43,7 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error) { GOptionContext *context = NULL; gboolean ret = FALSE; - GCancellable *cancellable = NULL; + __attribute__ ((unused)) GCancellable *cancellable = NULL; ot_lobj GFile *child = NULL; ot_lobj GFile *grandchild = NULL; ot_lobj OstreeRepo *repo = NULL; @@ -110,9 +110,6 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error) if (!ostree_repo_check (repo, error)) goto out; - if (!ostree_repo_regenerate_pack_index (repo, cancellable, error)) - goto out; - ret = TRUE; out: if (context) diff --git a/src/ostree/ot-builtin-pack.c b/src/ostree/ot-builtin-pack.c deleted file mode 100644 index 74564355..00000000 --- a/src/ostree/ot-builtin-pack.c +++ /dev/null @@ -1,967 +0,0 @@ -/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- - * - * Copyright (C) 2012 Colin Walters - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 02111-1307, USA. - * - * Author: Colin Walters - */ - -#include "config.h" - -#include "ot-builtins.h" -#include "ostree.h" - -#include -#include - -#include -#include - -#define OT_GZIP_COMPRESSION_LEVEL (8) - -static gboolean opt_analyze_only; -static gboolean opt_metadata_only; -static gboolean opt_content_only; -static gboolean opt_reindex_only; -static gboolean opt_delete_all_loose; -static gboolean opt_keep_all_loose; -static char* opt_pack_size = "50m"; -static char* opt_int_compression; -static char* opt_ext_compression; - -typedef enum { - OT_COMPRESSION_NONE, - OT_COMPRESSION_GZIP, - OT_COMPRESSION_XZ -} OtCompressionType; - -static GOptionEntry options[] = { - { "pack-size", 0, 0, G_OPTION_ARG_STRING, &opt_pack_size, "Maximum uncompressed size of packfiles in bytes; may be suffixed with k, m, or g (default: 50m)", "BYTES" }, - { "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION", "COMPRESSION" }, - { "external-compression", 0, 0, G_OPTION_ARG_STRING, &opt_ext_compression, "Compress entire packfiles using COMPRESSION", "COMPRESSION" }, - { "metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_metadata_only, "Only pack metadata objects", NULL }, - { "content-only", 0, 0, G_OPTION_ARG_NONE, &opt_content_only, "Only pack content objects", NULL }, - { "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL }, - { "reindex-only", 0, 0, G_OPTION_ARG_NONE, &opt_reindex_only, "Regenerate pack index", NULL }, - { "delete-all-loose", 0, 0, G_OPTION_ARG_NONE, &opt_delete_all_loose, "Delete all loose objects (default: delete unreferenced loose)", NULL }, - { "keep-all-loose", 0, 0, G_OPTION_ARG_NONE, &opt_keep_all_loose, "Don't delete any loose objects (default: delete unreferenced loose)", NULL }, - { NULL } -}; - -typedef struct { - OstreeRepo *repo; - - guint64 pack_size; - OtCompressionType int_compression; - OtCompressionType ext_compression; - - gboolean had_error; - GError **error; -} OtRepackData; - -typedef struct { - GOutputStream *out; - GPtrArray *compressor_argv; - GPid compress_child_pid; -} OtBuildRepackFile; - -static gint -compare_object_data_by_size (gconstpointer ap, - gconstpointer bp) -{ - GVariant *a = *(void **)ap; - GVariant *b = *(void **)bp; - guint64 a_size; - guint64 b_size; - - g_variant_get_child (a, 2, "t", &a_size); - g_variant_get_child (b, 2, "t", &b_size); - if (a == b) - return 0; - else if (a > b) - return 1; - else - return -1; -} - -static gboolean -write_bytes_update_checksum (GOutputStream *output, - gconstpointer bytes, - gsize len, - GChecksum *checksum, - guint64 *inout_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gsize bytes_written; - - if (len > 0) - { - g_checksum_update (checksum, (guchar*) bytes, len); - if (!g_output_stream_write_all (output, bytes, len, &bytes_written, - cancellable, error)) - goto out; - g_assert_cmpint (bytes_written, ==, len); - *inout_offset += bytes_written; - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -write_padding (GOutputStream *output, - guint alignment, - GChecksum *checksum, - guint64 *inout_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint bits; - guint padding_len; - guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0}; - - if (alignment == 8) - bits = ((*inout_offset) & 7); - else - bits = ((*inout_offset) & 3); - - if (bits > 0) - { - padding_len = alignment - bits; - if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding_len, - checksum, inout_offset, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - return ret; -} - -static gint -compare_index_content (gconstpointer ap, - gconstpointer bp) -{ - gpointer a = *((gpointer*)ap); - gpointer b = *((gpointer*)bp); - GVariant *a_v = a; - GVariant *b_v = b; - guchar a_objtype; - guchar b_objtype; - guint64 a_offset; - guint64 b_offset; - int c; - ot_lvariant GVariant *a_csum_bytes = NULL; - ot_lvariant GVariant *b_csum_bytes = NULL; - - g_variant_get (a_v, "(y@ayt)", &a_objtype, &a_csum_bytes, &a_offset); - g_variant_get (b_v, "(y@ayt)", &b_objtype, &b_csum_bytes, &b_offset); - c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (a_csum_bytes), - ostree_checksum_bytes_peek (b_csum_bytes)); - if (c == 0) - { - if (a_objtype < b_objtype) - c = -1; - else if (a_objtype > b_objtype) - c = 1; - } - return c; -} - -static gboolean -delete_loose_object (OtRepackData *data, - const char *checksum, - OstreeObjectType objtype, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean do_delete = FALSE; - GError *temp_error = NULL; - ot_lobj GFile *object_path = NULL; - ot_lobj GFile *file_content_object_path = NULL; - - object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); - - if (objtype == OSTREE_OBJECT_TYPE_FILE) - { - ot_lobj GFileInfo *file_info = NULL; - - if (ostree_repo_get_mode (data->repo) == OSTREE_REPO_MODE_BARE) - { - file_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, &temp_error); - } - else - { - ot_lobj GFile *content_object_path = NULL; - - content_object_path = ostree_repo_get_archive_content_path (data->repo, checksum); - - file_info = g_file_query_info (content_object_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, &temp_error); - } - - if (!file_info) - { - if (!g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_propagate_error (error, temp_error); - goto out; - } - else - { - g_clear_error (&temp_error); - } - } - - do_delete = opt_delete_all_loose - || (file_info && g_file_info_get_attribute_uint32 (file_info, "unix::nlink") <= 1); - } - else - do_delete = TRUE; - - if (do_delete) - { - if (!ot_gfile_unlink (object_path, cancellable, error)) - { - g_prefix_error (error, "Failed to delete loose object '%s'", - ot_gfile_get_path_cached (object_path)); - goto out; - } - - if (objtype == OSTREE_OBJECT_TYPE_FILE - && ostree_repo_get_mode (data->repo) == OSTREE_REPO_MODE_ARCHIVE) - { - ot_lobj GFile *content_object_path = NULL; - - content_object_path = ostree_repo_get_archive_content_path (data->repo, checksum); - - /* Ignoring errors for now; later should only be trying to - * delete files with content. - */ - (void) ot_gfile_unlink (content_object_path, NULL, NULL); - } - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -pack_one_meta_object (OtRepackData *data, - const char *checksum, - OstreeObjectType objtype, - GVariant **out_packed_object, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GFile *object_path = NULL; - ot_lvariant GVariant *metadata_v = NULL; - ot_lvariant GVariant *ret_packed_object = NULL; - - object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); - - if (!ot_util_variant_map (object_path, ostree_metadata_variant_type (objtype), - TRUE, &metadata_v, error)) - goto out; - - ret_packed_object = g_variant_new ("(y@ayv)", (guchar) objtype, - ostree_checksum_to_bytes_v (checksum), - metadata_v); - - ret = TRUE; - ot_transfer_out_value (out_packed_object, &ret_packed_object); - out: - return ret; -} - -static gboolean -pack_one_data_object (OtRepackData *data, - const char *checksum, - OstreeObjectType objtype, - guint64 expected_objsize, - GVariant **out_packed_object, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guchar entry_flags = 0; - GInputStream *read_object_in; /* nofree */ - ot_lobj GInputStream *input = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lvariant GVariant *xattrs = NULL; - ot_lobj GMemoryOutputStream *object_data_stream = NULL; - ot_lobj GConverter *compressor = NULL; - ot_lobj GConverterInputStream *compressed_object_input = NULL; - ot_lvariant GVariant *file_header = NULL; - ot_lvariant GVariant *ret_packed_object = NULL; - - switch (data->int_compression) - { - case OT_COMPRESSION_GZIP: - { - entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP; - break; - } - default: - { - g_assert_not_reached (); - } - } - - if (!ostree_repo_load_file (data->repo, checksum, &input, &file_info, &xattrs, - cancellable, error)) - goto out; - - file_header = ostree_file_header_new (file_info, xattrs); - - object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free); - if (input != NULL) - { - if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP) - { - compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL); - compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM, - "converter", compressor, - "base-stream", input, - "close-base-stream", TRUE, - NULL); - read_object_in = (GInputStream*)compressed_object_input; - } - else - { - read_object_in = (GInputStream*)input; - } - - if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in, - G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, - cancellable, error)) - goto out; - } - - { - guchar *data = g_memory_output_stream_get_data (object_data_stream); - gsize data_len = g_memory_output_stream_get_data_size (object_data_stream); - ret_packed_object = g_variant_new ("(@ayy@(uuuusa(ayay))@ay)", - ostree_checksum_to_bytes_v (checksum), - entry_flags, - file_header, - ot_gvariant_new_bytearray (data, data_len)); - } - - ret = TRUE; - ot_transfer_out_value (out_packed_object, &ret_packed_object); - out: - return ret; -} - -static gboolean -create_pack_file (OtRepackData *data, - gboolean is_meta, - GPtrArray *objects, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - guint64 offset; - gsize bytes_written; - ot_lobj GFile *pack_dir = NULL; - ot_lobj GFile *index_temppath = NULL; - ot_lobj GOutputStream *index_out = NULL; - ot_lobj GFile *pack_temppath = NULL; - ot_lobj GOutputStream *pack_out = NULL; - ot_lptrarray GPtrArray *index_content_list = NULL; - ot_lvariant GVariant *pack_header = NULL; - ot_lvariant GVariant *index_content = NULL; - ot_lfree char *pack_name = NULL; - ot_lobj GFile *pack_file_path = NULL; - ot_lobj GFile *pack_index_path = NULL; - GVariantBuilder index_content_builder; - GChecksum *pack_checksum = NULL; - - if (g_cancellable_set_error_if_cancelled (cancellable, error)) - return FALSE; - - if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo), - "pack-index", NULL, - &index_temppath, - &index_out, - cancellable, error)) - goto out; - - if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo), - "pack-content", NULL, - &pack_temppath, - &pack_out, - cancellable, error)) - goto out; - - index_content_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - - offset = 0; - pack_checksum = g_checksum_new (G_CHECKSUM_SHA256); - - pack_header = g_variant_new ("(s@a{sv}t)", - is_meta ? "OSTv0PACKMETAFILE" : "OSTv0PACKDATAFILE", - g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), - (guint64)objects->len); - - if (!ostree_write_variant_with_size (pack_out, pack_header, offset, &bytes_written, pack_checksum, - cancellable, error)) - goto out; - offset += bytes_written; - - for (i = 0; i < objects->len; i++) - { - GVariant *object_data = objects->pdata[i]; - const char *checksum; - guint32 objtype_u32; - OstreeObjectType objtype; - guint64 expected_objsize; - ot_lvariant GVariant *packed_object = NULL; - ot_lvariant GVariant *index_entry = NULL; - - g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize); - - objtype = (OstreeObjectType) objtype_u32; - - if (is_meta) - { - if (!pack_one_meta_object (data, checksum, objtype, &packed_object, - cancellable, error)) - goto out; - } - else - { - if (!pack_one_data_object (data, checksum, objtype, expected_objsize, - &packed_object, cancellable, error)) - goto out; - } - - if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error)) - goto out; - - /* offset points to aligned header size */ - index_entry = g_variant_new ("(y@ayt)", - (guchar)objtype, - ostree_checksum_to_bytes_v (checksum), - GUINT64_TO_BE (offset)); - g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry)); - index_entry = NULL; - - bytes_written = 0; - if (!ostree_write_variant_with_size (pack_out, packed_object, offset, &bytes_written, - pack_checksum, cancellable, error)) - goto out; - offset += bytes_written; - } - - if (!g_output_stream_close (pack_out, cancellable, error)) - goto out; - - g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(yayt)")); - g_ptr_array_sort (index_content_list, compare_index_content); - for (i = 0; i < index_content_list->len; i++) - { - GVariant *index_item = index_content_list->pdata[i]; - g_variant_builder_add_value (&index_content_builder, index_item); - } - index_content = g_variant_new ("(s@a{sv}@a(yayt))", - "OSTv0PACKINDEX", - g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), - g_variant_builder_end (&index_content_builder)); - - if (!g_output_stream_write_all (index_out, - g_variant_get_data (index_content), - g_variant_get_size (index_content), - &bytes_written, - cancellable, - error)) - goto out; - - if (!g_output_stream_close (index_out, cancellable, error)) - goto out; - - if (!ostree_repo_add_pack_file (data->repo, - g_checksum_get_string (pack_checksum), - is_meta, - index_temppath, - pack_temppath, - cancellable, - error)) - goto out; - - if (!ostree_repo_regenerate_pack_index (data->repo, cancellable, error)) - goto out; - - g_print ("Created %s pack file '%s' with %u objects\n", is_meta ? "metadata" : "content", - g_checksum_get_string (pack_checksum), objects->len); - - if (!opt_keep_all_loose) - { - for (i = 0; i < objects->len; i++) - { - GVariant *object_data = objects->pdata[i]; - const char *checksum; - guint32 objtype_u32; - OstreeObjectType objtype; - guint64 expected_objsize; - - g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize); - - objtype = (OstreeObjectType) objtype_u32; - - if (!delete_loose_object (data, checksum, objtype, cancellable, error)) - goto out; - } - } - - ret = TRUE; - out: - if (index_temppath) - (void) unlink (ot_gfile_get_path_cached (index_temppath)); - if (pack_temppath) - (void) unlink (ot_gfile_get_path_cached (pack_temppath)); - if (pack_checksum) - g_checksum_free (pack_checksum); - return ret; -} - -static void -cluster_one_object_chain (OtRepackData *data, - GPtrArray *object_list, - GPtrArray *inout_clusters) -{ - guint i; - guint64 current_size; - guint current_offset; - - current_size = 0; - current_offset = 0; - for (i = 0; i < object_list->len; i++) - { - GVariant *objdata = object_list->pdata[i]; - guint64 objsize; - - g_variant_get_child (objdata, 2, "t", &objsize); - - if (current_size + objsize > data->pack_size || i == (object_list->len - 1)) - { - guint j; - GPtrArray *current; - - if (current_offset < i) - { - current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - for (j = current_offset; j <= i; j++) - { - g_ptr_array_add (current, g_variant_ref (object_list->pdata[j])); - } - g_ptr_array_add (inout_clusters, current); - current_size = objsize; - current_offset = i+1; - } - } - else if (objsize > data->pack_size) - { - break; - } - else - { - current_size += objsize; - } - } -} - -/** - * cluster_objects_stupidly: - * @objects: Map from serialized object name to objdata - * @out_meta_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). - * @out_data_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). - * - * Just sorts by size currently. Also filters out non-regular object - * content. - */ -static gboolean -cluster_objects_stupidly (OtRepackData *data, - GHashTable *objects, - GPtrArray **out_meta_clusters, - GPtrArray **out_data_clusters, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - GHashTableIter hash_iter; - gpointer key, value; - ot_lptrarray GPtrArray *ret_meta_clusters = NULL; - ot_lptrarray GPtrArray *ret_data_clusters = NULL; - ot_lptrarray GPtrArray *meta_object_list = NULL; - ot_lptrarray GPtrArray *data_object_list = NULL; - - meta_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - data_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - - g_hash_table_iter_init (&hash_iter, objects); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - GVariant *serialized_key = key; - const char *checksum; - OstreeObjectType objtype; - guint64 size; - GVariant *v; - ot_lobj GFile *object_path = NULL; - ot_lobj GFileInfo *object_info = NULL; - - ostree_object_name_deserialize (serialized_key, &checksum, &objtype); - - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - { - object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); - - object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); - if (!object_info) - goto out; - } - else - { - if (!ostree_repo_load_file (data->repo, checksum, NULL, &object_info, NULL, - cancellable, error)) - goto out; - } - - size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE); - - v = g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size)); - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - g_ptr_array_add (meta_object_list, v); - else - g_ptr_array_add (data_object_list, v); - } - - g_ptr_array_sort (meta_object_list, compare_object_data_by_size); - g_ptr_array_sort (data_object_list, compare_object_data_by_size); - - ret_meta_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref); - ret_data_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref); - - cluster_one_object_chain (data, meta_object_list, ret_meta_clusters); - cluster_one_object_chain (data, data_object_list, ret_data_clusters); - - ret = TRUE; - ot_transfer_out_value (out_meta_clusters, &ret_meta_clusters); - ot_transfer_out_value (out_data_clusters, &ret_data_clusters); - out: - return ret; -} - -static gboolean -parse_size_spec_with_suffix (const char *spec, - guint64 *out_size, - GError **error) -{ - gboolean ret = FALSE; - char *endptr = NULL; - guint64 ret_size; - - ret_size = g_ascii_strtoull (spec, &endptr, 10); - - if (endptr && *endptr) - { - char suffix = *endptr; - - switch (suffix) - { - case 'k': - case 'K': - { - ret_size *= 1024; - break; - } - case 'm': - case 'M': - { - ret_size *= (1024 * 1024); - break; - } - case 'g': - case 'G': - { - ret_size *= (1024 * 1024 * 1024); - break; - } - default: - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid size suffix '%c'", suffix); - goto out; - } - } - - ret = TRUE; - *out_size = ret_size; - out: - return ret; -} - -static gboolean -parse_compression_string (const char *compstr, - OtCompressionType *out_comptype, - GError **error) -{ - gboolean ret = FALSE; - OtCompressionType ret_comptype; - - if (compstr == NULL) - ret_comptype = OT_COMPRESSION_NONE; - else if (strcmp (compstr, "gzip") == 0) - ret_comptype = OT_COMPRESSION_GZIP; - else if (strcmp (compstr, "xz") == 0) - ret_comptype = OT_COMPRESSION_XZ; - else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid compression '%s'", compstr); - goto out; - } - - ret = TRUE; - *out_comptype = ret_comptype; - out: - return ret; -} - -static gboolean -do_stats_gather_loose (OtRepackData *data, - GHashTable *objects, - GHashTable **out_loose, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint n_loose = 0; - guint n_loose_and_packed = 0; - guint n_packed = 0; - guint n_dup_packed = 0; - guint n_commits = 0; - guint n_dirmeta = 0; - guint n_dirtree = 0; - guint n_files = 0; - GHashTableIter hash_iter; - gpointer key, value; - ot_lhash GHashTable *ret_loose = NULL; - - ret_loose = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, - (GDestroyNotify) g_variant_unref, - NULL); - - g_hash_table_iter_init (&hash_iter, objects); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - GVariant *serialized_key = key; - GVariant *objdata = value; - const char *checksum; - OstreeObjectType objtype; - gboolean is_loose; - gboolean is_packed; - ot_lvariant GVariant *pack_array = NULL; - - ostree_object_name_deserialize (serialized_key, &checksum, &objtype); - - g_variant_get (objdata, "(b@as)", &is_loose, &pack_array); - - is_packed = g_variant_n_children (pack_array) > 0; - - if (is_loose && is_packed) - { - n_loose_and_packed++; - } - else if (is_loose) - { - if (!(opt_metadata_only && !OSTREE_OBJECT_TYPE_IS_META(objtype)) - && !(opt_content_only && OSTREE_OBJECT_TYPE_IS_META(objtype))) - { - GVariant *copy = g_variant_ref (serialized_key); - g_hash_table_replace (ret_loose, copy, copy); - } - n_loose++; - } - else if (g_variant_n_children (pack_array) > 1) - { - n_dup_packed++; - } - else - { - n_packed++; - } - - switch (objtype) - { - case OSTREE_OBJECT_TYPE_COMMIT: - n_commits++; - break; - case OSTREE_OBJECT_TYPE_DIR_TREE: - n_dirtree++; - break; - case OSTREE_OBJECT_TYPE_DIR_META: - n_dirmeta++; - break; - case OSTREE_OBJECT_TYPE_FILE: - n_files++; - break; - default: - g_assert_not_reached (); - } - } - - g_print ("Commits: %u\n", n_commits); - g_print ("Tree contents: %u\n", n_dirtree); - g_print ("Tree meta: %u\n", n_dirmeta); - g_print ("Files: %u\n", n_files); - g_print ("\n"); - g_print ("Loose+packed objects: %u\n", n_loose_and_packed); - g_print ("Loose-only objects: %u\n", n_loose); - g_print ("Duplicate packed objects: %u\n", n_dup_packed); - g_print ("Packed-only objects: %u\n", n_packed); - - ret = TRUE; - ot_transfer_out_value (out_loose, &ret_loose); - /* out: */ - return ret; -} - -static gboolean -do_incremental_pack (OtRepackData *data, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - guint i; - ot_lhash GHashTable *objects = NULL; - ot_lptrarray GPtrArray *meta_clusters = NULL; - ot_lptrarray GPtrArray *data_clusters = NULL; - ot_lhash GHashTable *loose_objects = NULL; - - if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, - cancellable, error)) - goto out; - - if (!do_stats_gather_loose (data, objects, &loose_objects, cancellable, error)) - goto out; - - g_print ("\n"); - g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size); - - if (!cluster_objects_stupidly (data, loose_objects, &meta_clusters, &data_clusters, - cancellable, error)) - goto out; - - if (meta_clusters->len > 0 || data_clusters->len > 0) - g_print ("Going to create %u meta packfiles, %u data packfiles\n", - meta_clusters->len, data_clusters->len); - else - g_print ("Nothing to do\n"); - - if (!opt_analyze_only) - { - for (i = 0; i < meta_clusters->len; i++) - { - GPtrArray *cluster = meta_clusters->pdata[i]; - - if (!create_pack_file (data, TRUE, cluster, cancellable, error)) - goto out; - } - for (i = 0; i < data_clusters->len; i++) - { - GPtrArray *cluster = data_clusters->pdata[i]; - - if (!create_pack_file (data, FALSE, cluster, cancellable, error)) - goto out; - } - } - - ret = TRUE; - out: - return ret; -} - -gboolean -ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error) -{ - gboolean ret = FALSE; - GOptionContext *context; - GCancellable *cancellable = NULL; - OtRepackData data; - ot_lobj OstreeRepo *repo = NULL; - - memset (&data, 0, sizeof (data)); - - context = g_option_context_new ("- Recompress objects"); - g_option_context_add_main_entries (context, options, NULL); - - if (!g_option_context_parse (context, &argc, &argv, error)) - goto out; - - if (opt_metadata_only && opt_content_only) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "--content-only cannot be specified with --metadata-only"); - goto out; - } - - repo = ostree_repo_new (repo_path); - if (!ostree_repo_check (repo, error)) - goto out; - - data.repo = repo; - data.error = error; - - if (!parse_size_spec_with_suffix (opt_pack_size, &data.pack_size, error)) - goto out; - /* Default internal compression to gzip */ - if (!parse_compression_string (opt_int_compression ? opt_int_compression : "gzip", &data.int_compression, error)) - goto out; - if (!parse_compression_string (opt_ext_compression, &data.ext_compression, error)) - goto out; - - if (opt_reindex_only) - { - if (!ostree_repo_regenerate_pack_index (repo, cancellable, error)) - goto out; - } - else - { - if (!do_incremental_pack (&data, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - if (context) - g_option_context_free (context); - return ret; -} diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index 2dec7627..7d6e05d0 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -67,7 +67,7 @@ import_one_object (OtLocalCloneData *data, cancellable, error)) goto out; - if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum, FALSE, + if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum, file_object, length, cancellable, error)) goto out; @@ -81,7 +81,7 @@ import_one_object (OtLocalCloneData *data, input = ot_variant_read (metadata); if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype, - checksum, FALSE, input, + checksum, input, cancellable, error)) goto out; } diff --git a/src/ostree/ot-builtin-unpack.c b/src/ostree/ot-builtin-unpack.c deleted file mode 100644 index 53e66e6c..00000000 --- a/src/ostree/ot-builtin-unpack.c +++ /dev/null @@ -1,266 +0,0 @@ -/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- - * - * Copyright (C) 2012 Colin Walters - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 02111-1307, USA. - * - * Author: Colin Walters - */ - -#include "config.h" - -#include "ot-builtins.h" -#include "ostree.h" - -#include -#include - -#include -#include - -static gboolean opt_keep_packs; - -static GOptionEntry options[] = { - { "keep-packs", 0, 0, G_OPTION_ARG_NONE, &opt_keep_packs, "Don't delete pack files", NULL }, - { NULL } -}; - -typedef struct { - OstreeRepo *repo; -} OtUnpackData; - -static gboolean -unpack_one_object (OstreeRepo *repo, - const char *checksum, - OstreeObjectType objtype, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lobj GInputStream *input = NULL; - ot_lobj GFileInfo *file_info = NULL; - ot_lvariant GVariant *xattrs = NULL; - ot_lvariant GVariant *meta = NULL; - - if (objtype == OSTREE_OBJECT_TYPE_FILE) - { - ot_lobj GInputStream *file_object = NULL; - guint64 length; - - if (!ostree_repo_load_file (repo, checksum, - &input, &file_info, &xattrs, - cancellable, error)) - goto out; - - if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, &length, - cancellable, error)) - goto out; - - if (!ostree_repo_stage_file_object_trusted (repo, checksum, TRUE, file_object, length, - cancellable, error)) - goto out; - } - else - { - if (!ostree_repo_load_variant (repo, objtype, checksum, &meta, error)) - goto out; - - input = ot_variant_read (meta); - - if (!ostree_repo_stage_object_trusted (repo, objtype, checksum, TRUE, - input, cancellable, error)) - goto out; - } - - ret = TRUE; - out: - return ret; -} - -static gboolean -delete_one_packfile (OstreeRepo *repo, - const char *pack_checksum, - gboolean is_meta, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - ot_lfree char *data_name = NULL; - ot_lobj GFile *data_path = NULL; - ot_lfree char *index_name = NULL; - ot_lobj GFile *index_path = NULL; - - index_name = ostree_get_relative_pack_index_path (is_meta, pack_checksum); - index_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), index_name); - data_name = ostree_get_relative_pack_data_path (is_meta, pack_checksum); - data_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), data_name); - - if (!ot_gfile_unlink (index_path, cancellable, error)) - { - g_prefix_error (error, "Failed to delete pack index '%s': ", ot_gfile_get_path_cached (index_path)); - goto out; - } - if (!ot_gfile_unlink (data_path, cancellable, error)) - { - g_prefix_error (error, "Failed to delete pack data '%s': ", ot_gfile_get_path_cached (data_path)); - goto out; - } - - ret = TRUE; - out: - return ret; -} - -gboolean -ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error) -{ - gboolean ret = FALSE; - GOptionContext *context; - GCancellable *cancellable = NULL; - gboolean in_transaction = FALSE; - OtUnpackData data; - gpointer key, value; - guint64 unpacked_object_count = 0; - GHashTableIter hash_iter; - ot_lobj OstreeRepo *repo = NULL; - ot_lhash GHashTable *objects = NULL; - ot_lptrarray GPtrArray *clusters = NULL; - ot_lhash GHashTable *meta_packfiles_to_delete = NULL; - ot_lhash GHashTable *data_packfiles_to_delete = NULL; - ot_lobj GFile *objpath = NULL; - - memset (&data, 0, sizeof (data)); - - context = g_option_context_new ("- Uncompress objects"); - g_option_context_add_main_entries (context, options, NULL); - - if (!g_option_context_parse (context, &argc, &argv, error)) - goto out; - - repo = ostree_repo_new (repo_path); - if (!ostree_repo_check (repo, error)) - goto out; - - if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "Can't unpack bare repositories yet"); - goto out; - } - - data.repo = repo; - - if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error)) - goto out; - - if (!ostree_repo_prepare_transaction (repo, cancellable, error)) - goto out; - - in_transaction = TRUE; - - meta_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - data_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - - g_hash_table_iter_init (&hash_iter, objects); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - GVariant *objkey = key; - GVariant *objdata; - const char *checksum; - const char *pack_checksum; - OstreeObjectType objtype; - gboolean is_loose; - gboolean is_packed = FALSE; - GVariantIter *pack_array_iter; - GHashTable *target_hash; - - ostree_object_name_deserialize (objkey, &checksum, &objtype); - - objdata = g_hash_table_lookup (objects, objkey); - g_assert (objdata); - - g_variant_get (objdata, "(bas)", &is_loose, &pack_array_iter); - - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - target_hash = meta_packfiles_to_delete; - else - target_hash = data_packfiles_to_delete; - - while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum)) - { - is_packed = TRUE; - if (!g_hash_table_lookup (target_hash, pack_checksum)) - { - gchar *duped_checksum = g_strdup (pack_checksum); - g_hash_table_replace (target_hash, duped_checksum, duped_checksum); - } - } - g_variant_iter_free (pack_array_iter); - - if (is_packed) - { - if (!unpack_one_object (repo, checksum, objtype, cancellable, error)) - goto out; - - unpacked_object_count++; - } - } - - if (!ostree_repo_commit_transaction (repo, cancellable, error)) - goto out; - - if (!opt_keep_packs) - { - if (g_hash_table_size (meta_packfiles_to_delete) == 0 - && g_hash_table_size (data_packfiles_to_delete) == 0) - g_print ("No pack files; nothing to do\n"); - - g_hash_table_iter_init (&hash_iter, meta_packfiles_to_delete); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *pack_checksum = key; - - if (!delete_one_packfile (repo, pack_checksum, TRUE, cancellable, error)) - goto out; - - g_print ("Deleted packfile '%s'\n", pack_checksum); - } - - g_hash_table_iter_init (&hash_iter, data_packfiles_to_delete); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *pack_checksum = key; - - if (!delete_one_packfile (repo, pack_checksum, FALSE, cancellable, error)) - goto out; - - g_print ("Deleted packfile '%s'\n", pack_checksum); - } - - if (!ostree_repo_regenerate_pack_index (repo, cancellable, error)) - goto out; - } - - g_print ("Unpacked %" G_GUINT64_FORMAT " objects\n", unpacked_object_count); - - ret = TRUE; - out: - if (in_transaction) - (void) ostree_repo_abort_transaction (repo, cancellable, NULL); - if (context) - g_option_context_free (context); - return ret; -} diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index aa7a645d..b3d446ea 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -41,10 +41,8 @@ gboolean ostree_builtin_ls (int argc, char **argv, GFile *repo_path, GError **er gboolean ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_show (int argc, char **argv, GFile *repo_path, GError **error); -gboolean ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_rev_parse (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_remote (int argc, char **argv, GFile *repo_path, GError **error); -gboolean ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error); gboolean ostree_builtin_write_refs (int argc, char **argv, GFile *repo_path, GError **error); G_END_DECLS diff --git a/tests/t0001-archive.sh b/tests/t0001-archive.sh index 3a3e706d..0e9f88cf 100755 --- a/tests/t0001-archive.sh +++ b/tests/t0001-archive.sh @@ -21,7 +21,7 @@ set -e . libtest.sh -echo '1..22' +echo '1..11' setup_test_repository "archive" echo "ok setup" @@ -68,45 +68,6 @@ $OSTREE cat test2 /baz/cow > cow-contents assert_file_has_content cow-contents "moo" echo "ok cat-file" -cd ${test_tmpdir} -$OSTREE pack -echo "ok pack" - cd ${test_tmpdir} $OSTREE fsck echo "ok fsck" - -$OSTREE checkout test2 checkout-test2-from-packed -echo "ok checkout union 1" - -cd ${test_tmpdir} -$OSTREE fsck -echo "ok fsck" - -$OSTREE pack --analyze-only -echo "ok pack analyze" - -$OSTREE unpack -echo "ok unpack" - -cd ${test_tmpdir} -$OSTREE fsck -echo "ok fsck" - -cd ${test_tmpdir} -$OSTREE checkout test2 checkout-test2-from-unpacked -echo "ok checkout union 2" - -$OSTREE pack --metadata-only -echo "ok pack metadata" - -$OSTREE fsck -echo "ok fsck" - -cd ${test_tmpdir} -rm -rf checkout-test2 -$OSTREE checkout test2 checkout-test2 -echo "ok checkout metadata-packed" - -$OSTREE unpack -echo "ok unpack" diff --git a/tests/t0010-pull.sh b/tests/t0010-pull.sh index 57416c60..1fe002ee 100755 --- a/tests/t0010-pull.sh +++ b/tests/t0010-pull.sh @@ -21,7 +21,7 @@ set -e . libtest.sh -echo '1..4' +echo '1..2' setup_fake_remote_repo1 cd ${test_tmpdir} @@ -38,21 +38,3 @@ cd checkout-origin-main assert_file_has_content firstfile '^first$' assert_file_has_content baz/cow '^moo$' echo "ok pull contents" - -cd ${test_tmpdir} -ostree --repo=$(pwd)/ostree-srv/gnomerepo pack -rm -rf repo -mkdir repo -${CMD_PREFIX} ostree --repo=repo init -${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo -${CMD_PREFIX} ostree-pull --repo=repo origin main -${CMD_PREFIX} ostree --repo=repo fsck -echo "ok pull packed" - -cd ${test_tmpdir} -rm -rf checkout-origin-main -$OSTREE checkout origin/main checkout-origin-main -cd checkout-origin-main -assert_file_has_content firstfile '^first$' -assert_file_has_content baz/cow '^moo$' -echo "ok pull contents packed"