core: Drop packfiles as they are now

They're not a large efficiency win at the moment, because we don't
do any delta compression.

At the moment, they simply served to compress data, but we will change
the archive mode to do that by default.
This commit is contained in:
Colin Walters committed 2012-09-23 16:02:03 -04:00
1 parent aa9aaa9a62
commit 2a0601efc7
15 files changed
+37 -3694

No files matched your search

-332
View File
@@ -944,58 +944,6 @@ ostree_get_relative_archive_content_path (const char *checksum)
return g_string_free (path, FALSE);
}
static char *
get_pack_name (gboolean is_meta,
gboolean is_index,
const char *prefix,
const char *checksum)
{
GString *path;
g_assert (strlen (checksum) == 64);
path = g_string_new (prefix);
if (is_meta)
g_string_append (path, "ostmetapack-");
else
g_string_append (path, "ostdatapack-");
g_string_append (path, checksum);
if (is_index)
g_string_append (path, ".index");
else
g_string_append (path, ".data");
return g_string_free (path, FALSE);
}
char *
ostree_get_pack_index_name (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, TRUE, "", checksum);
}
char *
ostree_get_pack_data_name (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, FALSE, "", checksum);
}
char *
ostree_get_relative_pack_index_path (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, TRUE, "objects/pack/", checksum);
}
char *
ostree_get_relative_pack_data_path (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, FALSE, "objects/pack/", checksum);
}
gboolean
ostree_file_header_parse (GVariant *metadata,
GFileInfo **out_file_info,
@@ -1343,194 +1291,6 @@ ostree_create_temp_dir (GFile *dir,
return ret;
}
gboolean
ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len,
guint64 offset,
gboolean trusted,
gboolean is_meta,
GVariant **out_entry,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint64 entry_start;
guint64 entry_end;
guint32 entry_len;
ot_lvariant GVariant *ret_entry = NULL;
if (G_UNLIKELY (!(offset <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range offset %" G_GUINT64_FORMAT,
offset);
goto out;
}
if (G_UNLIKELY (!((offset & 0x3) == 0)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT,
offset);
goto out;
}
entry_start = ALIGN_VALUE (offset + 4, 8);
if (G_UNLIKELY (!(entry_start <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
entry_start);
goto out;
}
g_assert ((((guint64)pack_data+offset) & 0x3) == 0);
entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
entry_end = entry_start + entry_len;
if (G_UNLIKELY (!(entry_end <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range entry length %u",
entry_len);
goto out;
}
ret_entry = g_variant_new_from_data (is_meta ? OSTREE_PACK_META_FILE_VARIANT_FORMAT :
OSTREE_PACK_DATA_FILE_VARIANT_FORMAT,
pack_data+entry_start, entry_len,
trusted, NULL, NULL);
g_variant_ref_sink (ret_entry);
ret = TRUE;
ot_transfer_out_value (out_entry, &ret_entry);
out:
return ret;
}
gboolean
ostree_parse_file_pack_entry (GVariant *pack_entry,
GInputStream **out_input,
GFileInfo **out_info,
GVariant **out_xattrs,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guchar entry_flags;
ot_lvariant GVariant *file_header = NULL;
ot_lvariant GVariant *pack_data = NULL;
ot_lobj GInputStream *memory_input = NULL;
ot_lobj GInputStream *ret_input = NULL;
ot_lobj GFileInfo *ret_info = NULL;
ot_lvariant GVariant *ret_xattrs = NULL;
g_variant_get_child (pack_entry, 1, "y", &entry_flags);
g_variant_get_child (pack_entry, 2, "@(uuuusa(ayay))", &file_header);
g_variant_get_child (pack_entry, 3, "@ay", &pack_data);
if (!ostree_file_header_parse (file_header, &ret_info, &ret_xattrs,
error))
goto out;
g_file_info_set_size (ret_info, g_variant_get_size (pack_data));
if (g_file_info_get_file_type (ret_info) == G_FILE_TYPE_REGULAR)
{
memory_input = ot_variant_read (pack_data);
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
ot_lobj GConverter *decompressor = NULL;
decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP);
ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", decompressor,
"base-stream", memory_input,
"close-base-stream", TRUE,
NULL);
}
else
{
ret_input = g_object_ref (memory_input);
}
}
ret = TRUE;
ot_transfer_out_value (out_input, &ret_input);
ot_transfer_out_value (out_info, &ret_info);
ot_transfer_out_value (out_xattrs, &ret_xattrs);
out:
return ret;
}
gboolean
ostree_pack_index_search (GVariant *index,
GVariant *csum_v,
OstreeObjectType objtype,
guint64 *out_offset)
{
gboolean ret = FALSE;
gsize imax, imin;
gsize n;
guint32 target_objtype;
const guchar *csum;
ot_lvariant GVariant *index_contents = NULL;
csum = ostree_checksum_bytes_peek (csum_v);
index_contents = g_variant_get_child_value (index, 2);
target_objtype = (guint32) objtype;
n = g_variant_n_children (index_contents);
if (n == 0)
goto out;
imax = n - 1;
imin = 0;
while (imax >= imin)
{
GVariant *cur_csum_bytes;
guchar cur_objtype;
guint64 cur_offset;
gsize imid;
int c;
imid = (imin + imax) / 2;
g_variant_get_child (index_contents, imid, "(y@ayt)", &cur_objtype,
&cur_csum_bytes, &cur_offset);
c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum);
if (c == 0)
{
if (cur_objtype < target_objtype)
c = -1;
else if (cur_objtype > target_objtype)
c = 1;
}
g_variant_unref (cur_csum_bytes);
if (c < 0)
imin = imid + 1;
else if (c > 0)
{
if (imid == 0)
goto out;
imax = imid - 1;
}
else
{
if (out_offset)
*out_offset = GUINT64_FROM_BE (cur_offset);
ret = TRUE;
goto out;
}
}
out:
return ret;
}
gboolean
ostree_validate_structureof_objtype (guchar objtype,
GError **error)
@@ -1766,95 +1526,3 @@ ostree_validate_structureof_dirmeta (GVariant *dirmeta,
out:
return ret;
}
gboolean
ostree_validate_structureof_pack_index (GVariant *index,
GError **error)
{
gboolean ret = FALSE;
const char *header;
guchar objtype_u8;
guint64 offset;
ot_lvariant GVariant *csum_v = NULL;
GVariantIter *content_iter = NULL;
if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error))
goto out;
g_variant_get_child (index, 0, "&s", &header);
if (strcmp (header, "OSTv0PACKINDEX") != 0)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid pack index; doesn't match header");
goto out;
}
g_variant_get_child (index, 2, "a(yayt)", &content_iter);
while (g_variant_iter_loop (content_iter, "(y@ayt)",
&objtype_u8, &csum_v, &offset))
{
if (!ostree_validate_structureof_objtype (objtype_u8, error))
goto out;
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
}
csum_v = NULL;
ret = TRUE;
out:
if (content_iter)
g_variant_iter_free (content_iter);
return ret;
}
gboolean
ostree_validate_structureof_pack_superindex (GVariant *superindex,
GError **error)
{
gboolean ret = FALSE;
const char *header;
ot_lvariant GVariant *csum_v = NULL;
ot_lvariant GVariant *bloom = NULL;
GVariantIter *content_iter = NULL;
if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error))
goto out;
g_variant_get_child (superindex, 0, "&s", &header);
if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid pack superindex; doesn't match header");
goto out;
}
g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_v, &bloom))
{
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
}
csum_v = NULL;
bloom = NULL;
g_variant_iter_free (content_iter);
g_variant_get_child (superindex, 3, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_v, &bloom))
{
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
}
csum_v = NULL;
bloom = NULL;
ret = TRUE;
out:
if (content_iter)
g_variant_iter_free (content_iter);
return ret;
}
-79
View File
@@ -87,48 +87,6 @@ typedef enum {
*/
#define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE ("(a{sv}aya(say)sstayay)")
/* Pack super index
* s - OSTv0SUPERPACKINDEX
* a{sv} - Metadata
* a(ayay) - metadata packs (pack file checksum, bloom filter)
* a(ayay) - data packs (pack file checksum, bloom filter)
*/
#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay)a(ayay))")
/* Pack index
* s - OSTv0PACKINDEX
* a{sv} - Metadata
* a(yayt) - (objtype, checksum, offset into packfile)
*/
#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(yayt))")
typedef enum {
OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0)
} OstreePackFileEntryFlag;
/* Data Pack files
* s - OSTv0PACKDATAFILE
* a{sv} - Metadata
* t - number of entries
*
* Repeating pair of:
* <padding to alignment of 8>
* ( ayy(uuuusa(ayay))ay) ) - checksum, flags, file meta, data
*/
#define OSTREE_PACK_DATA_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(ayy(uuuusa(ayay))ay)")
/* Meta Pack files
* s - OSTv0PACKMETAFILE
* a{sv} - Metadata
* t - number of entries
*
* Repeating pair of:
* <padding to alignment of 8>
* ( yayv ) - objtype, checksum, data
*/
#define OSTREE_PACK_META_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yayv)")
const GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype);
gboolean ostree_validate_checksum_string (const char *sha256,
@@ -173,16 +131,6 @@ char *ostree_get_relative_object_path (const char *checksum,
char *ostree_get_relative_archive_content_path (const char *checksum);
char *ostree_get_pack_index_name (gboolean is_meta,
const char *checksum);
char *ostree_get_pack_data_name (gboolean is_meta,
const char *checksum);
char *ostree_get_relative_pack_index_path (gboolean is_meta,
const char *checksum);
char *ostree_get_relative_pack_data_path (gboolean is_meta,
const char *checksum);
gboolean ostree_get_xattrs_for_file (GFile *f,
GVariant **out_xattrs,
GCancellable *cancellable,
@@ -304,27 +252,6 @@ gboolean ostree_create_temp_dir (GFile *dir,
GCancellable *cancellable,
GError **error);
gboolean ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len,
guint64 object_offset,
gboolean trusted,
gboolean is_meta,
GVariant **out_entry,
GCancellable *cancellable,
GError **error);
gboolean ostree_parse_file_pack_entry (GVariant *pack_entry,
GInputStream **out_input,
GFileInfo **out_info,
GVariant **out_xattrs,
GCancellable *cancellable,
GError **error);
gboolean ostree_pack_index_search (GVariant *index,
GVariant *csum_bytes,
OstreeObjectType objtype,
guint64 *out_offset);
/** VALIDATION **/
gboolean ostree_validate_structureof_objtype (guchar objtype,
@@ -348,10 +275,4 @@ gboolean ostree_validate_structureof_dirtree (GVariant *index,
gboolean ostree_validate_structureof_dirmeta (GVariant *index,
GError **error);
gboolean ostree_validate_structureof_pack_index (GVariant *index,
GError **error);
gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex,
GError **error);
#endif /* _OSTREE_REPO */
File diff suppressed because it is too large. Load diff
-88
View File
@@ -111,14 +111,12 @@ gboolean ostree_repo_stage_file_object (OstreeRepo *self,
gboolean ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
gboolean store_if_packed,
GInputStream *content,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_stage_file_object_trusted (OstreeRepo *self,
const char *checksum,
gboolean store_if_packed,
GInputStream *content,
guint64 content_length,
GCancellable *cancellable,
@@ -159,27 +157,6 @@ gboolean ostree_repo_load_variant_if_exists (OstreeRepo *self,
GVariant **out_variant,
GError **error);
gboolean ostree_repo_load_pack_index (OstreeRepo *self,
const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_load_pack_data (OstreeRepo *self,
const char *pack_checksum,
guchar **out_data,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_map_pack_file (OstreeRepo *self,
const char *sha256,
gboolean is_meta,
guchar **out_data,
guint64 *out_len,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_load_file (OstreeRepo *self,
const char *entry_sha256,
GInputStream **out_input,
@@ -248,65 +225,6 @@ gboolean ostree_repo_stage_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_add_pack_file (OstreeRepo *self,
const char *checksum,
gboolean is_meta,
GFile *pack_index_path,
GFile *pack_data_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
const char *remote_name,
GFile *superindex_path,
GPtrArray **out_cached_meta_indexes,
GPtrArray **out_cached_data_indexes,
GPtrArray **out_uncached_meta_indexes,
GPtrArray **out_uncached_data_indexes,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile *cached_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile *cached_path,
GCancellable *cancellable,
GError **error);
typedef enum {
OSTREE_REPO_CHECKOUT_MODE_NONE = 0,
OSTREE_REPO_CHECKOUT_MODE_USER = 1
@@ -359,12 +277,6 @@ gboolean ostree_repo_list_objects (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_list_pack_indexes (OstreeRepo *self,
GPtrArray **out_meta_indexes,
GPtrArray **out_data_indexes,
GCancellable *cancellable,
GError **error);
G_END_DECLS
#endif /* _OSTREE_REPO */