packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

core: Split pack files into data/metadata 

This will allow us to download metadata first (separately), and in the
future we can have an improved metadata pack format that compresses
heavily.
This commit is contained in:
Colin Walters 2012-04-09 22:46:08 -04:00
parent 11c0fbc738
commit 5a82b141ae
8 changed files with 922 additions and 555 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
src/libostree/ostree-core.c
View File

@ -710,6 +710,58 @@ ostree_get_relative_object_path (const char *checksum,
return g_string_free (path, FALSE); return g_string_free (path, FALSE);
} }
static char *
get_pack_name (gboolean is_meta,
gboolean is_index,
const char *prefix,
const char *checksum)
{
GString *path;
g_assert (strlen (checksum) == 64);
path = g_string_new (prefix);
if (is_meta)
g_string_append (path, "ostmetapack-");
else
g_string_append (path, "ostdatapack-");
g_string_append (path, checksum);
if (is_index)
g_string_append (path, ".index");
else
g_string_append (path, ".data");
return g_string_free (path, FALSE);
}
char *
ostree_get_pack_index_name (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, TRUE, "", checksum);
}
char *
ostree_get_pack_data_name (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, FALSE, "", checksum);
}
char *
ostree_get_relative_pack_index_path (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, TRUE, "objects/pack/", checksum);
}
char *
ostree_get_relative_pack_data_path (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, FALSE, "objects/pack/", checksum);
}
GVariant * GVariant *
ostree_create_archive_file_metadata (GFileInfo *finfo, ostree_create_archive_file_metadata (GFileInfo *finfo,
GVariant *xattrs) GVariant *xattrs)
@ -1169,6 +1221,7 @@ ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len, guint64 pack_len,
guint64 offset, guint64 offset,
gboolean trusted, gboolean trusted,
gboolean is_meta,
GVariant **out_entry, GVariant **out_entry,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -1215,7 +1268,8 @@ ostree_read_pack_entry_raw (guchar *pack_data,
goto out; goto out;
} }
ret_entry = g_variant_new_from_data (OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT, ret_entry = g_variant_new_from_data (is_meta ? OSTREE_PACK_META_FILE_VARIANT_FORMAT :
OSTREE_PACK_DATA_FILE_VARIANT_FORMAT,
pack_data+entry_start, entry_len, pack_data+entry_start, entry_len,
trusted, NULL, NULL); trusted, NULL, NULL);
g_variant_ref_sink (ret_entry); g_variant_ref_sink (ret_entry);
@ -1274,32 +1328,13 @@ ostree_read_pack_entry_variant (GVariant *pack_entry,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
guint32 actual_type;
ot_lobj GInputStream *stream = NULL;
ot_lvariant GVariant *container_variant = NULL;
ot_lvariant GVariant *ret_variant = NULL; ot_lvariant GVariant *ret_variant = NULL;
stream = ostree_read_pack_entry_as_stream (pack_entry); g_variant_get_child (pack_entry, 2, "v", &ret_variant);
if (!ot_util_variant_from_stream (stream, OSTREE_SERIALIZED_VARIANT_FORMAT,
trusted, &container_variant, cancellable, error))
goto out;
g_variant_get (container_variant, "(uv)",
&actual_type, &ret_variant);
actual_type = GUINT32_FROM_BE (actual_type);
if (actual_type != expected_objtype)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted metadata object in pack file; found type %u, expected %u",
actual_type, (guint32)expected_objtype);
goto out;
}
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_variant, &ret_variant); ot_transfer_out_value (out_variant, &ret_variant);
out: /* out: */
return ret; return ret;
} }
@ -1332,16 +1367,15 @@ ostree_pack_index_search (GVariant *index,
while (imax >= imin) while (imax >= imin)
{ {
GVariant *cur_csum_bytes; GVariant *cur_csum_bytes;
guint32 cur_objtype; guchar cur_objtype;
guint64 cur_offset; guint64 cur_offset;
gsize imid; gsize imid;
int c; int c;
imid = (imin + imax) / 2; imid = (imin + imax) / 2;
g_variant_get_child (index_contents, imid, "(u@ayt)", &cur_objtype, g_variant_get_child (index_contents, imid, "(y@ayt)", &cur_objtype,
&cur_csum_bytes, &cur_offset); &cur_csum_bytes, &cur_offset);
cur_objtype = GUINT32_FROM_BE (cur_objtype);
c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum); c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum);
if (c == 0) if (c == 0)
@ -1375,12 +1409,12 @@ ostree_pack_index_search (GVariant *index,
} }
gboolean gboolean
ostree_validate_structureof_objtype (guint32 objtype, ostree_validate_structureof_objtype (guchar objtype,
GError **error) GError **error)
{ {
objtype = GUINT32_FROM_BE (objtype); OstreeObjectType objtype_v = (OstreeObjectType) objtype;
if (objtype < OSTREE_OBJECT_TYPE_RAW_FILE if (objtype_v < OSTREE_OBJECT_TYPE_RAW_FILE
|| objtype > OSTREE_OBJECT_TYPE_COMMIT) || objtype_v > OSTREE_OBJECT_TYPE_COMMIT)
{ {
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid object type '%u'", objtype); "Invalid object type '%u'", objtype);
@ -1542,7 +1576,8 @@ validate_stat_mode_perms (guint32 mode,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
guint32 otherbits = (~S_IFMT & ~S_IRWXU & ~S_IRWXG & ~S_IRWXO); guint32 otherbits = (~S_IFMT & ~S_IRWXU & ~S_IRWXG & ~S_IRWXO &
~S_ISUID & ~S_ISGID & ~S_ISVTX);
if (mode & otherbits) if (mode & otherbits)
{ {
@ -1615,7 +1650,7 @@ ostree_validate_structureof_pack_index (GVariant *index,
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
const char *header; const char *header;
guint32 objtype; guchar objtype_u8;
guint64 offset; guint64 offset;
ot_lvariant GVariant *csum_v = NULL; ot_lvariant GVariant *csum_v = NULL;
GVariantIter *content_iter = NULL; GVariantIter *content_iter = NULL;
@ -1632,12 +1667,12 @@ ostree_validate_structureof_pack_index (GVariant *index,
goto out; goto out;
} }
g_variant_get_child (index, 2, "a(uayt)", &content_iter); g_variant_get_child (index, 2, "a(yayt)", &content_iter);
while (g_variant_iter_loop (content_iter, "(u@ayt)", while (g_variant_iter_loop (content_iter, "(y@ayt)",
&objtype, &csum_v, &offset)) &objtype_u8, &csum_v, &offset))
{ {
if (!ostree_validate_structureof_objtype (objtype, error)) if (!ostree_validate_structureof_objtype (objtype_u8, error))
goto out; goto out;
if (!ostree_validate_structureof_csum_v (csum_v, error)) if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out; goto out;
@ -1674,7 +1709,15 @@ ostree_validate_structureof_pack_superindex (GVariant *superindex,
} }
g_variant_get_child (superindex, 2, "a(ayay)", &content_iter); g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_v, &bloom))
{
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
}
csum_v = NULL;
g_variant_get_child (superindex, 3, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)", while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_v, &bloom)) &csum_v, &bloom))
{ {

41
src/libostree/ostree-core.h
View File

@ -93,34 +93,44 @@ typedef enum {
/* Pack super index /* Pack super index
* s - OSTv0SUPERPACKINDEX * s - OSTv0SUPERPACKINDEX
* a{sv} - Metadata * a{sv} - Metadata
* a(ayay) - (pack file checksum, bloom filter) * a(ayay) - metadata packs (pack file checksum, bloom filter)
* a(ayay) - data packs (pack file checksum, bloom filter)
*/ */
#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay))") #define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay)a(ayay))")
/* Pack index /* Pack index
* s - OSTv0PACKINDEX * s - OSTv0PACKINDEX
* a{sv} - Metadata * a{sv} - Metadata
* a(uayt) - (objtype, checksum, offset into packfile) * a(yayt) - (objtype, checksum, offset into packfile)
*/ */
#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(uayt))") #define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(yayt))")
typedef enum { typedef enum {
OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0, OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0) OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0)
} OstreePackFileEntryFlag; } OstreePackFileEntryFlag;
/* Pack files /* Data Pack files
* s - OSTv0PACKFILE * s - OSTv0PACKDATAFILE
* a{sv} - Metadata * a{sv} - Metadata
* t - number of entries * t - number of entries
* *
* Repeating pair of: * Repeating pair of:
* <padding to alignment of 8> * <padding to alignment of 8>
* ( uyayay ) - objtype, flags, checksum, data * ( yyayay ) - objtype, flags, checksum, data
*/ */
#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}t)") #define OSTREE_PACK_DATA_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yyayay)")
#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(uyayay)") /* Meta Pack files
* s - OSTv0PACKMETAFILE
* a{sv} - Metadata
* t - number of entries
*
* Repeating pair of:
* <padding to alignment of 8>
* ( yayv ) - objtype, checksum, data
*/
#define OSTREE_PACK_META_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yayv)")
gboolean ostree_validate_checksum_string (const char *sha256, gboolean ostree_validate_checksum_string (const char *sha256,
GError **error); GError **error);
@ -162,6 +172,16 @@ void ostree_object_from_string (const char *str,
char *ostree_get_relative_object_path (const char *checksum, char *ostree_get_relative_object_path (const char *checksum,
OstreeObjectType type); OstreeObjectType type);
char *ostree_get_pack_index_name (gboolean is_meta,
const char *checksum);
char *ostree_get_pack_data_name (gboolean is_meta,
const char *checksum);
char *ostree_get_relative_pack_index_path (gboolean is_meta,
const char *checksum);
char *ostree_get_relative_pack_data_path (gboolean is_meta,
const char *checksum);
gboolean ostree_get_xattrs_for_file (GFile *f, gboolean ostree_get_xattrs_for_file (GFile *f,
GVariant **out_xattrs, GVariant **out_xattrs,
GCancellable *cancellable, GCancellable *cancellable,
@ -260,6 +280,7 @@ gboolean ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len, guint64 pack_len,
guint64 object_offset, guint64 object_offset,
gboolean trusted, gboolean trusted,
gboolean is_meta,
GVariant **out_entry, GVariant **out_entry,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
@ -280,7 +301,7 @@ gboolean ostree_pack_index_search (GVariant *index,
/** VALIDATION **/ /** VALIDATION **/
gboolean ostree_validate_structureof_objtype (guint32 objtype, gboolean ostree_validate_structureof_objtype (guchar objtype,
GError **error); GError **error);
gboolean ostree_validate_structureof_csum_v (GVariant *checksum, gboolean ostree_validate_structureof_csum_v (GVariant *checksum,

605
src/libostree/ostree-repo.c
View File

@ -1507,6 +1507,7 @@ get_checksum_from_pack_name (const char *name)
static gboolean static gboolean
list_pack_indexes_from_dir (OstreeRepo *self, list_pack_indexes_from_dir (OstreeRepo *self,
gboolean is_meta,
GPtrArray **out_indexes, GPtrArray **out_indexes,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -1518,7 +1519,7 @@ list_pack_indexes_from_dir (OstreeRepo *self,
ot_lptrarray GPtrArray *ret_indexes = NULL; ot_lptrarray GPtrArray *ret_indexes = NULL;
if (!list_files_in_dir_matching (priv->pack_dir, if (!list_files_in_dir_matching (priv->pack_dir,
"ostpack-", ".index", is_meta ? "ostmetapack-" : "ostdatapack-", ".index",
&index_files, &index_files,
cancellable, error)) cancellable, error))
goto out; goto out;
@ -1539,24 +1540,27 @@ list_pack_indexes_from_dir (OstreeRepo *self,
static gboolean static gboolean
list_pack_checksums_from_superindex_file (GFile *superindex_path, list_pack_checksums_from_superindex_file (GFile *superindex_path,
GPtrArray **out_indexes, GPtrArray **out_meta_indexes,
GPtrArray **out_data_indexes,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
const char *magic; const char *magic;
ot_lptrarray GPtrArray *ret_indexes = NULL; ot_lptrarray GPtrArray *ret_meta_indexes = NULL;
ot_lptrarray GPtrArray *ret_data_indexes = NULL;
ot_lvariant GVariant *superindex_variant = NULL; ot_lvariant GVariant *superindex_variant = NULL;
ot_lvariant GVariant *checksum = NULL; ot_lvariant GVariant *checksum = NULL;
ot_lvariant GVariant *bloom = NULL; ot_lvariant GVariant *bloom = NULL;
GVariantIter *variant_iter = NULL; GVariantIter *meta_variant_iter = NULL;
GVariantIter *data_variant_iter = NULL;
if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
&superindex_variant, error)) &superindex_variant, error))
goto out; goto out;
g_variant_get (superindex_variant, "(&s@a{sv}a(ayay))", g_variant_get (superindex_variant, "(&s@a{sv}a(ayay)a(ayay))",
&magic, NULL, &variant_iter); &magic, NULL, &meta_variant_iter, &data_variant_iter);
if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0) if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0)
{ {
@ -1565,47 +1569,62 @@ list_pack_checksums_from_superindex_file (GFile *superindex_path,
goto out; goto out;
} }
ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
while (g_variant_iter_loop (meta_variant_iter, "(@ay@ay)",
while (g_variant_iter_loop (variant_iter, "(@ay@ay)",
&checksum, &bloom)) &checksum, &bloom))
g_ptr_array_add (ret_indexes, ostree_checksum_from_bytes_v (checksum)); g_ptr_array_add (ret_meta_indexes, ostree_checksum_from_bytes_v (checksum));
checksum = NULL;
bloom = NULL;
ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
while (g_variant_iter_loop (data_variant_iter, "(@ay@ay)",
&checksum, &bloom))
g_ptr_array_add (ret_data_indexes, ostree_checksum_from_bytes_v (checksum));
checksum = NULL; checksum = NULL;
bloom = NULL; bloom = NULL;
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_indexes, &ret_indexes); ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes);
ot_transfer_out_value (out_data_indexes, &ret_data_indexes);
out: out:
if (variant_iter) if (meta_variant_iter)
g_variant_iter_free (variant_iter); g_variant_iter_free (meta_variant_iter);
if (data_variant_iter)
g_variant_iter_free (data_variant_iter);
return ret; return ret;
} }
gboolean gboolean
ostree_repo_list_pack_indexes (OstreeRepo *self, ostree_repo_list_pack_indexes (OstreeRepo *self,
GPtrArray **out_indexes, GPtrArray **out_meta_indexes,
GPtrArray **out_data_indexes,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self); OstreeRepoPrivate *priv = GET_PRIVATE (self);
ot_lobj GFile *superindex_path = NULL; ot_lobj GFile *superindex_path = NULL;
ot_lptrarray GPtrArray *ret_indexes = NULL; ot_lptrarray GPtrArray *ret_meta_indexes = NULL;
ot_lptrarray GPtrArray *ret_data_indexes = NULL;
superindex_path = g_file_get_child (priv->pack_dir, "index"); superindex_path = g_file_get_child (priv->pack_dir, "index");
if (g_file_query_exists (superindex_path, cancellable)) if (g_file_query_exists (superindex_path, cancellable))
{ {
if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_indexes, cancellable, error)) if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_meta_indexes,
&ret_data_indexes,
cancellable, error))
goto out; goto out;
} }
else else
{ {
ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
} }
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_indexes, &ret_indexes); ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes);
ot_transfer_out_value (out_data_indexes, &ret_data_indexes);
out: out:
return ret; return ret;
} }
@ -1631,6 +1650,35 @@ create_index_bloom (OstreeRepo *self,
return ret; return ret;
} }
static gboolean
append_index_builder (OstreeRepo *self,
GPtrArray *indexes,
GVariantBuilder *builder,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
for (i = 0; i < indexes->len; i++)
{
const char *pack_checksum = indexes->pdata[i];
ot_lvariant GVariant *bloom = NULL;
if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error))
goto out;
g_variant_builder_add (builder,
"(@ay@ay)",
ostree_checksum_to_bytes_v (pack_checksum),
bloom);
}
ret = TRUE;
out:
return ret;
}
/** /**
* Regenerate the pack superindex file based on the set of pack * Regenerate the pack superindex file based on the set of pack
* indexes currently in the filesystem. * indexes currently in the filesystem.
@ -1641,69 +1689,80 @@ ostree_repo_regenerate_pack_index (OstreeRepo *self,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
guint i;
OstreeRepoPrivate *priv = GET_PRIVATE (self); OstreeRepoPrivate *priv = GET_PRIVATE (self);
ot_lobj GFile *index_path = NULL; ot_lobj GFile *superindex_path = NULL;
ot_lptrarray GPtrArray *pack_indexes = NULL; ot_lptrarray GPtrArray *pack_indexes = NULL;
ot_lvariant GVariant *index_variant = NULL; ot_lvariant GVariant *superindex_variant = NULL;
GVariantBuilder *index_content_builder = NULL; GVariantBuilder *meta_index_content_builder = NULL;
GVariantBuilder *data_index_content_builder = NULL;
if (!list_pack_indexes_from_dir (self, &pack_indexes, cancellable, error)) superindex_path = g_file_get_child (priv->pack_dir, "index");
ot_clear_ptrarray (&pack_indexes);
if (!list_pack_indexes_from_dir (self, TRUE, &pack_indexes,
cancellable, error))
goto out;
meta_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)"));
if (!append_index_builder (self, pack_indexes, meta_index_content_builder,
cancellable, error))
goto out; goto out;
index_path = g_file_get_child (priv->pack_dir, "index"); ot_clear_ptrarray (&pack_indexes);
if (!list_pack_indexes_from_dir (self, FALSE, &pack_indexes,
cancellable, error))
goto out;
data_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)"));
if (!append_index_builder (self, pack_indexes, data_index_content_builder,
cancellable, error))
goto out;
index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); superindex_variant = g_variant_new ("(s@a{sv}@a(ayay)@a(ayay))",
"OSTv0SUPERPACKINDEX",
for (i = 0; i < pack_indexes->len; i++) g_variant_new_array (G_VARIANT_TYPE ("{sv}"),
{ NULL, 0),
const char *pack_checksum = pack_indexes->pdata[i]; g_variant_builder_end (meta_index_content_builder),
GVariant *bloom; g_variant_builder_end (data_index_content_builder));
g_variant_ref_sink (superindex_variant);
if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error)) if (!ot_util_variant_save (superindex_path, superindex_variant,
goto out;
g_variant_builder_add (index_content_builder,
"(@ay@ay)",
ostree_checksum_to_bytes_v (pack_checksum),
bloom);
g_variant_unref (bloom);
}
index_variant = g_variant_new ("(s@a{sv}@a(ayay))",
"OSTv0SUPERPACKINDEX",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"),
NULL, 0),
g_variant_builder_end (index_content_builder));
g_variant_ref_sink (index_variant);
if (!ot_util_variant_save (index_path, index_variant,
cancellable, error)) cancellable, error))
goto out; goto out;
ret = TRUE; ret = TRUE;
out: out:
if (index_content_builder) if (meta_index_content_builder)
g_variant_builder_unref (index_content_builder); g_variant_builder_unref (meta_index_content_builder);
if (data_index_content_builder)
g_variant_builder_unref (data_index_content_builder);
return ret; return ret;
} }
static GFile * static GFile *
get_pack_index_name_from_checksum (GFile *parent, const char *pack_checksum) get_pack_index_path (GFile *parent,
gboolean is_meta,
const char *checksum)
{ {
return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".index", NULL); char *path = ostree_get_pack_index_name (is_meta, checksum);
GFile *ret = g_file_resolve_relative_path (parent, path);
g_free (path);
return ret;
} }
static GFile * static GFile *
get_pack_data_name_from_checksum (GFile *parent, const char *pack_checksum) get_pack_data_path (GFile *parent,
gboolean is_meta,
const char *checksum)
{ {
return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".data", NULL); char *path = ostree_get_pack_data_name (is_meta, checksum);
GFile *ret = g_file_resolve_relative_path (parent, path);
g_free (path);
return ret;
} }
gboolean gboolean
ostree_repo_add_pack_file (OstreeRepo *self, ostree_repo_add_pack_file (OstreeRepo *self,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GFile *index_path, GFile *index_path,
GFile *data_path, GFile *data_path,
GCancellable *cancellable, GCancellable *cancellable,
@ -1717,11 +1776,11 @@ ostree_repo_add_pack_file (OstreeRepo *self,
if (!ot_gfile_ensure_directory (priv->pack_dir, FALSE, error)) if (!ot_gfile_ensure_directory (priv->pack_dir, FALSE, error))
goto out; goto out;
pack_data_path = get_pack_data_name_from_checksum (priv->pack_dir, pack_checksum); pack_data_path = get_pack_data_path (priv->pack_dir, is_meta, pack_checksum);
if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error)) if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error))
goto out; goto out;
pack_index_path = get_pack_index_name_from_checksum (priv->pack_dir, pack_checksum); pack_index_path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum);
if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error)) if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error))
goto out; goto out;
@ -1752,6 +1811,80 @@ ensure_remote_cache_dir (OstreeRepo *self,
return ret; return ret;
} }
static gboolean
delete_no_longer_referenced (OstreeRepo *self,
GFile *cache_path,
const char *prefix,
const char *suffix,
GHashTable *new_files,
GPtrArray *inout_cached,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
ot_lptrarray GPtrArray *current_files = NULL;
ot_lfree char *pack_checksum = NULL;
if (!list_files_in_dir_matching (cache_path,
prefix, suffix,
&current_files,
cancellable, error))
goto out;
for (i = 0; i < current_files->len; i++)
{
GFile *file = current_files->pdata[i];
g_free (pack_checksum);
pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (file));
if (!g_hash_table_lookup (new_files, pack_checksum))
{
if (!ot_gfile_unlink (file, cancellable, error))
goto out;
}
if (inout_cached)
{
g_ptr_array_add (inout_cached, pack_checksum);
pack_checksum = NULL; /* transfer ownership */
}
}
ret = TRUE;
out:
return ret;
}
static void
gather_uncached (GHashTable *new_files,
GPtrArray *cached,
GPtrArray *inout_uncached)
{
guint i;
GHashTableIter hash_iter;
gpointer key, value;
g_hash_table_iter_init (&hash_iter, new_files);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *cur_pack_checksum = key;
gboolean found = FALSE;
for (i = 0; i < cached->len; i++)
{
const char *checksum = cached->pdata[i];
if (strcmp (cur_pack_checksum, checksum) == 0)
{
found = TRUE;
break;
}
}
if (!found)
g_ptr_array_add (inout_uncached, g_strdup (cur_pack_checksum));
}
}
/** /**
* Take a pack superindex file @superindex_path, and clean up any * Take a pack superindex file @superindex_path, and clean up any
* no-longer-referenced pack files in the lookaside cache for * no-longer-referenced pack files in the lookaside cache for
@ -1766,23 +1899,27 @@ gboolean
ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
const char *remote_name, const char *remote_name,
GFile *superindex_path, GFile *superindex_path,
GPtrArray **out_cached_indexes, GPtrArray **out_cached_meta_indexes,
GPtrArray **out_uncached_indexes, GPtrArray **out_cached_data_indexes,
GPtrArray **out_uncached_meta_indexes,
GPtrArray **out_uncached_data_indexes,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
GHashTableIter hash_iter;
gpointer key, value;
guint i;
ot_lvariant GVariant *superindex_variant = NULL; ot_lvariant GVariant *superindex_variant = NULL;
ot_lobj GFile *cache_path = NULL; ot_lobj GFile *cache_path = NULL;
ot_lobj GFile *superindex_cache_path = NULL; ot_lobj GFile *superindex_cache_path = NULL;
ot_lptrarray GPtrArray *index_files = NULL; ot_lptrarray GPtrArray *meta_index_files = NULL;
ot_lptrarray GPtrArray *data_files = NULL; ot_lptrarray GPtrArray *data_index_files = NULL;
ot_lhash GHashTable *new_pack_indexes = NULL; ot_lptrarray GPtrArray *meta_data_files = NULL;
ot_lptrarray GPtrArray *ret_cached_indexes = NULL; ot_lptrarray GPtrArray *data_data_files = NULL;
ot_lptrarray GPtrArray *ret_uncached_indexes = NULL; ot_lhash GHashTable *new_pack_meta_indexes = NULL;
ot_lhash GHashTable *new_pack_data_indexes = NULL;
ot_lptrarray GPtrArray *ret_cached_meta_indexes = NULL;
ot_lptrarray GPtrArray *ret_cached_data_indexes = NULL;
ot_lptrarray GPtrArray *ret_uncached_meta_indexes = NULL;
ot_lptrarray GPtrArray *ret_uncached_data_indexes = NULL;
ot_lvariant GVariant *csum_bytes = NULL; ot_lvariant GVariant *csum_bytes = NULL;
ot_lvariant GVariant *bloom = NULL; ot_lvariant GVariant *bloom = NULL;
ot_lfree char *pack_checksum = NULL; ot_lfree char *pack_checksum = NULL;
@ -1791,8 +1928,10 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error)) if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error))
goto out; goto out;
ret_cached_indexes = g_ptr_array_new_with_free_func (g_free); ret_cached_meta_indexes = g_ptr_array_new_with_free_func (g_free);
ret_uncached_indexes = g_ptr_array_new_with_free_func (g_free); ret_cached_data_indexes = g_ptr_array_new_with_free_func (g_free);
ret_uncached_meta_indexes = g_ptr_array_new_with_free_func (g_free);
ret_uncached_data_indexes = g_ptr_array_new_with_free_func (g_free);
if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
&superindex_variant, error)) &superindex_variant, error))
@ -1801,59 +1940,45 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
if (!ostree_validate_structureof_pack_superindex (superindex_variant, error)) if (!ostree_validate_structureof_pack_superindex (superindex_variant, error))
goto out; goto out;
new_pack_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); new_pack_meta_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
new_pack_data_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_variant_get_child (superindex_variant, 2, "a(ayay)", g_variant_get_child (superindex_variant, 2, "a(ayay)",
&superindex_contents_iter); &superindex_contents_iter);
while (g_variant_iter_loop (superindex_contents_iter, while (g_variant_iter_loop (superindex_contents_iter,
"(@ay@ay)", &csum_bytes, &bloom)) "(@ay@ay)", &csum_bytes, &bloom))
{ {
pack_checksum = ostree_checksum_from_bytes_v (csum_bytes); pack_checksum = ostree_checksum_from_bytes_v (csum_bytes);
g_hash_table_insert (new_pack_indexes, pack_checksum, pack_checksum); g_hash_table_insert (new_pack_meta_indexes, pack_checksum, pack_checksum);
pack_checksum = NULL; /* transfer ownership */ pack_checksum = NULL; /* transfer ownership */
} }
if (!list_files_in_dir_matching (cache_path, g_variant_get_child (superindex_variant, 3, "a(ayay)",
"ostpack-", ".index", &superindex_contents_iter);
&index_files, while (g_variant_iter_loop (superindex_contents_iter,
cancellable, error)) "(@ay@ay)", &csum_bytes, &bloom))
{
pack_checksum = ostree_checksum_from_bytes_v (csum_bytes);
g_hash_table_insert (new_pack_data_indexes, pack_checksum, pack_checksum);
pack_checksum = NULL; /* transfer ownership */
}
if (!delete_no_longer_referenced (self, cache_path,
"ostmetapack-", ".index",
new_pack_meta_indexes,
ret_cached_meta_indexes,
cancellable, error))
goto out; goto out;
for (i = 0; i < index_files->len; i++)
{
GFile *index_file = index_files->pdata[i];
g_free (pack_checksum);
pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (index_file));
if (!g_hash_table_lookup (new_pack_indexes, pack_checksum))
{
if (!ot_gfile_unlink (index_file, cancellable, error))
goto out;
}
g_ptr_array_add (ret_cached_indexes, pack_checksum);
pack_checksum = NULL; /* transfer ownership */
}
g_hash_table_iter_init (&hash_iter, new_pack_indexes); if (!delete_no_longer_referenced (self, cache_path,
while (g_hash_table_iter_next (&hash_iter, &key, &value)) "ostdatapack-", ".index",
{ new_pack_data_indexes,
const char *cur_pack_checksum = key; ret_cached_data_indexes,
gboolean found = FALSE; cancellable, error))
goto out;
for (i = 0; i < ret_cached_indexes->len; i++) gather_uncached (new_pack_meta_indexes, ret_cached_meta_indexes, ret_uncached_meta_indexes);
{ gather_uncached (new_pack_data_indexes, ret_cached_data_indexes, ret_uncached_data_indexes);
if (strcmp (cur_pack_checksum, ret_cached_indexes->pdata[i]) == 0)
{
found = TRUE;
break;
}
}
if (!found)
g_ptr_array_add (ret_uncached_indexes, g_strdup (cur_pack_checksum));
}
superindex_cache_path = g_file_get_child (cache_path, "index"); superindex_cache_path = g_file_get_child (cache_path, "index");
if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error)) if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error))
@ -1861,29 +1986,22 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
/* Now also delete stale pack files */ /* Now also delete stale pack files */
if (!list_files_in_dir_matching (cache_path, if (!delete_no_longer_referenced (self, cache_path,
"ostpack-", ".data", "ostmetapack-", ".data",
&data_files, new_pack_meta_indexes, NULL,
cancellable, error)) cancellable, error))
goto out;
if (!delete_no_longer_referenced (self, cache_path,
"ostdatapack-", ".data",
new_pack_data_indexes, NULL,
cancellable, error))
goto out; goto out;
for (i = 0; i < data_files->len; i++)
{
GFile *data_file = data_files->pdata[i];
g_free (pack_checksum);
pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (data_file));
if (!g_hash_table_lookup (new_pack_indexes, pack_checksum))
{
if (!ot_gfile_unlink (data_file, cancellable, error))
goto out;
}
}
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_cached_indexes, &ret_cached_indexes); ot_transfer_out_value (out_cached_meta_indexes, &ret_cached_meta_indexes);
ot_transfer_out_value (out_uncached_indexes, &ret_uncached_indexes); ot_transfer_out_value (out_cached_data_indexes, &ret_cached_data_indexes);
ot_transfer_out_value (out_uncached_meta_indexes, &ret_uncached_data_indexes);
ot_transfer_out_value (out_uncached_data_indexes, &ret_uncached_data_indexes);
out: out:
if (superindex_contents_iter) if (superindex_contents_iter)
g_variant_iter_free (superindex_contents_iter); g_variant_iter_free (superindex_contents_iter);
@ -1905,11 +2023,24 @@ ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self,
goto out; goto out;
if (!list_files_in_dir_matching (cache_path, if (!list_files_in_dir_matching (cache_path,
"ostpack-", ".data", "ostmetapack-", ".data",
&data_files, &data_files,
cancellable, error)) cancellable, error))
goto out; goto out;
for (i = 0; i < data_files->len; i++)
{
GFile *data_file = data_files->pdata[i];
if (!ot_gfile_unlink (data_file, cancellable, error))
goto out;
}
ot_clear_ptrarray (&data_files);
if (!list_files_in_dir_matching (cache_path,
"ostdatapack-", ".data",
&data_files,
cancellable, error))
goto out;
for (i = 0; i < data_files->len; i++) for (i = 0; i < data_files->len; i++)
{ {
GFile *data_file = data_files->pdata[i]; GFile *data_file = data_files->pdata[i];
@ -1931,6 +2062,7 @@ gboolean
ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name, const char *remote_name,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant, GVariant **out_variant,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -1944,7 +2076,7 @@ ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
cancellable, error)) cancellable, error))
goto out; goto out;
cached_pack_path = get_pack_index_name_from_checksum (cache_dir, pack_checksum); cached_pack_path = get_pack_index_path (cache_dir, is_meta, pack_checksum);
if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT, if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT,
&ret_variant, error)) &ret_variant, error))
goto out; goto out;
@ -1964,6 +2096,7 @@ gboolean
ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name, const char *remote_name,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GFile *cached_path, GFile *cached_path,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -1989,7 +2122,7 @@ ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
goto out; goto out;
target_path = get_pack_index_name_from_checksum (cachedir, pack_checksum); target_path = get_pack_index_path (cachedir, is_meta, pack_checksum);
if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error)) if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error))
goto out; goto out;
@ -2007,6 +2140,7 @@ gboolean
ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name, const char *remote_name,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path, GFile **out_cached_path,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -2020,7 +2154,7 @@ ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
cancellable, error)) cancellable, error))
goto out; goto out;
cached_pack_path = get_pack_data_name_from_checksum (cache_dir, pack_checksum); cached_pack_path = get_pack_data_path (cache_dir, is_meta, pack_checksum);
if (g_file_query_exists (cached_pack_path, cancellable)) if (g_file_query_exists (cached_pack_path, cancellable))
{ {
ret_cached_path = cached_pack_path; ret_cached_path = cached_pack_path;
@ -2044,6 +2178,7 @@ gboolean
ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name, const char *remote_name,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GFile *cached_path, GFile *cached_path,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -2055,8 +2190,7 @@ ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
goto out; goto out;
target_path = get_pack_data_name_from_checksum (cachedir, pack_checksum); target_path = get_pack_data_path (cachedir, is_meta, pack_checksum);
if (!ot_gfile_rename (cached_path, target_path, cancellable, error)) if (!ot_gfile_rename (cached_path, target_path, cancellable, error))
goto out; goto out;
@ -2908,37 +3042,10 @@ list_loose_objects (OstreeRepo *self,
return ret; return ret;
} }
GFile *
ostree_repo_get_pack_index_path (OstreeRepo *self,
const char *checksum)
{
char *name;
GFile *ret;
name = g_strconcat ("ostpack-", checksum, ".index", NULL);
ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name);
g_free (name);
return ret;
}
GFile *
ostree_repo_get_pack_data_path (OstreeRepo *self,
const char *checksum)
{
char *name;
GFile *ret;
name = g_strconcat ("ostpack-", checksum, ".data", NULL);
ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name);
g_free (name);
return ret;
}
gboolean gboolean
ostree_repo_load_pack_index (OstreeRepo *self, ostree_repo_load_pack_index (OstreeRepo *self,
const char *sha256, const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant, GVariant **out_variant,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -2948,21 +3055,21 @@ ostree_repo_load_pack_index (OstreeRepo *self,
ot_lvariant GVariant *ret_variant = NULL; ot_lvariant GVariant *ret_variant = NULL;
ot_lobj GFile *path = NULL; ot_lobj GFile *path = NULL;
ret_variant = g_hash_table_lookup (priv->pack_index_mappings, sha256); ret_variant = g_hash_table_lookup (priv->pack_index_mappings, pack_checksum);
if (ret_variant) if (ret_variant)
{ {
g_variant_ref (ret_variant); g_variant_ref (ret_variant);
} }
else else
{ {
path = ostree_repo_get_pack_index_path (self, sha256); path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum);
if (!map_variant_file_check_header_string (path, if (!map_variant_file_check_header_string (path,
OSTREE_PACK_INDEX_VARIANT_FORMAT, OSTREE_PACK_INDEX_VARIANT_FORMAT,
"OSTv0PACKINDEX", "OSTv0PACKINDEX",
&ret_variant, &ret_variant,
cancellable, error)) cancellable, error))
goto out; goto out;
g_hash_table_insert (priv->pack_index_mappings, g_strdup (sha256), g_hash_table_insert (priv->pack_index_mappings, g_strdup (pack_checksum),
g_variant_ref (ret_variant)); g_variant_ref (ret_variant));
} }
@ -2973,19 +3080,16 @@ ostree_repo_load_pack_index (OstreeRepo *self,
} }
/** /**
* ostree_repo_map_pack_file:
* @self:
* @sha256: Checksum of pack file * @sha256: Checksum of pack file
* @out_data: (out): Pointer to pack file data * @out_data: (out): Pointer to pack file data
* @cancellable:
* @error:
* *
* Ensure that the given pack file is mapped into * Ensure that the given pack file is mapped into
* memory. * memory.
*/ */
gboolean gboolean
ostree_repo_map_pack_file (OstreeRepo *self, ostree_repo_map_pack_file (OstreeRepo *self,
const char *sha256, const char *pack_checksum,
gboolean is_meta,
guchar **out_data, guchar **out_data,
guint64 *out_len, guint64 *out_len,
GCancellable *cancellable, GCancellable *cancellable,
@ -2998,16 +3102,16 @@ ostree_repo_map_pack_file (OstreeRepo *self,
GMappedFile *map = NULL; GMappedFile *map = NULL;
ot_lobj GFile *path = NULL; ot_lobj GFile *path = NULL;
map = g_hash_table_lookup (priv->pack_data_mappings, sha256); map = g_hash_table_lookup (priv->pack_data_mappings, pack_checksum);
if (map == NULL) if (map == NULL)
{ {
path = ostree_repo_get_pack_data_path (self, sha256); path = get_pack_data_path (priv->pack_dir, is_meta, pack_checksum);
map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error); map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error);
if (!map) if (!map)
goto out; goto out;
g_hash_table_insert (priv->pack_data_mappings, g_strdup (sha256), map); g_hash_table_insert (priv->pack_data_mappings, g_strdup (pack_checksum), map);
ret_data = g_mapped_file_get_contents (map); ret_data = g_mapped_file_get_contents (map);
} }
@ -3058,53 +3162,55 @@ ostree_repo_load_file (OstreeRepo *self,
error)) error))
goto out; goto out;
/* Blah, right now we need to look up the content too to get the file size */ if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR)
if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
checksum, &content_loose_path,
&content_pack_checksum, &content_pack_offset,
cancellable, error))
goto out;
if (content_loose_path)
{ {
content_loose_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO, /* Blah, right now we need to look up the content too to get the file size */
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
if (!content_loose_info) checksum, &content_loose_path,
&content_pack_checksum, &content_pack_offset,
cancellable, error))
goto out; goto out;
g_file_info_set_attribute_uint64 (ret_file_info, if (content_loose_path)
"standard::size", {
g_file_info_get_attribute_uint64 (content_loose_info, "standard::size")); content_loose_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO,
} G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
/* fixme - don't have file size for packed =/ */ if (!content_loose_info)
goto out;
g_file_info_set_attribute_uint64 (ret_file_info,
"standard::size",
g_file_info_get_attribute_uint64 (content_loose_info, "standard::size"));
}
/* fixme - don't have file size for packed =/ */
/* Now, look for the content */ /* Now, look for the content */
if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR if (out_input)
&& out_input)
{
if (content_pack_checksum != NULL)
{ {
if (!ostree_repo_map_pack_file (self, content_pack_checksum, if (content_pack_checksum != NULL)
&content_pack_data, &content_pack_len, {
cancellable, error)) if (!ostree_repo_map_pack_file (self, content_pack_checksum, FALSE,
goto out; &content_pack_data, &content_pack_len,
if (!ostree_read_pack_entry_raw (content_pack_data, content_pack_len, cancellable, error))
content_pack_offset, TRUE, goto out;
&packed_object, cancellable, error)) if (!ostree_read_pack_entry_raw (content_pack_data, content_pack_len,
goto out; content_pack_offset, TRUE, FALSE,
ret_input = ostree_read_pack_entry_as_stream (packed_object); &packed_object, cancellable, error))
} goto out;
else if (content_loose_path != NULL) ret_input = ostree_read_pack_entry_as_stream (packed_object);
{ }
ret_input = (GInputStream*)g_file_read (content_loose_path, cancellable, error); else if (content_loose_path != NULL)
if (!ret_input) {
goto out; ret_input = (GInputStream*)g_file_read (content_loose_path, cancellable, error);
} if (!ret_input)
else goto out;
{ }
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, else
"Couldn't find object '%s'", checksum); {
goto out; g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"Couldn't find object '%s'", checksum);
goto out;
}
} }
} }
} }
@ -3142,12 +3248,14 @@ ostree_repo_load_file (OstreeRepo *self,
static gboolean static gboolean
list_objects_in_index (OstreeRepo *self, list_objects_in_index (OstreeRepo *self,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GHashTable *inout_objects, GHashTable *inout_objects,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
guint32 objtype_u32; OstreeRepoPrivate *priv = GET_PRIVATE (self);
guint32 objtype_u8;
guint64 offset; guint64 offset;
ot_lobj GFile *index_path = NULL; ot_lobj GFile *index_path = NULL;
ot_lvariant GVariant *index_variant = NULL; ot_lvariant GVariant *index_variant = NULL;
@ -3156,15 +3264,16 @@ list_objects_in_index (OstreeRepo *self,
ot_lfree char *checksum = NULL; ot_lfree char *checksum = NULL;
GVariantIter content_iter; GVariantIter content_iter;
index_path = ostree_repo_get_pack_index_path (self, pack_checksum); index_path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum);
if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error)) if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta,
&index_variant, cancellable, error))
goto out; goto out;
contents = g_variant_get_child_value (index_variant, 2); contents = g_variant_get_child_value (index_variant, 2);
g_variant_iter_init (&content_iter, contents); g_variant_iter_init (&content_iter, contents);
while (g_variant_iter_loop (&content_iter, "(u@ayt)", &objtype_u32, &csum_bytes, &offset)) while (g_variant_iter_loop (&content_iter, "(y@ayt)", &objtype_u8, &csum_bytes, &offset))
{ {
GVariant *obj_key; GVariant *obj_key;
GVariant *objdata; GVariant *objdata;
@ -3172,7 +3281,7 @@ list_objects_in_index (OstreeRepo *self,
GVariantBuilder pack_contents_builder; GVariantBuilder pack_contents_builder;
gboolean is_loose; gboolean is_loose;
objtype = (OstreeObjectType) GUINT32_FROM_BE (objtype_u32); objtype = (OstreeObjectType) objtype_u8;
offset = GUINT64_FROM_BE (offset); offset = GUINT64_FROM_BE (offset);
g_variant_builder_init (&pack_contents_builder, g_variant_builder_init (&pack_contents_builder,
@ -3221,19 +3330,27 @@ list_packed_objects (OstreeRepo *self,
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
guint i; guint i;
ot_lptrarray GPtrArray *index_checksums = NULL; ot_lptrarray GPtrArray *meta_index_checksums = NULL;
ot_lptrarray GPtrArray *data_index_checksums = NULL;
if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error)) if (!ostree_repo_list_pack_indexes (self, &meta_index_checksums, &data_index_checksums,
cancellable, error))
goto out; goto out;
for (i = 0; i < index_checksums->len; i++) for (i = 0; i < meta_index_checksums->len; i++)
{ {
const char *checksum = index_checksums->pdata[i]; const char *checksum = meta_index_checksums->pdata[i];
if (!list_objects_in_index (self, checksum, TRUE, inout_objects, cancellable, error))
if (!list_objects_in_index (self, checksum, inout_objects, cancellable, error))
goto out; goto out;
} }
for (i = 0; i < data_index_checksums->len; i++)
{
const char *checksum = data_index_checksums->pdata[i];
if (!list_objects_in_index (self, checksum, FALSE, inout_objects, cancellable, error))
goto out;
}
ret = TRUE; ret = TRUE;
out: out:
return ret; return ret;
@ -3251,27 +3368,37 @@ find_object_in_packs (OstreeRepo *self,
gboolean ret = FALSE; gboolean ret = FALSE;
guint i; guint i;
guint64 ret_pack_offset; guint64 ret_pack_offset;
gboolean is_meta;
ot_lptrarray GPtrArray *index_checksums = NULL; ot_lptrarray GPtrArray *index_checksums = NULL;
ot_lfree char *ret_pack_checksum = NULL; ot_lfree char *ret_pack_checksum = NULL;
ot_lobj GFile *index_path = NULL;
ot_lvariant GVariant *csum_bytes = NULL; ot_lvariant GVariant *csum_bytes = NULL;
ot_lvariant GVariant *index_variant = NULL; ot_lvariant GVariant *index_variant = NULL;
csum_bytes = ostree_checksum_to_bytes_v (checksum); csum_bytes = ostree_checksum_to_bytes_v (checksum);
if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error)) is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
goto out;
if (is_meta)
{
if (!ostree_repo_list_pack_indexes (self, &index_checksums, NULL,
cancellable, error))
goto out;
}
else
{
if (!ostree_repo_list_pack_indexes (self, NULL, &index_checksums,
cancellable, error))
goto out;
}
for (i = 0; i < index_checksums->len; i++) for (i = 0; i < index_checksums->len; i++)
{ {
const char *pack_checksum = index_checksums->pdata[i]; const char *pack_checksum = index_checksums->pdata[i];
guint64 offset; guint64 offset;
g_clear_object (&index_path);
index_path = ostree_repo_get_pack_index_path (self, pack_checksum);
ot_clear_gvariant (&index_variant); ot_clear_gvariant (&index_variant);
if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error)) if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, &index_variant,
cancellable, error))
goto out; goto out;
if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset)) if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset))
@ -3388,12 +3515,12 @@ ostree_repo_load_variant (OstreeRepo *self,
} }
else if (pack_checksum != NULL) else if (pack_checksum != NULL)
{ {
if (!ostree_repo_map_pack_file (self, pack_checksum, &pack_data, &pack_len, if (!ostree_repo_map_pack_file (self, pack_checksum, TRUE, &pack_data, &pack_len,
cancellable, error)) cancellable, error))
goto out; goto out;
if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset, if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset,
TRUE, &packed_object, cancellable, error)) TRUE, TRUE, &packed_object, cancellable, error))
goto out; goto out;
if (!ostree_read_pack_entry_variant (packed_object, objtype, TRUE, if (!ostree_read_pack_entry_variant (packed_object, objtype, TRUE,

26
src/libostree/ostree-repo.h
View File

@ -150,19 +150,21 @@ gboolean ostree_repo_load_variant (OstreeRepo *self,
GError **error); GError **error);
gboolean ostree_repo_load_pack_index (OstreeRepo *self, gboolean ostree_repo_load_pack_index (OstreeRepo *self,
const char *sha256, const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant, GVariant **out_variant,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
gboolean ostree_repo_load_pack_data (OstreeRepo *self, gboolean ostree_repo_load_pack_data (OstreeRepo *self,
const char *sha256, const char *pack_checksum,
guchar **out_data, guchar **out_data,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
gboolean ostree_repo_map_pack_file (OstreeRepo *self, gboolean ostree_repo_map_pack_file (OstreeRepo *self,
const char *sha256, const char *sha256,
gboolean is_meta,
guchar **out_data, guchar **out_data,
guint64 *out_len, guint64 *out_len,
GCancellable *cancellable, GCancellable *cancellable,
@ -241,6 +243,7 @@ gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self,
gboolean ostree_repo_add_pack_file (OstreeRepo *self, gboolean ostree_repo_add_pack_file (OstreeRepo *self,
const char *checksum, const char *checksum,
gboolean is_meta,
GFile *pack_index_path, GFile *pack_index_path,
GFile *pack_data_path, GFile *pack_data_path,
GCancellable *cancellable, GCancellable *cancellable,
@ -249,8 +252,10 @@ gboolean ostree_repo_add_pack_file (OstreeRepo *self,
gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
const char *remote_name, const char *remote_name,
GFile *superindex_path, GFile *superindex_path,
GPtrArray **out_cached_indexes, GPtrArray **out_cached_meta_indexes,
GPtrArray **out_uncached_indexes, GPtrArray **out_cached_data_indexes,
GPtrArray **out_uncached_meta_indexes,
GPtrArray **out_uncached_data_indexes,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
@ -262,6 +267,7 @@ gboolean ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self,
gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name, const char *remote_name,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant, GVariant **out_variant,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
@ -269,6 +275,7 @@ gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name, const char *remote_name,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GFile *cached_path, GFile *cached_path,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
@ -276,6 +283,7 @@ gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name, const char *remote_name,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path, GFile **out_cached_path,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
@ -283,6 +291,7 @@ gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name, const char *remote_name,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GFile *cached_path, GFile *cached_path,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
@ -334,16 +343,11 @@ gboolean ostree_repo_list_objects (OstreeRepo *self,
GError **error); GError **error);
gboolean ostree_repo_list_pack_indexes (OstreeRepo *self, gboolean ostree_repo_list_pack_indexes (OstreeRepo *self,
GPtrArray **out_indexes, GPtrArray **out_meta_indexes,
GPtrArray **out_data_indexes,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
GFile * ostree_repo_get_pack_index_path (OstreeRepo *self,
const char *checksum);
GFile * ostree_repo_get_pack_data_path (OstreeRepo *self,
const char *checksum);
G_END_DECLS G_END_DECLS
#endif /* _OSTREE_REPO */ #endif /* _OSTREE_REPO */

90
src/ostree/ostree-pull.c
View File

@ -62,7 +62,8 @@ typedef struct {
SoupURI *base_uri; SoupURI *base_uri;
gboolean fetched_packs; gboolean fetched_packs;
GPtrArray *cached_pack_indexes; GPtrArray *cached_meta_pack_indexes;
GPtrArray *cached_data_pack_indexes;
GHashTable *file_checksums_to_fetch; GHashTable *file_checksums_to_fetch;
@ -266,6 +267,7 @@ fetch_uri_contents_utf8 (OtPullData *pull_data,
static gboolean static gboolean
fetch_one_pack_file (OtPullData *pull_data, fetch_one_pack_file (OtPullData *pull_data,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path, GFile **out_cached_path,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -277,26 +279,26 @@ fetch_one_pack_file (OtPullData *pull_data,
SoupURI *pack_uri = NULL; SoupURI *pack_uri = NULL;
if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, &ret_cached_path, pack_checksum, is_meta, &ret_cached_path,
cancellable, error)) cancellable, error))
goto out; goto out;
if (ret_cached_path == NULL) if (ret_cached_path == NULL)
{ {
pack_name = g_strconcat ("ostpack-", pack_checksum, ".data", NULL); pack_name = ostree_get_pack_data_name (is_meta, pack_checksum);
pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL); pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL);
if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error)) if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error))
goto out; goto out;
if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, tmp_path, pack_checksum, is_meta, tmp_path,
cancellable, error)) cancellable, error))
goto out; goto out;
} }
if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, &ret_cached_path, pack_checksum, is_meta, &ret_cached_path,
cancellable, error)) cancellable, error))
goto out; goto out;
@ -322,19 +324,25 @@ find_object_in_remote_packs (OtPullData *pull_data,
gboolean ret = FALSE; gboolean ret = FALSE;
guint64 offset; guint64 offset;
guint i; guint i;
GPtrArray *iter;
ot_lvariant GVariant *mapped_pack = NULL; ot_lvariant GVariant *mapped_pack = NULL;
ot_lvariant GVariant *csum_bytes = NULL; ot_lvariant GVariant *csum_bytes = NULL;
ot_lfree char *ret_pack_checksum = NULL; ot_lfree char *ret_pack_checksum = NULL;
csum_bytes = ostree_checksum_to_bytes_v (checksum); csum_bytes = ostree_checksum_to_bytes_v (checksum);
for (i = 0; i < pull_data->cached_pack_indexes->len; i++) if (OSTREE_OBJECT_TYPE_IS_META (objtype))
iter = pull_data->cached_meta_pack_indexes;
else
iter = pull_data->cached_data_pack_indexes;
for (i = 0; i < iter->len; i++)
{ {
const char *pack_checksum = pull_data->cached_pack_indexes->pdata[i]; const char *pack_checksum = iter->pdata[i];
ot_clear_gvariant (&mapped_pack); ot_clear_gvariant (&mapped_pack);
if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
pack_checksum, &mapped_pack, pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype),
&mapped_pack,
cancellable, error)) cancellable, error))
goto out; goto out;
@ -354,17 +362,18 @@ find_object_in_remote_packs (OtPullData *pull_data,
} }
static gboolean static gboolean
fetch_one_cache_index (OtPullData *pull_data, fetch_one_cache_index (OtPullData *pull_data,
const char *pack_checksum, const char *pack_checksum,
GCancellable *cancellable, gboolean is_meta,
GError **error) GCancellable *cancellable,
GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
ot_lobj GFile *tmp_path = NULL; ot_lobj GFile *tmp_path = NULL;
ot_lfree char *pack_index_name = NULL; ot_lfree char *pack_index_name = NULL;
SoupURI *index_uri = NULL; SoupURI *index_uri = NULL;
pack_index_name = g_strconcat ("ostpack-", pack_checksum, ".index", NULL); pack_index_name = ostree_get_pack_index_name (is_meta, pack_checksum);
index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL); index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL);
if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path, if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path,
@ -372,7 +381,7 @@ fetch_one_cache_index (OtPullData *pull_data,
goto out; goto out;
if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
pack_checksum, tmp_path, pack_checksum, is_meta, tmp_path,
cancellable, error)) cancellable, error))
goto out; goto out;
@ -398,8 +407,10 @@ fetch_and_cache_pack_indexes (OtPullData *pull_data,
gboolean ret = FALSE; gboolean ret = FALSE;
guint i; guint i;
ot_lobj GFile *superindex_tmppath = NULL; ot_lobj GFile *superindex_tmppath = NULL;
ot_lptrarray GPtrArray *cached_indexes = NULL; ot_lptrarray GPtrArray *cached_meta_indexes = NULL;
ot_lptrarray GPtrArray *uncached_indexes = NULL; ot_lptrarray GPtrArray *cached_data_indexes = NULL;
ot_lptrarray GPtrArray *uncached_meta_indexes = NULL;
ot_lptrarray GPtrArray *uncached_data_indexes = NULL;
ot_lvariant GVariant *superindex_variant = NULL; ot_lvariant GVariant *superindex_variant = NULL;
GVariantIter *contents_iter = NULL; GVariantIter *contents_iter = NULL;
SoupURI *superindex_uri = NULL; SoupURI *superindex_uri = NULL;
@ -412,22 +423,33 @@ fetch_and_cache_pack_indexes (OtPullData *pull_data,
if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name, if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name,
superindex_tmppath, superindex_tmppath,
&cached_indexes, &uncached_indexes, &cached_meta_indexes,
&cached_data_indexes,
&uncached_meta_indexes,
&uncached_data_indexes,
cancellable, error)) cancellable, error))
goto out; goto out;
for (i = 0; i < cached_indexes->len; i++) for (i = 0; i < cached_meta_indexes->len; i++)
g_ptr_array_add (pull_data->cached_pack_indexes, g_ptr_array_add (pull_data->cached_meta_pack_indexes,
g_strdup (cached_indexes->pdata[i])); g_strdup (cached_meta_indexes->pdata[i]));
for (i = 0; i < cached_data_indexes->len; i++)
g_ptr_array_add (pull_data->cached_data_pack_indexes,
g_strdup (cached_data_indexes->pdata[i]));
for (i = 0; i < uncached_indexes->len; i++) for (i = 0; i < uncached_meta_indexes->len; i++)
{ {
const char *pack_checksum = uncached_indexes->pdata[i]; const char *pack_checksum = uncached_meta_indexes->pdata[i];
if (!fetch_one_cache_index (pull_data, pack_checksum, TRUE, cancellable, error))
if (!fetch_one_cache_index (pull_data, pack_checksum, cancellable, error))
goto out; goto out;
g_ptr_array_add (pull_data->cached_meta_pack_indexes, g_strdup (pack_checksum));
g_ptr_array_add (pull_data->cached_pack_indexes, g_strdup (pack_checksum)); }
for (i = 0; i < uncached_data_indexes->len; i++)
{
const char *pack_checksum = uncached_data_indexes->pdata[i];
if (!fetch_one_cache_index (pull_data, pack_checksum, FALSE, cancellable, error))
goto out;
g_ptr_array_add (pull_data->cached_data_pack_indexes, g_strdup (pack_checksum));
} }
ret = TRUE; ret = TRUE;
@ -485,6 +507,7 @@ fetch_object_if_not_stored (OtPullData *pull_data,
gboolean ret = FALSE; gboolean ret = FALSE;
guint64 pack_offset = 0; guint64 pack_offset = 0;
gboolean is_stored; gboolean is_stored;
gboolean is_meta;
ot_lobj GInputStream *ret_input = NULL; ot_lobj GInputStream *ret_input = NULL;
ot_lobj GFile *temp_path = NULL; ot_lobj GFile *temp_path = NULL;
ot_lobj GFile *stored_path = NULL; ot_lobj GFile *stored_path = NULL;
@ -494,6 +517,8 @@ fetch_object_if_not_stored (OtPullData *pull_data,
ot_lvariant GVariant *pack_entry = NULL; ot_lvariant GVariant *pack_entry = NULL;
GMappedFile *pack_map = NULL; GMappedFile *pack_map = NULL;
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
if (!ostree_repo_find_object (pull_data->repo, objtype, checksum, if (!ostree_repo_find_object (pull_data->repo, objtype, checksum,
&stored_path, &local_pack_checksum, NULL, &stored_path, &local_pack_checksum, NULL,
cancellable, error)) cancellable, error))
@ -505,7 +530,8 @@ fetch_object_if_not_stored (OtPullData *pull_data,
if (!pull_data->fetched_packs) if (!pull_data->fetched_packs)
{ {
pull_data->fetched_packs = TRUE; pull_data->fetched_packs = TRUE;
pull_data->cached_pack_indexes = g_ptr_array_new_with_free_func (g_free); pull_data->cached_meta_pack_indexes = g_ptr_array_new_with_free_func (g_free);
pull_data->cached_data_pack_indexes = g_ptr_array_new_with_free_func (g_free);
if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error)) if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error))
goto out; goto out;
@ -521,8 +547,8 @@ fetch_object_if_not_stored (OtPullData *pull_data,
{ {
g_assert (!is_stored); g_assert (!is_stored);
if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path, if (!fetch_one_pack_file (pull_data, remote_pack_checksum, is_meta,
cancellable, error)) &pack_path, cancellable, error))
goto out; goto out;
pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_path), FALSE, error); pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_path), FALSE, error);
@ -531,7 +557,7 @@ fetch_object_if_not_stored (OtPullData *pull_data,
if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
g_mapped_file_get_length (pack_map), g_mapped_file_get_length (pack_map),
pack_offset, FALSE, &pack_entry, pack_offset, FALSE, is_meta, &pack_entry,
cancellable, error)) cancellable, error))
goto out; goto out;
@ -1147,8 +1173,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
g_clear_object (&pull_data->session); g_clear_object (&pull_data->session);
if (pull_data->base_uri) if (pull_data->base_uri)
soup_uri_free (pull_data->base_uri); soup_uri_free (pull_data->base_uri);
if (pull_data->cached_pack_indexes) ot_clear_ptrarray (&pull_data->cached_meta_pack_indexes);
g_ptr_array_unref (pull_data->cached_pack_indexes); ot_clear_ptrarray (&pull_data->cached_data_pack_indexes);
if (summary_uri) if (summary_uri)
soup_uri_free (summary_uri); soup_uri_free (summary_uri);
return ret; return ret;

170
src/ostree/ot-builtin-fsck.c
View File

@ -43,88 +43,76 @@ typedef struct {
} OtFsckData; } OtFsckData;
static gboolean static gboolean
fsck_pack_files (OtFsckData *data, fsck_one_pack_file (OtFsckData *data,
GCancellable *cancellable, const char *pack_checksum,
GError **error) gboolean is_meta,
GCancellable *cancellable,
GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
guint i; guchar objtype_u8;
guint32 objtype;
guint64 offset; guint64 offset;
guint64 pack_size; guint64 pack_size;
ot_lptrarray GPtrArray *pack_indexes = NULL; ot_lfree char *path = NULL;
ot_lobj GFileInfo *pack_info = NULL;
ot_lobj GInputStream *input = NULL;
ot_lvariant GVariant *index_variant = NULL; ot_lvariant GVariant *index_variant = NULL;
ot_lobj GFile *pack_index_path = NULL; ot_lobj GFile *pack_index_path = NULL;
ot_lobj GFile *pack_data_path = NULL; ot_lobj GFile *pack_data_path = NULL;
ot_lobj GFileInfo *pack_info = NULL;
ot_lobj GInputStream *input = NULL;
GChecksum *pack_content_checksum = NULL; GChecksum *pack_content_checksum = NULL;
GVariantIter *index_content_iter = NULL; GVariantIter *index_content_iter = NULL;
if (!ostree_repo_list_pack_indexes (data->repo, &pack_indexes, cancellable, error)) g_free (path);
path = ostree_get_relative_pack_index_path (is_meta, pack_checksum);
pack_index_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path);
if (!ot_util_variant_map (pack_index_path,
OSTREE_PACK_INDEX_VARIANT_FORMAT,
&index_variant, error))
goto out;
if (!ostree_validate_structureof_pack_index (index_variant, error))
goto out; goto out;
for (i = 0; i < pack_indexes->len; i++) g_free (path);
{ path = ostree_get_relative_pack_data_path (is_meta, pack_checksum);
const char *checksum = pack_indexes->pdata[i]; pack_data_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path);
g_clear_object (&pack_index_path);
pack_index_path = ostree_repo_get_pack_index_path (data->repo, checksum);
ot_clear_gvariant (&index_variant);
if (!ot_util_variant_map (pack_index_path,
OSTREE_PACK_INDEX_VARIANT_FORMAT,
&index_variant, error))
goto out;
if (!ostree_validate_structureof_pack_index (index_variant, error)) input = (GInputStream*)g_file_read (pack_data_path, cancellable, error);
goto out; if (!input)
goto out;
g_clear_object (&pack_data_path); pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO,
pack_data_path = ostree_repo_get_pack_data_path (data->repo, checksum); cancellable, error);
if (!pack_info)
g_clear_object (&input); goto out;
input = (GInputStream*)g_file_read (pack_data_path, cancellable, error); pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size");
if (!input)
goto out;
g_clear_object (&pack_info);
pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO,
cancellable, error);
if (!pack_info)
goto out;
pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size");
if (pack_content_checksum) if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error))
g_checksum_free (pack_content_checksum); goto out;
if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error))
goto out;
if (strcmp (g_checksum_get_string (pack_content_checksum), checksum) != 0) if (strcmp (g_checksum_get_string (pack_content_checksum), pack_checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', expected checksum %s",
pack_checksum, g_checksum_get_string (pack_content_checksum));
goto out;
}
g_variant_get_child (index_variant, 2, "a(yayt)", &index_content_iter);
while (g_variant_iter_loop (index_content_iter, "(y@ayt)",
&objtype_u8, NULL, &offset))
{
offset = GUINT64_FROM_BE (offset);
if (offset > pack_size)
{ {
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', expected checksum %s", "corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT,
checksum, g_checksum_get_string (pack_content_checksum)); pack_checksum,
offset, pack_size);
goto out; goto out;
} }
g_variant_get_child (index_variant, 2, "a(uayt)", &index_content_iter);
while (g_variant_iter_loop (index_content_iter, "(u@ayt)",
&objtype, NULL, &offset))
{
offset = GUINT64_FROM_BE (offset);
if (offset > pack_size)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT,
checksum,
offset, pack_size);
goto out;
}
}
data->n_pack_files++;
} }
ret = TRUE; ret = TRUE;
@ -135,6 +123,44 @@ fsck_pack_files (OtFsckData *data,
return ret; return ret;
} }
static gboolean
fsck_pack_files (OtFsckData *data,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
ot_lptrarray GPtrArray *meta_pack_indexes = NULL;
ot_lptrarray GPtrArray *data_pack_indexes = NULL;
if (!ostree_repo_list_pack_indexes (data->repo, &meta_pack_indexes, &data_pack_indexes,
cancellable, error))
goto out;
for (i = 0; i < meta_pack_indexes->len; i++)
{
const char *pack_checksum = meta_pack_indexes->pdata[i];
if (!fsck_one_pack_file (data, pack_checksum, TRUE, cancellable, error))
goto out;
data->n_pack_files++;
}
for (i = 0; i < data_pack_indexes->len; i++)
{
const char *pack_checksum = data_pack_indexes->pdata[i];
if (!fsck_one_pack_file (data, pack_checksum, FALSE, cancellable, error))
goto out;
data->n_pack_files++;
}
ret = TRUE;
out:
return ret;
}
static gboolean static gboolean
fsck_reachable_objects_from_commits (OtFsckData *data, fsck_reachable_objects_from_commits (OtFsckData *data,
GHashTable *commits, GHashTable *commits,
@ -198,17 +224,26 @@ fsck_reachable_objects_from_commits (OtFsckData *data,
if (objtype == OSTREE_OBJECT_TYPE_COMMIT) if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
{ {
if (!ostree_validate_structureof_commit (metadata, error)) if (!ostree_validate_structureof_commit (metadata, error))
goto out; {
g_prefix_error (error, "While validating commit metadata '%s': ", checksum);
goto out;
}
} }
else if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE) else if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE)
{ {
if (!ostree_validate_structureof_dirtree (metadata, error)) if (!ostree_validate_structureof_dirtree (metadata, error))
goto out; {
g_prefix_error (error, "While validating directory tree '%s': ", checksum);
goto out;
}
} }
else if (objtype == OSTREE_OBJECT_TYPE_DIR_META) else if (objtype == OSTREE_OBJECT_TYPE_DIR_META)
{ {
if (!ostree_validate_structureof_dirmeta (metadata, error)) if (!ostree_validate_structureof_dirmeta (metadata, error))
goto out; {
g_prefix_error (error, "While validating directory metadata '%s': ", checksum);
goto out;
}
} }
else else
g_assert_not_reached (); g_assert_not_reached ();
@ -236,7 +271,10 @@ fsck_reachable_objects_from_commits (OtFsckData *data,
mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
if (!ostree_validate_structureof_file_mode (mode, error)) if (!ostree_validate_structureof_file_mode (mode, error))
goto out; {
g_prefix_error (error, "While validating file '%s': ", checksum);
goto out;
}
} }
else else
{ {

387
src/ostree/ot-builtin-pack.c
View File

@ -197,18 +197,16 @@ compare_index_content (gconstpointer ap,
gpointer b = *((gpointer*)bp); gpointer b = *((gpointer*)bp);
GVariant *a_v = a; GVariant *a_v = a;
GVariant *b_v = b; GVariant *b_v = b;
guint32 a_objtype; guchar a_objtype;
guint32 b_objtype; guchar b_objtype;
guint64 a_offset; guint64 a_offset;
guint64 b_offset; guint64 b_offset;
int c; int c;
ot_lvariant GVariant *a_csum_bytes = NULL; ot_lvariant GVariant *a_csum_bytes = NULL;
ot_lvariant GVariant *b_csum_bytes = NULL; ot_lvariant GVariant *b_csum_bytes = NULL;
g_variant_get (a_v, "(u@ayt)", &a_objtype, &a_csum_bytes, &a_offset); g_variant_get (a_v, "(y@ayt)", &a_objtype, &a_csum_bytes, &a_offset);
g_variant_get (b_v, "(u@ayt)", &b_objtype, &b_csum_bytes, &b_offset); g_variant_get (b_v, "(y@ayt)", &b_objtype, &b_csum_bytes, &b_offset);
a_objtype = GUINT32_FROM_BE (a_objtype);
b_objtype = GUINT32_FROM_BE (b_objtype);
c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (a_csum_bytes), c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (a_csum_bytes),
ostree_checksum_bytes_peek (b_csum_bytes)); ostree_checksum_bytes_peek (b_csum_bytes));
if (c == 0) if (c == 0)
@ -270,8 +268,121 @@ delete_loose_object (OtRepackData *data,
return ret; return ret;
} }
static gboolean
pack_one_meta_object (OtRepackData *data,
const char *checksum,
OstreeObjectType objtype,
GVariant **out_packed_object,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lobj GFile *object_path = NULL;
ot_lvariant GVariant *metadata_v = NULL;
ot_lvariant GVariant *ret_packed_object = NULL;
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
if (!ostree_map_metadata_file (object_path, objtype, &metadata_v, error))
goto out;
ret_packed_object = g_variant_new ("(y@ayv)", (guchar) objtype,
ostree_checksum_to_bytes_v (checksum),
metadata_v);
ret = TRUE;
ot_transfer_out_value (out_packed_object, &ret_packed_object);
out:
return ret;
}
static gboolean
pack_one_data_object (OtRepackData *data,
const char *checksum,
OstreeObjectType objtype,
guint64 expected_objsize,
GVariant **out_packed_object,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint64 objsize;
guchar entry_flags = 0;
GInputStream *read_object_in; /* nofree */
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInputStream *object_input = NULL;
ot_lobj GFileInfo *object_file_info = NULL;
ot_lobj GMemoryOutputStream *object_data_stream = NULL;
ot_lobj GConverter *compressor = NULL;
ot_lobj GConverterInputStream *compressed_object_input = NULL;
ot_lvariant GVariant *ret_packed_object = NULL;
switch (data->int_compression)
{
case OT_COMPRESSION_GZIP:
{
entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
break;
}
default:
{
g_assert_not_reached ();
}
}
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
object_input = g_file_read (object_path, cancellable, error);
if (!object_input)
goto out;
object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error);
if (!object_file_info)
goto out;
objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_assert_cmpint (objsize, ==, expected_objsize);
object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL);
compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", compressor,
"base-stream", object_input,
"close-base-stream", TRUE,
NULL);
read_object_in = (GInputStream*)compressed_object_input;
}
else
{
read_object_in = (GInputStream*)object_input;
}
if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in,
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
cancellable, error))
goto out;
{
guchar *data = g_memory_output_stream_get_data (object_data_stream);
gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
ret_packed_object = g_variant_new ("(yy@ay@ay)", (guchar)objtype, entry_flags,
ostree_checksum_to_bytes_v (checksum),
ot_gvariant_new_bytearray (data, data_len));
}
ret = TRUE;
ot_transfer_out_value (out_packed_object, &ret_packed_object);
out:
return ret;
}
static gboolean static gboolean
create_pack_file (OtRepackData *data, create_pack_file (OtRepackData *data,
gboolean is_meta,
GPtrArray *objects, GPtrArray *objects,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
@ -285,19 +396,12 @@ create_pack_file (OtRepackData *data,
ot_lobj GOutputStream *index_out = NULL; ot_lobj GOutputStream *index_out = NULL;
ot_lobj GFile *pack_temppath = NULL; ot_lobj GFile *pack_temppath = NULL;
ot_lobj GOutputStream *pack_out = NULL; ot_lobj GOutputStream *pack_out = NULL;
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInfo *object_file_info = NULL;
ot_lobj GFileInputStream *object_input = NULL;
ot_lobj GConverter *compressor = NULL;
ot_lobj GConverterInputStream *compressed_object_input = NULL;
ot_lptrarray GPtrArray *index_content_list = NULL; ot_lptrarray GPtrArray *index_content_list = NULL;
ot_lvariant GVariant *pack_header = NULL; ot_lvariant GVariant *pack_header = NULL;
ot_lvariant GVariant *packed_object = NULL;
ot_lvariant GVariant *index_content = NULL; ot_lvariant GVariant *index_content = NULL;
ot_lfree char *pack_name = NULL; ot_lfree char *pack_name = NULL;
ot_lobj GFile *pack_file_path = NULL; ot_lobj GFile *pack_file_path = NULL;
ot_lobj GFile *pack_index_path = NULL; ot_lobj GFile *pack_index_path = NULL;
GMemoryOutputStream *object_data_stream = NULL;
GVariantBuilder index_content_builder; GVariantBuilder index_content_builder;
GChecksum *pack_checksum = NULL; GChecksum *pack_checksum = NULL;
@ -324,7 +428,7 @@ create_pack_file (OtRepackData *data,
pack_checksum = g_checksum_new (G_CHECKSUM_SHA256); pack_checksum = g_checksum_new (G_CHECKSUM_SHA256);
pack_header = g_variant_new ("(s@a{sv}t)", pack_header = g_variant_new ("(s@a{sv}t)",
"OSTv0PACKFILE", is_meta ? "OSTv0PACKMETAFILE" : "OSTv0PACKDATAFILE",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
(guint64)objects->len); (guint64)objects->len);
@ -339,92 +443,37 @@ create_pack_file (OtRepackData *data,
guint32 objtype_u32; guint32 objtype_u32;
OstreeObjectType objtype; OstreeObjectType objtype;
guint64 expected_objsize; guint64 expected_objsize;
guint64 objsize; ot_lvariant GVariant *packed_object = NULL;
GInputStream *read_object_in; ot_lvariant GVariant *index_entry = NULL;
guchar entry_flags = 0;
GVariant *index_entry;
g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize); g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
objtype = (OstreeObjectType) objtype_u32; objtype = (OstreeObjectType) objtype_u32;
switch (data->int_compression) if (is_meta)
{ {
case OT_COMPRESSION_GZIP: if (!pack_one_meta_object (data, checksum, objtype, &packed_object,
{ cancellable, error))
entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP; goto out;
break;
}
default:
{
g_assert_not_reached ();
}
}
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
g_clear_object (&object_input);
object_input = g_file_read (object_path, cancellable, error);
if (!object_input)
goto out;
g_clear_object (&object_file_info);
object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error);
if (!object_file_info)
goto out;
objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_assert_cmpint (objsize, ==, expected_objsize);
g_clear_object (&object_data_stream);
object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
g_clear_object (&compressor);
compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL);
g_clear_object (&compressed_object_input);
compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", compressor,
"base-stream", object_input,
"close-base-stream", TRUE,
NULL);
read_object_in = (GInputStream*)compressed_object_input;
} }
else else
{ {
read_object_in = (GInputStream*)object_input; if (!pack_one_data_object (data, checksum, objtype, expected_objsize,
&packed_object, cancellable, error))
goto out;
} }
if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in,
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
cancellable, error))
goto out;
ot_clear_gvariant (&packed_object);
{
guchar *data = g_memory_output_stream_get_data (object_data_stream);
gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
packed_object = g_variant_new ("(uy@ay@ay)", GUINT32_TO_BE ((guint32)objtype),
entry_flags,
ostree_checksum_to_bytes_v (checksum),
ot_gvariant_new_bytearray (data, data_len));
g_clear_object (&object_data_stream);
}
if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error)) if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
goto out; goto out;
/* offset points to aligned header size */ /* offset points to aligned header size */
index_entry = g_variant_new ("(u@ayt)", index_entry = g_variant_new ("(y@ayt)",
GUINT32_TO_BE ((guint32)objtype), (guchar)objtype,
ostree_checksum_to_bytes_v (checksum), ostree_checksum_to_bytes_v (checksum),
GUINT64_TO_BE (offset)); GUINT64_TO_BE (offset));
g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry)); g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry));
index_entry = NULL;
if (!write_variant_with_size (pack_out, packed_object, pack_checksum, if (!write_variant_with_size (pack_out, packed_object, pack_checksum,
&offset, cancellable, error)) &offset, cancellable, error))
goto out; goto out;
@ -433,14 +482,14 @@ create_pack_file (OtRepackData *data,
if (!g_output_stream_close (pack_out, cancellable, error)) if (!g_output_stream_close (pack_out, cancellable, error))
goto out; goto out;
g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(uayt)")); g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(yayt)"));
g_ptr_array_sort (index_content_list, compare_index_content); g_ptr_array_sort (index_content_list, compare_index_content);
for (i = 0; i < index_content_list->len; i++) for (i = 0; i < index_content_list->len; i++)
{ {
GVariant *index_item = index_content_list->pdata[i]; GVariant *index_item = index_content_list->pdata[i];
g_variant_builder_add_value (&index_content_builder, index_item); g_variant_builder_add_value (&index_content_builder, index_item);
} }
index_content = g_variant_new ("(s@a{sv}@a(uayt))", index_content = g_variant_new ("(s@a{sv}@a(yayt))",
"OSTv0PACKINDEX", "OSTv0PACKINDEX",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
g_variant_builder_end (&index_content_builder)); g_variant_builder_end (&index_content_builder));
@ -458,6 +507,7 @@ create_pack_file (OtRepackData *data,
if (!ostree_repo_add_pack_file (data->repo, if (!ostree_repo_add_pack_file (data->repo,
g_checksum_get_string (pack_checksum), g_checksum_get_string (pack_checksum),
is_meta,
index_temppath, index_temppath,
pack_temppath, pack_temppath,
cancellable, cancellable,
@ -499,67 +549,14 @@ create_pack_file (OtRepackData *data,
return ret; return ret;
} }
/** static void
* cluster_objects_stupidly: cluster_one_object_chain (OtRepackData *data,
* @objects: Map from serialized object name to objdata GPtrArray *object_list,
* @out_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). GPtrArray *inout_clusters)
*
* Just sorts by size currently. Also filters out non-regular object
* content.
*/
static gboolean
cluster_objects_stupidly (OtRepackData *data,
GHashTable *objects,
GPtrArray **out_clusters,
GCancellable *cancellable,
GError **error)
{ {
gboolean ret = FALSE;
guint i; guint i;
guint64 current_size; guint64 current_size;
guint current_offset; guint current_offset;
GHashTableIter hash_iter;
gpointer key, value;
ot_lptrarray GPtrArray *ret_clusters = NULL;
ot_lptrarray GPtrArray *object_list = NULL;
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInfo *object_info = NULL;
object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
const char *checksum;
OstreeObjectType objtype;
guint64 size;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
g_clear_object (&object_info);
object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!object_info)
goto out;
if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR)
continue;
size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_ptr_array_add (object_list,
g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size)));
}
g_ptr_array_sort (object_list, compare_object_data_by_size);
ret_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
current_size = 0; current_size = 0;
current_offset = 0; current_offset = 0;
@ -582,7 +579,7 @@ cluster_objects_stupidly (OtRepackData *data,
{ {
g_ptr_array_add (current, g_variant_ref (object_list->pdata[j])); g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
} }
g_ptr_array_add (ret_clusters, current); g_ptr_array_add (inout_clusters, current);
current_size = objsize; current_size = objsize;
current_offset = i+1; current_offset = i+1;
} }
@ -596,9 +593,83 @@ cluster_objects_stupidly (OtRepackData *data,
current_size += objsize; current_size += objsize;
} }
} }
}
/**
* cluster_objects_stupidly:
* @objects: Map from serialized object name to objdata
* @out_meta_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
* @out_data_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
*
* Just sorts by size currently. Also filters out non-regular object
* content.
*/
static gboolean
cluster_objects_stupidly (OtRepackData *data,
GHashTable *objects,
GPtrArray **out_meta_clusters,
GPtrArray **out_data_clusters,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GHashTableIter hash_iter;
gpointer key, value;
ot_lptrarray GPtrArray *ret_meta_clusters = NULL;
ot_lptrarray GPtrArray *ret_data_clusters = NULL;
ot_lptrarray GPtrArray *meta_object_list = NULL;
ot_lptrarray GPtrArray *data_object_list = NULL;
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInfo *object_info = NULL;
meta_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
data_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
const char *checksum;
OstreeObjectType objtype;
guint64 size;
GVariant *v;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
g_clear_object (&object_info);
object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!object_info)
goto out;
if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR)
continue;
size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
v = g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size));
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
g_ptr_array_add (meta_object_list, v);
else
g_ptr_array_add (data_object_list, v);
}
g_ptr_array_sort (meta_object_list, compare_object_data_by_size);
g_ptr_array_sort (data_object_list, compare_object_data_by_size);
ret_meta_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
ret_data_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
cluster_one_object_chain (data, meta_object_list, ret_meta_clusters);
cluster_one_object_chain (data, data_object_list, ret_data_clusters);
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_clusters, &ret_clusters); ot_transfer_out_value (out_meta_clusters, &ret_meta_clusters);
ot_transfer_out_value (out_data_clusters, &ret_data_clusters);
out: out:
return ret; return ret;
} }
@ -792,7 +863,8 @@ do_incremental_pack (OtRepackData *data,
gboolean ret = FALSE; gboolean ret = FALSE;
guint i; guint i;
ot_lhash GHashTable *objects = NULL; ot_lhash GHashTable *objects = NULL;
ot_lptrarray GPtrArray *clusters = NULL; ot_lptrarray GPtrArray *meta_clusters = NULL;
ot_lptrarray GPtrArray *data_clusters = NULL;
ot_lhash GHashTable *loose_objects = NULL; ot_lhash GHashTable *loose_objects = NULL;
if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
@ -805,21 +877,30 @@ do_incremental_pack (OtRepackData *data,
g_print ("\n"); g_print ("\n");
g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size); g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size);
if (!cluster_objects_stupidly (data, loose_objects, &clusters, cancellable, error)) if (!cluster_objects_stupidly (data, loose_objects, &meta_clusters, &data_clusters,
cancellable, error))
goto out; goto out;
if (clusters->len > 0) if (meta_clusters->len > 0 || data_clusters->len > 0)
g_print ("Going to create %u packfiles\n", clusters->len); g_print ("Going to create %u meta packfiles, %u data packfiles\n",
meta_clusters->len, data_clusters->len);
else else
g_print ("Nothing to do\n"); g_print ("Nothing to do\n");
for (i = 0; i < clusters->len; i++) if (!opt_analyze_only)
{ {
GPtrArray *cluster = clusters->pdata[i]; for (i = 0; i < meta_clusters->len; i++)
if (!opt_analyze_only)
{ {
if (!create_pack_file (data, cluster, cancellable, error)) GPtrArray *cluster = meta_clusters->pdata[i];
if (!create_pack_file (data, TRUE, cluster, cancellable, error))
goto out;
}
for (i = 0; i < data_clusters->len; i++)
{
GPtrArray *cluster = data_clusters->pdata[i];
if (!create_pack_file (data, FALSE, cluster, cancellable, error))
goto out; goto out;
} }
} }

45
src/ostree/ot-builtin-unpack.c
View File

@ -144,15 +144,20 @@ unpack_one_object (OstreeRepo *repo,
static gboolean static gboolean
delete_one_packfile (OstreeRepo *repo, delete_one_packfile (OstreeRepo *repo,
const char *pack_checksum, const char *pack_checksum,
gboolean is_meta,
GCancellable *cancellable, GCancellable *cancellable,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
ot_lfree char *data_name = NULL;
ot_lobj GFile *data_path = NULL; ot_lobj GFile *data_path = NULL;
ot_lfree char *index_name = NULL;
ot_lobj GFile *index_path = NULL; ot_lobj GFile *index_path = NULL;
index_path = ostree_repo_get_pack_index_path (repo, pack_checksum); index_name = ostree_get_relative_pack_index_path (is_meta, pack_checksum);
data_path = ostree_repo_get_pack_data_path (repo, pack_checksum); index_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), index_name);
data_name = ostree_get_relative_pack_data_path (is_meta, pack_checksum);
data_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), data_name);
if (!ot_gfile_unlink (index_path, cancellable, error)) if (!ot_gfile_unlink (index_path, cancellable, error))
{ {
@ -185,7 +190,8 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
ot_lhash GHashTable *objects = NULL; ot_lhash GHashTable *objects = NULL;
ot_lptrarray GPtrArray *clusters = NULL; ot_lptrarray GPtrArray *clusters = NULL;
ot_lhash GHashTable *packed_objects = NULL; ot_lhash GHashTable *packed_objects = NULL;
ot_lhash GHashTable *packfiles_to_delete = NULL; ot_lhash GHashTable *meta_packfiles_to_delete = NULL;
ot_lhash GHashTable *data_packfiles_to_delete = NULL;
ot_lobj GFile *objpath = NULL; ot_lobj GFile *objpath = NULL;
memset (&data, 0, sizeof (data)); memset (&data, 0, sizeof (data));
@ -220,7 +226,8 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
in_transaction = TRUE; in_transaction = TRUE;
packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); meta_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
data_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_hash_table_iter_init (&hash_iter, packed_objects); g_hash_table_iter_init (&hash_iter, packed_objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value)) while (g_hash_table_iter_next (&hash_iter, &key, &value))
@ -232,6 +239,9 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
OstreeObjectType objtype; OstreeObjectType objtype;
gboolean is_loose; gboolean is_loose;
GVariantIter *pack_array_iter; GVariantIter *pack_array_iter;
GHashTable *target_hash;
ostree_object_name_deserialize (objkey, &checksum, &objtype);
objdata = g_hash_table_lookup (objects, objkey); objdata = g_hash_table_lookup (objects, objkey);
g_assert (objdata); g_assert (objdata);
@ -240,12 +250,17 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
g_assert (!is_loose); g_assert (!is_loose);
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
target_hash = meta_packfiles_to_delete;
else
target_hash = data_packfiles_to_delete;
while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum)) while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum))
{ {
if (!g_hash_table_lookup (packfiles_to_delete, pack_checksum)) if (!g_hash_table_lookup (target_hash, pack_checksum))
{ {
gchar *duped_checksum = g_strdup (pack_checksum); gchar *duped_checksum = g_strdup (pack_checksum);
g_hash_table_replace (packfiles_to_delete, duped_checksum, duped_checksum); g_hash_table_replace (target_hash, duped_checksum, duped_checksum);
} }
} }
g_variant_iter_free (pack_array_iter); g_variant_iter_free (pack_array_iter);
@ -261,15 +276,27 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_repo_commit_transaction (repo, cancellable, error)) if (!ostree_repo_commit_transaction (repo, cancellable, error))
goto out; goto out;
if (g_hash_table_size (packfiles_to_delete) == 0) if (g_hash_table_size (meta_packfiles_to_delete) == 0
&& g_hash_table_size (data_packfiles_to_delete) == 0)
g_print ("No pack files; nothing to do\n"); g_print ("No pack files; nothing to do\n");
g_hash_table_iter_init (&hash_iter, packfiles_to_delete); g_hash_table_iter_init (&hash_iter, meta_packfiles_to_delete);
while (g_hash_table_iter_next (&hash_iter, &key, &value)) while (g_hash_table_iter_next (&hash_iter, &key, &value))
{ {
const char *pack_checksum = key; const char *pack_checksum = key;
if (!delete_one_packfile (repo, pack_checksum, cancellable, error)) if (!delete_one_packfile (repo, pack_checksum, TRUE, cancellable, error))
goto out;
g_print ("Deleted packfile '%s'\n", pack_checksum);
}
g_hash_table_iter_init (&hash_iter, data_packfiles_to_delete);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *pack_checksum = key;
if (!delete_one_packfile (repo, pack_checksum, FALSE, cancellable, error))
goto out; goto out;
g_print ("Deleted packfile '%s'\n", pack_checksum); g_print ("Deleted packfile '%s'\n", pack_checksum);