packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

core: Drop packfiles as they are now 

They're not a large efficiency win at the moment, because we don't
do any delta compression.

At the moment, they simply served to compress data, but we will change
the archive mode to do that by default.
This commit is contained in:
Colin Walters 2012-09-23 15:16:33 -04:00
parent aa9aaa9a62
commit 2a0601efc7
15 changed files with 37 additions and 3694 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile-ostree.am
View File

@ -37,8 +37,6 @@ ostree_SOURCES = src/ostree/main.c \
src/ostree/ot-builtin-ls.c \
src/ostree/ot-builtin-prune.c \
src/ostree/ot-builtin-remote.c \
src/ostree/ot-builtin-pack.c \
src/ostree/ot-builtin-unpack.c \
src/ostree/ot-builtin-rev-parse.c \
src/ostree/ot-builtin-show.c \
src/ostree/ot-builtin-write-refs.c \

332
src/libostree/ostree-core.c
View File

@ -944,58 +944,6 @@ ostree_get_relative_archive_content_path (const char *checksum)
return g_string_free (path, FALSE);
}
static char *
get_pack_name (gboolean is_meta,
gboolean is_index,
const char *prefix,
const char *checksum)
{
GString *path;
g_assert (strlen (checksum) == 64);
path = g_string_new (prefix);
if (is_meta)
g_string_append (path, "ostmetapack-");
else
g_string_append (path, "ostdatapack-");
g_string_append (path, checksum);
if (is_index)
g_string_append (path, ".index");
else
g_string_append (path, ".data");
return g_string_free (path, FALSE);
}
char *
ostree_get_pack_index_name (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, TRUE, "", checksum);
}
char *
ostree_get_pack_data_name (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, FALSE, "", checksum);
}
char *
ostree_get_relative_pack_index_path (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, TRUE, "objects/pack/", checksum);
}
char *
ostree_get_relative_pack_data_path (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, FALSE, "objects/pack/", checksum);
}
gboolean
ostree_file_header_parse (GVariant *metadata,
GFileInfo **out_file_info,
@ -1343,194 +1291,6 @@ ostree_create_temp_dir (GFile *dir,
return ret;
}
gboolean
ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len,
guint64 offset,
gboolean trusted,
gboolean is_meta,
GVariant **out_entry,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint64 entry_start;
guint64 entry_end;
guint32 entry_len;
ot_lvariant GVariant *ret_entry = NULL;
if (G_UNLIKELY (!(offset <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range offset %" G_GUINT64_FORMAT,
offset);
goto out;
}
if (G_UNLIKELY (!((offset & 0x3) == 0)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT,
offset);
goto out;
}
entry_start = ALIGN_VALUE (offset + 4, 8);
if (G_UNLIKELY (!(entry_start <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
entry_start);
goto out;
}
g_assert ((((guint64)pack_data+offset) & 0x3) == 0);
entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
entry_end = entry_start + entry_len;
if (G_UNLIKELY (!(entry_end <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range entry length %u",
entry_len);
goto out;
}
ret_entry = g_variant_new_from_data (is_meta ? OSTREE_PACK_META_FILE_VARIANT_FORMAT :
OSTREE_PACK_DATA_FILE_VARIANT_FORMAT,
pack_data+entry_start, entry_len,
trusted, NULL, NULL);
g_variant_ref_sink (ret_entry);
ret = TRUE;
ot_transfer_out_value (out_entry, &ret_entry);
out:
return ret;
}
gboolean
ostree_parse_file_pack_entry (GVariant *pack_entry,
GInputStream **out_input,
GFileInfo **out_info,
GVariant **out_xattrs,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guchar entry_flags;
ot_lvariant GVariant *file_header = NULL;
ot_lvariant GVariant *pack_data = NULL;
ot_lobj GInputStream *memory_input = NULL;
ot_lobj GInputStream *ret_input = NULL;
ot_lobj GFileInfo *ret_info = NULL;
ot_lvariant GVariant *ret_xattrs = NULL;
g_variant_get_child (pack_entry, 1, "y", &entry_flags);
g_variant_get_child (pack_entry, 2, "@(uuuusa(ayay))", &file_header);
g_variant_get_child (pack_entry, 3, "@ay", &pack_data);
if (!ostree_file_header_parse (file_header, &ret_info, &ret_xattrs,
error))
goto out;
g_file_info_set_size (ret_info, g_variant_get_size (pack_data));
if (g_file_info_get_file_type (ret_info) == G_FILE_TYPE_REGULAR)
{
memory_input = ot_variant_read (pack_data);
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
ot_lobj GConverter *decompressor = NULL;
decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP);
ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", decompressor,
"base-stream", memory_input,
"close-base-stream", TRUE,
NULL);
}
else
{
ret_input = g_object_ref (memory_input);
}
}
ret = TRUE;
ot_transfer_out_value (out_input, &ret_input);
ot_transfer_out_value (out_info, &ret_info);
ot_transfer_out_value (out_xattrs, &ret_xattrs);
out:
return ret;
}
gboolean
ostree_pack_index_search (GVariant *index,
GVariant *csum_v,
OstreeObjectType objtype,
guint64 *out_offset)
{
gboolean ret = FALSE;
gsize imax, imin;
gsize n;
guint32 target_objtype;
const guchar *csum;
ot_lvariant GVariant *index_contents = NULL;
csum = ostree_checksum_bytes_peek (csum_v);
index_contents = g_variant_get_child_value (index, 2);
target_objtype = (guint32) objtype;
n = g_variant_n_children (index_contents);
if (n == 0)
goto out;
imax = n - 1;
imin = 0;
while (imax >= imin)
{
GVariant *cur_csum_bytes;
guchar cur_objtype;
guint64 cur_offset;
gsize imid;
int c;
imid = (imin + imax) / 2;
g_variant_get_child (index_contents, imid, "(y@ayt)", &cur_objtype,
&cur_csum_bytes, &cur_offset);
c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum);
if (c == 0)
{
if (cur_objtype < target_objtype)
c = -1;
else if (cur_objtype > target_objtype)
c = 1;
}
g_variant_unref (cur_csum_bytes);
if (c < 0)
imin = imid + 1;
else if (c > 0)
{
if (imid == 0)
goto out;
imax = imid - 1;
}
else
{
if (out_offset)
*out_offset = GUINT64_FROM_BE (cur_offset);
ret = TRUE;
goto out;
}
}
out:
return ret;
}
gboolean
ostree_validate_structureof_objtype (guchar objtype,
GError **error)
@ -1766,95 +1526,3 @@ ostree_validate_structureof_dirmeta (GVariant *dirmeta,
out:
return ret;
}
gboolean
ostree_validate_structureof_pack_index (GVariant *index,
GError **error)
{
gboolean ret = FALSE;
const char *header;
guchar objtype_u8;
guint64 offset;
ot_lvariant GVariant *csum_v = NULL;
GVariantIter *content_iter = NULL;
if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error))
goto out;
g_variant_get_child (index, 0, "&s", &header);
if (strcmp (header, "OSTv0PACKINDEX") != 0)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid pack index; doesn't match header");
goto out;
}
g_variant_get_child (index, 2, "a(yayt)", &content_iter);
while (g_variant_iter_loop (content_iter, "(y@ayt)",
&objtype_u8, &csum_v, &offset))
{
if (!ostree_validate_structureof_objtype (objtype_u8, error))
goto out;
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
}
csum_v = NULL;
ret = TRUE;
out:
if (content_iter)
g_variant_iter_free (content_iter);
return ret;
}
gboolean
ostree_validate_structureof_pack_superindex (GVariant *superindex,
GError **error)
{
gboolean ret = FALSE;
const char *header;
ot_lvariant GVariant *csum_v = NULL;
ot_lvariant GVariant *bloom = NULL;
GVariantIter *content_iter = NULL;
if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error))
goto out;
g_variant_get_child (superindex, 0, "&s", &header);
if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid pack superindex; doesn't match header");
goto out;
}
g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_v, &bloom))
{
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
}
csum_v = NULL;
bloom = NULL;
g_variant_iter_free (content_iter);
g_variant_get_child (superindex, 3, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_v, &bloom))
{
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
}
csum_v = NULL;
bloom = NULL;
ret = TRUE;
out:
if (content_iter)
g_variant_iter_free (content_iter);
return ret;
}

79
src/libostree/ostree-core.h
View File

@ -87,48 +87,6 @@ typedef enum {
*/
#define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE ("(a{sv}aya(say)sstayay)")
/* Pack super index
* s - OSTv0SUPERPACKINDEX
* a{sv} - Metadata
* a(ayay) - metadata packs (pack file checksum, bloom filter)
* a(ayay) - data packs (pack file checksum, bloom filter)
*/
#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay)a(ayay))")
/* Pack index
* s - OSTv0PACKINDEX
* a{sv} - Metadata
* a(yayt) - (objtype, checksum, offset into packfile)
*/
#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(yayt))")
typedef enum {
OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0)
} OstreePackFileEntryFlag;
/* Data Pack files
* s - OSTv0PACKDATAFILE
* a{sv} - Metadata
* t - number of entries
*
* Repeating pair of:
* <padding to alignment of 8>
* ( ayy(uuuusa(ayay))ay) ) - checksum, flags, file meta, data
*/
#define OSTREE_PACK_DATA_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(ayy(uuuusa(ayay))ay)")
/* Meta Pack files
* s - OSTv0PACKMETAFILE
* a{sv} - Metadata
* t - number of entries
*
* Repeating pair of:
* <padding to alignment of 8>
* ( yayv ) - objtype, checksum, data
*/
#define OSTREE_PACK_META_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yayv)")
const GVariantType *ostree_metadata_variant_type (OstreeObjectType objtype);
gboolean ostree_validate_checksum_string (const char *sha256,
@ -173,16 +131,6 @@ char *ostree_get_relative_object_path (const char *checksum,
char *ostree_get_relative_archive_content_path (const char *checksum);
char *ostree_get_pack_index_name (gboolean is_meta,
const char *checksum);
char *ostree_get_pack_data_name (gboolean is_meta,
const char *checksum);
char *ostree_get_relative_pack_index_path (gboolean is_meta,
const char *checksum);
char *ostree_get_relative_pack_data_path (gboolean is_meta,
const char *checksum);
gboolean ostree_get_xattrs_for_file (GFile *f,
GVariant **out_xattrs,
GCancellable *cancellable,
@ -304,27 +252,6 @@ gboolean ostree_create_temp_dir (GFile *dir,
GCancellable *cancellable,
GError **error);
gboolean ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len,
guint64 object_offset,
gboolean trusted,
gboolean is_meta,
GVariant **out_entry,
GCancellable *cancellable,
GError **error);
gboolean ostree_parse_file_pack_entry (GVariant *pack_entry,
GInputStream **out_input,
GFileInfo **out_info,
GVariant **out_xattrs,
GCancellable *cancellable,
GError **error);
gboolean ostree_pack_index_search (GVariant *index,
GVariant *csum_bytes,
OstreeObjectType objtype,
guint64 *out_offset);
/** VALIDATION **/
gboolean ostree_validate_structureof_objtype (guchar objtype,
@ -348,10 +275,4 @@ gboolean ostree_validate_structureof_dirtree (GVariant *index,
gboolean ostree_validate_structureof_dirmeta (GVariant *index,
GError **error);
gboolean ostree_validate_structureof_pack_index (GVariant *index,
GError **error);
gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex,
GError **error);
#endif /* _OSTREE_REPO */

1289
src/libostree/ostree-repo.c
View File

File diff suppressed because it is too large Load Diff

88
src/libostree/ostree-repo.h
View File

@ -111,14 +111,12 @@ gboolean ostree_repo_stage_file_object (OstreeRepo *self,
gboolean ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
gboolean store_if_packed,
GInputStream *content,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_stage_file_object_trusted (OstreeRepo *self,
const char *checksum,
gboolean store_if_packed,
GInputStream *content,
guint64 content_length,
GCancellable *cancellable,
@ -159,27 +157,6 @@ gboolean ostree_repo_load_variant_if_exists (OstreeRepo *self,
GVariant **out_variant,
GError **error);
gboolean ostree_repo_load_pack_index (OstreeRepo *self,
const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_load_pack_data (OstreeRepo *self,
const char *pack_checksum,
guchar **out_data,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_map_pack_file (OstreeRepo *self,
const char *sha256,
gboolean is_meta,
guchar **out_data,
guint64 *out_len,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_load_file (OstreeRepo *self,
const char *entry_sha256,
GInputStream **out_input,
@ -248,65 +225,6 @@ gboolean ostree_repo_stage_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_add_pack_file (OstreeRepo *self,
const char *checksum,
gboolean is_meta,
GFile *pack_index_path,
GFile *pack_data_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
const char *remote_name,
GFile *superindex_path,
GPtrArray **out_cached_meta_indexes,
GPtrArray **out_cached_data_indexes,
GPtrArray **out_uncached_meta_indexes,
GPtrArray **out_uncached_data_indexes,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile *cached_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile *cached_path,
GCancellable *cancellable,
GError **error);
typedef enum {
OSTREE_REPO_CHECKOUT_MODE_NONE = 0,
OSTREE_REPO_CHECKOUT_MODE_USER = 1
@ -359,12 +277,6 @@ gboolean ostree_repo_list_objects (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_list_pack_indexes (OstreeRepo *self,
GPtrArray **out_meta_indexes,
GPtrArray **out_data_indexes,
GCancellable *cancellable,
GError **error);
G_END_DECLS
#endif /* _OSTREE_REPO */

2
src/ostree/main.c
View File

@ -42,14 +42,12 @@ static OstreeCommand commands[] = {
{ "init", ostree_builtin_init, 0 },
{ "log", ostree_builtin_log, 0 },
{ "ls", ostree_builtin_ls, 0 },
{ "pack", ostree_builtin_pack, 0 },
{ "prune", ostree_builtin_prune, 0 },
{ "pull", NULL, 0 },
{ "pull-local", ostree_builtin_pull_local, 0 },
{ "remote", ostree_builtin_remote, 0 },
{ "rev-parse", ostree_builtin_rev_parse, 0 },
{ "show", ostree_builtin_show, 0 },
{ "unpack", ostree_builtin_unpack, 0 },
{ "write-refs", ostree_builtin_write_refs, 0 },
{ NULL }
};

508
src/ostree/ostree-pull.c
View File

@ -67,13 +67,11 @@
#include "ostree-fetcher.h"
gboolean verbose;
gint opt_packfile_threshold = 66;
gboolean opt_related;
gint opt_depth;
static GOptionEntry options[] = {
{ "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose, "Show more information", NULL },
{ "packfile-threshold", 't', 0, G_OPTION_ARG_INT, &opt_packfile_threshold, "Only download packfiles if more than PERCENT objects are needed (default: 66)", "PERCENT" },
{ "related", 0, 0, G_OPTION_ARG_NONE, &opt_related, "Download related commits", NULL },
{ "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Download parent commits up to this depth (default: 0)", NULL },
{ NULL },
@ -85,10 +83,6 @@ typedef struct {
OstreeFetcher *fetcher;
SoupURI *base_uri;
gboolean fetched_packs;
GPtrArray *cached_meta_pack_indexes;
GPtrArray *cached_data_pack_indexes;
GHashTable *file_checksums_to_fetch;
GMainLoop *loop;
@ -325,234 +319,6 @@ fetch_uri_contents_utf8 (OtPullData *pull_data,
return ret;
}
static gboolean
fetch_one_pack_file (OtPullData *pull_data,
const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lobj GFile *ret_cached_path = NULL;
ot_lobj GFile *tmp_path = NULL;
ot_lfree char *pack_name = NULL;
SoupURI *pack_uri = NULL;
if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, is_meta, &ret_cached_path,
cancellable, error))
goto out;
if (ret_cached_path == NULL)
{
pack_name = ostree_get_pack_data_name (is_meta, pack_checksum);
pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL);
if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error))
goto out;
if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, is_meta, tmp_path,
cancellable, error))
goto out;
}
g_clear_object (&ret_cached_path);
if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, is_meta, &ret_cached_path,
cancellable, error))
goto out;
g_assert (ret_cached_path != NULL);
ret = TRUE;
ot_transfer_out_value (out_cached_path, &ret_cached_path);
out:
if (pack_uri)
soup_uri_free (pack_uri);
return ret;
}
static gboolean
find_object_in_one_remote_pack (OtPullData *pull_data,
GVariant *csum_bytes_v,
OstreeObjectType objtype,
const char *pack_checksum,
gboolean *out_exists,
guint64 *out_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint64 ret_offset;
gboolean ret_exists;
ot_lvariant GVariant *mapped_pack = NULL;
if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype),
&mapped_pack,
cancellable, error))
goto out;
ret_exists = ostree_pack_index_search (mapped_pack, csum_bytes_v, objtype, &ret_offset);
ret = TRUE;
if (out_exists)
*out_exists = ret_exists;
if (out_offset)
*out_offset = ret_offset;
out:
return ret;
}
static gboolean
find_object_in_remote_packs (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
char **out_pack_checksum,
guint64 *out_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint64 ret_offset = 0;
guint i;
GPtrArray *iter;
ot_lvariant GVariant *csum_bytes_v = NULL;
ot_lfree char *ret_pack_checksum = NULL;
csum_bytes_v = ostree_checksum_to_bytes_v (checksum);
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
iter = pull_data->cached_meta_pack_indexes;
else
iter = pull_data->cached_data_pack_indexes;
for (i = 0; i < iter->len; i++)
{
const char *pack_checksum = iter->pdata[i];
gboolean exists;
if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, objtype,
pack_checksum, &exists, &ret_offset,
cancellable, error))
goto out;
if (exists)
{
ret_pack_checksum = g_strdup (pack_checksum);
break;
}
}
ret = TRUE;
ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
if (out_offset)
*out_offset = ret_offset;
out:
return ret;
}
static gboolean
fetch_one_cache_index (OtPullData *pull_data,
const char *pack_checksum,
gboolean is_meta,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lobj GFile *tmp_path = NULL;
ot_lfree char *pack_index_name = NULL;
SoupURI *index_uri = NULL;
pack_index_name = ostree_get_pack_index_name (is_meta, pack_checksum);
index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL);
if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path,
cancellable, error))
goto out;
if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
pack_checksum, is_meta, tmp_path,
cancellable, error))
goto out;
if (!ot_gfile_unlink (tmp_path, cancellable, error))
goto out;
g_clear_object (&tmp_path);
ret = TRUE;
out:
if (tmp_path != NULL)
(void) ot_gfile_unlink (tmp_path, NULL, NULL);
if (index_uri)
soup_uri_free (index_uri);
return ret;
}
static gboolean
fetch_and_cache_pack_indexes (OtPullData *pull_data,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
ot_lobj GFile *superindex_tmppath = NULL;
ot_lptrarray GPtrArray *cached_meta_indexes = NULL;
ot_lptrarray GPtrArray *cached_data_indexes = NULL;
ot_lptrarray GPtrArray *uncached_meta_indexes = NULL;
ot_lptrarray GPtrArray *uncached_data_indexes = NULL;
ot_lvariant GVariant *superindex_variant = NULL;
GVariantIter *contents_iter = NULL;
SoupURI *superindex_uri = NULL;
superindex_uri = suburi_new (pull_data->base_uri, "objects", "pack", "index", NULL);
if (!fetch_uri (pull_data, superindex_uri, "index-",
&superindex_tmppath, cancellable, error))
goto out;
if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name,
superindex_tmppath,
&cached_meta_indexes,
&cached_data_indexes,
&uncached_meta_indexes,
&uncached_data_indexes,
cancellable, error))
goto out;
for (i = 0; i < cached_meta_indexes->len; i++)
g_ptr_array_add (pull_data->cached_meta_pack_indexes,
g_strdup (cached_meta_indexes->pdata[i]));
for (i = 0; i < cached_data_indexes->len; i++)
g_ptr_array_add (pull_data->cached_data_pack_indexes,
g_strdup (cached_data_indexes->pdata[i]));
for (i = 0; i < uncached_meta_indexes->len; i++)
{
const char *pack_checksum = uncached_meta_indexes->pdata[i];
if (!fetch_one_cache_index (pull_data, pack_checksum, TRUE, cancellable, error))
goto out;
g_ptr_array_add (pull_data->cached_meta_pack_indexes, g_strdup (pack_checksum));
}
for (i = 0; i < uncached_data_indexes->len; i++)
{
const char *pack_checksum = uncached_data_indexes->pdata[i];
if (!fetch_one_cache_index (pull_data, pack_checksum, FALSE, cancellable, error))
goto out;
g_ptr_array_add (pull_data->cached_data_pack_indexes, g_strdup (pack_checksum));
}
ret = TRUE;
out:
if (superindex_uri)
soup_uri_free (superindex_uri);
if (contents_iter)
g_variant_iter_free (contents_iter);
return ret;
}
static gboolean
fetch_loose_object (OtPullData *pull_data,
const char *checksum,
@ -581,89 +347,6 @@ fetch_loose_object (OtPullData *pull_data,
return ret;
}
static gboolean
find_object_ensure_indexes (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
gboolean *out_is_stored,
char **out_remote_pack_checksum,
guint64 *out_remote_pack_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gboolean ret_is_stored;
ot_lfree char *ret_remote_pack_checksum = NULL;
if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &ret_is_stored,
cancellable, error))
goto out;
if (!ret_is_stored)
{
if (!pull_data->fetched_packs)
{
pull_data->fetched_packs = TRUE;
pull_data->cached_meta_pack_indexes = g_ptr_array_new_with_free_func (g_free);
pull_data->cached_data_pack_indexes = g_ptr_array_new_with_free_func (g_free);
if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error))
goto out;
}
if (!find_object_in_remote_packs (pull_data, checksum, objtype,
&ret_remote_pack_checksum, out_remote_pack_offset,
cancellable, error))
goto out;
}
ret = TRUE;
if (out_is_stored)
*out_is_stored = ret_is_stored;
ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum);
out:
return ret;
}
static gboolean
find_object_ensure_pack_data (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
gboolean *out_is_stored,
GFile **out_remote_pack_path,
guint64 *out_remote_pack_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gboolean ret_is_stored;
ot_lfree char *remote_pack_checksum = NULL;
ot_lobj GFile *ret_remote_pack_path = NULL;
if (!find_object_ensure_indexes (pull_data, checksum, objtype, &ret_is_stored,
&remote_pack_checksum, out_remote_pack_offset,
cancellable, error))
goto out;
if (!ret_is_stored)
{
if (remote_pack_checksum)
{
if (!fetch_one_pack_file (pull_data, remote_pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype),
&ret_remote_pack_path, cancellable, error))
goto out;
}
}
ret = TRUE;
if (out_is_stored)
*out_is_stored = ret_is_stored;
ot_transfer_out_value (out_remote_pack_path, &ret_remote_pack_path);
/* offset set above */
out:
return ret;
}
static gboolean
fetch_and_store_metadata (OtPullData *pull_data,
const char *checksum,
@ -673,42 +356,19 @@ fetch_and_store_metadata (OtPullData *pull_data,
GError **error)
{
gboolean ret = FALSE;
guint64 pack_offset = 0;
gboolean is_stored;
ot_lvariant GVariant *ret_variant = NULL;
ot_lobj GFile *remote_pack_path = NULL;
ot_lobj GFile *temp_path = NULL;
ot_lobj GInputStream *input = NULL;
ot_lvariant GVariant *pack_entry = NULL;
ot_lvariant GVariant *metadata = NULL;
GMappedFile *pack_map = NULL;
g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype));
if (!find_object_ensure_pack_data (pull_data, checksum, objtype,
&is_stored, &remote_pack_path, &pack_offset,
cancellable, error))
if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored,
cancellable, error))
goto out;
if (remote_pack_path != NULL)
{
g_assert (!is_stored);
pack_map = g_mapped_file_new (ot_gfile_get_path_cached (remote_pack_path), FALSE, error);
if (!pack_map)
goto out;
if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
g_mapped_file_get_length (pack_map),
pack_offset, FALSE, TRUE, &pack_entry,
cancellable, error))
goto out;
g_variant_get_child (pack_entry, 2, "v", &metadata);
input = ot_variant_read (metadata);
}
else if (!is_stored)
if (!is_stored)
{
if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error))
goto out;
@ -716,11 +376,7 @@ fetch_and_store_metadata (OtPullData *pull_data,
input = (GInputStream*)g_file_read (temp_path, cancellable, error);
if (!input)
goto out;
}
if (input)
{
g_assert (remote_pack_path != NULL || !is_stored);
if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, input,
cancellable, error))
goto out;
@ -735,8 +391,6 @@ fetch_and_store_metadata (OtPullData *pull_data,
out:
if (temp_path)
(void) ot_gfile_unlink (temp_path, NULL, NULL);
if (pack_map)
g_mapped_file_unref (pack_map);
return ret;
}
@ -753,7 +407,6 @@ fetch_and_store_tree_metadata_recurse (OtPullData *pull_data,
ot_lvariant GVariant *files_variant = NULL;
ot_lvariant GVariant *dirs_variant = NULL;
ot_lobj GFile *stored_path = NULL;
ot_lfree char *pack_checksum = NULL;
if (depth > OSTREE_MAX_RECURSION)
{
@ -936,66 +589,6 @@ fetch_ref_contents (OtPullData *pull_data,
return ret;
}
static gboolean
store_file_from_pack (OtPullData *pull_data,
const char *checksum,
const char *pack_checksum,
GFile *pack_file,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gboolean exists;
guint64 pack_offset;
ot_lobj GFile *remote_pack_path = NULL;
ot_lobj GFile *temp_path = NULL;
ot_lvariant GVariant *pack_entry = NULL;
ot_lobj GInputStream *input = NULL;
ot_lobj GInputStream *file_object_input = NULL;
ot_lobj GFileInfo *file_info = NULL;
ot_lvariant GVariant *xattrs = NULL;
ot_lvariant GVariant *csum_bytes_v = NULL;
GMappedFile *pack_map = NULL;
csum_bytes_v = ostree_checksum_to_bytes_v (checksum);
if (!find_object_in_one_remote_pack (pull_data, csum_bytes_v, OSTREE_OBJECT_TYPE_FILE,
pack_checksum, &exists, &pack_offset,
cancellable, error))
goto out;
g_assert (exists);
pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_file), FALSE, error);
if (!pack_map)
goto out;
if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
g_mapped_file_get_length (pack_map),
pack_offset, FALSE, FALSE, &pack_entry,
cancellable, error))
goto out;
if (!ostree_parse_file_pack_entry (pack_entry, &input, &file_info, &xattrs,
cancellable, error))
goto out;
if (!ostree_raw_file_to_content_stream (input, file_info, xattrs,
&file_object_input, NULL, cancellable, error))
goto out;
if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum,
file_object_input,
cancellable, error))
goto out;
ret = TRUE;
out:
if (pack_map)
g_mapped_file_unref (pack_map);
return ret;
}
typedef struct {
OtPullData *pull_data;
@ -1072,7 +665,7 @@ content_fetch_on_checksum_complete (GObject *object,
}
if (!ostree_repo_stage_file_object_trusted (data->pull_data->repo, checksum,
FALSE, file_object_input, length,
file_object_input, length,
cancellable, error))
goto out;
@ -1231,42 +824,23 @@ fetch_content (OtPullData *pull_data,
gpointer key, value;
ot_lobj GFile *temp_path = NULL;
ot_lobj GFile *content_temp_path = NULL;
ot_lhash GHashTable *data_packs_to_fetch = NULL;
ot_lhash GHashTable *loose_files = NULL;
SoupURI *content_uri = NULL;
guint n_objects_to_fetch = 0;
data_packs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free,
(GDestroyNotify) g_ptr_array_unref);
loose_files = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_hash_table_iter_init (&hash_iter, pull_data->file_checksums_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *checksum = key;
GPtrArray *files_to_fetch;
gboolean is_stored;
ot_lfree char *remote_pack_checksum = NULL;
if (!find_object_ensure_indexes (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE,
&is_stored, &remote_pack_checksum, NULL,
cancellable, error))
if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, &is_stored,
cancellable, error))
goto out;
if (remote_pack_checksum)
{
files_to_fetch = g_hash_table_lookup (data_packs_to_fetch, remote_pack_checksum);
if (files_to_fetch == NULL)
{
files_to_fetch = g_ptr_array_new_with_free_func (g_free);
g_hash_table_insert (data_packs_to_fetch, remote_pack_checksum, files_to_fetch);
/* transfer ownership */
remote_pack_checksum = NULL;
}
g_ptr_array_add (files_to_fetch, g_strdup (checksum));
n_objects_to_fetch++;
}
else if (!is_stored)
if (!is_stored)
{
char *key = g_strdup (checksum);
g_hash_table_insert (loose_files, key, key);
@ -1274,71 +848,9 @@ fetch_content (OtPullData *pull_data,
}
}
g_hash_table_iter_init (&hash_iter, data_packs_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *pack_checksum = key;
GPtrArray *files = value;
ot_lvariant GVariant *mapped_pack = NULL;
ot_lvariant GVariant *content_list = NULL;
gboolean fetch;
if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
pack_checksum, FALSE,
&mapped_pack,
cancellable, error))
goto out;
content_list = g_variant_get_child_value (mapped_pack, 2);
fetch = (((double)files->len) / g_variant_n_children (content_list)) > ((double)opt_packfile_threshold / 100);
if (!fetch)
{
guint i;
for (i = 0; i < files->len; i++)
{
g_hash_table_insert (loose_files, files->pdata[i], files->pdata[i]);
files->pdata[i] = NULL; /* steal data */
}
g_hash_table_iter_remove (&hash_iter);
}
}
if (n_objects_to_fetch > 0)
g_print ("%u content objects to fetch\n", n_objects_to_fetch);
if (g_hash_table_size (data_packs_to_fetch) > 0)
g_print ("Fetching %u content packs\n",
g_hash_table_size (data_packs_to_fetch));
g_hash_table_iter_init (&hash_iter, data_packs_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *pack_checksum = key;
GPtrArray *file_checksums = value;
guint i;
ot_lobj GFile *pack_path = NULL;
if (!fetch_one_pack_file (pull_data, pack_checksum, FALSE,
&pack_path, cancellable, error))
goto out;
g_print ("Storing %u objects from content pack %s\n", file_checksums->len,
pack_checksum);
for (i = 0; i < file_checksums->len; i++)
{
const char *checksum = file_checksums->pdata[i];
if (!store_file_from_pack (pull_data, checksum, pack_checksum, pack_path,
cancellable, error))
goto out;
}
if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, FALSE, NULL,
cancellable, error))
goto out;
}
if (g_hash_table_size (loose_files) > 0)
g_print ("Fetching %u loose objects\n",
g_hash_table_size (loose_files));
@ -1729,10 +1241,6 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
g_print ("remote %s is now %s\n", remote_ref, checksum);
}
if (!ostree_repo_clean_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
cancellable, error))
goto out;
bytes_transferred = ostree_fetcher_bytes_transferred (pull_data->fetcher);
if (bytes_transferred > 0)
{
@ -1751,8 +1259,6 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
if (pull_data->base_uri)
soup_uri_free (pull_data->base_uri);
g_clear_pointer (&pull_data->file_checksums_to_fetch, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->cached_meta_pack_indexes, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->cached_data_pack_indexes, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref);
if (summary_uri)
soup_uri_free (summary_uri);

126
src/ostree/ot-builtin-fsck.c
View File

@ -39,129 +39,8 @@ static GOptionEntry options[] = {
typedef struct {
OstreeRepo *repo;
guint n_pack_files;
} OtFsckData;
static gboolean
fsck_one_pack_file (OtFsckData *data,
const char *pack_checksum,
gboolean is_meta,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guchar objtype_u8;
guint64 offset;
guint64 pack_size;
ot_lfree char *path = NULL;
ot_lobj GFileInfo *pack_info = NULL;
ot_lobj GInputStream *input = NULL;
ot_lvariant GVariant *index_variant = NULL;
ot_lobj GFile *pack_index_path = NULL;
ot_lobj GFile *pack_data_path = NULL;
ot_lfree guchar *pack_content_csum = NULL;
ot_lfree char *tmp_checksum = NULL;
GVariantIter *index_content_iter = NULL;
g_free (path);
path = ostree_get_relative_pack_index_path (is_meta, pack_checksum);
pack_index_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path);
if (!ot_util_variant_map (pack_index_path,
OSTREE_PACK_INDEX_VARIANT_FORMAT, FALSE,
&index_variant, error))
goto out;
if (!ostree_validate_structureof_pack_index (index_variant, error))
goto out;
g_free (path);
path = ostree_get_relative_pack_data_path (is_meta, pack_checksum);
pack_data_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path);
input = (GInputStream*)g_file_read (pack_data_path, cancellable, error);
if (!input)
goto out;
pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO,
cancellable, error);
if (!pack_info)
goto out;
pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size");
if (!ot_gio_checksum_stream (input, &pack_content_csum, cancellable, error))
goto out;
tmp_checksum = ostree_checksum_from_bytes (pack_content_csum);
if (strcmp (tmp_checksum, pack_checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', actual checksum is %s",
pack_checksum, tmp_checksum);
goto out;
}
g_variant_get_child (index_variant, 2, "a(yayt)", &index_content_iter);
while (g_variant_iter_loop (index_content_iter, "(y@ayt)",
&objtype_u8, NULL, &offset))
{
offset = GUINT64_FROM_BE (offset);
if (offset > pack_size)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT,
pack_checksum,
offset, pack_size);
goto out;
}
}
ret = TRUE;
out:
if (index_content_iter)
g_variant_iter_free (index_content_iter);
return ret;
}
static gboolean
fsck_pack_files (OtFsckData *data,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
ot_lptrarray GPtrArray *meta_pack_indexes = NULL;
ot_lptrarray GPtrArray *data_pack_indexes = NULL;
if (!ostree_repo_list_pack_indexes (data->repo, &meta_pack_indexes, &data_pack_indexes,
cancellable, error))
goto out;
for (i = 0; i < meta_pack_indexes->len; i++)
{
const char *pack_checksum = meta_pack_indexes->pdata[i];
if (!fsck_one_pack_file (data, pack_checksum, TRUE, cancellable, error))
goto out;
data->n_pack_files++;
}
for (i = 0; i < data_pack_indexes->len; i++)
{
const char *pack_checksum = data_pack_indexes->pdata[i];
if (!fsck_one_pack_file (data, pack_checksum, FALSE, cancellable, error))
goto out;
data->n_pack_files++;
}
ret = TRUE;
out:
return ret;
}
static gboolean
fsck_reachable_objects_from_commits (OtFsckData *data,
GHashTable *commits,
@ -355,11 +234,6 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error)
if (!fsck_reachable_objects_from_commits (&data, commits, cancellable, error))
goto out;
g_print ("Verifying structure of pack files...\n");
if (!fsck_pack_files (&data, cancellable, error))
goto out;
ret = TRUE;
out:
if (context)

5
src/ostree/ot-builtin-init.c
View File

@ -43,7 +43,7 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
{
GOptionContext *context = NULL;
gboolean ret = FALSE;
GCancellable *cancellable = NULL;
__attribute__ ((unused)) GCancellable *cancellable = NULL;
ot_lobj GFile *child = NULL;
ot_lobj GFile *grandchild = NULL;
ot_lobj OstreeRepo *repo = NULL;
@ -110,9 +110,6 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_repo_check (repo, error))
goto out;
if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
goto out;
ret = TRUE;
out:
if (context)

967
src/ostree/ot-builtin-pack.c
View File

@ -1,967 +0,0 @@
/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
*
* Copyright (C) 2012 Colin Walters <walters@verbum.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* Author: Colin Walters <walters@verbum.org>
*/
#include "config.h"
#include "ot-builtins.h"
#include "ostree.h"
#include <glib/gi18n.h>
#include <glib/gprintf.h>
#include <gio/gunixinputstream.h>
#include <gio/gunixoutputstream.h>
#define OT_GZIP_COMPRESSION_LEVEL (8)
static gboolean opt_analyze_only;
static gboolean opt_metadata_only;
static gboolean opt_content_only;
static gboolean opt_reindex_only;
static gboolean opt_delete_all_loose;
static gboolean opt_keep_all_loose;
static char* opt_pack_size = "50m";
static char* opt_int_compression;
static char* opt_ext_compression;
typedef enum {
OT_COMPRESSION_NONE,
OT_COMPRESSION_GZIP,
OT_COMPRESSION_XZ
} OtCompressionType;
static GOptionEntry options[] = {
{ "pack-size", 0, 0, G_OPTION_ARG_STRING, &opt_pack_size, "Maximum uncompressed size of packfiles in bytes; may be suffixed with k, m, or g (default: 50m)", "BYTES" },
{ "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION", "COMPRESSION" },
{ "external-compression", 0, 0, G_OPTION_ARG_STRING, &opt_ext_compression, "Compress entire packfiles using COMPRESSION", "COMPRESSION" },
{ "metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_metadata_only, "Only pack metadata objects", NULL },
{ "content-only", 0, 0, G_OPTION_ARG_NONE, &opt_content_only, "Only pack content objects", NULL },
{ "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL },
{ "reindex-only", 0, 0, G_OPTION_ARG_NONE, &opt_reindex_only, "Regenerate pack index", NULL },
{ "delete-all-loose", 0, 0, G_OPTION_ARG_NONE, &opt_delete_all_loose, "Delete all loose objects (default: delete unreferenced loose)", NULL },
{ "keep-all-loose", 0, 0, G_OPTION_ARG_NONE, &opt_keep_all_loose, "Don't delete any loose objects (default: delete unreferenced loose)", NULL },
{ NULL }
};
typedef struct {
OstreeRepo *repo;
guint64 pack_size;
OtCompressionType int_compression;
OtCompressionType ext_compression;
gboolean had_error;
GError **error;
} OtRepackData;
typedef struct {
GOutputStream *out;
GPtrArray *compressor_argv;
GPid compress_child_pid;
} OtBuildRepackFile;
static gint
compare_object_data_by_size (gconstpointer ap,
gconstpointer bp)
{
GVariant *a = *(void **)ap;
GVariant *b = *(void **)bp;
guint64 a_size;
guint64 b_size;
g_variant_get_child (a, 2, "t", &a_size);
g_variant_get_child (b, 2, "t", &b_size);
if (a == b)
return 0;
else if (a > b)
return 1;
else
return -1;
}
static gboolean
write_bytes_update_checksum (GOutputStream *output,
gconstpointer bytes,
gsize len,
GChecksum *checksum,
guint64 *inout_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gsize bytes_written;
if (len > 0)
{
g_checksum_update (checksum, (guchar*) bytes, len);
if (!g_output_stream_write_all (output, bytes, len, &bytes_written,
cancellable, error))
goto out;
g_assert_cmpint (bytes_written, ==, len);
*inout_offset += bytes_written;
}
ret = TRUE;
out:
return ret;
}
static gboolean
write_padding (GOutputStream *output,
guint alignment,
GChecksum *checksum,
guint64 *inout_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint bits;
guint padding_len;
guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0};
if (alignment == 8)
bits = ((*inout_offset) & 7);
else
bits = ((*inout_offset) & 3);
if (bits > 0)
{
padding_len = alignment - bits;
if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding_len,
checksum, inout_offset, cancellable, error))
goto out;
}
ret = TRUE;
out:
return ret;
}
static gint
compare_index_content (gconstpointer ap,
gconstpointer bp)
{
gpointer a = *((gpointer*)ap);
gpointer b = *((gpointer*)bp);
GVariant *a_v = a;
GVariant *b_v = b;
guchar a_objtype;
guchar b_objtype;
guint64 a_offset;
guint64 b_offset;
int c;
ot_lvariant GVariant *a_csum_bytes = NULL;
ot_lvariant GVariant *b_csum_bytes = NULL;
g_variant_get (a_v, "(y@ayt)", &a_objtype, &a_csum_bytes, &a_offset);
g_variant_get (b_v, "(y@ayt)", &b_objtype, &b_csum_bytes, &b_offset);
c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (a_csum_bytes),
ostree_checksum_bytes_peek (b_csum_bytes));
if (c == 0)
{
if (a_objtype < b_objtype)
c = -1;
else if (a_objtype > b_objtype)
c = 1;
}
return c;
}
static gboolean
delete_loose_object (OtRepackData *data,
const char *checksum,
OstreeObjectType objtype,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gboolean do_delete = FALSE;
GError *temp_error = NULL;
ot_lobj GFile *object_path = NULL;
ot_lobj GFile *file_content_object_path = NULL;
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
if (objtype == OSTREE_OBJECT_TYPE_FILE)
{
ot_lobj GFileInfo *file_info = NULL;
if (ostree_repo_get_mode (data->repo) == OSTREE_REPO_MODE_BARE)
{
file_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, &temp_error);
}
else
{
ot_lobj GFile *content_object_path = NULL;
content_object_path = ostree_repo_get_archive_content_path (data->repo, checksum);
file_info = g_file_query_info (content_object_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, &temp_error);
}
if (!file_info)
{
if (!g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
{
g_propagate_error (error, temp_error);
goto out;
}
else
{
g_clear_error (&temp_error);
}
}
do_delete = opt_delete_all_loose
|| (file_info && g_file_info_get_attribute_uint32 (file_info, "unix::nlink") <= 1);
}
else
do_delete = TRUE;
if (do_delete)
{
if (!ot_gfile_unlink (object_path, cancellable, error))
{
g_prefix_error (error, "Failed to delete loose object '%s'",
ot_gfile_get_path_cached (object_path));
goto out;
}
if (objtype == OSTREE_OBJECT_TYPE_FILE
&& ostree_repo_get_mode (data->repo) == OSTREE_REPO_MODE_ARCHIVE)
{
ot_lobj GFile *content_object_path = NULL;
content_object_path = ostree_repo_get_archive_content_path (data->repo, checksum);
/* Ignoring errors for now; later should only be trying to
* delete files with content.
*/
(void) ot_gfile_unlink (content_object_path, NULL, NULL);
}
}
ret = TRUE;
out:
return ret;
}
static gboolean
pack_one_meta_object (OtRepackData *data,
const char *checksum,
OstreeObjectType objtype,
GVariant **out_packed_object,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lobj GFile *object_path = NULL;
ot_lvariant GVariant *metadata_v = NULL;
ot_lvariant GVariant *ret_packed_object = NULL;
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
if (!ot_util_variant_map (object_path, ostree_metadata_variant_type (objtype),
TRUE, &metadata_v, error))
goto out;
ret_packed_object = g_variant_new ("(y@ayv)", (guchar) objtype,
ostree_checksum_to_bytes_v (checksum),
metadata_v);
ret = TRUE;
ot_transfer_out_value (out_packed_object, &ret_packed_object);
out:
return ret;
}
static gboolean
pack_one_data_object (OtRepackData *data,
const char *checksum,
OstreeObjectType objtype,
guint64 expected_objsize,
GVariant **out_packed_object,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guchar entry_flags = 0;
GInputStream *read_object_in; /* nofree */
ot_lobj GInputStream *input = NULL;
ot_lobj GFileInfo *file_info = NULL;
ot_lvariant GVariant *xattrs = NULL;
ot_lobj GMemoryOutputStream *object_data_stream = NULL;
ot_lobj GConverter *compressor = NULL;
ot_lobj GConverterInputStream *compressed_object_input = NULL;
ot_lvariant GVariant *file_header = NULL;
ot_lvariant GVariant *ret_packed_object = NULL;
switch (data->int_compression)
{
case OT_COMPRESSION_GZIP:
{
entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
break;
}
default:
{
g_assert_not_reached ();
}
}
if (!ostree_repo_load_file (data->repo, checksum, &input, &file_info, &xattrs,
cancellable, error))
goto out;
file_header = ostree_file_header_new (file_info, xattrs);
object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
if (input != NULL)
{
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL);
compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", compressor,
"base-stream", input,
"close-base-stream", TRUE,
NULL);
read_object_in = (GInputStream*)compressed_object_input;
}
else
{
read_object_in = (GInputStream*)input;
}
if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in,
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
cancellable, error))
goto out;
}
{
guchar *data = g_memory_output_stream_get_data (object_data_stream);
gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
ret_packed_object = g_variant_new ("(@ayy@(uuuusa(ayay))@ay)",
ostree_checksum_to_bytes_v (checksum),
entry_flags,
file_header,
ot_gvariant_new_bytearray (data, data_len));
}
ret = TRUE;
ot_transfer_out_value (out_packed_object, &ret_packed_object);
out:
return ret;
}
static gboolean
create_pack_file (OtRepackData *data,
gboolean is_meta,
GPtrArray *objects,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
guint64 offset;
gsize bytes_written;
ot_lobj GFile *pack_dir = NULL;
ot_lobj GFile *index_temppath = NULL;
ot_lobj GOutputStream *index_out = NULL;
ot_lobj GFile *pack_temppath = NULL;
ot_lobj GOutputStream *pack_out = NULL;
ot_lptrarray GPtrArray *index_content_list = NULL;
ot_lvariant GVariant *pack_header = NULL;
ot_lvariant GVariant *index_content = NULL;
ot_lfree char *pack_name = NULL;
ot_lobj GFile *pack_file_path = NULL;
ot_lobj GFile *pack_index_path = NULL;
GVariantBuilder index_content_builder;
GChecksum *pack_checksum = NULL;
if (g_cancellable_set_error_if_cancelled (cancellable, error))
return FALSE;
if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
"pack-index", NULL,
&index_temppath,
&index_out,
cancellable, error))
goto out;
if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
"pack-content", NULL,
&pack_temppath,
&pack_out,
cancellable, error))
goto out;
index_content_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
offset = 0;
pack_checksum = g_checksum_new (G_CHECKSUM_SHA256);
pack_header = g_variant_new ("(s@a{sv}t)",
is_meta ? "OSTv0PACKMETAFILE" : "OSTv0PACKDATAFILE",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
(guint64)objects->len);
if (!ostree_write_variant_with_size (pack_out, pack_header, offset, &bytes_written, pack_checksum,
cancellable, error))
goto out;
offset += bytes_written;
for (i = 0; i < objects->len; i++)
{
GVariant *object_data = objects->pdata[i];
const char *checksum;
guint32 objtype_u32;
OstreeObjectType objtype;
guint64 expected_objsize;
ot_lvariant GVariant *packed_object = NULL;
ot_lvariant GVariant *index_entry = NULL;
g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
objtype = (OstreeObjectType) objtype_u32;
if (is_meta)
{
if (!pack_one_meta_object (data, checksum, objtype, &packed_object,
cancellable, error))
goto out;
}
else
{
if (!pack_one_data_object (data, checksum, objtype, expected_objsize,
&packed_object, cancellable, error))
goto out;
}
if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
goto out;
/* offset points to aligned header size */
index_entry = g_variant_new ("(y@ayt)",
(guchar)objtype,
ostree_checksum_to_bytes_v (checksum),
GUINT64_TO_BE (offset));
g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry));
index_entry = NULL;
bytes_written = 0;
if (!ostree_write_variant_with_size (pack_out, packed_object, offset, &bytes_written,
pack_checksum, cancellable, error))
goto out;
offset += bytes_written;
}
if (!g_output_stream_close (pack_out, cancellable, error))
goto out;
g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(yayt)"));
g_ptr_array_sort (index_content_list, compare_index_content);
for (i = 0; i < index_content_list->len; i++)
{
GVariant *index_item = index_content_list->pdata[i];
g_variant_builder_add_value (&index_content_builder, index_item);
}
index_content = g_variant_new ("(s@a{sv}@a(yayt))",
"OSTv0PACKINDEX",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
g_variant_builder_end (&index_content_builder));
if (!g_output_stream_write_all (index_out,
g_variant_get_data (index_content),
g_variant_get_size (index_content),
&bytes_written,
cancellable,
error))
goto out;
if (!g_output_stream_close (index_out, cancellable, error))
goto out;
if (!ostree_repo_add_pack_file (data->repo,
g_checksum_get_string (pack_checksum),
is_meta,
index_temppath,
pack_temppath,
cancellable,
error))
goto out;
if (!ostree_repo_regenerate_pack_index (data->repo, cancellable, error))
goto out;
g_print ("Created %s pack file '%s' with %u objects\n", is_meta ? "metadata" : "content",
g_checksum_get_string (pack_checksum), objects->len);
if (!opt_keep_all_loose)
{
for (i = 0; i < objects->len; i++)
{
GVariant *object_data = objects->pdata[i];
const char *checksum;
guint32 objtype_u32;
OstreeObjectType objtype;
guint64 expected_objsize;
g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
objtype = (OstreeObjectType) objtype_u32;
if (!delete_loose_object (data, checksum, objtype, cancellable, error))
goto out;
}
}
ret = TRUE;
out:
if (index_temppath)
(void) unlink (ot_gfile_get_path_cached (index_temppath));
if (pack_temppath)
(void) unlink (ot_gfile_get_path_cached (pack_temppath));
if (pack_checksum)
g_checksum_free (pack_checksum);
return ret;
}
static void
cluster_one_object_chain (OtRepackData *data,
GPtrArray *object_list,
GPtrArray *inout_clusters)
{
guint i;
guint64 current_size;
guint current_offset;
current_size = 0;
current_offset = 0;
for (i = 0; i < object_list->len; i++)
{
GVariant *objdata = object_list->pdata[i];
guint64 objsize;
g_variant_get_child (objdata, 2, "t", &objsize);
if (current_size + objsize > data->pack_size || i == (object_list->len - 1))
{
guint j;
GPtrArray *current;
if (current_offset < i)
{
current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
for (j = current_offset; j <= i; j++)
{
g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
}
g_ptr_array_add (inout_clusters, current);
current_size = objsize;
current_offset = i+1;
}
}
else if (objsize > data->pack_size)
{
break;
}
else
{
current_size += objsize;
}
}
}
/**
* cluster_objects_stupidly:
* @objects: Map from serialized object name to objdata
* @out_meta_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
* @out_data_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
*
* Just sorts by size currently. Also filters out non-regular object
* content.
*/
static gboolean
cluster_objects_stupidly (OtRepackData *data,
GHashTable *objects,
GPtrArray **out_meta_clusters,
GPtrArray **out_data_clusters,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GHashTableIter hash_iter;
gpointer key, value;
ot_lptrarray GPtrArray *ret_meta_clusters = NULL;
ot_lptrarray GPtrArray *ret_data_clusters = NULL;
ot_lptrarray GPtrArray *meta_object_list = NULL;
ot_lptrarray GPtrArray *data_object_list = NULL;
meta_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
data_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
const char *checksum;
OstreeObjectType objtype;
guint64 size;
GVariant *v;
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInfo *object_info = NULL;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
{
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!object_info)
goto out;
}
else
{
if (!ostree_repo_load_file (data->repo, checksum, NULL, &object_info, NULL,
cancellable, error))
goto out;
}
size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
v = g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size));
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
g_ptr_array_add (meta_object_list, v);
else
g_ptr_array_add (data_object_list, v);
}
g_ptr_array_sort (meta_object_list, compare_object_data_by_size);
g_ptr_array_sort (data_object_list, compare_object_data_by_size);
ret_meta_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
ret_data_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
cluster_one_object_chain (data, meta_object_list, ret_meta_clusters);
cluster_one_object_chain (data, data_object_list, ret_data_clusters);
ret = TRUE;
ot_transfer_out_value (out_meta_clusters, &ret_meta_clusters);
ot_transfer_out_value (out_data_clusters, &ret_data_clusters);
out:
return ret;
}
static gboolean
parse_size_spec_with_suffix (const char *spec,
guint64 *out_size,
GError **error)
{
gboolean ret = FALSE;
char *endptr = NULL;
guint64 ret_size;
ret_size = g_ascii_strtoull (spec, &endptr, 10);
if (endptr && *endptr)
{
char suffix = *endptr;
switch (suffix)
{
case 'k':
case 'K':
{
ret_size *= 1024;
break;
}
case 'm':
case 'M':
{
ret_size *= (1024 * 1024);
break;
}
case 'g':
case 'G':
{
ret_size *= (1024 * 1024 * 1024);
break;
}
default:
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid size suffix '%c'", suffix);
goto out;
}
}
ret = TRUE;
*out_size = ret_size;
out:
return ret;
}
static gboolean
parse_compression_string (const char *compstr,
OtCompressionType *out_comptype,
GError **error)
{
gboolean ret = FALSE;
OtCompressionType ret_comptype;
if (compstr == NULL)
ret_comptype = OT_COMPRESSION_NONE;
else if (strcmp (compstr, "gzip") == 0)
ret_comptype = OT_COMPRESSION_GZIP;
else if (strcmp (compstr, "xz") == 0)
ret_comptype = OT_COMPRESSION_XZ;
else
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid compression '%s'", compstr);
goto out;
}
ret = TRUE;
*out_comptype = ret_comptype;
out:
return ret;
}
static gboolean
do_stats_gather_loose (OtRepackData *data,
GHashTable *objects,
GHashTable **out_loose,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint n_loose = 0;
guint n_loose_and_packed = 0;
guint n_packed = 0;
guint n_dup_packed = 0;
guint n_commits = 0;
guint n_dirmeta = 0;
guint n_dirtree = 0;
guint n_files = 0;
GHashTableIter hash_iter;
gpointer key, value;
ot_lhash GHashTable *ret_loose = NULL;
ret_loose = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify) g_variant_unref,
NULL);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
GVariant *objdata = value;
const char *checksum;
OstreeObjectType objtype;
gboolean is_loose;
gboolean is_packed;
ot_lvariant GVariant *pack_array = NULL;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_variant_get (objdata, "(b@as)", &is_loose, &pack_array);
is_packed = g_variant_n_children (pack_array) > 0;
if (is_loose && is_packed)
{
n_loose_and_packed++;
}
else if (is_loose)
{
if (!(opt_metadata_only && !OSTREE_OBJECT_TYPE_IS_META(objtype))
&& !(opt_content_only && OSTREE_OBJECT_TYPE_IS_META(objtype)))
{
GVariant *copy = g_variant_ref (serialized_key);
g_hash_table_replace (ret_loose, copy, copy);
}
n_loose++;
}
else if (g_variant_n_children (pack_array) > 1)
{
n_dup_packed++;
}
else
{
n_packed++;
}
switch (objtype)
{
case OSTREE_OBJECT_TYPE_COMMIT:
n_commits++;
break;
case OSTREE_OBJECT_TYPE_DIR_TREE:
n_dirtree++;
break;
case OSTREE_OBJECT_TYPE_DIR_META:
n_dirmeta++;
break;
case OSTREE_OBJECT_TYPE_FILE:
n_files++;
break;
default:
g_assert_not_reached ();
}
}
g_print ("Commits: %u\n", n_commits);
g_print ("Tree contents: %u\n", n_dirtree);
g_print ("Tree meta: %u\n", n_dirmeta);
g_print ("Files: %u\n", n_files);
g_print ("\n");
g_print ("Loose+packed objects: %u\n", n_loose_and_packed);
g_print ("Loose-only objects: %u\n", n_loose);
g_print ("Duplicate packed objects: %u\n", n_dup_packed);
g_print ("Packed-only objects: %u\n", n_packed);
ret = TRUE;
ot_transfer_out_value (out_loose, &ret_loose);
/* out: */
return ret;
}
static gboolean
do_incremental_pack (OtRepackData *data,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
ot_lhash GHashTable *objects = NULL;
ot_lptrarray GPtrArray *meta_clusters = NULL;
ot_lptrarray GPtrArray *data_clusters = NULL;
ot_lhash GHashTable *loose_objects = NULL;
if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
cancellable, error))
goto out;
if (!do_stats_gather_loose (data, objects, &loose_objects, cancellable, error))
goto out;
g_print ("\n");
g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size);
if (!cluster_objects_stupidly (data, loose_objects, &meta_clusters, &data_clusters,
cancellable, error))
goto out;
if (meta_clusters->len > 0 || data_clusters->len > 0)
g_print ("Going to create %u meta packfiles, %u data packfiles\n",
meta_clusters->len, data_clusters->len);
else
g_print ("Nothing to do\n");
if (!opt_analyze_only)
{
for (i = 0; i < meta_clusters->len; i++)
{
GPtrArray *cluster = meta_clusters->pdata[i];
if (!create_pack_file (data, TRUE, cluster, cancellable, error))
goto out;
}
for (i = 0; i < data_clusters->len; i++)
{
GPtrArray *cluster = data_clusters->pdata[i];
if (!create_pack_file (data, FALSE, cluster, cancellable, error))
goto out;
}
}
ret = TRUE;
out:
return ret;
}
gboolean
ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error)
{
gboolean ret = FALSE;
GOptionContext *context;
GCancellable *cancellable = NULL;
OtRepackData data;
ot_lobj OstreeRepo *repo = NULL;
memset (&data, 0, sizeof (data));
context = g_option_context_new ("- Recompress objects");
g_option_context_add_main_entries (context, options, NULL);
if (!g_option_context_parse (context, &argc, &argv, error))
goto out;
if (opt_metadata_only && opt_content_only)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"--content-only cannot be specified with --metadata-only");
goto out;
}
repo = ostree_repo_new (repo_path);
if (!ostree_repo_check (repo, error))
goto out;
data.repo = repo;
data.error = error;
if (!parse_size_spec_with_suffix (opt_pack_size, &data.pack_size, error))
goto out;
/* Default internal compression to gzip */
if (!parse_compression_string (opt_int_compression ? opt_int_compression : "gzip", &data.int_compression, error))
goto out;
if (!parse_compression_string (opt_ext_compression, &data.ext_compression, error))
goto out;
if (opt_reindex_only)
{
if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
goto out;
}
else
{
if (!do_incremental_pack (&data, cancellable, error))
goto out;
}
ret = TRUE;
out:
if (context)
g_option_context_free (context);
return ret;
}

4
src/ostree/ot-builtin-pull-local.c
View File

@ -67,7 +67,7 @@ import_one_object (OtLocalCloneData *data,
cancellable, error))
goto out;
if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum, FALSE,
if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum,
file_object, length,
cancellable, error))
goto out;
@ -81,7 +81,7 @@ import_one_object (OtLocalCloneData *data,
input = ot_variant_read (metadata);
if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype,
checksum, FALSE, input,
checksum, input,
cancellable, error))
goto out;
}

266
src/ostree/ot-builtin-unpack.c
View File

@ -1,266 +0,0 @@
/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
*
* Copyright (C) 2012 Colin Walters <walters@verbum.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* Author: Colin Walters <walters@verbum.org>
*/
#include "config.h"
#include "ot-builtins.h"
#include "ostree.h"
#include <glib/gi18n.h>
#include <glib/gprintf.h>
#include <gio/gunixinputstream.h>
#include <gio/gunixoutputstream.h>
static gboolean opt_keep_packs;
static GOptionEntry options[] = {
{ "keep-packs", 0, 0, G_OPTION_ARG_NONE, &opt_keep_packs, "Don't delete pack files", NULL },
{ NULL }
};
typedef struct {
OstreeRepo *repo;
} OtUnpackData;
static gboolean
unpack_one_object (OstreeRepo *repo,
const char *checksum,
OstreeObjectType objtype,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lobj GInputStream *input = NULL;
ot_lobj GFileInfo *file_info = NULL;
ot_lvariant GVariant *xattrs = NULL;
ot_lvariant GVariant *meta = NULL;
if (objtype == OSTREE_OBJECT_TYPE_FILE)
{
ot_lobj GInputStream *file_object = NULL;
guint64 length;
if (!ostree_repo_load_file (repo, checksum,
&input, &file_info, &xattrs,
cancellable, error))
goto out;
if (!ostree_raw_file_to_content_stream (input, file_info, xattrs, &file_object, &length,
cancellable, error))
goto out;
if (!ostree_repo_stage_file_object_trusted (repo, checksum, TRUE, file_object, length,
cancellable, error))
goto out;
}
else
{
if (!ostree_repo_load_variant (repo, objtype, checksum, &meta, error))
goto out;
input = ot_variant_read (meta);
if (!ostree_repo_stage_object_trusted (repo, objtype, checksum, TRUE,
input, cancellable, error))
goto out;
}
ret = TRUE;
out:
return ret;
}
static gboolean
delete_one_packfile (OstreeRepo *repo,
const char *pack_checksum,
gboolean is_meta,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lfree char *data_name = NULL;
ot_lobj GFile *data_path = NULL;
ot_lfree char *index_name = NULL;
ot_lobj GFile *index_path = NULL;
index_name = ostree_get_relative_pack_index_path (is_meta, pack_checksum);
index_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), index_name);
data_name = ostree_get_relative_pack_data_path (is_meta, pack_checksum);
data_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), data_name);
if (!ot_gfile_unlink (index_path, cancellable, error))
{
g_prefix_error (error, "Failed to delete pack index '%s': ", ot_gfile_get_path_cached (index_path));
goto out;
}
if (!ot_gfile_unlink (data_path, cancellable, error))
{
g_prefix_error (error, "Failed to delete pack data '%s': ", ot_gfile_get_path_cached (data_path));
goto out;
}
ret = TRUE;
out:
return ret;
}
gboolean
ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
{
gboolean ret = FALSE;
GOptionContext *context;
GCancellable *cancellable = NULL;
gboolean in_transaction = FALSE;
OtUnpackData data;
gpointer key, value;
guint64 unpacked_object_count = 0;
GHashTableIter hash_iter;
ot_lobj OstreeRepo *repo = NULL;
ot_lhash GHashTable *objects = NULL;
ot_lptrarray GPtrArray *clusters = NULL;
ot_lhash GHashTable *meta_packfiles_to_delete = NULL;
ot_lhash GHashTable *data_packfiles_to_delete = NULL;
ot_lobj GFile *objpath = NULL;
memset (&data, 0, sizeof (data));
context = g_option_context_new ("- Uncompress objects");
g_option_context_add_main_entries (context, options, NULL);
if (!g_option_context_parse (context, &argc, &argv, error))
goto out;
repo = ostree_repo_new (repo_path);
if (!ostree_repo_check (repo, error))
goto out;
if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"Can't unpack bare repositories yet");
goto out;
}
data.repo = repo;
if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error))
goto out;
if (!ostree_repo_prepare_transaction (repo, cancellable, error))
goto out;
in_transaction = TRUE;
meta_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
data_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *objkey = key;
GVariant *objdata;
const char *checksum;
const char *pack_checksum;
OstreeObjectType objtype;
gboolean is_loose;
gboolean is_packed = FALSE;
GVariantIter *pack_array_iter;
GHashTable *target_hash;
ostree_object_name_deserialize (objkey, &checksum, &objtype);
objdata = g_hash_table_lookup (objects, objkey);
g_assert (objdata);
g_variant_get (objdata, "(bas)", &is_loose, &pack_array_iter);
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
target_hash = meta_packfiles_to_delete;
else
target_hash = data_packfiles_to_delete;
while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum))
{
is_packed = TRUE;
if (!g_hash_table_lookup (target_hash, pack_checksum))
{
gchar *duped_checksum = g_strdup (pack_checksum);
g_hash_table_replace (target_hash, duped_checksum, duped_checksum);
}
}
g_variant_iter_free (pack_array_iter);
if (is_packed)
{
if (!unpack_one_object (repo, checksum, objtype, cancellable, error))
goto out;
unpacked_object_count++;
}
}
if (!ostree_repo_commit_transaction (repo, cancellable, error))
goto out;
if (!opt_keep_packs)
{
if (g_hash_table_size (meta_packfiles_to_delete) == 0
&& g_hash_table_size (data_packfiles_to_delete) == 0)
g_print ("No pack files; nothing to do\n");
g_hash_table_iter_init (&hash_iter, meta_packfiles_to_delete);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *pack_checksum = key;
if (!delete_one_packfile (repo, pack_checksum, TRUE, cancellable, error))
goto out;
g_print ("Deleted packfile '%s'\n", pack_checksum);
}
g_hash_table_iter_init (&hash_iter, data_packfiles_to_delete);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *pack_checksum = key;
if (!delete_one_packfile (repo, pack_checksum, FALSE, cancellable, error))
goto out;
g_print ("Deleted packfile '%s'\n", pack_checksum);
}
if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
goto out;
}
g_print ("Unpacked %" G_GUINT64_FORMAT " objects\n", unpacked_object_count);
ret = TRUE;
out:
if (in_transaction)
(void) ostree_repo_abort_transaction (repo, cancellable, NULL);
if (context)
g_option_context_free (context);
return ret;
}

2
src/ostree/ot-builtins.h
View File

@ -41,10 +41,8 @@ gboolean ostree_builtin_ls (int argc, char **argv, GFile *repo_path, GError **er
gboolean ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_show (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_rev_parse (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_remote (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_write_refs (int argc, char **argv, GFile *repo_path, GError **error);
G_END_DECLS

41
tests/t0001-archive.sh
View File

@ -21,7 +21,7 @@ set -e
. libtest.sh
echo '1..22'
echo '1..11'
setup_test_repository "archive"
echo "ok setup"
@ -68,45 +68,6 @@ $OSTREE cat test2 /baz/cow > cow-contents
assert_file_has_content cow-contents "moo"
echo "ok cat-file"
cd ${test_tmpdir}
$OSTREE pack
echo "ok pack"
cd ${test_tmpdir}
$OSTREE fsck
echo "ok fsck"
$OSTREE checkout test2 checkout-test2-from-packed
echo "ok checkout union 1"
cd ${test_tmpdir}
$OSTREE fsck
echo "ok fsck"
$OSTREE pack --analyze-only
echo "ok pack analyze"
$OSTREE unpack
echo "ok unpack"
cd ${test_tmpdir}
$OSTREE fsck
echo "ok fsck"
cd ${test_tmpdir}
$OSTREE checkout test2 checkout-test2-from-unpacked
echo "ok checkout union 2"
$OSTREE pack --metadata-only
echo "ok pack metadata"
$OSTREE fsck
echo "ok fsck"
cd ${test_tmpdir}
rm -rf checkout-test2
$OSTREE checkout test2 checkout-test2
echo "ok checkout metadata-packed"
$OSTREE unpack
echo "ok unpack"

20
tests/t0010-pull.sh
View File

@ -21,7 +21,7 @@ set -e
. libtest.sh
echo '1..4'
echo '1..2'
setup_fake_remote_repo1
cd ${test_tmpdir}
@ -38,21 +38,3 @@ cd checkout-origin-main
assert_file_has_content firstfile '^first$'
assert_file_has_content baz/cow '^moo$'
echo "ok pull contents"
cd ${test_tmpdir}
ostree --repo=$(pwd)/ostree-srv/gnomerepo pack
rm -rf repo
mkdir repo
${CMD_PREFIX} ostree --repo=repo init
${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo
${CMD_PREFIX} ostree-pull --repo=repo origin main
${CMD_PREFIX} ostree --repo=repo fsck
echo "ok pull packed"
cd ${test_tmpdir}
rm -rf checkout-origin-main
$OSTREE checkout origin/main checkout-origin-main
cd checkout-origin-main
assert_file_has_content firstfile '^first$'
assert_file_has_content baz/cow '^moo$'
echo "ok pull contents packed"