Static deltas support
https://bugzilla.gnome.org/show_bug.cgi?id=721799
This commit is contained in:
Colin Walters
Giuseppe Scrivano
parent
a23b3b332e
commit
ca678224be
|
|
@ -96,6 +96,10 @@ char *
|
|||
_ostree_get_relative_static_delta_path (const char *from,
|
||||
const char *to);
|
||||
|
||||
char *
|
||||
_ostree_get_relative_static_delta_detachedmeta_path (const char *from,
|
||||
const char *to);
|
||||
|
||||
char *
|
||||
_ostree_get_relative_static_delta_part_path (const char *from,
|
||||
const char *to,
|
||||
|
|
@ -114,5 +118,9 @@ _ostree_loose_path_with_suffix (char *buf,
|
|||
OstreeRepoMode repo_mode,
|
||||
const char *suffix);
|
||||
|
||||
GVariant *
|
||||
_ostree_detached_metadata_append_gpg_sig (GVariant *existing_metadata,
|
||||
GBytes *signature_bytes);
|
||||
|
||||
G_END_DECLS
|
||||
|
||||
|
|
|
|||
|
|
@ -1321,11 +1321,31 @@ _ostree_get_relative_object_path (const char *checksum,
|
|||
return g_string_free (path, FALSE);
|
||||
}
|
||||
|
||||
static char *
|
||||
get_delta_path (const char *from,
|
||||
const char *to,
|
||||
const char *target)
|
||||
{
|
||||
char prefix[3];
|
||||
prefix[0] = from[0];
|
||||
prefix[1] = from[1];
|
||||
prefix[2] = '\0';
|
||||
from += 2;
|
||||
return g_strconcat ("deltas/", prefix, "/", from, "-", to, "/", target, NULL);
|
||||
}
|
||||
|
||||
char *
|
||||
_ostree_get_relative_static_delta_path (const char *from,
|
||||
const char *to)
|
||||
{
|
||||
return g_strdup_printf ("deltas/%s-%s/meta", from, to);
|
||||
return get_delta_path (from, to, "superblock");
|
||||
}
|
||||
|
||||
char *
|
||||
_ostree_get_relative_static_delta_detachedmeta_path (const char *from,
|
||||
const char *to)
|
||||
{
|
||||
return get_delta_path (from, to, "meta");
|
||||
}
|
||||
|
||||
char *
|
||||
|
|
@ -1333,7 +1353,8 @@ _ostree_get_relative_static_delta_part_path (const char *from,
|
|||
const char *to,
|
||||
guint i)
|
||||
{
|
||||
return g_strdup_printf ("deltas/%s-%s/%u", from, to, i);
|
||||
gs_free char *partstr = g_strdup_printf ("%u", i);
|
||||
return get_delta_path (from, to, partstr);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1763,3 +1784,31 @@ ostree_commit_get_timestamp (GVariant *commit_variant)
|
|||
g_variant_get_child (commit_variant, 5, "t", &ret);
|
||||
return GUINT64_FROM_BE (ret);
|
||||
}
|
||||
|
||||
GVariant *
|
||||
_ostree_detached_metadata_append_gpg_sig (GVariant *existing_metadata,
|
||||
GBytes *signature_bytes)
|
||||
{
|
||||
GVariantBuilder *builder = NULL;
|
||||
gs_unref_variant GVariant *signaturedata = NULL;
|
||||
gs_unref_variant_builder GVariantBuilder *signature_builder = NULL;
|
||||
|
||||
if (existing_metadata)
|
||||
{
|
||||
builder = ot_util_variant_builder_from_variant (existing_metadata, G_VARIANT_TYPE ("a{sv}"));
|
||||
signaturedata = g_variant_lookup_value (existing_metadata, "ostree.gpgsigs", G_VARIANT_TYPE ("aay"));
|
||||
if (signaturedata)
|
||||
signature_builder = ot_util_variant_builder_from_variant (signaturedata, G_VARIANT_TYPE ("aay"));
|
||||
}
|
||||
if (!builder)
|
||||
builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
|
||||
if (!signature_builder)
|
||||
signature_builder = g_variant_builder_new (G_VARIANT_TYPE ("aay"));
|
||||
|
||||
g_variant_builder_add (signature_builder, "@ay", ot_gvariant_new_ay_bytes (signature_bytes));
|
||||
|
||||
g_variant_builder_add (builder, "{sv}", "ostree.gpgsigs", g_variant_builder_end (signature_builder));
|
||||
|
||||
return g_variant_ref_sink (g_variant_builder_end (builder));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -176,5 +176,14 @@ _ostree_repo_get_remote_boolean_option (OstreeRepo *self,
|
|||
gboolean *out_value,
|
||||
GError **error);
|
||||
|
||||
gboolean
|
||||
_ostree_repo_gpg_verify_file_with_metadata (OstreeRepo *self,
|
||||
GFile *path,
|
||||
GVariant *metadata,
|
||||
GFile *keyringdir,
|
||||
GFile *extra_keyring,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
|
||||
G_END_DECLS
|
||||
|
||||
|
|
|
|||
|
|
@ -63,8 +63,11 @@ typedef struct {
|
|||
guint n_outstanding_metadata_write_requests;
|
||||
guint n_outstanding_content_fetches;
|
||||
guint n_outstanding_content_write_requests;
|
||||
guint n_outstanding_deltapart_fetches;
|
||||
guint n_outstanding_deltapart_write_requests;
|
||||
gint n_requested_metadata;
|
||||
gint n_requested_content;
|
||||
guint n_fetched_deltaparts;
|
||||
guint n_fetched_metadata;
|
||||
guint n_fetched_content;
|
||||
|
||||
|
|
@ -88,6 +91,12 @@ typedef struct {
|
|||
gboolean is_detached_meta;
|
||||
} FetchObjectData;
|
||||
|
||||
typedef struct {
|
||||
OtPullData *pull_data;
|
||||
GVariant *objects;
|
||||
char *expected_checksum;
|
||||
} FetchStaticDeltaData;
|
||||
|
||||
static SoupURI *
|
||||
suburi_new (SoupURI *base,
|
||||
const char *first,
|
||||
|
|
@ -158,9 +167,11 @@ update_progress (gpointer user_data)
|
|||
return FALSE;
|
||||
|
||||
outstanding_writes = pull_data->n_outstanding_content_write_requests +
|
||||
pull_data->n_outstanding_metadata_write_requests;
|
||||
pull_data->n_outstanding_metadata_write_requests +
|
||||
pull_data->n_outstanding_deltapart_write_requests;
|
||||
outstanding_fetches = pull_data->n_outstanding_content_fetches +
|
||||
pull_data->n_outstanding_metadata_fetches;
|
||||
pull_data->n_outstanding_metadata_fetches +
|
||||
pull_data->n_outstanding_deltapart_fetches;
|
||||
bytes_transferred = _ostree_fetcher_bytes_transferred (pull_data->fetcher);
|
||||
fetched = pull_data->n_fetched_metadata + pull_data->n_fetched_content;
|
||||
requested = pull_data->n_requested_metadata + pull_data->n_requested_content;
|
||||
|
|
@ -211,9 +222,11 @@ check_outstanding_requests_handle_error (OtPullData *pull_data,
|
|||
GError *error)
|
||||
{
|
||||
gboolean current_fetch_idle = (pull_data->n_outstanding_metadata_fetches == 0 &&
|
||||
pull_data->n_outstanding_content_fetches == 0);
|
||||
pull_data->n_outstanding_content_fetches == 0 &&
|
||||
pull_data->n_outstanding_deltapart_fetches == 0);
|
||||
gboolean current_write_idle = (pull_data->n_outstanding_metadata_write_requests == 0 &&
|
||||
pull_data->n_outstanding_content_write_requests == 0);
|
||||
pull_data->n_outstanding_content_write_requests == 0 &&
|
||||
pull_data->n_outstanding_deltapart_write_requests == 0 );
|
||||
gboolean current_idle = current_fetch_idle && current_write_idle;
|
||||
|
||||
throw_async_error (pull_data, error);
|
||||
|
|
@ -729,7 +742,7 @@ meta_fetch_on_complete (GObject *object,
|
|||
g_assert (pull_data->n_outstanding_metadata_fetches > 0);
|
||||
pull_data->n_outstanding_metadata_fetches--;
|
||||
pull_data->n_fetched_metadata++;
|
||||
throw_async_error (pull_data, local_error);
|
||||
check_outstanding_requests_handle_error (pull_data, local_error);
|
||||
if (local_error)
|
||||
{
|
||||
g_variant_unref (fetch_data->object);
|
||||
|
|
@ -737,6 +750,104 @@ meta_fetch_on_complete (GObject *object,
|
|||
}
|
||||
}
|
||||
|
||||
static void
|
||||
fetch_static_delta_data_free (gpointer data)
|
||||
{
|
||||
FetchStaticDeltaData *fetch_data = data;
|
||||
g_free (fetch_data->expected_checksum);
|
||||
g_variant_unref (fetch_data->objects);
|
||||
g_free (fetch_data);
|
||||
}
|
||||
|
||||
static void
|
||||
on_static_delta_written (GObject *object,
|
||||
GAsyncResult *result,
|
||||
gpointer user_data)
|
||||
{
|
||||
FetchStaticDeltaData *fetch_data = user_data;
|
||||
OtPullData *pull_data = fetch_data->pull_data;
|
||||
GError *local_error = NULL;
|
||||
GError **error = &local_error;
|
||||
|
||||
g_debug ("execute static delta part %s complete", fetch_data->expected_checksum);
|
||||
|
||||
if (!_ostree_static_delta_part_execute_finish (pull_data->repo, result, error))
|
||||
goto out;
|
||||
|
||||
out:
|
||||
g_assert (pull_data->n_outstanding_deltapart_write_requests > 0);
|
||||
pull_data->n_outstanding_deltapart_write_requests--;
|
||||
check_outstanding_requests_handle_error (pull_data, local_error);
|
||||
/* Always free state */
|
||||
fetch_static_delta_data_free (fetch_data);
|
||||
}
|
||||
|
||||
static void
|
||||
static_deltapart_fetch_on_complete (GObject *object,
|
||||
GAsyncResult *result,
|
||||
gpointer user_data)
|
||||
{
|
||||
FetchStaticDeltaData *fetch_data = user_data;
|
||||
OtPullData *pull_data = fetch_data->pull_data;
|
||||
gs_unref_variant GVariant *metadata = NULL;
|
||||
gs_unref_object GFile *temp_path = NULL;
|
||||
gs_unref_object GInputStream *in = NULL;
|
||||
gs_free char *actual_checksum = NULL;
|
||||
gs_free guint8 *csum = NULL;
|
||||
GError *local_error = NULL;
|
||||
GError **error = &local_error;
|
||||
|
||||
g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
|
||||
|
||||
temp_path = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
|
||||
if (!temp_path)
|
||||
goto out;
|
||||
|
||||
in = (GInputStream*)g_file_read (temp_path, pull_data->cancellable, error);
|
||||
if (!in)
|
||||
goto out;
|
||||
|
||||
/* TODO - consider making async */
|
||||
if (!ot_gio_checksum_stream (in, &csum, pull_data->cancellable, error))
|
||||
goto out;
|
||||
|
||||
actual_checksum = ostree_checksum_from_bytes (csum);
|
||||
|
||||
if (strcmp (actual_checksum, fetch_data->expected_checksum) != 0)
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"Corrupted static delta part; checksum expected='%s' actual='%s'",
|
||||
fetch_data->expected_checksum, actual_checksum);
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Might as well close the fd here */
|
||||
(void) g_input_stream_close (in, NULL, NULL);
|
||||
|
||||
{
|
||||
gs_unref_bytes GBytes *delta_data
|
||||
= gs_file_map_readonly (temp_path, pull_data->cancellable, error);
|
||||
if (!delta_data)
|
||||
goto out;
|
||||
|
||||
_ostree_static_delta_part_execute_async (pull_data->repo,
|
||||
fetch_data->objects,
|
||||
delta_data,
|
||||
pull_data->cancellable,
|
||||
on_static_delta_written,
|
||||
fetch_data);
|
||||
pull_data->n_outstanding_deltapart_write_requests++;
|
||||
}
|
||||
|
||||
out:
|
||||
g_assert (pull_data->n_outstanding_deltapart_fetches > 0);
|
||||
pull_data->n_outstanding_deltapart_fetches--;
|
||||
pull_data->n_fetched_deltaparts++;
|
||||
check_outstanding_requests_handle_error (pull_data, local_error);
|
||||
if (local_error)
|
||||
fetch_static_delta_data_free (fetch_data);
|
||||
}
|
||||
|
||||
static gboolean
|
||||
scan_commit_object (OtPullData *pull_data,
|
||||
const char *checksum,
|
||||
|
|
@ -1076,58 +1187,280 @@ load_remote_repo_config (OtPullData *pull_data,
|
|||
return ret;
|
||||
}
|
||||
|
||||
#if 0
|
||||
static gboolean
|
||||
request_static_delta_meta_sync (OtPullData *pull_data,
|
||||
const char *ref,
|
||||
fetch_metadata_to_verify_delta_superblock (OtPullData *pull_data,
|
||||
const char *from_revision,
|
||||
const char *checksum,
|
||||
GVariant **out_delta_meta,
|
||||
GBytes *superblock_data,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_free char *from_revision = NULL;
|
||||
gs_free char *meta_path = _ostree_get_relative_static_delta_detachedmeta_path (from_revision, checksum);
|
||||
gs_unref_bytes GBytes *detached_meta_data = NULL;
|
||||
SoupURI *target_uri = NULL;
|
||||
gs_unref_variant GVariant *ret_delta_meta = NULL;
|
||||
gs_unref_object GFile *temp_input_path = NULL;
|
||||
gs_unref_object GOutputStream *temp_input_stream = NULL;
|
||||
gs_unref_object GInputStream *superblock_in = NULL;
|
||||
gs_unref_variant GVariant *metadata = NULL;
|
||||
|
||||
if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE, &from_revision, error))
|
||||
target_uri = suburi_new (pull_data->base_uri, meta_path, NULL);
|
||||
|
||||
if (!fetch_uri_contents_membuf_sync (pull_data, target_uri, FALSE, FALSE,
|
||||
&detached_meta_data,
|
||||
pull_data->cancellable, error))
|
||||
{
|
||||
g_prefix_error (error, "GPG verification enabled, but failed to fetch metadata: ");
|
||||
goto out;
|
||||
}
|
||||
|
||||
superblock_in = g_memory_input_stream_new_from_bytes (superblock_data);
|
||||
|
||||
if (!gs_file_open_in_tmpdir (pull_data->repo->tmp_dir, 0644,
|
||||
&temp_input_path, &temp_input_stream,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (from_revision == NULL)
|
||||
{
|
||||
initiate_commit_scan (pull_data, checksum);
|
||||
}
|
||||
else
|
||||
{
|
||||
gs_free char *delta_name = _ostree_get_relative_static_delta_path (from_revision, checksum);
|
||||
if (0 > g_output_stream_splice (temp_input_stream, superblock_in,
|
||||
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
|
||||
G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
metadata = ot_variant_new_from_bytes (G_VARIANT_TYPE ("a{sv}"),
|
||||
detached_meta_data,
|
||||
FALSE);
|
||||
|
||||
if (!_ostree_repo_gpg_verify_file_with_metadata (pull_data->repo, temp_input_path,
|
||||
metadata, NULL, NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
request_static_delta_superblock_sync (OtPullData *pull_data,
|
||||
const char *from_revision,
|
||||
const char *to_revision,
|
||||
GVariant **out_delta_superblock,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_variant GVariant *ret_delta_superblock = NULL;
|
||||
gs_free char *delta_name = _ostree_get_relative_static_delta_path (from_revision, to_revision);
|
||||
gs_unref_bytes GBytes *delta_superblock_data = NULL;
|
||||
gs_unref_bytes GBytes *delta_meta_data = NULL;
|
||||
gs_unref_variant GVariant *delta_meta = NULL;
|
||||
gs_unref_variant GVariant *delta_superblock = NULL;
|
||||
SoupURI *target_uri = NULL;
|
||||
|
||||
target_uri = suburi_new (pull_data->base_uri, delta_name, NULL);
|
||||
|
||||
if (!fetch_uri_contents_membuf_sync (pull_data, target_uri, FALSE, TRUE,
|
||||
&delta_meta_data,
|
||||
&delta_superblock_data,
|
||||
pull_data->cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (delta_meta_data)
|
||||
if (delta_superblock_data)
|
||||
{
|
||||
ret_delta_meta = ot_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_META_FORMAT,
|
||||
delta_meta_data, FALSE);
|
||||
if (pull_data->gpg_verify)
|
||||
{
|
||||
if (!fetch_metadata_to_verify_delta_superblock (pull_data,
|
||||
from_revision,
|
||||
to_revision,
|
||||
delta_superblock_data,
|
||||
pull_data->cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
|
||||
ret_delta_superblock = ot_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
|
||||
delta_superblock_data, FALSE);
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
gs_transfer_out_value (out_delta_superblock, &ret_delta_superblock);
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
process_one_static_delta_fallback (OtPullData *pull_data,
|
||||
GVariant *fallback_object,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_variant GVariant *csum_v = NULL;
|
||||
gs_free char *checksum = NULL;
|
||||
guint8 objtype_y;
|
||||
OstreeObjectType objtype;
|
||||
gboolean is_stored;
|
||||
guint64 compressed_size, uncompressed_size;
|
||||
|
||||
g_variant_get (fallback_object, "(y@aytt)",
|
||||
&objtype_y, &csum_v, &compressed_size, &uncompressed_size);
|
||||
if (!ostree_validate_structureof_objtype (objtype_y, error))
|
||||
goto out;
|
||||
if (!ostree_validate_structureof_csum_v (csum_v, error))
|
||||
goto out;
|
||||
|
||||
objtype = (OstreeObjectType)objtype_y;
|
||||
checksum = ostree_checksum_from_bytes_v (csum_v);
|
||||
|
||||
if (!ostree_repo_has_object (pull_data->repo, objtype, checksum,
|
||||
&is_stored,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!is_stored)
|
||||
{
|
||||
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
|
||||
{
|
||||
if (!g_hash_table_lookup (pull_data->requested_metadata, checksum))
|
||||
{
|
||||
gboolean do_fetch_detached;
|
||||
g_hash_table_insert (pull_data->requested_metadata, checksum, checksum);
|
||||
|
||||
do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
|
||||
enqueue_one_object_request (pull_data, checksum, objtype, do_fetch_detached);
|
||||
checksum = NULL; /* Transfer ownership */
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!g_hash_table_lookup (pull_data->requested_content, checksum))
|
||||
{
|
||||
g_hash_table_insert (pull_data->requested_content, checksum, checksum);
|
||||
enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, FALSE);
|
||||
checksum = NULL; /* Transfer ownership */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
gs_transfer_out_value (out_delta_meta, &ret_delta_meta);
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
static void
|
||||
process_one_static_delta_meta (OtPullData *pull_data,
|
||||
GVariant *delta_meta)
|
||||
static gboolean
|
||||
process_one_static_delta (OtPullData *pull_data,
|
||||
const char *from_revision,
|
||||
const char *to_revision,
|
||||
GVariant *delta_superblock,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_variant GVariant *headers = NULL;
|
||||
gs_unref_variant GVariant *fallback_objects = NULL;
|
||||
guint i, n;
|
||||
|
||||
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
|
||||
headers = g_variant_get_child_value (delta_superblock, 6);
|
||||
fallback_objects = g_variant_get_child_value (delta_superblock, 7);
|
||||
|
||||
/* First process the fallbacks */
|
||||
n = g_variant_n_children (fallback_objects);
|
||||
for (i = 0; i < n; i++)
|
||||
{
|
||||
gs_unref_variant GVariant *fallback_object =
|
||||
g_variant_get_child_value (fallback_objects, i);
|
||||
|
||||
if (!process_one_static_delta_fallback (pull_data,
|
||||
fallback_object,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Write the to-commit object */
|
||||
{
|
||||
gs_unref_variant GVariant *to_csum_v = NULL;
|
||||
gs_free char *to_checksum = NULL;
|
||||
gs_unref_variant GVariant *to_commit = NULL;
|
||||
gboolean have_to_commit;
|
||||
|
||||
to_csum_v = g_variant_get_child_value (delta_superblock, 3);
|
||||
if (!ostree_validate_structureof_csum_v (to_csum_v, error))
|
||||
goto out;
|
||||
to_checksum = ostree_checksum_from_bytes_v (to_csum_v);
|
||||
|
||||
if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
|
||||
&have_to_commit, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!have_to_commit)
|
||||
{
|
||||
FetchObjectData *fetch_data = g_new0 (FetchObjectData, 1);
|
||||
fetch_data->pull_data = pull_data;
|
||||
fetch_data->object = ostree_object_name_serialize (to_checksum, OSTREE_OBJECT_TYPE_COMMIT);
|
||||
fetch_data->is_detached_meta = FALSE;
|
||||
|
||||
to_commit = g_variant_get_child_value (delta_superblock, 4);
|
||||
|
||||
ostree_repo_write_metadata_async (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
|
||||
to_commit,
|
||||
pull_data->cancellable,
|
||||
on_metadata_writed, fetch_data);
|
||||
pull_data->n_outstanding_metadata_write_requests++;
|
||||
}
|
||||
}
|
||||
|
||||
n = g_variant_n_children (headers);
|
||||
for (i = 0; i < n; i++)
|
||||
{
|
||||
const guchar *csum;
|
||||
gs_unref_variant GVariant *header = NULL;
|
||||
gboolean have_all = FALSE;
|
||||
SoupURI *target_uri = NULL;
|
||||
gs_free char *deltapart_path = NULL;
|
||||
FetchStaticDeltaData *fetch_data;
|
||||
gs_unref_variant GVariant *csum_v = NULL;
|
||||
gs_unref_variant GVariant *objects = NULL;
|
||||
guint64 size, usize;
|
||||
|
||||
header = g_variant_get_child_value (headers, i);
|
||||
g_variant_get (header, "(@aytt@ay)", &csum_v, &size, &usize, &objects);
|
||||
|
||||
csum = ostree_checksum_bytes_peek_validate (csum_v, error);
|
||||
if (!csum)
|
||||
goto out;
|
||||
|
||||
if (!_ostree_repo_static_delta_part_have_all_objects (pull_data->repo,
|
||||
objects,
|
||||
&have_all,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (have_all)
|
||||
{
|
||||
g_debug ("Have all objects from static delta %s-%s part %u",
|
||||
from_revision, to_revision,
|
||||
i);
|
||||
continue;
|
||||
}
|
||||
|
||||
fetch_data = g_new0 (FetchStaticDeltaData, 1);
|
||||
fetch_data->pull_data = pull_data;
|
||||
fetch_data->objects = g_variant_ref (objects);
|
||||
fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
|
||||
|
||||
deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
|
||||
|
||||
target_uri = suburi_new (pull_data->base_uri, deltapart_path, NULL);
|
||||
_ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size,
|
||||
pull_data->cancellable,
|
||||
static_deltapart_fetch_on_complete,
|
||||
fetch_data);
|
||||
pull_data->n_outstanding_deltapart_fetches++;
|
||||
soup_uri_free (target_uri);
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* documented in ostree-repo.c */
|
||||
|
|
@ -1185,6 +1518,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
|||
gboolean tls_permissive = FALSE;
|
||||
OstreeFetcherConfigFlags fetcher_flags = 0;
|
||||
guint i;
|
||||
gs_free char *remote_key = NULL;
|
||||
gs_free char *path = NULL;
|
||||
gs_free char *baseurl = NULL;
|
||||
gs_free char *metalink_url_str = NULL;
|
||||
|
|
@ -1560,15 +1894,37 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
|||
g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
|
||||
while (g_hash_table_iter_next (&hash_iter, &key, &value))
|
||||
{
|
||||
const char *checksum = value;
|
||||
if (!scan_one_metadata_object (pull_data, checksum, OSTREE_OBJECT_TYPE_COMMIT,
|
||||
0, pull_data->cancellable, error))
|
||||
gs_free char *from_revision = NULL;
|
||||
const char *ref = key;
|
||||
const char *to_revision = value;
|
||||
GVariant *delta_superblock = NULL;
|
||||
|
||||
if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE,
|
||||
&from_revision, error))
|
||||
goto out;
|
||||
|
||||
if (from_revision && g_strcmp0 (from_revision, to_revision) != 0)
|
||||
{
|
||||
if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision,
|
||||
&delta_superblock, cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
|
||||
for (i = 0; i < pull_data->static_delta_metas->len; i++)
|
||||
if (!delta_superblock)
|
||||
{
|
||||
process_one_static_delta_meta (pull_data, pull_data->static_delta_metas->pdata[i]);
|
||||
g_debug ("no delta superblock for %s-%s", from_revision, to_revision);
|
||||
if (!scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT,
|
||||
0, pull_data->cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
else
|
||||
{
|
||||
g_debug ("processing delta superblock for %s-%s", from_revision, to_revision);
|
||||
if (!process_one_static_delta (pull_data, from_revision, to_revision,
|
||||
delta_superblock,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
idle_src = g_idle_source_new ();
|
||||
|
|
@ -1633,10 +1989,12 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
|||
else
|
||||
shift = 1024;
|
||||
|
||||
msg = g_strdup_printf ("%u metadata, %u content objects fetched; %" G_GUINT64_FORMAT " %s transferred in %u seconds",
|
||||
msg = g_strdup_printf ("%u metadata, %u content objects fetched; %" G_GUINT64_FORMAT " %s; %u delta parts fetched, "
|
||||
"transferred in %u seconds",
|
||||
pull_data->n_fetched_metadata, pull_data->n_fetched_content,
|
||||
(guint64)(bytes_transferred / shift),
|
||||
shift == 1 ? "B" : "KiB",
|
||||
pull_data->n_fetched_deltaparts,
|
||||
(guint) ((end_time - pull_data->start_time) / G_USEC_PER_SEC));
|
||||
ostree_async_progress_set_status (pull_data->progress, msg);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,10 +24,14 @@
|
|||
|
||||
#include "ostree-core-private.h"
|
||||
#include "ostree-repo-private.h"
|
||||
#include "ostree-lzma-compressor.h"
|
||||
#include "ostree-repo-static-delta-private.h"
|
||||
#include "ostree-diff.h"
|
||||
#include "otutil.h"
|
||||
#include "ostree-varint.h"
|
||||
#include "bupsplit.h"
|
||||
|
||||
#define ROLLSUM_BLOB_MAX (8192*4)
|
||||
|
||||
typedef struct {
|
||||
guint64 uncompressed_size;
|
||||
|
|
@ -38,6 +42,9 @@ typedef struct {
|
|||
|
||||
typedef struct {
|
||||
GPtrArray *parts;
|
||||
GPtrArray *fallback_objects;
|
||||
guint64 loose_compressed_size;
|
||||
guint64 max_usize_bytes;
|
||||
} OstreeStaticDeltaBuilder;
|
||||
|
||||
static void
|
||||
|
|
@ -70,7 +77,6 @@ objtype_checksum_array_new (GPtrArray *objects)
|
|||
guint i;
|
||||
GByteArray *ret = g_byte_array_new ();
|
||||
|
||||
g_assert (objects->len > 0);
|
||||
for (i = 0; i < objects->len; i++)
|
||||
{
|
||||
GVariant *serialized_key = objects->pdata[i];
|
||||
|
|
@ -91,97 +97,44 @@ objtype_checksum_array_new (GPtrArray *objects)
|
|||
}
|
||||
|
||||
static gboolean
|
||||
generate_delta_lowlatency (OstreeRepo *repo,
|
||||
const char *from,
|
||||
const char *to,
|
||||
process_one_object (OstreeRepo *repo,
|
||||
OstreeStaticDeltaBuilder *builder,
|
||||
OstreeStaticDeltaPartBuilder **current_part_val,
|
||||
const char *checksum,
|
||||
OstreeObjectType objtype,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
GHashTableIter hashiter;
|
||||
gpointer key, value;
|
||||
OstreeStaticDeltaPartBuilder *current_part = NULL;
|
||||
gs_unref_object GFile *root_from = NULL;
|
||||
gs_unref_object GFile *root_to = NULL;
|
||||
gs_unref_ptrarray GPtrArray *modified = NULL;
|
||||
gs_unref_ptrarray GPtrArray *removed = NULL;
|
||||
gs_unref_ptrarray GPtrArray *added = NULL;
|
||||
gs_unref_hashtable GHashTable *to_reachable_objects = NULL;
|
||||
gs_unref_hashtable GHashTable *from_reachable_objects = NULL;
|
||||
gs_unref_hashtable GHashTable *new_reachable_objects = NULL;
|
||||
|
||||
if (!ostree_repo_read_commit (repo, from, &root_from, NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
if (!ostree_repo_read_commit (repo, to, &root_to, NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
/* Gather a filesystem level diff; when we do heuristics to ship
|
||||
* just parts of changed files, we can make use of this data.
|
||||
*/
|
||||
modified = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_diff_item_unref);
|
||||
removed = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref);
|
||||
added = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref);
|
||||
if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE, root_from, root_to, modified, removed, added,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_traverse_commit (repo, from, -1, &from_reachable_objects,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_traverse_commit (repo, to, -1, &to_reachable_objects,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
new_reachable_objects = ostree_repo_traverse_new_reachable ();
|
||||
|
||||
g_hash_table_iter_init (&hashiter, to_reachable_objects);
|
||||
while (g_hash_table_iter_next (&hashiter, &key, &value))
|
||||
{
|
||||
GVariant *serialized_key = key;
|
||||
|
||||
if (g_hash_table_contains (from_reachable_objects, serialized_key))
|
||||
continue;
|
||||
|
||||
g_hash_table_insert (new_reachable_objects, g_variant_ref (serialized_key), serialized_key);
|
||||
}
|
||||
|
||||
current_part = allocate_part (builder);
|
||||
|
||||
g_hash_table_iter_init (&hashiter, new_reachable_objects);
|
||||
while (g_hash_table_iter_next (&hashiter, &key, &value))
|
||||
{
|
||||
GVariant *serialized_key = key;
|
||||
const char *checksum;
|
||||
OstreeObjectType objtype;
|
||||
guint64 content_size;
|
||||
gsize object_payload_start;
|
||||
gs_unref_object GInputStream *content_stream = NULL;
|
||||
gsize bytes_read;
|
||||
const guint readlen = 4096;
|
||||
|
||||
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
|
||||
guint64 compressed_size;
|
||||
OstreeStaticDeltaPartBuilder *current_part = *current_part_val;
|
||||
|
||||
if (!ostree_repo_load_object_stream (repo, objtype, checksum,
|
||||
&content_stream, &content_size,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
current_part->uncompressed_size += content_size;
|
||||
|
||||
/* Ensure we have at least one object per delta, even if a given
|
||||
* object is larger.
|
||||
*/
|
||||
/* Check to see if this delta is maximum size */
|
||||
if (current_part->objects->len > 0 &&
|
||||
current_part->payload->len + content_size > OSTREE_STATIC_DELTA_PART_MAX_SIZE_BYTES)
|
||||
current_part->payload->len + content_size > builder->max_usize_bytes)
|
||||
{
|
||||
current_part = allocate_part (builder);
|
||||
*current_part_val = current_part = allocate_part (builder);
|
||||
}
|
||||
|
||||
g_ptr_array_add (current_part->objects, g_variant_ref (serialized_key));
|
||||
if (!ostree_repo_query_object_storage_size (repo, objtype, checksum,
|
||||
&compressed_size,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
builder->loose_compressed_size += compressed_size;
|
||||
|
||||
current_part->uncompressed_size += content_size;
|
||||
|
||||
g_ptr_array_add (current_part->objects, ostree_object_name_serialize (checksum, objtype));
|
||||
|
||||
object_payload_start = current_part->payload->len;
|
||||
|
||||
|
|
@ -210,11 +163,179 @@ generate_delta_lowlatency (OstreeRepo *repo,
|
|||
current_part->payload->len += bytes_read;
|
||||
}
|
||||
|
||||
/* A little lame here to duplicate the content size - but if in the
|
||||
* future we do rsync-style rolling checksums, then we'll have
|
||||
* multiple write calls.
|
||||
*/
|
||||
_ostree_write_varuint64 (current_part->operations, content_size);
|
||||
g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_WRITE);
|
||||
_ostree_write_varuint64 (current_part->operations, object_payload_start);
|
||||
_ostree_write_varuint64 (current_part->operations, content_size);
|
||||
g_printerr ("write %" G_GUINT64_FORMAT " %" G_GUINT64_FORMAT "\n", (guint64) object_payload_start, (guint64)(content_size));
|
||||
g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_CLOSE);
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
generate_delta_lowlatency (OstreeRepo *repo,
|
||||
const char *from,
|
||||
const char *to,
|
||||
OstreeStaticDeltaBuilder *builder,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
GHashTableIter hashiter;
|
||||
gpointer key, value;
|
||||
guint i;
|
||||
OstreeStaticDeltaPartBuilder *current_part = NULL;
|
||||
gs_unref_object GFile *root_from = NULL;
|
||||
gs_unref_object GFile *root_to = NULL;
|
||||
gs_unref_ptrarray GPtrArray *modified = NULL;
|
||||
gs_unref_ptrarray GPtrArray *removed = NULL;
|
||||
gs_unref_ptrarray GPtrArray *added = NULL;
|
||||
gs_unref_hashtable GHashTable *to_reachable_objects = NULL;
|
||||
gs_unref_hashtable GHashTable *from_reachable_objects = NULL;
|
||||
gs_unref_hashtable GHashTable *new_reachable_metadata = NULL;
|
||||
gs_unref_hashtable GHashTable *new_reachable_content = NULL;
|
||||
gs_unref_hashtable GHashTable *modified_content_objects = NULL;
|
||||
gs_unref_hashtable GHashTable *content_object_to_size = NULL;
|
||||
|
||||
if (!ostree_repo_read_commit (repo, from, &root_from, NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
if (!ostree_repo_read_commit (repo, to, &root_to, NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
/* Gather a filesystem level diff; when we do heuristics to ship
|
||||
* just parts of changed files, we can make use of this data.
|
||||
*/
|
||||
modified = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_diff_item_unref);
|
||||
removed = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref);
|
||||
added = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref);
|
||||
if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE, root_from, root_to, modified, removed, added,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
modified_content_objects = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
|
||||
NULL,
|
||||
(GDestroyNotify) g_variant_unref);
|
||||
for (i = 0; i < modified->len; i++)
|
||||
{
|
||||
OstreeDiffItem *diffitem = modified->pdata[i];
|
||||
GVariant *objname = ostree_object_name_serialize (diffitem->target_checksum,
|
||||
OSTREE_OBJECT_TYPE_FILE);
|
||||
g_hash_table_add (modified_content_objects, objname);
|
||||
}
|
||||
|
||||
if (!ostree_repo_traverse_commit (repo, from, -1, &from_reachable_objects,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_traverse_commit (repo, to, -1, &to_reachable_objects,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
new_reachable_metadata = ostree_repo_traverse_new_reachable ();
|
||||
new_reachable_content = ostree_repo_traverse_new_reachable ();
|
||||
|
||||
g_hash_table_iter_init (&hashiter, to_reachable_objects);
|
||||
while (g_hash_table_iter_next (&hashiter, &key, &value))
|
||||
{
|
||||
GVariant *serialized_key = key;
|
||||
const char *checksum;
|
||||
OstreeObjectType objtype;
|
||||
|
||||
if (g_hash_table_contains (from_reachable_objects, serialized_key))
|
||||
continue;
|
||||
|
||||
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
|
||||
|
||||
g_variant_ref (serialized_key);
|
||||
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
|
||||
g_hash_table_add (new_reachable_metadata, serialized_key);
|
||||
else
|
||||
g_hash_table_add (new_reachable_content, serialized_key);
|
||||
}
|
||||
|
||||
g_printerr ("modified: %u removed: %u added: %u\n",
|
||||
modified->len, removed->len, added->len);
|
||||
g_printerr ("new reachable: metadata=%u content=%u\n",
|
||||
g_hash_table_size (new_reachable_metadata),
|
||||
g_hash_table_size (new_reachable_content));
|
||||
|
||||
/* We already ship the to commit in the superblock, don't ship it twice */
|
||||
g_hash_table_remove (new_reachable_metadata,
|
||||
ostree_object_name_serialize (to, OSTREE_OBJECT_TYPE_COMMIT));
|
||||
|
||||
/* Scan for large objects, so we can fall back to plain HTTP-based
|
||||
* fetch. In the future this should come after an rsync-style
|
||||
* rolling delta check for modified files.
|
||||
*/
|
||||
g_hash_table_iter_init (&hashiter, new_reachable_content);
|
||||
while (g_hash_table_iter_next (&hashiter, &key, &value))
|
||||
{
|
||||
GVariant *serialized_key = key;
|
||||
const char *checksum;
|
||||
OstreeObjectType objtype;
|
||||
guint64 uncompressed_size;
|
||||
gboolean fallback = FALSE;
|
||||
|
||||
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
|
||||
|
||||
if (!ostree_repo_load_object_stream (repo, objtype, checksum,
|
||||
NULL, &uncompressed_size,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
if (uncompressed_size > builder->max_usize_bytes)
|
||||
fallback = TRUE;
|
||||
|
||||
if (fallback)
|
||||
{
|
||||
gs_free char *size = g_format_size (uncompressed_size);
|
||||
g_printerr ("fallback for %s (%s)\n",
|
||||
ostree_object_to_string (checksum, objtype), size);
|
||||
g_ptr_array_add (builder->fallback_objects,
|
||||
g_variant_ref (serialized_key));
|
||||
g_hash_table_iter_remove (&hashiter);
|
||||
}
|
||||
}
|
||||
|
||||
current_part = allocate_part (builder);
|
||||
|
||||
/* Pack the metadata first */
|
||||
g_hash_table_iter_init (&hashiter, new_reachable_metadata);
|
||||
while (g_hash_table_iter_next (&hashiter, &key, &value))
|
||||
{
|
||||
GVariant *serialized_key = key;
|
||||
const char *checksum;
|
||||
OstreeObjectType objtype;
|
||||
|
||||
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
|
||||
|
||||
if (!process_one_object (repo, builder, ¤t_part,
|
||||
checksum, objtype,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Now content */
|
||||
g_hash_table_iter_init (&hashiter, new_reachable_content);
|
||||
while (g_hash_table_iter_next (&hashiter, &key, &value))
|
||||
{
|
||||
GVariant *serialized_key = key;
|
||||
const char *checksum;
|
||||
OstreeObjectType objtype;
|
||||
|
||||
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
|
||||
|
||||
if (!process_one_object (repo, builder, ¤t_part,
|
||||
checksum, objtype,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
|
|
@ -222,6 +343,71 @@ generate_delta_lowlatency (OstreeRepo *repo,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
get_fallback_headers (OstreeRepo *self,
|
||||
OstreeStaticDeltaBuilder *builder,
|
||||
GVariant **out_headers,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
guint i;
|
||||
gs_unref_variant GVariant *ret_headers = NULL;
|
||||
gs_unref_variant_builder GVariantBuilder *fallback_builder = NULL;
|
||||
|
||||
fallback_builder = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT));
|
||||
|
||||
for (i = 0; i < builder->fallback_objects->len; i++)
|
||||
{
|
||||
GVariant *serialized = builder->fallback_objects->pdata[i];
|
||||
const char *checksum;
|
||||
OstreeObjectType objtype;
|
||||
guint64 compressed_size;
|
||||
guint64 uncompressed_size;
|
||||
|
||||
ostree_object_name_deserialize (serialized, &checksum, &objtype);
|
||||
|
||||
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
|
||||
{
|
||||
if (!ostree_repo_load_object_stream (self, objtype, checksum,
|
||||
NULL, &uncompressed_size,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
compressed_size = uncompressed_size;
|
||||
}
|
||||
else
|
||||
{
|
||||
gs_unref_object GFileInfo *file_info = NULL;
|
||||
|
||||
if (!ostree_repo_query_object_storage_size (self, OSTREE_OBJECT_TYPE_FILE,
|
||||
checksum,
|
||||
&compressed_size,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_load_file (self, checksum,
|
||||
NULL, &file_info, NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
uncompressed_size = g_file_info_get_size (file_info);
|
||||
}
|
||||
|
||||
g_variant_builder_add_value (fallback_builder,
|
||||
g_variant_new ("(y@aytt)",
|
||||
objtype,
|
||||
ostree_checksum_to_bytes_v (checksum),
|
||||
compressed_size, uncompressed_size));
|
||||
}
|
||||
|
||||
ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder));
|
||||
|
||||
ret = TRUE;
|
||||
gs_transfer_out_value (out_headers, &ret_headers);
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* ostree_repo_static_delta_generate:
|
||||
* @self: Repo
|
||||
|
|
@ -229,6 +415,7 @@ generate_delta_lowlatency (OstreeRepo *repo,
|
|||
* @from: ASCII SHA256 checksum of origin
|
||||
* @to: ASCII SHA256 checksum of target
|
||||
* @metadata: (allow-none): Optional metadata
|
||||
* @params: (allow-none): Parameters, see below
|
||||
* @cancellable: Cancellable
|
||||
* @error: Error
|
||||
*
|
||||
|
|
@ -236,6 +423,11 @@ generate_delta_lowlatency (OstreeRepo *repo,
|
|||
* the objects in @to. This delta is an optimization over fetching
|
||||
* individual objects, and can be conveniently stored and applied
|
||||
* offline.
|
||||
*
|
||||
* The @params argument should be an a{sv}. The following attributes
|
||||
* are known:
|
||||
* - max-usize: u: Maximum size in megabytes of a delta part
|
||||
* - compression: y: Compression type: 0=none, x=lzma, g=gzip
|
||||
*/
|
||||
gboolean
|
||||
ostree_repo_static_delta_generate (OstreeRepo *self,
|
||||
|
|
@ -243,22 +435,37 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
|
|||
const char *from,
|
||||
const char *to,
|
||||
GVariant *metadata,
|
||||
GVariant *params,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
OstreeStaticDeltaBuilder builder = { 0, };
|
||||
guint i;
|
||||
guint max_usize;
|
||||
GVariant *metadata_source;
|
||||
guint64 total_compressed_size = 0;
|
||||
guint64 total_uncompressed_size = 0;
|
||||
gs_unref_variant_builder GVariantBuilder *part_headers = NULL;
|
||||
gs_unref_ptrarray GPtrArray *part_tempfiles = NULL;
|
||||
gs_unref_variant GVariant *delta_descriptor = NULL;
|
||||
gs_unref_variant GVariant *to_commit = NULL;
|
||||
gs_free char *descriptor_relpath = NULL;
|
||||
gs_unref_object GFile *descriptor_path = NULL;
|
||||
gs_unref_object GFile *descriptor_dir = NULL;
|
||||
gs_unref_variant GVariant *tmp_metadata = NULL;
|
||||
gs_unref_variant GVariant *fallback_headers = NULL;
|
||||
|
||||
builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
|
||||
builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
|
||||
|
||||
if (!g_variant_lookup (params, "max-usize", "u", &max_usize))
|
||||
max_usize = 32;
|
||||
builder.max_usize_bytes = ((guint64)max_usize) * 1000 * 1000;
|
||||
|
||||
if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, to,
|
||||
&to_commit, error))
|
||||
goto out;
|
||||
|
||||
/* Ignore optimization flags */
|
||||
if (!generate_delta_lowlatency (self, from, to, &builder,
|
||||
|
|
@ -282,10 +489,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
|
|||
gs_unref_object GInputStream *part_payload_in = NULL;
|
||||
gs_unref_object GMemoryOutputStream *part_payload_out = NULL;
|
||||
gs_unref_object GConverterOutputStream *part_payload_compressor = NULL;
|
||||
gs_unref_object GConverter *zlib_compressor = NULL;
|
||||
gs_unref_object GConverter *compressor = NULL;
|
||||
gs_unref_variant GVariant *delta_part_content = NULL;
|
||||
gs_unref_variant GVariant *delta_part = NULL;
|
||||
gs_unref_variant GVariant *delta_part_header = NULL;
|
||||
guint8 compression_type_char;
|
||||
|
||||
payload_b = g_string_free_to_bytes (part_builder->payload);
|
||||
part_builder->payload = NULL;
|
||||
|
|
@ -298,11 +506,12 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
|
|||
ot_gvariant_new_ay_bytes (operations_b));
|
||||
g_variant_ref_sink (delta_part_content);
|
||||
|
||||
/* Hardcode gzip for now */
|
||||
zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9);
|
||||
/* Hardcode xz for now */
|
||||
compressor = (GConverter*)_ostree_lzma_compressor_new (NULL);
|
||||
compression_type_char = 'x';
|
||||
part_payload_in = ot_variant_read (delta_part_content);
|
||||
part_payload_out = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
|
||||
part_payload_compressor = (GConverterOutputStream*)g_converter_output_stream_new ((GOutputStream*)part_payload_out, zlib_compressor);
|
||||
part_payload_compressor = (GConverterOutputStream*)g_converter_output_stream_new ((GOutputStream*)part_payload_out, compressor);
|
||||
|
||||
if (0 > g_output_stream_splice ((GOutputStream*)part_payload_compressor, part_payload_in,
|
||||
G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET | G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
|
||||
|
|
@ -311,7 +520,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
|
|||
|
||||
/* FIXME - avoid duplicating memory here */
|
||||
delta_part = g_variant_new ("(y@ay)",
|
||||
(guint8)'g',
|
||||
compression_type_char,
|
||||
ot_gvariant_new_ay_bytes (g_memory_output_stream_steal_as_bytes (part_payload_out)));
|
||||
|
||||
if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
|
||||
|
|
@ -333,6 +542,14 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
|
|||
ot_gvariant_new_ay_bytes (objtype_checksum_array));
|
||||
g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
|
||||
g_ptr_array_add (part_tempfiles, g_object_ref (part_tempfile));
|
||||
|
||||
total_compressed_size += g_variant_get_size (delta_part);
|
||||
total_uncompressed_size += part_builder->uncompressed_size;
|
||||
|
||||
g_printerr ("part %u n:%u compressed:%" G_GUINT64_FORMAT " uncompressed:%" G_GUINT64_FORMAT "\n",
|
||||
i, part_builder->objects->len,
|
||||
g_variant_get_size (delta_part),
|
||||
part_builder->uncompressed_size);
|
||||
}
|
||||
|
||||
descriptor_relpath = _ostree_get_relative_static_delta_path (from, to);
|
||||
|
|
@ -365,21 +582,42 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
|
|||
metadata_source = tmp_metadata;
|
||||
}
|
||||
|
||||
if (!get_fallback_headers (self, &builder, &fallback_headers,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
/* Generate OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
|
||||
{
|
||||
GDateTime *now = g_date_time_new_now_utc ();
|
||||
delta_descriptor = g_variant_new ("(@(a(ss)a(say))taya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT ")",
|
||||
/* floating */ GVariant *from_csum_v =
|
||||
ostree_checksum_to_bytes_v (from);
|
||||
/* floating */ GVariant *to_csum_v =
|
||||
ostree_checksum_to_bytes_v (to);
|
||||
delta_descriptor = g_variant_new ("(@(a(ss)a(say))t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay"
|
||||
"a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT
|
||||
"@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")",
|
||||
metadata_source,
|
||||
GUINT64_TO_BE (g_date_time_to_unix (now)),
|
||||
from_csum_v,
|
||||
to_csum_v,
|
||||
to_commit,
|
||||
g_variant_builder_new (G_VARIANT_TYPE ("ay")),
|
||||
part_headers);
|
||||
part_headers,
|
||||
fallback_headers);
|
||||
g_date_time_unref (now);
|
||||
}
|
||||
|
||||
g_printerr ("delta uncompressed=%" G_GUINT64_FORMAT " compressed=%" G_GUINT64_FORMAT " loose=%" G_GUINT64_FORMAT "\n",
|
||||
total_uncompressed_size,
|
||||
total_compressed_size,
|
||||
builder.loose_compressed_size);
|
||||
|
||||
if (!ot_util_variant_save (descriptor_path, delta_descriptor, cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
g_clear_pointer (&builder.parts, g_ptr_array_unref);
|
||||
g_clear_pointer (&builder.fallback_objects, g_ptr_array_unref);
|
||||
return ret;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ ostree_repo_list_static_delta_names (OstreeRepo *self,
|
|||
name = gs_file_get_basename_cached (child);
|
||||
|
||||
{
|
||||
gs_unref_object GFile *meta_path = g_file_get_child (child, "meta");
|
||||
gs_unref_object GFile *meta_path = g_file_get_child (child, "superblock");
|
||||
|
||||
if (g_file_query_exists (meta_path, NULL))
|
||||
{
|
||||
|
|
@ -115,8 +115,8 @@ ostree_repo_list_static_delta_names (OstreeRepo *self,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
have_all_objects (OstreeRepo *repo,
|
||||
gboolean
|
||||
_ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo,
|
||||
GVariant *checksum_array,
|
||||
gboolean *out_have_all,
|
||||
GCancellable *cancellable,
|
||||
|
|
@ -160,31 +160,6 @@ have_all_objects (OstreeRepo *repo,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
zlib_uncompress_data (GBytes *data,
|
||||
GBytes **out_uncompressed,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_object GMemoryInputStream *memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes (data);
|
||||
gs_unref_object GMemoryOutputStream *memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
|
||||
gs_unref_object GConverter *zlib_decomp =
|
||||
(GConverter*) g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
|
||||
gs_unref_object GInputStream *convin = g_converter_input_stream_new ((GInputStream*)memin, zlib_decomp);
|
||||
|
||||
if (0 > g_output_stream_splice ((GOutputStream*)memout, convin,
|
||||
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
|
||||
G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
*out_uncompressed = g_memory_output_stream_steal_as_bytes (memout);
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* ostree_repo_static_delta_execute_offline:
|
||||
* @self: Repo
|
||||
|
|
@ -196,7 +171,7 @@ zlib_uncompress_data (GBytes *data,
|
|||
* Given a directory representing an already-downloaded static delta
|
||||
* on disk, apply it, generating a new commit. The directory must be
|
||||
* named with the form "FROM-TO", where both are checksums, and it
|
||||
* must contain a file named "meta", along with at least one part.
|
||||
* must contain a file named "superblock", along with at least one part.
|
||||
*/
|
||||
gboolean
|
||||
ostree_repo_static_delta_execute_offline (OstreeRepo *self,
|
||||
|
|
@ -207,15 +182,52 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
|
|||
{
|
||||
gboolean ret = FALSE;
|
||||
guint i, n;
|
||||
gs_unref_object GFile *meta_file = g_file_get_child (dir, "meta");
|
||||
gs_unref_object GFile *meta_file = g_file_get_child (dir, "superblock");
|
||||
gs_unref_variant GVariant *meta = NULL;
|
||||
gs_unref_variant GVariant *headers = NULL;
|
||||
gs_unref_variant GVariant *fallback = NULL;
|
||||
|
||||
if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_META_FORMAT),
|
||||
if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
|
||||
FALSE, &meta, error))
|
||||
goto out;
|
||||
|
||||
headers = g_variant_get_child_value (meta, 3);
|
||||
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
|
||||
|
||||
/* Write the to-commit object */
|
||||
{
|
||||
gs_unref_variant GVariant *to_csum_v = NULL;
|
||||
gs_free char *to_checksum = NULL;
|
||||
gs_unref_variant GVariant *to_commit = NULL;
|
||||
gboolean have_to_commit;
|
||||
|
||||
to_csum_v = g_variant_get_child_value (meta, 3);
|
||||
if (!ostree_validate_structureof_csum_v (to_csum_v, error))
|
||||
goto out;
|
||||
to_checksum = ostree_checksum_from_bytes_v (to_csum_v);
|
||||
|
||||
if (!ostree_repo_has_object (self, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
|
||||
&have_to_commit, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!have_to_commit)
|
||||
{
|
||||
to_commit = g_variant_get_child_value (meta, 4);
|
||||
if (!ostree_repo_write_metadata (self, OSTREE_OBJECT_TYPE_COMMIT,
|
||||
to_checksum, to_commit, NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
fallback = g_variant_get_child_value (meta, 7);
|
||||
if (g_variant_n_children (fallback) > 0)
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"Cannot execute delta offline: contains nonempty http fallback entries");
|
||||
goto out;
|
||||
}
|
||||
|
||||
headers = g_variant_get_child_value (meta, 6);
|
||||
n = g_variant_n_children (headers);
|
||||
for (i = 0; i < n; i++)
|
||||
{
|
||||
|
|
@ -227,14 +239,14 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
|
|||
gs_unref_variant GVariant *csum_v = NULL;
|
||||
gs_unref_variant GVariant *objects = NULL;
|
||||
gs_unref_object GFile *part_path = NULL;
|
||||
gs_unref_variant GVariant *part = NULL;
|
||||
gs_unref_object GInputStream *raw_in = NULL;
|
||||
gs_unref_object GInputStream *in = NULL;
|
||||
|
||||
header = g_variant_get_child_value (headers, i);
|
||||
g_variant_get (header, "(@aytt@ay)", &csum_v, &size, &usize, &objects);
|
||||
|
||||
if (!have_all_objects (self, objects, &have_all, cancellable, error))
|
||||
if (!_ostree_repo_static_delta_part_have_all_objects (self, objects, &have_all,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
/* If we already have these objects, don't bother executing the
|
||||
|
|
@ -255,32 +267,16 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
|
|||
|
||||
if (!skip_validation)
|
||||
{
|
||||
gs_unref_object GInputStream *tmp_in = NULL;
|
||||
gs_free guchar *actual_checksum = NULL;
|
||||
|
||||
tmp_in = (GInputStream*)g_file_read (part_path, cancellable, error);
|
||||
if (!tmp_in)
|
||||
goto out;
|
||||
|
||||
if (!ot_gio_checksum_stream (tmp_in, &actual_checksum,
|
||||
gs_free char *expected_checksum = ostree_checksum_from_bytes (csum);
|
||||
if (!_ostree_static_delta_part_validate (self, part_path, i,
|
||||
expected_checksum,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (ostree_cmp_checksum_bytes (csum, actual_checksum) != 0)
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"Checksum mismatch in static delta %s part %u",
|
||||
gs_file_get_path_cached (dir), i);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error);
|
||||
gs_unref_bytes GBytes *bytes = NULL;
|
||||
gs_unref_bytes GBytes *payload = NULL;
|
||||
gsize partlen;
|
||||
const guint8*partdata;
|
||||
|
||||
if (!mfile)
|
||||
goto out;
|
||||
|
|
@ -288,37 +284,8 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
|
|||
bytes = g_mapped_file_get_bytes (mfile);
|
||||
g_mapped_file_unref (mfile);
|
||||
|
||||
partdata = g_bytes_get_data (bytes, &partlen);
|
||||
|
||||
if (partlen < 1)
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"Corrupted 0 length byte part %s/%i",
|
||||
gs_file_get_basename_cached (dir),
|
||||
i);
|
||||
goto out;
|
||||
}
|
||||
|
||||
switch (partdata[0])
|
||||
{
|
||||
case 0:
|
||||
payload = g_bytes_new_from_bytes (bytes, 1, partlen - 1);
|
||||
break;
|
||||
case 'g':
|
||||
{
|
||||
gs_unref_bytes GBytes *subbytes = g_bytes_new_from_bytes (bytes, 1, partlen - 1);
|
||||
if (!zlib_uncompress_data (subbytes, &payload,
|
||||
if (!_ostree_static_delta_part_execute (self, objects, bytes,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
part = ot_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT),
|
||||
payload, FALSE);
|
||||
|
||||
|
||||
if (!_ostree_static_delta_part_execute (self, objects, part, cancellable, error))
|
||||
{
|
||||
g_prefix_error (error, "executing delta part %i: ", i);
|
||||
goto out;
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ G_BEGIN_DECLS
|
|||
/**
|
||||
* OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT:
|
||||
*
|
||||
* y compression type (0: none, 'z': zlib)
|
||||
* ---
|
||||
* ay data source
|
||||
* ay operations
|
||||
*/
|
||||
|
|
@ -51,17 +53,34 @@ G_BEGIN_DECLS
|
|||
|
||||
#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(ayttay)"
|
||||
|
||||
|
||||
/**
|
||||
* OSTREE_STATIC_DELTA_META_FORMAT:
|
||||
* OSTREE_STATIC_DELTA_FALLBACK_FORMAT:
|
||||
*
|
||||
* y: objtype
|
||||
* ay: checksum
|
||||
* t: compressed size
|
||||
* t: uncompressed size
|
||||
*
|
||||
* Object to fetch invididually; includes compressed/uncompressed size.
|
||||
*/
|
||||
#define OSTREE_STATIC_DELTA_FALLBACK_FORMAT "(yaytt)"
|
||||
|
||||
/**
|
||||
* OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT:
|
||||
*
|
||||
* A .delta object is a custom binary format. It has the following high
|
||||
* level form:
|
||||
*
|
||||
* delta-descriptor:
|
||||
* metadata: a{sv}
|
||||
* timestamp: guint64
|
||||
* t: timestamp
|
||||
* from: ay checksum
|
||||
* to: ay checksum
|
||||
* commit: new commit object
|
||||
* ARRAY[(csum from, csum to)]: ay
|
||||
* ARRAY[delta-meta-entry]
|
||||
* array[fallback]
|
||||
*
|
||||
* The metadata would include things like a version number, as well as
|
||||
* extended verification data like a GPG signature.
|
||||
|
|
@ -70,22 +89,51 @@ G_BEGIN_DECLS
|
|||
* fetched and applied before this one. This is a fairly generic
|
||||
* recursion mechanism that would potentially allow saving significant
|
||||
* storage space on the server.
|
||||
*
|
||||
* The heart of the static delta: the array of delta parts.
|
||||
*
|
||||
* Finally, we have the fallback array, which is the set of objects to
|
||||
* fetch individually - the compiler determined it wasn't worth
|
||||
* duplicating the space.
|
||||
*/
|
||||
#define OSTREE_STATIC_DELTA_META_FORMAT "(a{sv}taya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT ")"
|
||||
#define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}tayay" OSTREE_COMMIT_GVARIANT_STRING "aya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")"
|
||||
|
||||
gboolean _ostree_static_delta_part_validate (OstreeRepo *repo,
|
||||
GFile *part_path,
|
||||
guint part_offset,
|
||||
const char *expected_checksum,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
|
||||
gboolean _ostree_static_delta_part_execute (OstreeRepo *repo,
|
||||
GVariant *header,
|
||||
GBytes *partdata,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
|
||||
gboolean _ostree_static_delta_part_execute_raw (OstreeRepo *repo,
|
||||
GVariant *header,
|
||||
GVariant *part,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
|
||||
void _ostree_static_delta_part_execute_async (OstreeRepo *repo,
|
||||
GVariant *header,
|
||||
GBytes *partdata,
|
||||
GCancellable *cancellable,
|
||||
GAsyncReadyCallback callback,
|
||||
gpointer user_data);
|
||||
|
||||
gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo,
|
||||
GAsyncResult *result,
|
||||
GError **error);
|
||||
|
||||
typedef enum {
|
||||
OSTREE_STATIC_DELTA_OP_FETCH = 1,
|
||||
OSTREE_STATIC_DELTA_OP_WRITE = 2,
|
||||
OSTREE_STATIC_DELTA_OP_GUNZIP = 3,
|
||||
OSTREE_STATIC_DELTA_OP_CLOSE = 4,
|
||||
OSTREE_STATIC_DELTA_OP_READOBJECT = 5,
|
||||
OSTREE_STATIC_DELTA_OP_READPAYLOAD = 6
|
||||
OSTREE_STATIC_DELTA_OP_WRITE = 1,
|
||||
OSTREE_STATIC_DELTA_OP_GUNZIP = 2,
|
||||
OSTREE_STATIC_DELTA_OP_CLOSE = 3,
|
||||
OSTREE_STATIC_DELTA_OP_READOBJECT = 4,
|
||||
OSTREE_STATIC_DELTA_OP_READPAYLOAD = 5
|
||||
} OstreeStaticDeltaOpCode;
|
||||
|
||||
gboolean
|
||||
|
|
@ -93,5 +141,12 @@ _ostree_static_delta_parse_checksum_array (GVariant *array,
|
|||
guint8 **out_checksums_array,
|
||||
guint *out_n_checksums,
|
||||
GError **error);
|
||||
|
||||
gboolean
|
||||
_ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo,
|
||||
GVariant *checksum_array,
|
||||
gboolean *out_have_all,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
G_END_DECLS
|
||||
|
||||
|
|
|
|||
|
|
@ -22,8 +22,13 @@
|
|||
|
||||
#include <string.h>
|
||||
|
||||
#include <glib-unix.h>
|
||||
#include <gio/gunixinputstream.h>
|
||||
#include <gio/gunixoutputstream.h>
|
||||
|
||||
#include "ostree-repo-private.h"
|
||||
#include "ostree-repo-static-delta-private.h"
|
||||
#include "ostree-lzma-decompressor.h"
|
||||
#include "otutil.h"
|
||||
#include "ostree-varint.h"
|
||||
|
||||
|
|
@ -31,6 +36,7 @@
|
|||
G_STATIC_ASSERT (sizeof (guint) >= sizeof (guint32));
|
||||
|
||||
typedef struct {
|
||||
OstreeRepo *repo;
|
||||
guint checksum_index;
|
||||
const guint8 *checksums;
|
||||
guint n_checksums;
|
||||
|
|
@ -38,6 +44,12 @@ typedef struct {
|
|||
const guint8 *opdata;
|
||||
guint oplen;
|
||||
|
||||
gboolean object_start;
|
||||
guint outstanding_content_writes;
|
||||
GMainContext *content_writing_context;
|
||||
gboolean caught_error;
|
||||
GError **async_error;
|
||||
|
||||
OstreeObjectType output_objtype;
|
||||
const guint8 *output_target;
|
||||
GFile *output_tmp_path;
|
||||
|
|
@ -48,6 +60,11 @@ typedef struct {
|
|||
guint64 payload_size;
|
||||
} StaticDeltaExecutionState;
|
||||
|
||||
typedef struct {
|
||||
StaticDeltaExecutionState *state;
|
||||
char checksum[65];
|
||||
} StaticDeltaContentWrite;
|
||||
|
||||
typedef gboolean (*DispatchOpFunc) (OstreeRepo *repo,
|
||||
StaticDeltaExecutionState *state,
|
||||
GCancellable *cancellable,
|
||||
|
|
@ -64,123 +81,22 @@ typedef struct {
|
|||
GCancellable *cancellable, \
|
||||
GError **error);
|
||||
|
||||
OPPROTO(fetch)
|
||||
OPPROTO(write)
|
||||
OPPROTO(gunzip)
|
||||
OPPROTO(close)
|
||||
#undef OPPROTO
|
||||
|
||||
static OstreeStaticDeltaOperation op_dispatch_table[] = {
|
||||
{ "fetch", dispatch_fetch },
|
||||
{ "write", dispatch_write },
|
||||
{ "gunzip", dispatch_gunzip },
|
||||
{ "close", dispatch_close },
|
||||
{ NULL }
|
||||
};
|
||||
|
||||
static gboolean
|
||||
open_output_target_csum (OstreeRepo *repo,
|
||||
StaticDeltaExecutionState *state,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
guint8 *objcsum;
|
||||
|
||||
g_assert (state->checksums != NULL);
|
||||
g_assert (state->output_target == NULL);
|
||||
g_assert (state->output_tmp_path == NULL);
|
||||
g_assert (state->output_tmp_stream == NULL);
|
||||
g_assert (state->checksum_index < state->n_checksums);
|
||||
|
||||
objcsum = (guint8*)state->checksums + (state->checksum_index * OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN);
|
||||
|
||||
if (G_UNLIKELY(!ostree_validate_structureof_objtype (*objcsum, error)))
|
||||
goto out;
|
||||
|
||||
state->output_objtype = (OstreeObjectType) *objcsum;
|
||||
state->output_target = objcsum + 1;
|
||||
if (!gs_file_open_in_tmpdir (repo->tmp_dir, 0644,
|
||||
&state->output_tmp_path, &state->output_tmp_stream,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
gboolean
|
||||
_ostree_static_delta_part_execute (OstreeRepo *repo,
|
||||
GVariant *objects,
|
||||
GVariant *part,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
guint8 *checksums_data;
|
||||
gs_unref_variant GVariant *checksums = NULL;
|
||||
gs_unref_variant GVariant *payload = NULL;
|
||||
gs_unref_variant GVariant *ops = NULL;
|
||||
StaticDeltaExecutionState statedata = { 0, };
|
||||
StaticDeltaExecutionState *state = &statedata;
|
||||
guint n_executed = 0;
|
||||
|
||||
if (!_ostree_static_delta_parse_checksum_array (objects,
|
||||
&checksums_data,
|
||||
&state->n_checksums,
|
||||
error))
|
||||
goto out;
|
||||
|
||||
state->checksums = checksums_data;
|
||||
g_assert (state->n_checksums > 0);
|
||||
if (!open_output_target_csum (repo, state, cancellable, error))
|
||||
goto out;
|
||||
|
||||
g_variant_get (part, "(@ay@ay)", &payload, &ops);
|
||||
|
||||
state->payload_data = g_variant_get_data (payload);
|
||||
state->payload_size = g_variant_get_size (payload);
|
||||
|
||||
state->oplen = g_variant_n_children (ops);
|
||||
state->opdata = g_variant_get_data (ops);
|
||||
while (state->oplen > 0)
|
||||
{
|
||||
guint8 opcode = state->opdata[0];
|
||||
OstreeStaticDeltaOperation *op;
|
||||
|
||||
if (G_UNLIKELY (opcode == 0 || opcode >= G_N_ELEMENTS (op_dispatch_table)))
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
|
||||
"Out of range opcode %u at offset %u", opcode, n_executed);
|
||||
goto out;
|
||||
}
|
||||
op = &op_dispatch_table[opcode-1];
|
||||
g_printerr ("dispatch %u\n", opcode-1);
|
||||
state->oplen--;
|
||||
state->opdata++;
|
||||
if (!op->func (repo, state, cancellable, error))
|
||||
goto out;
|
||||
|
||||
n_executed++;
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
dispatch_fetch (OstreeRepo *repo,
|
||||
StaticDeltaExecutionState *state,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
|
||||
"Static delta fetch opcode is not implemented in this version");
|
||||
return FALSE;
|
||||
}
|
||||
static void
|
||||
on_content_written (GObject *src,
|
||||
GAsyncResult *result,
|
||||
gpointer user_data);
|
||||
|
||||
static gboolean
|
||||
read_varuint64 (StaticDeltaExecutionState *state,
|
||||
|
|
@ -199,6 +115,366 @@ read_varuint64 (StaticDeltaExecutionState *state,
|
|||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
open_output_target (StaticDeltaExecutionState *state,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
guint8 *objcsum;
|
||||
char checksum[65];
|
||||
guint64 object_size;
|
||||
gs_unref_object GInputStream *content_in_stream = NULL;
|
||||
|
||||
g_assert (state->checksums != NULL);
|
||||
g_assert (state->output_target == NULL);
|
||||
g_assert (state->output_tmp_path == NULL);
|
||||
g_assert (state->output_tmp_stream == NULL);
|
||||
g_assert (state->checksum_index < state->n_checksums);
|
||||
|
||||
objcsum = (guint8*)state->checksums + (state->checksum_index * OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN);
|
||||
|
||||
if (G_UNLIKELY(!ostree_validate_structureof_objtype (*objcsum, error)))
|
||||
goto out;
|
||||
|
||||
state->output_objtype = (OstreeObjectType) *objcsum;
|
||||
state->output_target = objcsum + 1;
|
||||
|
||||
ostree_checksum_inplace_from_bytes (state->output_target, checksum);
|
||||
|
||||
/* Object size is the first element of the opstream */
|
||||
if (!read_varuint64 (state, &object_size, error))
|
||||
goto out;
|
||||
|
||||
if (OSTREE_OBJECT_TYPE_IS_META (state->output_objtype))
|
||||
{
|
||||
if (!gs_file_open_in_tmpdir (state->repo->tmp_dir, 0644,
|
||||
&state->output_tmp_path, &state->output_tmp_stream,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
else
|
||||
{
|
||||
int pipefds[2];
|
||||
|
||||
if (!g_unix_open_pipe (pipefds, FD_CLOEXEC, error))
|
||||
goto out;
|
||||
|
||||
content_in_stream = g_unix_input_stream_new (pipefds[0], TRUE);
|
||||
state->output_tmp_stream = g_unix_output_stream_new (pipefds[1], TRUE);
|
||||
|
||||
if (!state->content_writing_context)
|
||||
state->content_writing_context = g_main_context_new();
|
||||
|
||||
g_main_context_push_thread_default (state->content_writing_context);
|
||||
|
||||
{
|
||||
StaticDeltaContentWrite *writedata = g_new0 (StaticDeltaContentWrite, 1);
|
||||
writedata->state = state;
|
||||
memcpy (writedata->checksum, checksum, sizeof (writedata->checksum));
|
||||
ostree_repo_write_content_async (state->repo, checksum,
|
||||
content_in_stream,
|
||||
object_size,
|
||||
cancellable,
|
||||
on_content_written,
|
||||
writedata);
|
||||
}
|
||||
state->outstanding_content_writes++;
|
||||
|
||||
g_main_context_pop_thread_default (state->content_writing_context);
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
gboolean
|
||||
_ostree_static_delta_part_validate (OstreeRepo *repo,
|
||||
GFile *part_path,
|
||||
guint part_offset,
|
||||
const char *expected_checksum,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_object GInputStream *tmp_in = NULL;
|
||||
gs_free guchar *actual_checksum_bytes = NULL;
|
||||
gs_free gchar *actual_checksum = NULL;
|
||||
|
||||
tmp_in = (GInputStream*)g_file_read (part_path, cancellable, error);
|
||||
if (!tmp_in)
|
||||
goto out;
|
||||
|
||||
if (!ot_gio_checksum_stream (tmp_in, &actual_checksum_bytes,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
actual_checksum = ostree_checksum_from_bytes (actual_checksum_bytes);
|
||||
if (strcmp (actual_checksum, expected_checksum) != 0)
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"Checksum mismatch in static delta part %u; expected=%s actual=%s",
|
||||
part_offset, expected_checksum, actual_checksum);
|
||||
goto out;
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
gboolean
|
||||
_ostree_static_delta_part_execute_raw (OstreeRepo *repo,
|
||||
GVariant *objects,
|
||||
GVariant *part,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
guint8 *checksums_data;
|
||||
gs_unref_variant GVariant *checksums = NULL;
|
||||
gs_unref_variant GVariant *payload = NULL;
|
||||
gs_unref_variant GVariant *ops = NULL;
|
||||
StaticDeltaExecutionState statedata = { 0, };
|
||||
StaticDeltaExecutionState *state = &statedata;
|
||||
guint n_executed = 0;
|
||||
|
||||
state->repo = repo;
|
||||
state->async_error = error;
|
||||
|
||||
if (!_ostree_static_delta_parse_checksum_array (objects,
|
||||
&checksums_data,
|
||||
&state->n_checksums,
|
||||
error))
|
||||
goto out;
|
||||
|
||||
state->checksums = checksums_data;
|
||||
g_assert (state->n_checksums > 0);
|
||||
|
||||
g_variant_get (part, "(@ay@ay)", &payload, &ops);
|
||||
|
||||
state->payload_data = g_variant_get_data (payload);
|
||||
state->payload_size = g_variant_get_size (payload);
|
||||
|
||||
state->oplen = g_variant_n_children (ops);
|
||||
state->opdata = g_variant_get_data (ops);
|
||||
state->object_start = TRUE;
|
||||
while (state->oplen > 0)
|
||||
{
|
||||
guint8 opcode;
|
||||
OstreeStaticDeltaOperation *op;
|
||||
|
||||
if (state->object_start)
|
||||
{
|
||||
if (!open_output_target (state, cancellable, error))
|
||||
goto out;
|
||||
state->object_start = FALSE;
|
||||
}
|
||||
|
||||
opcode = state->opdata[0];
|
||||
|
||||
if (G_UNLIKELY (opcode == 0 || opcode >= G_N_ELEMENTS (op_dispatch_table)))
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
|
||||
"Out of range opcode %u at offset %u", opcode, n_executed);
|
||||
goto out;
|
||||
}
|
||||
op = &op_dispatch_table[opcode-1];
|
||||
g_printerr ("dispatch %u\n", opcode-1);
|
||||
state->oplen--;
|
||||
state->opdata++;
|
||||
if (!op->func (repo, state, cancellable, error))
|
||||
goto out;
|
||||
|
||||
n_executed++;
|
||||
}
|
||||
|
||||
while (state->outstanding_content_writes > 0)
|
||||
g_main_context_iteration (state->content_writing_context, TRUE);
|
||||
|
||||
if (state->caught_error)
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
g_clear_pointer (&state->content_writing_context, g_main_context_unref);
|
||||
g_clear_object (&state->output_tmp_path);
|
||||
g_clear_object (&state->output_tmp_stream);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
decompress_all (GConverter *converter,
|
||||
GBytes *data,
|
||||
GBytes **out_uncompressed,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_object GMemoryInputStream *memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes (data);
|
||||
gs_unref_object GMemoryOutputStream *memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
|
||||
gs_unref_object GInputStream *convin = g_converter_input_stream_new ((GInputStream*)memin, converter);
|
||||
|
||||
if (0 > g_output_stream_splice ((GOutputStream*)memout, convin,
|
||||
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
|
||||
G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
*out_uncompressed = g_memory_output_stream_steal_as_bytes (memout);
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
gboolean
|
||||
_ostree_static_delta_part_execute (OstreeRepo *repo,
|
||||
GVariant *header,
|
||||
GBytes *part_bytes,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gsize partlen;
|
||||
const guint8*partdata;
|
||||
gs_unref_bytes GBytes *part_payload_bytes = NULL;
|
||||
gs_unref_bytes GBytes *payload_data = NULL;
|
||||
gs_unref_variant GVariant *payload = NULL;
|
||||
guint8 comptype;
|
||||
|
||||
partdata = g_bytes_get_data (part_bytes, &partlen);
|
||||
|
||||
if (partlen < 1)
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"Corrupted 0 length delta part");
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* First byte is compression type */
|
||||
comptype = partdata[0];
|
||||
/* Then the rest may be compressed or uncompressed */
|
||||
part_payload_bytes = g_bytes_new_from_bytes (part_bytes, 1, partlen - 1);
|
||||
switch (comptype)
|
||||
{
|
||||
case 0:
|
||||
/* No compression */
|
||||
payload_data = g_bytes_ref (part_payload_bytes);
|
||||
break;
|
||||
case 'g':
|
||||
{
|
||||
gs_unref_object GConverter *decomp =
|
||||
(GConverter*) g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
|
||||
|
||||
if (!decompress_all (decomp, part_payload_bytes, &payload_data,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
break;
|
||||
case 'x':
|
||||
{
|
||||
gs_unref_object GConverter *decomp =
|
||||
(GConverter*) _ostree_lzma_decompressor_new ();
|
||||
|
||||
if (!decompress_all (decomp, part_payload_bytes, &payload_data,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"Invalid compression type '%u'", comptype);
|
||||
goto out;
|
||||
}
|
||||
|
||||
payload = ot_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT),
|
||||
payload_data, FALSE);
|
||||
if (!_ostree_static_delta_part_execute_raw (repo, header, payload,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
OstreeRepo *repo;
|
||||
GVariant *header;
|
||||
GBytes *partdata;
|
||||
GCancellable *cancellable;
|
||||
GSimpleAsyncResult *result;
|
||||
} StaticDeltaPartExecuteAsyncData;
|
||||
|
||||
static void
|
||||
static_delta_part_execute_async_data_free (gpointer user_data)
|
||||
{
|
||||
StaticDeltaPartExecuteAsyncData *data = user_data;
|
||||
|
||||
g_clear_object (&data->repo);
|
||||
g_variant_unref (data->header);
|
||||
g_bytes_unref (data->partdata);
|
||||
g_clear_object (&data->cancellable);
|
||||
g_free (data);
|
||||
}
|
||||
|
||||
static void
|
||||
static_delta_part_execute_thread (GSimpleAsyncResult *res,
|
||||
GObject *object,
|
||||
GCancellable *cancellable)
|
||||
{
|
||||
GError *error = NULL;
|
||||
StaticDeltaPartExecuteAsyncData *data;
|
||||
|
||||
data = g_simple_async_result_get_op_res_gpointer (res);
|
||||
if (!_ostree_static_delta_part_execute (data->repo,
|
||||
data->header,
|
||||
data->partdata,
|
||||
cancellable, &error))
|
||||
g_simple_async_result_take_error (res, error);
|
||||
}
|
||||
|
||||
void
|
||||
_ostree_static_delta_part_execute_async (OstreeRepo *repo,
|
||||
GVariant *header,
|
||||
GBytes *partdata,
|
||||
GCancellable *cancellable,
|
||||
GAsyncReadyCallback callback,
|
||||
gpointer user_data)
|
||||
{
|
||||
StaticDeltaPartExecuteAsyncData *asyncdata;
|
||||
|
||||
asyncdata = g_new0 (StaticDeltaPartExecuteAsyncData, 1);
|
||||
asyncdata->repo = g_object_ref (repo);
|
||||
asyncdata->header = g_variant_ref (header);
|
||||
asyncdata->partdata = g_bytes_ref (partdata);
|
||||
asyncdata->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
|
||||
|
||||
asyncdata->result = g_simple_async_result_new ((GObject*) repo,
|
||||
callback, user_data,
|
||||
_ostree_static_delta_part_execute_async);
|
||||
|
||||
g_simple_async_result_set_op_res_gpointer (asyncdata->result, asyncdata,
|
||||
static_delta_part_execute_async_data_free);
|
||||
g_simple_async_result_run_in_thread (asyncdata->result, static_delta_part_execute_thread, G_PRIORITY_DEFAULT, cancellable);
|
||||
g_object_unref (asyncdata->result);
|
||||
}
|
||||
|
||||
gboolean
|
||||
_ostree_static_delta_part_execute_finish (OstreeRepo *repo,
|
||||
GAsyncResult *result,
|
||||
GError **error)
|
||||
{
|
||||
GSimpleAsyncResult *simple = G_SIMPLE_ASYNC_RESULT (result);
|
||||
|
||||
g_warn_if_fail (g_simple_async_result_get_source_tag (simple) == _ostree_static_delta_part_execute_async);
|
||||
|
||||
if (g_simple_async_result_propagate_error (simple, error))
|
||||
return FALSE;
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
validate_ofs (StaticDeltaExecutionState *state,
|
||||
guint64 offset,
|
||||
|
|
@ -216,6 +492,40 @@ validate_ofs (StaticDeltaExecutionState *state,
|
|||
return TRUE;
|
||||
}
|
||||
|
||||
static void
|
||||
on_content_written (GObject *src,
|
||||
GAsyncResult *result,
|
||||
gpointer user_data)
|
||||
{
|
||||
StaticDeltaContentWrite *writedata = user_data;
|
||||
StaticDeltaExecutionState *state = writedata->state;
|
||||
GError *local_error = NULL;
|
||||
GError **error = &local_error;
|
||||
|
||||
if (!ostree_repo_write_content_finish ((OstreeRepo*)src, result, NULL, error))
|
||||
goto out;
|
||||
|
||||
g_print ("Wrote content object '%s'\n", writedata->checksum);
|
||||
|
||||
out:
|
||||
state->outstanding_content_writes--;
|
||||
if (state->outstanding_content_writes == 0)
|
||||
g_main_context_wakeup (state->content_writing_context);
|
||||
if (local_error)
|
||||
{
|
||||
if (!state->caught_error)
|
||||
{
|
||||
state->caught_error = TRUE;
|
||||
g_main_context_wakeup (state->content_writing_context);
|
||||
g_propagate_error (state->async_error, local_error);
|
||||
}
|
||||
else
|
||||
{
|
||||
g_error_free (local_error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static gboolean
|
||||
dispatch_write (OstreeRepo *repo,
|
||||
StaticDeltaExecutionState *state,
|
||||
|
|
@ -315,7 +625,6 @@ dispatch_close (OstreeRepo *repo,
|
|||
}
|
||||
|
||||
g_assert (state->output_tmp_stream);
|
||||
g_assert (state->output_tmp_path);
|
||||
|
||||
if (!g_output_stream_close (state->output_tmp_stream, cancellable, error))
|
||||
goto out;
|
||||
|
|
@ -328,6 +637,8 @@ dispatch_close (OstreeRepo *repo,
|
|||
{
|
||||
gs_unref_variant GVariant *metadata = NULL;
|
||||
|
||||
g_assert (state->output_tmp_path);
|
||||
|
||||
if (!ot_util_variant_map (state->output_tmp_path,
|
||||
ostree_metadata_variant_type (state->output_objtype),
|
||||
TRUE, &metadata, error))
|
||||
|
|
@ -342,36 +653,16 @@ dispatch_close (OstreeRepo *repo,
|
|||
}
|
||||
else
|
||||
{
|
||||
gs_unref_object GInputStream *in = NULL;
|
||||
gs_unref_object GFileInfo *info = NULL;
|
||||
|
||||
in = (GInputStream*)g_file_read (state->output_tmp_path, cancellable, error);
|
||||
if (!in)
|
||||
goto out;
|
||||
|
||||
info = g_file_input_stream_query_info ((GFileInputStream*)in, G_FILE_ATTRIBUTE_STANDARD_SIZE,
|
||||
cancellable, error);
|
||||
if (!info)
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_write_content (repo, tmp_checksum, in,
|
||||
g_file_info_get_size (info), NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
g_print ("Wrote content object '%s'\n",
|
||||
tmp_checksum);
|
||||
/* We already have an async write going, the close() above will
|
||||
* ensure it completes.
|
||||
*/
|
||||
}
|
||||
|
||||
state->output_target = NULL;
|
||||
g_clear_object (&state->output_tmp_path);
|
||||
|
||||
state->object_start = TRUE;
|
||||
state->checksum_index++;
|
||||
if (state->checksum_index < state->n_checksums)
|
||||
{
|
||||
if (!open_output_target_csum (repo, state, cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
|
|
|
|||
|
|
@ -2752,9 +2752,8 @@ ostree_repo_append_gpg_signature (OstreeRepo *self,
|
|||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_variant GVariant *metadata = NULL;
|
||||
gs_unref_variant GVariant *new_metadata = NULL;
|
||||
gs_unref_variant_builder GVariantBuilder *builder = NULL;
|
||||
gs_unref_variant_builder GVariantBuilder *signature_builder = NULL;
|
||||
gs_unref_variant GVariant *signaturedata = NULL;
|
||||
|
||||
if (!ostree_repo_read_commit_detached_metadata (self,
|
||||
commit_checksum,
|
||||
|
|
@ -2767,27 +2766,11 @@ ostree_repo_append_gpg_signature (OstreeRepo *self,
|
|||
goto out;
|
||||
}
|
||||
|
||||
if (metadata)
|
||||
{
|
||||
builder = ot_util_variant_builder_from_variant (metadata, G_VARIANT_TYPE ("a{sv}"));
|
||||
signaturedata = g_variant_lookup_value (metadata, "ostree.gpgsigs", G_VARIANT_TYPE ("aay"));
|
||||
if (signaturedata)
|
||||
signature_builder = ot_util_variant_builder_from_variant (signaturedata, G_VARIANT_TYPE ("aay"));
|
||||
}
|
||||
if (!builder)
|
||||
builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
|
||||
if (!signature_builder)
|
||||
signature_builder = g_variant_builder_new (G_VARIANT_TYPE ("aay"));
|
||||
|
||||
g_variant_builder_add (signature_builder, "@ay", ot_gvariant_new_ay_bytes (signature_bytes));
|
||||
|
||||
g_variant_builder_add (builder, "{sv}", "ostree.gpgsigs", g_variant_builder_end (signature_builder));
|
||||
|
||||
metadata = g_variant_builder_end (builder);
|
||||
new_metadata = _ostree_detached_metadata_append_gpg_sig (metadata, signature_bytes);
|
||||
|
||||
if (!ostree_repo_write_commit_detached_metadata (self,
|
||||
commit_checksum,
|
||||
metadata,
|
||||
new_metadata,
|
||||
cancellable,
|
||||
error))
|
||||
{
|
||||
|
|
@ -2801,34 +2784,21 @@ ostree_repo_append_gpg_signature (OstreeRepo *self,
|
|||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* ostree_repo_sign_commit:
|
||||
* @self: Self
|
||||
* @commit_checksum: SHA256 of given commit to sign
|
||||
* @key_id: Use this GPG key id
|
||||
* @homedir: (allow-none): GPG home directory, or %NULL
|
||||
* @cancellable: A #GCancellable
|
||||
* @error: a #GError
|
||||
*
|
||||
* Add a GPG signature to a commit.
|
||||
*/
|
||||
gboolean
|
||||
ostree_repo_sign_commit (OstreeRepo *self,
|
||||
const gchar *commit_checksum,
|
||||
static gboolean
|
||||
sign_data (OstreeRepo *self,
|
||||
GBytes *input_data,
|
||||
const gchar *key_id,
|
||||
const gchar *homedir,
|
||||
GBytes **out_signature,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
#ifdef HAVE_GPGME
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_object GFile *commit_path = NULL;
|
||||
gs_free gchar *commit_filename = NULL;
|
||||
gs_unref_object GFile *tmp_signature_file = NULL;
|
||||
gs_unref_object GOutputStream *tmp_signature_output = NULL;
|
||||
gs_unref_variant GVariant *commit_variant = NULL;
|
||||
gs_unref_bytes GBytes *signature_bytes = NULL;
|
||||
gpgme_ctx_t context = NULL;
|
||||
gs_unref_bytes GBytes *ret_signature = NULL;
|
||||
gpgme_engine_info_t info;
|
||||
gpgme_error_t err;
|
||||
gpgme_key_t key = NULL;
|
||||
|
|
@ -2837,10 +2807,6 @@ ostree_repo_sign_commit (OstreeRepo *self,
|
|||
int signature_fd = -1;
|
||||
GMappedFile *signature_file = NULL;
|
||||
|
||||
if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT,
|
||||
commit_checksum, &commit_variant, error))
|
||||
goto out;
|
||||
|
||||
if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
|
||||
&tmp_signature_file, &tmp_signature_output,
|
||||
cancellable, error))
|
||||
|
|
@ -2895,13 +2861,16 @@ ostree_repo_sign_commit (OstreeRepo *self,
|
|||
goto out;
|
||||
}
|
||||
|
||||
if ((err = gpgme_data_new_from_mem (&commit_buffer, g_variant_get_data (commit_variant),
|
||||
g_variant_get_size (commit_variant), FALSE)) != GPG_ERR_NO_ERROR)
|
||||
{
|
||||
gsize len;
|
||||
const char *buf = g_bytes_get_data (input_data, &len);
|
||||
if ((err = gpgme_data_new_from_mem (&commit_buffer, buf, len, FALSE)) != GPG_ERR_NO_ERROR)
|
||||
{
|
||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"Failed to create buffer from commit file");
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
signature_fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)tmp_signature_output);
|
||||
if (signature_fd < 0)
|
||||
|
|
@ -2932,13 +2901,10 @@ ostree_repo_sign_commit (OstreeRepo *self,
|
|||
signature_file = gs_file_map_noatime (tmp_signature_file, cancellable, error);
|
||||
if (!signature_file)
|
||||
goto out;
|
||||
signature_bytes = g_mapped_file_get_bytes (signature_file);
|
||||
|
||||
if (!ostree_repo_append_gpg_signature (self, commit_checksum, signature_bytes,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
ret_signature = g_mapped_file_get_bytes (signature_file);
|
||||
|
||||
ret = TRUE;
|
||||
gs_transfer_out_value (out_signature, &ret_signature);
|
||||
out:
|
||||
if (commit_buffer)
|
||||
gpgme_data_release (commit_buffer);
|
||||
|
|
@ -2958,7 +2924,134 @@ out:
|
|||
#endif
|
||||
}
|
||||
|
||||
static gboolean
|
||||
/**
|
||||
* ostree_repo_sign_commit:
|
||||
* @self: Self
|
||||
* @commit_checksum: SHA256 of given commit to sign
|
||||
* @key_id: Use this GPG key id
|
||||
* @homedir: (allow-none): GPG home directory, or %NULL
|
||||
* @cancellable: A #GCancellable
|
||||
* @error: a #GError
|
||||
*
|
||||
* Add a GPG signature to a commit.
|
||||
*/
|
||||
gboolean
|
||||
ostree_repo_sign_commit (OstreeRepo *self,
|
||||
const gchar *commit_checksum,
|
||||
const gchar *key_id,
|
||||
const gchar *homedir,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_bytes GBytes *commit_data = NULL;
|
||||
gs_unref_bytes GBytes *signature_data = NULL;
|
||||
gs_unref_variant GVariant *commit_variant = NULL;
|
||||
|
||||
if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT,
|
||||
commit_checksum, &commit_variant, error))
|
||||
goto out;
|
||||
|
||||
/* This has the same lifecycle as the variant, so we can just
|
||||
* use static.
|
||||
*/
|
||||
signature_data = g_bytes_new_static (g_variant_get_data (commit_variant),
|
||||
g_variant_get_size (commit_variant));
|
||||
|
||||
if (!sign_data (self, signature_data, key_id, homedir,
|
||||
&signature_data,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_append_gpg_signature (self, commit_checksum, signature_data,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* ostree_repo_sign_delta:
|
||||
* @self: Self
|
||||
* @from_commit: SHA256 of starting commit to sign
|
||||
* @to_commit: SHA256 of target commit to sign
|
||||
* @key_id: Use this GPG key id
|
||||
* @homedir: (allow-none): GPG home directory, or %NULL
|
||||
* @cancellable: A #GCancellable
|
||||
* @error: a #GError
|
||||
*
|
||||
* Add a GPG signature to a static delta.
|
||||
*/
|
||||
gboolean
|
||||
ostree_repo_sign_delta (OstreeRepo *self,
|
||||
const gchar *from_commit,
|
||||
const gchar *to_commit,
|
||||
const gchar *key_id,
|
||||
const gchar *homedir,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_bytes GBytes *delta_data = NULL;
|
||||
gs_unref_bytes GBytes *signature_data = NULL;
|
||||
gs_unref_variant GVariant *commit_variant = NULL;
|
||||
gs_free char *delta_path = NULL;
|
||||
gs_unref_object GFile *delta_file = NULL;
|
||||
gs_unref_object char *detached_metadata_relpath = NULL;
|
||||
gs_unref_object GFile *detached_metadata_path = NULL;
|
||||
gs_unref_variant GVariant *existing_detached_metadata = NULL;
|
||||
gs_unref_variant GVariant *normalized = NULL;
|
||||
gs_unref_variant GVariant *new_metadata = NULL;
|
||||
GError *temp_error = NULL;
|
||||
|
||||
detached_metadata_relpath =
|
||||
_ostree_get_relative_static_delta_detachedmeta_path (from_commit, to_commit);
|
||||
detached_metadata_path = g_file_resolve_relative_path (self->repodir, detached_metadata_relpath);
|
||||
|
||||
delta_path = _ostree_get_relative_static_delta_path (from_commit, to_commit);
|
||||
delta_file = g_file_resolve_relative_path (self->repodir, delta_path);
|
||||
delta_data = gs_file_map_readonly (delta_file, cancellable, error);
|
||||
if (!delta_data)
|
||||
goto out;
|
||||
|
||||
if (!ot_util_variant_map (detached_metadata_path, G_VARIANT_TYPE ("a{sv}"),
|
||||
TRUE, &existing_detached_metadata, &temp_error))
|
||||
{
|
||||
if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
|
||||
{
|
||||
g_clear_error (&temp_error);
|
||||
}
|
||||
else
|
||||
{
|
||||
g_propagate_error (error, temp_error);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
if (!sign_data (self, delta_data, key_id, homedir,
|
||||
&signature_data,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
new_metadata = _ostree_detached_metadata_append_gpg_sig (existing_detached_metadata, signature_data);
|
||||
|
||||
normalized = g_variant_get_normal_form (new_metadata);
|
||||
|
||||
if (!g_file_replace_contents (detached_metadata_path,
|
||||
g_variant_get_data (normalized),
|
||||
g_variant_get_size (normalized),
|
||||
NULL, FALSE, 0, NULL,
|
||||
cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
gboolean
|
||||
_ostree_repo_gpg_verify_file_with_metadata (OstreeRepo *self,
|
||||
GFile *path,
|
||||
GVariant *metadata,
|
||||
|
|
|
|||
|
|
@ -510,6 +510,7 @@ gboolean ostree_repo_static_delta_generate (OstreeRepo *self,
|
|||
const char *from,
|
||||
const char *to,
|
||||
GVariant *metadata,
|
||||
GVariant *params,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
|
||||
|
|
@ -598,6 +599,14 @@ gboolean ostree_repo_sign_commit (OstreeRepo *self,
|
|||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
|
||||
gboolean ostree_repo_sign_delta (OstreeRepo *self,
|
||||
const gchar *from_commit,
|
||||
const gchar *to_commit,
|
||||
const gchar *key_id,
|
||||
const gchar *homedir,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
|
||||
gboolean ostree_repo_append_gpg_signature (OstreeRepo *self,
|
||||
const gchar *commit_checksum,
|
||||
GBytes *signature_bytes,
|
||||
|
|
|
|||
|
|
@ -23,53 +23,75 @@
|
|||
#include "ot-main.h"
|
||||
#include "ot-builtins.h"
|
||||
#include "ostree.h"
|
||||
#include "ot-main.h"
|
||||
#include "otutil.h"
|
||||
|
||||
static char *opt_from_rev;
|
||||
static char *opt_to_rev;
|
||||
static char *opt_apply;
|
||||
static char **opt_key_ids;
|
||||
static char *opt_gpg_homedir;
|
||||
static char *opt_max_usize;
|
||||
|
||||
static GOptionEntry options[] = {
|
||||
#define BUILTINPROTO(name) static gboolean ot_static_delta_builtin_ ## name (int argc, char **argv, GCancellable *cancellable, GError **error)
|
||||
|
||||
BUILTINPROTO(list);
|
||||
BUILTINPROTO(generate);
|
||||
BUILTINPROTO(apply_offline);
|
||||
|
||||
#undef BUILTINPROTO
|
||||
|
||||
static OstreeCommand static_delta_subcommands[] = {
|
||||
{ "list", ot_static_delta_builtin_list },
|
||||
{ "generate", ot_static_delta_builtin_generate },
|
||||
{ "apply-offline", ot_static_delta_builtin_apply_offline },
|
||||
{ NULL, NULL }
|
||||
};
|
||||
|
||||
static GOptionEntry generate_options[] = {
|
||||
{ "from", 0, 0, G_OPTION_ARG_STRING, &opt_from_rev, "Create delta from revision REV", "REV" },
|
||||
{ "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" },
|
||||
{ "apply", 0, 0, G_OPTION_ARG_FILENAME, &opt_apply, "Apply delta from PATH", "PATH" },
|
||||
{ "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the delta with", "key-id"},
|
||||
{ "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "homedir"},
|
||||
{ "max-usize", 'u', 0, G_OPTION_ARG_STRING, &opt_max_usize, "Maximum uncompressed size in megabytes", NULL},
|
||||
{ NULL }
|
||||
};
|
||||
|
||||
gboolean
|
||||
ostree_builtin_static_delta (int argc, char **argv, GCancellable *cancellable, GError **error)
|
||||
static void
|
||||
static_delta_usage (char **argv,
|
||||
gboolean is_error)
|
||||
{
|
||||
OstreeCommand *command = static_delta_subcommands;
|
||||
void (*print_func) (const gchar *format, ...);
|
||||
|
||||
if (is_error)
|
||||
print_func = g_printerr;
|
||||
else
|
||||
print_func = g_print;
|
||||
|
||||
print_func ("usage: %s --repo=PATH static-delta\n",
|
||||
argv[0]);
|
||||
print_func ("Builtin commands:\n");
|
||||
|
||||
while (command->name)
|
||||
{
|
||||
print_func (" %s\n", command->name);
|
||||
command++;
|
||||
}
|
||||
}
|
||||
|
||||
static gboolean
|
||||
ot_static_delta_builtin_list (int argc, char **argv, GCancellable *cancellable, GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
gs_unref_ptrarray GPtrArray *delta_names = NULL;
|
||||
guint i;
|
||||
GOptionContext *context;
|
||||
gs_unref_object OstreeRepo *repo = NULL;
|
||||
gs_unref_ptrarray GPtrArray *delta_names = NULL;
|
||||
|
||||
context = g_option_context_new ("Manage static delta files");
|
||||
|
||||
if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
|
||||
context = g_option_context_new ("LIST - list delta files");
|
||||
if (!ostree_option_context_parse (context, NULL, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (opt_apply)
|
||||
{
|
||||
gs_unref_object GFile *path = g_file_new_for_path (opt_apply);
|
||||
|
||||
if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_static_delta_execute_offline (repo, path, TRUE, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (argc >= 2 && opt_to_rev == NULL)
|
||||
opt_to_rev = argv[1];
|
||||
|
||||
if (argc < 2 && opt_to_rev == NULL)
|
||||
{
|
||||
guint i;
|
||||
if (!ostree_repo_list_static_delta_names (repo, &delta_names, cancellable, error))
|
||||
goto out;
|
||||
|
||||
|
|
@ -84,13 +106,43 @@ ostree_builtin_static_delta (int argc, char **argv, GCancellable *cancellable, G
|
|||
g_print ("%s\n", (char*)delta_names->pdata[i]);
|
||||
}
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
if (context)
|
||||
g_option_context_free (context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellable, GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
GOptionContext *context;
|
||||
gs_unref_object OstreeRepo *repo = NULL;
|
||||
|
||||
context = g_option_context_new ("Generate static delta files");
|
||||
if (!ostree_option_context_parse (context, generate_options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (argc >= 2 && opt_to_rev == NULL)
|
||||
opt_to_rev = argv[1];
|
||||
|
||||
if (argc < 2 && opt_to_rev == NULL)
|
||||
{
|
||||
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"TO revision must be specified");
|
||||
goto out;
|
||||
}
|
||||
else if (opt_to_rev != NULL)
|
||||
else
|
||||
{
|
||||
const char *from_source;
|
||||
gs_free char *from_resolved = NULL;
|
||||
gs_free char *to_resolved = NULL;
|
||||
gs_free char *from_parent_str = NULL;
|
||||
gs_unref_variant_builder GVariantBuilder *parambuilder = NULL;
|
||||
|
||||
g_assert (opt_to_rev);
|
||||
|
||||
if (opt_from_rev == NULL)
|
||||
{
|
||||
|
|
@ -107,20 +159,125 @@ ostree_builtin_static_delta (int argc, char **argv, GCancellable *cancellable, G
|
|||
if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error))
|
||||
goto out;
|
||||
|
||||
parambuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
|
||||
if (opt_max_usize)
|
||||
g_variant_builder_add (parambuilder, "{sv}",
|
||||
"max-usize", g_variant_new_uint32 (g_ascii_strtoull (opt_max_usize, NULL, 10)));
|
||||
|
||||
g_print ("Generating static delta:\n");
|
||||
g_print (" From: %s\n", from_resolved);
|
||||
g_print (" To: %s\n", to_resolved);
|
||||
if (!ostree_repo_static_delta_generate (repo, OSTREE_STATIC_DELTA_GENERATE_OPT_MAJOR,
|
||||
from_resolved, to_resolved, NULL,
|
||||
g_variant_builder_end (parambuilder),
|
||||
cancellable, error))
|
||||
goto out;
|
||||
}
|
||||
else
|
||||
|
||||
if (opt_key_ids)
|
||||
{
|
||||
ot_util_usage_error (context, "--from=REV must be specified", error);
|
||||
char **iter;
|
||||
|
||||
for (iter = opt_key_ids; iter && *iter; iter++)
|
||||
{
|
||||
const char *keyid = *iter;
|
||||
|
||||
if (!ostree_repo_sign_delta (repo,
|
||||
from_resolved, to_resolved,
|
||||
keyid,
|
||||
opt_gpg_homedir,
|
||||
cancellable,
|
||||
error))
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
if (context)
|
||||
g_option_context_free (context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
ot_static_delta_builtin_apply_offline (int argc, char **argv, GCancellable *cancellable, GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
const char *patharg;
|
||||
gs_unref_object GFile *path = NULL;
|
||||
GOptionContext *context;
|
||||
gs_unref_object OstreeRepo *repo = NULL;
|
||||
|
||||
context = g_option_context_new ("DELTA - Apply delta file");
|
||||
if (!ostree_option_context_parse (context, NULL, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (argc < 2)
|
||||
{
|
||||
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"PATH must be specified");
|
||||
goto out;
|
||||
}
|
||||
|
||||
patharg = argv[1];
|
||||
path = g_file_new_for_path (patharg);
|
||||
|
||||
if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_static_delta_execute_offline (repo, path, TRUE, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
if (context)
|
||||
g_option_context_free (context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
gboolean
|
||||
ostree_builtin_static_delta (int argc, char **argv, GCancellable *cancellable, GError **error)
|
||||
{
|
||||
gboolean ret = FALSE;
|
||||
OstreeCommand *command;
|
||||
const char *cmdname;
|
||||
GOptionContext *context;
|
||||
gs_unref_object OstreeRepo *repo = NULL;
|
||||
|
||||
context = g_option_context_new ("DELTA - Apply delta file");
|
||||
if (!ostree_option_context_parse (context, NULL, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (argc < 2)
|
||||
{
|
||||
static_delta_usage (argv, TRUE);
|
||||
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"No command specified");
|
||||
goto out;
|
||||
}
|
||||
|
||||
cmdname = argv[1];
|
||||
command = static_delta_subcommands;
|
||||
while (command->name)
|
||||
{
|
||||
if (g_strcmp0 (cmdname, command->name) == 0)
|
||||
break;
|
||||
command++;
|
||||
}
|
||||
|
||||
if (!command->fn)
|
||||
{
|
||||
gs_free char *msg = g_strdup_printf ("Unknown command '%s'", cmdname);
|
||||
static_delta_usage (argv, TRUE);
|
||||
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, msg);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (!command->fn (argc-1, argv+1, cancellable, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
|
|
|
|||
|
|
@ -19,21 +19,31 @@
|
|||
|
||||
set -e
|
||||
|
||||
cd ${test_tmpdir}
|
||||
mkdir repo
|
||||
${CMD_PREFIX} ostree --repo=repo init
|
||||
${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
|
||||
function repo_init() {
|
||||
cd ${test_tmpdir}
|
||||
rm repo -rf
|
||||
mkdir repo
|
||||
${CMD_PREFIX} ostree --repo=repo init
|
||||
${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
|
||||
}
|
||||
|
||||
function verify_initial_contents() {
|
||||
rm checkout-origin-main -rf
|
||||
$OSTREE checkout origin/main checkout-origin-main
|
||||
cd checkout-origin-main
|
||||
assert_file_has_content firstfile '^first$'
|
||||
assert_file_has_content baz/cow '^moo$'
|
||||
}
|
||||
|
||||
# Try both syntaxes
|
||||
repo_init
|
||||
${CMD_PREFIX} ostree --repo=repo pull origin main
|
||||
${CMD_PREFIX} ostree --repo=repo pull origin:main
|
||||
${CMD_PREFIX} ostree --repo=repo fsck
|
||||
echo "ok pull"
|
||||
|
||||
cd ${test_tmpdir}
|
||||
$OSTREE checkout origin/main checkout-origin-main
|
||||
cd checkout-origin-main
|
||||
assert_file_has_content firstfile '^first$'
|
||||
assert_file_has_content baz/cow '^moo$'
|
||||
verify_initial_contents
|
||||
echo "ok pull contents"
|
||||
|
||||
cd ${test_tmpdir}
|
||||
|
|
@ -52,3 +62,63 @@ ${CMD_PREFIX} ostree --repo=repo fsck
|
|||
$OSTREE show --print-detached-metadata-key=SIGNATURE main > main-meta
|
||||
assert_file_has_content main-meta "HANCOCK"
|
||||
echo "ok pull detached metadata"
|
||||
|
||||
cd ${test_tmpdir}
|
||||
repo_init
|
||||
${CMD_PREFIX} ostree --repo=repo pull origin main
|
||||
${CMD_PREFIX} ostree --repo=repo fsck
|
||||
# Generate a delta from old to current, even though we aren't going to
|
||||
# use it.
|
||||
ostree --repo=ostree-srv/gnomerepo static-delta generate main
|
||||
|
||||
rm main-files -rf
|
||||
ostree --repo=ostree-srv/gnomerepo checkout main main-files
|
||||
cd main-files
|
||||
echo "an added file for static deltas" > added-file
|
||||
echo "modified file for static deltas" > baz/cow
|
||||
rm baz/saucer
|
||||
ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s 'static delta test'
|
||||
cd ..
|
||||
rm main-files -rf
|
||||
# Generate delta that we'll use
|
||||
ostree --repo=ostree-srv/gnomerepo static-delta generate main
|
||||
|
||||
cd ${test_tmpdir}
|
||||
${CMD_PREFIX} ostree --repo=repo pull origin main
|
||||
${CMD_PREFIX} ostree --repo=repo fsck
|
||||
|
||||
rm checkout-origin-main -rf
|
||||
$OSTREE checkout origin:main checkout-origin-main
|
||||
cd checkout-origin-main
|
||||
assert_file_has_content firstfile '^first$'
|
||||
assert_file_has_content baz/cow "modified file for static deltas"
|
||||
assert_not_has_file baz/saucer
|
||||
|
||||
echo "ok static delta"
|
||||
|
||||
cd ${test_tmpdir}
|
||||
rm main-files -rf
|
||||
ostree --repo=ostree-srv/gnomerepo checkout main main-files
|
||||
cd main-files
|
||||
# Make a file larger than 16M for testing
|
||||
dd if=/dev/zero of=test-bigfile count=1 seek=42678
|
||||
echo "further modified file for static deltas" > baz/cow
|
||||
ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s '2nd static delta test'
|
||||
cd ..
|
||||
rm main-files -rf
|
||||
ostree --repo=ostree-srv/gnomerepo static-delta generate main
|
||||
|
||||
cd ${test_tmpdir}
|
||||
${CMD_PREFIX} ostree --repo=repo pull origin main
|
||||
${CMD_PREFIX} ostree --repo=repo fsck
|
||||
|
||||
rm checkout-origin-main -rf
|
||||
$OSTREE checkout origin:main checkout-origin-main
|
||||
cd checkout-origin-main
|
||||
assert_has_file test-bigfile
|
||||
stat --format=%s test-bigfile > bigfile-size
|
||||
assert_file_has_content bigfile-size 21851648
|
||||
assert_file_has_content baz/cow "further modified file for static deltas"
|
||||
assert_not_has_file baz/saucer
|
||||
|
||||
echo "ok static delta 2"
|
||||
|
|
|
|||
|
|
@ -57,18 +57,20 @@ function permuteDirectory() {
|
|||
|
||||
permuteDirectory 1 files
|
||||
ostree --repo=repo commit -b test -s test --tree=dir=files
|
||||
ostree static-delta --repo=repo
|
||||
ostree static-delta --repo=repo list
|
||||
|
||||
origrev=$(ostree --repo=repo rev-parse test^)
|
||||
newrev=$(ostree --repo=repo rev-parse test)
|
||||
ostree static-delta --repo=repo --from=${origrev} --to=${newrev}
|
||||
ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev}
|
||||
|
||||
assert_has_dir repo/deltas/${origrev}-${newrev}
|
||||
origstart=$(echo ${origrev} | dd bs=1 count=2 2>/dev/null)
|
||||
origend=$(echo ${origrev} | dd bs=1 skip=2 2>/dev/null)
|
||||
assert_has_dir repo/deltas/${origstart}/${origend}-${newrev}
|
||||
|
||||
mkdir repo2
|
||||
ostree --repo=repo2 init --mode=archive-z2
|
||||
ostree --repo=repo2 pull-local repo ${origrev}
|
||||
|
||||
ostree --repo=repo2 static-delta --apply=repo/deltas/${origrev}-${newrev}
|
||||
ostree --repo=repo2 static-delta apply-offline repo/deltas/${origstart}/${origend}-${newrev}
|
||||
ostree --repo=repo2 fsck
|
||||
ostree --repo=repo2 show ${newrev}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,49 @@
|
|||
#include "bupsplit.h"
|
||||
|
||||
#define BLOB_MAX (8192*4)
|
||||
#define BLOB_READ_SIZE (1024*1024)
|
||||
|
||||
static GPtrArray *
|
||||
rollsum_checksums_for_data (GBytes *bytes)
|
||||
{
|
||||
const guint8 *start;
|
||||
gsize len;
|
||||
GPtrArray *ret = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
|
||||
|
||||
start = g_bytes_get_data (bytes, &len);
|
||||
while (TRUE)
|
||||
{
|
||||
int offset, bits;
|
||||
offset = bupsplit_find_ofs (start, MIN(G_MAXINT32, len), &bits);
|
||||
if (offset == 0)
|
||||
break;
|
||||
if (offset > BLOB_MAX)
|
||||
offset = BLOB_MAX;
|
||||
{
|
||||
gs_free char *blobcsum =
|
||||
g_compute_checksum_for_data (G_CHECKSUM_SHA256,
|
||||
start, offset);
|
||||
g_ptr_array_add (ret, g_variant_ref_sink (g_variant_new ("(st)",
|
||||
blobcsum, (guint64)offset)));
|
||||
}
|
||||
start += offset;
|
||||
len -= offset;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void
|
||||
print_rollsums (GPtrArray *rollsums)
|
||||
{
|
||||
guint i;
|
||||
for (i = 0; i < rollsums->len; i++)
|
||||
{
|
||||
GVariant *sum = rollsums->pdata[i];
|
||||
const char *csum;
|
||||
guint64 val;
|
||||
g_variant_get (sum, "(&st)", &csum, &val);
|
||||
g_print ("chunk %s %" G_GUINT64_FORMAT "\n", csum, val);
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main (int argc, char **argv)
|
||||
|
|
@ -38,29 +80,57 @@ main (int argc, char **argv)
|
|||
|
||||
g_setenv ("GIO_USE_VFS", "local", TRUE);
|
||||
|
||||
if (argc > 1)
|
||||
if (argc == 2)
|
||||
{
|
||||
const guint8 *start;
|
||||
gsize len;
|
||||
gs_unref_ptrarray GPtrArray *rollsums = NULL;
|
||||
|
||||
path = g_file_new_for_path (argv[1]);
|
||||
bytes = gs_file_map_readonly (path, cancellable, error);
|
||||
if (!bytes)
|
||||
goto out;
|
||||
|
||||
start = g_bytes_get_data (bytes, &len);
|
||||
while (TRUE)
|
||||
{
|
||||
int offset, bits;
|
||||
offset = bupsplit_find_ofs (start, MIN(G_MAXINT32, len), &bits);
|
||||
if (offset == 0)
|
||||
break;
|
||||
if (offset > BLOB_MAX)
|
||||
offset = BLOB_MAX;
|
||||
g_print ("%" G_GUINT64_FORMAT "\n", (guint64)offset);
|
||||
start += offset;
|
||||
len -= offset;
|
||||
rollsums = rollsum_checksums_for_data (bytes);
|
||||
print_rollsums (rollsums);
|
||||
}
|
||||
else if (argc > 2)
|
||||
{
|
||||
guint i;
|
||||
gs_unref_hashtable GHashTable *sums = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
|
||||
guint64 input_size = 0;
|
||||
guint64 rollsum_size = 0;
|
||||
|
||||
for (i = 1; i < argc; i++)
|
||||
{
|
||||
guint j;
|
||||
gs_unref_ptrarray GPtrArray *rollsums = NULL;
|
||||
|
||||
path = g_file_new_for_path (argv[i]);
|
||||
bytes = gs_file_map_readonly (path, cancellable, error);
|
||||
if (!bytes)
|
||||
goto out;
|
||||
|
||||
input_size += g_bytes_get_size (bytes);
|
||||
|
||||
g_print ("input: %s size: %" G_GUINT64_FORMAT "\n", argv[i], g_bytes_get_size (bytes));
|
||||
|
||||
rollsums = rollsum_checksums_for_data (bytes);
|
||||
print_rollsums (rollsums);
|
||||
for (j = 0; j < rollsums->len; j++)
|
||||
{
|
||||
GVariant *sum = rollsums->pdata[j];
|
||||
const char *csum;
|
||||
guint64 ofs;
|
||||
g_variant_get (sum, "(&st)", &csum, &ofs);
|
||||
if (!g_hash_table_contains (sums, csum))
|
||||
{
|
||||
g_hash_table_add (sums, g_strdup (csum));
|
||||
rollsum_size += ofs;
|
||||
}
|
||||
}
|
||||
}
|
||||
g_print ("rollsums:%u input:%" G_GUINT64_FORMAT " output: %" G_GUINT64_FORMAT " speedup:%f\n",
|
||||
g_hash_table_size (sums), input_size, rollsum_size,
|
||||
(((double)(input_size+1)) / ((double) rollsum_size + 1)));
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue