From 5b721a5b08c9dc46c120d5f54783fcd3652a3170 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 13 Jan 2015 15:09:43 +0100 Subject: [PATCH] Allow creating and pulling static deltas starting from "empty" You create these with something like: ostree static-delta generate --empty --to=master These will be automatically used during pull if no previous revision exists in the target repo. These work very much like the normal static deltas except they are named just by the "to" revision. I.e: deltas/94/f7d2dc23759dd21f9bd01e6705a8fdf98f90cad3e0109ba3f6c091c1a3774d for a from-scratch to 94f7d2dc23759dd21f9bd01e6705a8fdf98f90cad3e0109ba3f6c091c1a3774d delta. https://bugzilla.gnome.org/show_bug.cgi?id=721799 --- src/libostree/ostree-core.c | 21 +++++++++--- src/libostree/ostree-diff.c | 11 ++++++- src/libostree/ostree-repo-pull.c | 8 ++--- .../ostree-repo-static-delta-compilation.c | 33 +++++++++++-------- src/libostree/ostree-repo.c | 2 +- src/ostree/ot-builtin-static-delta.c | 17 +++++++--- tests/test-delta.sh | 16 +++++++-- 7 files changed, 78 insertions(+), 30 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 87f29108..2d70e226 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1348,11 +1348,22 @@ get_delta_path (const char *from, const char *target) { char prefix[3]; - prefix[0] = from[0]; - prefix[1] = from[1]; - prefix[2] = '\0'; - from += 2; - return g_strconcat ("deltas/", prefix, "/", from, "-", to, "/", target, NULL); + if (from == NULL) + { + prefix[0] = to[0]; + prefix[1] = to[1]; + prefix[2] = '\0'; + to += 2; + return g_strconcat ("deltas/", prefix, "/", to, "/", target, NULL); + } + else + { + prefix[0] = from[0]; + prefix[1] = from[1]; + prefix[2] = '\0'; + from += 2; + return g_strconcat ("deltas/", prefix, "/", from, "-", to, "/", target, NULL); + } } char * diff --git a/src/libostree/ostree-diff.c b/src/libostree/ostree-diff.c index 6749b654..150d3345 100644 --- a/src/libostree/ostree-diff.c +++ b/src/libostree/ostree-diff.c @@ -204,7 +204,7 @@ diff_add_dir_recurse (GFile *d, /** * ostree_diff_dirs: * @flags: Flags - * @a: First directory path + * @a: First directory path, or %NULL * @b: First directory path * @modified: (element-type OstreeDiffItem): Modified files * @removed: (element-type Gio.File): Removed files @@ -231,6 +231,15 @@ ostree_diff_dirs (OstreeDiffFlags flags, gs_unref_object GFileInfo *child_a_info = NULL; gs_unref_object GFileInfo *child_b_info = NULL; + if (a == NULL) + { + if (!diff_add_dir_recurse (b, added, cancellable, error)) + goto out; + + ret = TRUE; + goto out; + } + child_a_info = g_file_query_info (a, OSTREE_GIO_FAST_QUERYINFO, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f3b8253a..2cd40a3b 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1446,7 +1446,7 @@ process_one_static_delta (OtPullData *pull_data, if (have_all) { g_debug ("Have all objects from static delta %s-%s part %u", - from_revision, to_revision, + from_revision ? from_revision : "empty", to_revision, i); continue; } @@ -1912,7 +1912,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, &from_revision, error)) goto out; - if (from_revision && g_strcmp0 (from_revision, to_revision) != 0) + if (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0) { if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision, &delta_superblock, cancellable, error)) @@ -1921,14 +1921,14 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!delta_superblock) { - g_debug ("no delta superblock for %s-%s", from_revision, to_revision); + g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision); if (!scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, 0, pull_data->cancellable, error)) goto out; } else { - g_debug ("processing delta superblock for %s-%s", from_revision, to_revision); + g_debug ("processing delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision); if (!process_one_static_delta (pull_data, from_revision, to_revision, delta_superblock, cancellable, error)) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 5aca367e..5f082156 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -203,9 +203,12 @@ generate_delta_lowlatency (OstreeRepo *repo, gs_unref_hashtable GHashTable *modified_content_objects = NULL; gs_unref_hashtable GHashTable *content_object_to_size = NULL; - if (!ostree_repo_read_commit (repo, from, &root_from, NULL, - cancellable, error)) - goto out; + if (from != NULL) + { + if (!ostree_repo_read_commit (repo, from, &root_from, NULL, + cancellable, error)) + goto out; + } if (!ostree_repo_read_commit (repo, to, &root_to, NULL, cancellable, error)) goto out; @@ -231,9 +234,12 @@ generate_delta_lowlatency (OstreeRepo *repo, g_hash_table_add (modified_content_objects, objname); } - if (!ostree_repo_traverse_commit (repo, from, 0, &from_reachable_objects, - cancellable, error)) - goto out; + if (from) + { + if (!ostree_repo_traverse_commit (repo, from, 0, &from_reachable_objects, + cancellable, error)) + goto out; + } if (!ostree_repo_traverse_commit (repo, to, 0, &to_reachable_objects, cancellable, error)) @@ -249,7 +255,7 @@ generate_delta_lowlatency (OstreeRepo *repo, const char *checksum; OstreeObjectType objtype; - if (g_hash_table_contains (from_reachable_objects, serialized_key)) + if (from_reachable_objects && g_hash_table_contains (from_reachable_objects, serialized_key)) continue; ostree_object_name_deserialize (serialized_key, &checksum, &objtype); @@ -412,17 +418,17 @@ get_fallback_headers (OstreeRepo *self, * ostree_repo_static_delta_generate: * @self: Repo * @opt: High level optimization choice - * @from: ASCII SHA256 checksum of origin + * @from: ASCII SHA256 checksum of origin, or %NULL * @to: ASCII SHA256 checksum of target * @metadata: (allow-none): Optional metadata * @params: (allow-none): Parameters, see below * @cancellable: Cancellable * @error: Error * - * Generate a lookaside "static delta" from @from which can generate - * the objects in @to. This delta is an optimization over fetching - * individual objects, and can be conveniently stored and applied - * offline. + * Generate a lookaside "static delta" from @from (%NULL means + * from-empty) which can generate the objects in @to. This delta is + * an optimization over fetching individual objects, and can be + * conveniently stored and applied offline. * * The @params argument should be an a{sv}. The following attributes * are known: @@ -590,9 +596,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self, { GDateTime *now = g_date_time_new_now_utc (); /* floating */ GVariant *from_csum_v = - ostree_checksum_to_bytes_v (from); + from ? ostree_checksum_to_bytes_v (from) : ot_gvariant_new_bytearray ((guchar *)"", 0); /* floating */ GVariant *to_csum_v = ostree_checksum_to_bytes_v (to); + delta_descriptor = g_variant_new ("(@(a(ss)a(say))t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay" "a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")", diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 4d93a89b..22abe914 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3069,7 +3069,7 @@ out: /** * ostree_repo_sign_delta: * @self: Self - * @from_commit: SHA256 of starting commit to sign + * @from_commit: SHA256 of starting commit to sign, or %NULL * @to_commit: SHA256 of target commit to sign * @key_id: Use this GPG key id * @homedir: (allow-none): GPG home directory, or %NULL diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index b133c4c3..45ba6b67 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -31,6 +31,7 @@ static char *opt_to_rev; static char **opt_key_ids; static char *opt_gpg_homedir; static char *opt_max_usize; +static gboolean opt_empty; #define BUILTINPROTO(name) static gboolean ot_static_delta_builtin_ ## name (int argc, char **argv, GCancellable *cancellable, GError **error) @@ -50,6 +51,7 @@ static OstreeCommand static_delta_subcommands[] = { static GOptionEntry generate_options[] = { { "from", 0, 0, G_OPTION_ARG_STRING, &opt_from_rev, "Create delta from revision REV", "REV" }, + { "empty", 0, 0, G_OPTION_ARG_NONE, &opt_empty, "Create delta from scratch", NULL }, { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" }, { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the delta with", "key-id"}, { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "homedir"}, @@ -153,7 +155,11 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab g_assert (opt_to_rev); - if (opt_from_rev == NULL) + if (opt_empty) + { + from_source = NULL; + } + else if (opt_from_rev == NULL) { from_parent_str = g_strconcat (opt_to_rev, "^", NULL); from_source = from_parent_str; @@ -163,8 +169,11 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab from_source = opt_from_rev; } - if (!ostree_repo_resolve_rev (repo, from_source, FALSE, &from_resolved, error)) - goto out; + if (from_source) + { + if (!ostree_repo_resolve_rev (repo, from_source, FALSE, &from_resolved, error)) + goto out; + } if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error)) goto out; @@ -174,7 +183,7 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab "max-usize", g_variant_new_uint32 (g_ascii_strtoull (opt_max_usize, NULL, 10))); g_print ("Generating static delta:\n"); - g_print (" From: %s\n", from_resolved); + g_print (" From: %s\n", from_resolved ? from_resolved : "empty"); g_print (" To: %s\n", to_resolved); if (!ostree_repo_static_delta_generate (repo, OSTREE_STATIC_DELTA_GENERATE_OPT_MAJOR, from_resolved, to_resolved, NULL, diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 175daec2..c5981861 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -55,17 +55,23 @@ function permuteDirectory() { done } +origrev=$(ostree --repo=repo rev-parse test) + +ostree --repo=repo static-delta generate --empty --to=${origrev} +ostree --repo=repo static-delta list | grep ${origrev} || exit 1 + permuteDirectory 1 files ostree --repo=repo commit -b test -s test --tree=dir=files -ostree --repo=repo static-delta list -origrev=$(ostree --repo=repo rev-parse test^) newrev=$(ostree --repo=repo rev-parse test) ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} +ostree --repo=repo static-delta list | grep ${origrev}-${newrev} || exit 1 + origstart=$(echo ${origrev} | dd bs=1 count=2 2>/dev/null) origend=$(echo ${origrev} | dd bs=1 skip=2 2>/dev/null) assert_has_dir repo/deltas/${origstart}/${origend}-${newrev} +assert_has_dir repo/deltas/${origstart}/${origend} mkdir repo2 ostree --repo=repo2 init --mode=archive-z2 @@ -74,3 +80,9 @@ ostree --repo=repo2 pull-local repo ${origrev} ostree --repo=repo2 static-delta apply-offline repo/deltas/${origstart}/${origend}-${newrev} ostree --repo=repo2 fsck ostree --repo=repo2 show ${newrev} + +mkdir repo3 +ostree --repo=repo3 init --mode=archive-z2 +ostree --repo=repo3 static-delta apply-offline repo/deltas/${origstart}/${origend} +ostree --repo=repo3 fsck +ostree --repo=repo3 show ${origrev}