From 92cc3b5968ee51c3de91445aadcf345c386a0678 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 25 Jan 2015 23:51:41 -0500 Subject: [PATCH] deltas: Use base64 for csums, add version to parts --- src/libostree/ostree-core.c | 101 ++++++++++++++++-- src/libostree/ostree-core.h | 4 + src/libostree/ostree-repo-private.h | 2 + src/libostree/ostree-repo-pull.c | 10 +- .../ostree-repo-static-delta-compilation.c | 13 +-- src/libostree/ostree-repo-static-delta-core.c | 19 +++- .../ostree-repo-static-delta-private.h | 7 +- tests/test-delta.sh | 15 --- 8 files changed, 133 insertions(+), 38 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 4858b14c..570e8199 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1103,6 +1103,35 @@ ostree_object_name_deserialize (GVariant *variant, *out_objtype = (OstreeObjectType)objtype_u32; } +/** + * ostree_checksum_b64_inplace_to_bytes: (skip) + * @checksum: (array fixed-size=32): An binary checksum of length 32 + * @buf: Output location, must be at least 45 bytes in length + * + * Overwrite the contents of @buf with stringified version of @csum. + */ +void +ostree_checksum_b64_inplace_to_bytes (const char *checksum, + guchar *buf) +{ + int state = 0; + guint save = 0; + char tmpbuf[44]; + int i; + + for (i = 0; i < 43; i++) + { + char c = checksum[i]; + if (c == '_') + tmpbuf[i] = '/'; + else + tmpbuf[i] = c; + } + tmpbuf[43] = '='; + + g_base64_decode_step (tmpbuf, sizeof (tmpbuf), (guchar *) buf, &state, &save); +} + /** * ostree_checksum_inplace_to_bytes: * @checksum: a SHA256 string @@ -1186,6 +1215,48 @@ ostree_checksum_inplace_from_bytes (const guchar *csum, buf[j] = '\0'; } +/** + * ostree_checksum_b64_inplace_from_bytes: (skip) + * @csum: (array fixed-size=32): An binary checksum of length 32 + * @buf: Output location, must be at least 44 bytes in length + * + * Overwrite the contents of @buf with modified base64 encoding of @csum. + * The "modified" term refers to the fact that instead of '/', the '_' + * character is used. + */ +void +ostree_checksum_b64_inplace_from_bytes (const guchar *csum, + char *buf) +{ + char tmpbuf[44]; + int save = 0; + int state = 0; + gsize outlen; + int i; + + /* At some point, we can optimize this, but for now it's + * a lot easier to reuse GLib's base64 encoder and postprocess it + * to replace the '/' with '_'. + */ + outlen = g_base64_encode_step (csum, 32, FALSE, tmpbuf, &state, &save); + outlen += g_base64_encode_close (FALSE, tmpbuf+outlen, &state, &save); + g_assert (outlen == 44); + + for (i = 0; i < sizeof (tmpbuf); i++) + { + char c = tmpbuf[i]; + if (c == '=') + { + g_assert (i == 43); + buf[i] = '\0'; + } + else if (c == '/') + buf[i] = '_'; + else + buf[i] = c; + } +} + /** * ostree_checksum_from_bytes: * @csum: (array fixed-size=32): An binary checksum of length 32 @@ -1360,21 +1431,35 @@ get_delta_path (const char *from, const char *target) { char prefix[3]; + guint8 csum_to[32]; + char to_b64[44]; + guint8 csum_to_copy[32]; + + ostree_checksum_inplace_to_bytes (to, csum_to); + ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64); + ostree_checksum_b64_inplace_to_bytes (to_b64, csum_to_copy); + + g_assert (memcmp (csum_to, csum_to_copy, 32) == 0); + if (from == NULL) { - prefix[0] = to[0]; - prefix[1] = to[1]; + prefix[0] = to_b64[0]; + prefix[1] = to_b64[1]; prefix[2] = '\0'; - to += 2; - return g_strconcat ("deltas/", prefix, "/", to, "/", target, NULL); + return g_strconcat ("deltas/", prefix, "/", ((char*)to_b64)+2, "/", target, NULL); } else { - prefix[0] = from[0]; - prefix[1] = from[1]; + guint8 csum_from[32]; + char from_b64[44]; + + ostree_checksum_inplace_to_bytes (from, csum_from); + ostree_checksum_b64_inplace_from_bytes (csum_from, from_b64); + + prefix[0] = from_b64[0]; + prefix[1] = from_b64[1]; prefix[2] = '\0'; - from += 2; - return g_strconcat ("deltas/", prefix, "/", from, "-", to, "/", target, NULL); + return g_strconcat ("deltas/", prefix, "/", ((char*)from_b64)+2, "-", to_b64, "/", target, NULL); } } diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index b8b0f586..c0f8798a 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -164,12 +164,16 @@ gboolean ostree_validate_checksum_string (const char *sha256, guchar *ostree_checksum_to_bytes (const char *checksum); GVariant *ostree_checksum_to_bytes_v (const char *checksum); +void ostree_checksum_b64_inplace_to_bytes (const char *checksum, + guint8 *buf); char * ostree_checksum_from_bytes (const guchar *csum); char * ostree_checksum_from_bytes_v (GVariant *csum_v); void ostree_checksum_inplace_from_bytes (const guchar *csum, char *buf); +void ostree_checksum_b64_inplace_from_bytes (const guchar *csum, + char *buf); void ostree_checksum_inplace_to_bytes (const char *checksum, guchar *buf); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 745725e0..d7ea7acc 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -24,6 +24,8 @@ G_BEGIN_DECLS +#define OSTREE_DELTAPART_VERSION (0) + #define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" /** diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index c4249143..daf61377 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1513,9 +1513,17 @@ process_one_static_delta (OtPullData *pull_data, gs_unref_variant GVariant *csum_v = NULL; gs_unref_variant GVariant *objects = NULL; guint64 size, usize; + guint32 version; header = g_variant_get_child_value (headers, i); - g_variant_get (header, "(@aytt@ay)", &csum_v, &size, &usize, &objects); + g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects); + + if (version > OSTREE_DELTAPART_VERSION) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Delta part has too new version %u", version); + goto out; + } csum = ostree_checksum_bytes_peek_validate (csum_v, error); if (!csum) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 3e8f8b3b..8b5d8387 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -548,7 +548,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self, checksum_bytes = g_bytes_new (part_checksum, 32); objtype_checksum_array = objtype_checksum_array_new (part_builder->objects); - delta_part_header = g_variant_new ("(@aytt@ay)", + delta_part_header = g_variant_new ("(u@aytt@ay)", + OSTREE_DELTAPART_VERSION, ot_gvariant_new_ay_bytes (checksum_bytes), g_variant_get_size (delta_part), part_builder->uncompressed_size, @@ -586,13 +587,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, metadata_source = metadata; else { - GVariantBuilder tmpbuilder; - g_variant_builder_init (&tmpbuilder, G_VARIANT_TYPE ("(a(ss)a(say))")); - g_variant_builder_add (&tmpbuilder, "a(ss)", NULL); - g_variant_builder_add (&tmpbuilder, "a(say)", NULL); - tmp_metadata = g_variant_builder_end (&tmpbuilder); - g_variant_ref_sink (tmp_metadata); - metadata_source = tmp_metadata; + metadata_source = ot_gvariant_new_empty_string_dict (); } if (!get_fallback_headers (self, &builder, &fallback_headers, @@ -607,7 +602,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, /* floating */ GVariant *to_csum_v = ostree_checksum_to_bytes_v (to); - delta_descriptor = g_variant_new ("(@(a(ss)a(say))t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay" + delta_descriptor = g_variant_new ("(@a{sv}t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay" "a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")", metadata_source, diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index a0230016..38fcc43d 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -127,7 +127,24 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, if (g_file_query_exists (meta_path, NULL)) { - g_ptr_array_add (ret_deltas, g_strconcat (name1, name2, NULL)); + gs_free char *buf = g_strconcat (name1, name2, NULL); + GString *out = g_string_new (""); + char checksum[65]; + guchar csum[32]; + const char *dash = strchr (buf, '-'); + + ostree_checksum_b64_inplace_to_bytes (buf, csum); + ostree_checksum_inplace_from_bytes (csum, checksum); + g_string_append (out, checksum); + if (dash) + { + g_string_append_c (out, '-'); + ostree_checksum_b64_inplace_to_bytes (dash+1, csum); + ostree_checksum_inplace_from_bytes (csum, checksum); + g_string_append (out, checksum); + } + + g_ptr_array_add (ret_deltas, g_string_free (out, FALSE)); } } } diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index 67458b0b..f3215229 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -42,6 +42,7 @@ G_BEGIN_DECLS /** * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT: * + * u: version * ay checksum * guint64 size: Total size of delta (sum of parts) * guint64 usize: Uncompressed size of resulting objects on disk @@ -51,7 +52,7 @@ G_BEGIN_DECLS * represents an OSTree object which will be created by the deltapart. */ -#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(ayttay)" +#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(uayttay)" /** @@ -131,9 +132,7 @@ gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo, typedef enum { OSTREE_STATIC_DELTA_OP_WRITE = 1, OSTREE_STATIC_DELTA_OP_GUNZIP = 2, - OSTREE_STATIC_DELTA_OP_CLOSE = 3, - OSTREE_STATIC_DELTA_OP_READOBJECT = 4, - OSTREE_STATIC_DELTA_OP_READPAYLOAD = 5 + OSTREE_STATIC_DELTA_OP_CLOSE = 3 } OstreeStaticDeltaOpCode; gboolean diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 6420b21f..988d7ab6 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -72,21 +72,6 @@ if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded" fi -origstart=$(echo ${origrev} | dd bs=1 count=2 2>/dev/null) -origend=$(echo ${origrev} | dd bs=1 skip=2 2>/dev/null) -assert_has_dir repo/deltas/${origstart}/${origend}-${newrev} -assert_has_dir repo/deltas/${origstart}/${origend} - mkdir repo2 ostree --repo=repo2 init --mode=archive-z2 ostree --repo=repo2 pull-local repo ${origrev} - -ostree --repo=repo2 static-delta apply-offline repo/deltas/${origstart}/${origend}-${newrev} -ostree --repo=repo2 fsck -ostree --repo=repo2 show ${newrev} - -mkdir repo3 -ostree --repo=repo3 init --mode=archive-z2 -ostree --repo=repo3 static-delta apply-offline repo/deltas/${origstart}/${origend} -ostree --repo=repo3 fsck -ostree --repo=repo3 show ${origrev}