packaging/ostree
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

deltas: Use endianness marker when parsing

Browse Source 
Extend the `static-delta show` and `pull` commands to use the
endianness information (if available).
This commit is contained in:
Colin Walters committed 2016-02-26 08:19:01 -05:00
1 parent 277220aaa6
commit 04d77da005
7 files changed
+201 -36
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/libostree/ostree-repo-pull.c
+12 -1
View File
@@ -1443,6 +1443,7 @@ request_static_delta_superblock_sync (OtPullData *pull_data,
static gboolean
process_one_static_delta_fallback (OtPullData *pull_data,
gboolean delta_byteswap,
GVariant *fallback_object,
GCancellable *cancellable,
GError **error)
@@ -1462,6 +1463,9 @@ process_one_static_delta_fallback (OtPullData *pull_data,
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size);
uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size);
pull_data->total_deltapart_size += compressed_size;
pull_data->total_deltapart_usize += uncompressed_size;
@@ -1518,11 +1522,14 @@ process_one_static_delta (OtPullData *pull_data,
GError **error)
{
gboolean ret = FALSE;
gboolean delta_byteswap;
g_autoptr(GVariant) metadata = NULL;
g_autoptr(GVariant) headers = NULL;
g_autoptr(GVariant) fallback_objects = NULL;
guint i, n;
delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock);
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
metadata = g_variant_get_child_value (delta_superblock, 0);
headers = g_variant_get_child_value (delta_superblock, 6);
@@ -1535,7 +1542,7 @@ process_one_static_delta (OtPullData *pull_data,
g_autoptr(GVariant) fallback_object =
g_variant_get_child_value (fallback_objects, i);
if (!process_one_static_delta_fallback (pull_data,
if (!process_one_static_delta_fallback (pull_data, delta_byteswap,
fallback_object,
cancellable, error))
goto out;
@@ -1609,6 +1616,10 @@ process_one_static_delta (OtPullData *pull_data,
header = g_variant_get_child_value (headers, i);
g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
version = maybe_swap_endian_u32 (delta_byteswap, version);
size = maybe_swap_endian_u64 (delta_byteswap, size);
usize = maybe_swap_endian_u64 (delta_byteswap, usize);
if (version > OSTREE_DELTAPART_VERSION)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
src/libostree/ostree-repo-static-delta-compilation.c
+15 -5
View File
@@ -59,6 +59,7 @@ typedef struct {
guint n_rollsum;
guint n_bsdiff;
guint n_fallback;
gboolean swap_endian;
} OstreeStaticDeltaBuilder;
typedef enum {
@@ -1191,7 +1192,8 @@ get_fallback_headers (OstreeRepo *self,
g_variant_new ("(y@aytt)",
objtype,
ostree_checksum_to_bytes_v (checksum),
compressed_size, uncompressed_size));
maybe_swap_endian_u64 (builder->swap_endian, compressed_size),
maybe_swap_endian_u64 (builder->swap_endian, uncompressed_size)));
}
ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder));
@@ -1228,6 +1230,7 @@ get_fallback_headers (OstreeRepo *self,
* - bsdiff-enabled: b: Enable bsdiff compression. Default TRUE.
* - inline-parts: b: Put part data in header, to get a single file delta. Default FALSE.
* - verbose: b: Print diagnostic messages. Default FALSE.
* - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or G_LITTLE_ENDIAN)
* - filename: ay: Save delta superblock to this filename, and parts in the same directory. Default saves to repository.
*/
gboolean
@@ -1262,6 +1265,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
g_autoptr(GVariant) fallback_headers = NULL;
g_autoptr(GVariant) detached = NULL;
gboolean inline_parts;
guint endianness = G_BYTE_ORDER;
g_autoptr(GFile) tmp_dir = NULL;
builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -1277,6 +1281,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
max_chunk_size = 32;
builder.max_chunk_size_bytes = ((guint64)max_chunk_size) * 1000 * 1000;
(void) g_variant_lookup (params, "endianness", "u", &endianness);
g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE);
builder.swap_endian = endianness != G_BYTE_ORDER;
{ gboolean use_bsdiff;
if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff))
use_bsdiff = TRUE;
@@ -1325,7 +1334,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
}
{ guint8 endianness_char;
switch (G_BYTE_ORDER)
switch (endianness)
{
case G_LITTLE_ENDIAN:
endianness_char = 'l';
@@ -1433,10 +1443,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN);
objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
delta_part_header = g_variant_new ("(u@aytt@ay)",
OSTREE_DELTAPART_VERSION,
maybe_swap_endian_u32 (builder.swap_endian, OSTREE_DELTAPART_VERSION),
ot_gvariant_new_ay_bytes (checksum_bytes),
(guint64) g_variant_get_size (delta_part),
part_builder->uncompressed_size,
maybe_swap_endian_u64 (builder.swap_endian, (guint64) g_variant_get_size (delta_part)),
maybe_swap_endian_u64 (builder.swap_endian, part_builder->uncompressed_size),
ot_gvariant_new_ay_bytes (objtype_checksum_array));
g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
src/libostree/ostree-repo-static-delta-core.c
+70 -22
View File
@@ -589,6 +589,7 @@ _ostree_static_delta_part_open (GInputStream *part_in,
static gboolean
show_one_part (OstreeRepo *self,
gboolean swap_endian,
const char *from,
const char *to,
GVariant *meta_entries,
@@ -608,6 +609,8 @@ show_one_part (OstreeRepo *self,
gint part_fd = -1;
g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects);
size = maybe_swap_endian_u64 (swap_endian, size);
usize = maybe_swap_endian_u64 (swap_endian, usize);
*total_size_ref += size;
*total_usize_ref += usize;
g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n",
@@ -666,6 +669,45 @@ show_one_part (OstreeRepo *self,
return ret;
}
OstreeDeltaEndianness
_ostree_delta_get_endianness (GVariant *superblock)
{
guint8 endianness_char;
g_autoptr(GVariant) delta_meta = NULL;
g_autoptr(GVariantDict) delta_metadict = NULL;
delta_meta = g_variant_get_child_value (superblock, 0);
delta_metadict = g_variant_dict_new (delta_meta);
if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
{
switch (endianness_char)
{
case 'l':
return OSTREE_DELTA_ENDIAN_LITTLE;
case 'B':
return OSTREE_DELTA_ENDIAN_BIG;
default:
return OSTREE_DELTA_ENDIAN_INVALID;
}
}
return OSTREE_DELTA_ENDIAN_UNKNOWN;
}
gboolean
_ostree_delta_needs_byteswap (GVariant *superblock)
{
switch (_ostree_delta_get_endianness (superblock))
{
case OSTREE_DELTA_ENDIAN_BIG:
return G_BYTE_ORDER == G_LITTLE_ENDIAN;
case OSTREE_DELTA_ENDIAN_LITTLE:
return G_BYTE_ORDER == G_BIG_ENDIAN;
default:
return FALSE;
}
}
gboolean
_ostree_repo_static_delta_dump (OstreeRepo *self,
const char *delta_id,
@@ -678,11 +720,11 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
g_autofree char *superblock_path = NULL;
glnx_fd_close int superblock_fd = -1;
g_autoptr(GVariant) delta_superblock = NULL;
g_autoptr(GVariant) delta_meta = NULL;
g_autoptr(GVariantDict) delta_metadict = NULL;
guint64 total_size = 0, total_usize = 0;
guint64 total_fallback_size = 0, total_fallback_usize = 0;
guint i;
OstreeDeltaEndianness endianness;
gboolean swap_endian = FALSE;
_ostree_parse_delta_name (delta_id, &from, &to);
superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
@@ -692,30 +734,34 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
TRUE, &delta_superblock, error))
goto out;
delta_meta = g_variant_get_child_value (delta_superblock, 0);
delta_metadict = g_variant_dict_new (delta_meta);
g_print ("%s\n", g_variant_print (delta_superblock, 1));
g_print ("Delta: %s\n", delta_id);
{ guint8 endianness_char;
const char *endianness_description;
{ const char *endianness_description;
if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
endianness = _ostree_delta_get_endianness (delta_superblock);
switch (endianness)
{
switch (endianness_char)
{
case 'l':
endianness_description = "little";
break;
case 'B':
endianness_description = "big";
break;
default:
endianness_description = "invalid";
break;
}
case OSTREE_DELTA_ENDIAN_BIG:
endianness_description = "big";
if (G_BYTE_ORDER == G_LITTLE_ENDIAN)
swap_endian = TRUE;
break;
case OSTREE_DELTA_ENDIAN_LITTLE:
endianness_description = "little";
if (G_BYTE_ORDER == G_BIG_ENDIAN)
swap_endian = TRUE;
break;
case OSTREE_DELTA_ENDIAN_UNKNOWN:
endianness_description = "unknown";
break;
case OSTREE_DELTA_ENDIAN_INVALID:
endianness_description = "invalid";
break;
default:
g_assert_not_reached ();
}
else
endianness_description = "unknown";
g_print ("Endianness: %s\n", endianness_description);
}
@@ -739,6 +785,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
{
guint64 size, usize;
g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize);
size = maybe_swap_endian_u64 (swap_endian, size);
usize = maybe_swap_endian_u64 (swap_endian, usize);
total_fallback_size += size;
total_fallback_usize += usize;
}
@@ -757,7 +805,7 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
for (i = 0; i < n_parts; i++)
{
if (!show_one_part (self, from, to, meta_entries, i,
if (!show_one_part (self, swap_endian, from, to, meta_entries, i,
&total_size, &total_usize,
cancellable, error))
goto out;
src/libostree/ostree-repo-static-delta-private.h
+40 -6
View File
@@ -46,10 +46,10 @@ G_BEGIN_DECLS
/**
* OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
*
* u: version
* u: version (non-canonical endian)
* ay checksum
* guint64 size: Total size of delta (sum of parts)
* guint64 usize: Uncompressed size of resulting objects on disk
* guint64 size: Total size of delta (sum of parts) (non-canonical endian)
* guint64 usize: Uncompressed size of resulting objects on disk (non-canonical endian)
* ARRAY[(guint8 objtype, csum object)]
*
* The checksum is of the delta payload, and each entry in the array
@@ -64,8 +64,8 @@ G_BEGIN_DECLS
*
* y: objtype
* ay: checksum
* t: compressed size
* t: uncompressed size
* t: compressed size (non-canonical endian)
* t: uncompressed size (non-canonical endian)
*
* Object to fetch invididually; includes compressed/uncompressed size.
*/
@@ -79,7 +79,7 @@ G_BEGIN_DECLS
*
* delta-descriptor:
* metadata: a{sv}
* t: timestamp
* t: timestamp (big endian)
* from: ay checksum
* to: ay checksum
* commit: new commit object
@@ -196,4 +196,38 @@ _ostree_repo_static_delta_dump (OstreeRepo *repo,
GCancellable *cancellable,
GError **error);
/* Used for static deltas which due to a historical mistake are
* inconsistent endian.
*
* https://bugzilla.gnome.org/show_bug.cgi?id=762515
*/
static inline guint32
maybe_swap_endian_u32 (gboolean swap,
guint32 v)
{
if (!swap)
return v;
return GUINT32_SWAP_LE_BE (v);
}
static inline guint64
maybe_swap_endian_u64 (gboolean swap,
guint64 v)
{
if (!swap)
return v;
return GUINT64_SWAP_LE_BE (v);
}
typedef enum {
OSTREE_DELTA_ENDIAN_BIG,
OSTREE_DELTA_ENDIAN_LITTLE,
OSTREE_DELTA_ENDIAN_UNKNOWN,
OSTREE_DELTA_ENDIAN_INVALID
} OstreeDeltaEndianness;
OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock);
gboolean _ostree_delta_needs_byteswap (GVariant *superblock);
G_END_DECLS