deltas: Use endianness marker when parsing

Extend the `static-delta show` and `pull` commands to use the
endianness information (if available).

src/libostree/ostree-repo-pull.c

@@ -1443,6 +1443,7 @@ request_static_delta_superblock_sync (OtPullData  *pull_data,
 
 static gboolean
 process_one_static_delta_fallback (OtPullData   *pull_data,
+                                   gboolean      delta_byteswap,
                                    GVariant     *fallback_object,
                                    GCancellable *cancellable,
                                    GError      **error)
@@ -1462,6 +1463,9 @@ process_one_static_delta_fallback (OtPullData   *pull_data,
   if (!ostree_validate_structureof_csum_v (csum_v, error))
     goto out;
 
+  compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size);
+  uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size);
+
   pull_data->total_deltapart_size += compressed_size;
   pull_data->total_deltapart_usize += uncompressed_size;
 
@@ -1518,11 +1522,14 @@ process_one_static_delta (OtPullData   *pull_data,
                           GError      **error)
 {
   gboolean ret = FALSE;
+  gboolean delta_byteswap;
   g_autoptr(GVariant) metadata = NULL;
   g_autoptr(GVariant) headers = NULL;
   g_autoptr(GVariant) fallback_objects = NULL;
   guint i, n;
 
+  delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock);
+
   /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
   metadata = g_variant_get_child_value (delta_superblock, 0);
   headers = g_variant_get_child_value (delta_superblock, 6);
@@ -1535,7 +1542,7 @@ process_one_static_delta (OtPullData   *pull_data,
       g_autoptr(GVariant) fallback_object =
         g_variant_get_child_value (fallback_objects, i);
 
-      if (!process_one_static_delta_fallback (pull_data,
+      if (!process_one_static_delta_fallback (pull_data, delta_byteswap,
                                               fallback_object,
                                               cancellable, error))
         goto out;
@@ -1609,6 +1616,10 @@ process_one_static_delta (OtPullData   *pull_data,
       header = g_variant_get_child_value (headers, i);
       g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
 
+      version = maybe_swap_endian_u32 (delta_byteswap, version);
+      size = maybe_swap_endian_u64 (delta_byteswap, size);
+      usize = maybe_swap_endian_u64 (delta_byteswap, usize);
+
       if (version > OSTREE_DELTAPART_VERSION)
         {
           g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,

src/libostree/ostree-repo-static-delta-compilation.c

@@ -59,6 +59,7 @@ typedef struct {
   guint n_rollsum;
   guint n_bsdiff;
   guint n_fallback;
+  gboolean swap_endian;
 } OstreeStaticDeltaBuilder;
 
 typedef enum {
@@ -1191,7 +1192,8 @@ get_fallback_headers (OstreeRepo               *self,
                                    g_variant_new ("(y@aytt)",
                                                   objtype,
                                                   ostree_checksum_to_bytes_v (checksum),
-                                                  compressed_size, uncompressed_size));
+                                                  maybe_swap_endian_u64 (builder->swap_endian, compressed_size),
+                                                  maybe_swap_endian_u64 (builder->swap_endian, uncompressed_size)));
     }
 
   ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder));
@@ -1228,6 +1230,7 @@ get_fallback_headers (OstreeRepo               *self,
  *   - bsdiff-enabled: b: Enable bsdiff compression.  Default TRUE.
  *   - inline-parts: b: Put part data in header, to get a single file delta.  Default FALSE.
  *   - verbose: b: Print diagnostic messages.  Default FALSE.
+ *   - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or G_LITTLE_ENDIAN)
  *   - filename: ay: Save delta superblock to this filename, and parts in the same directory.  Default saves to repository.
  */
 gboolean
@@ -1262,6 +1265,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
   g_autoptr(GVariant) fallback_headers = NULL;
   g_autoptr(GVariant) detached = NULL;
   gboolean inline_parts;
+  guint endianness = G_BYTE_ORDER; 
   g_autoptr(GFile) tmp_dir = NULL;
   builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
   builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -1277,6 +1281,11 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
     max_chunk_size = 32;
   builder.max_chunk_size_bytes = ((guint64)max_chunk_size) * 1000 * 1000;
 
+  (void) g_variant_lookup (params, "endianness", "u", &endianness);
+  g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE);
+
+  builder.swap_endian = endianness != G_BYTE_ORDER;
+
   { gboolean use_bsdiff;
     if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff))
       use_bsdiff = TRUE;
@@ -1325,7 +1334,8 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
     }
 
   { guint8 endianness_char;
-    switch (G_BYTE_ORDER)
+    
+    switch (endianness)
       {
       case G_LITTLE_ENDIAN:
         endianness_char = 'l';
@@ -1433,10 +1443,10 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
       checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN);
       objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
       delta_part_header = g_variant_new ("(u@aytt@ay)",
-                                         OSTREE_DELTAPART_VERSION,
+                                         maybe_swap_endian_u32 (builder.swap_endian, OSTREE_DELTAPART_VERSION),
                                          ot_gvariant_new_ay_bytes (checksum_bytes),
-                                         (guint64) g_variant_get_size (delta_part),
+                                         maybe_swap_endian_u64 (builder.swap_endian, (guint64) g_variant_get_size (delta_part)),
-                                         part_builder->uncompressed_size,
+                                         maybe_swap_endian_u64 (builder.swap_endian, part_builder->uncompressed_size),
                                          ot_gvariant_new_ay_bytes (objtype_checksum_array));
 
       g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));

src/libostree/ostree-repo-static-delta-core.c

@@ -589,6 +589,7 @@ _ostree_static_delta_part_open (GInputStream   *part_in,
 
 static gboolean
 show_one_part (OstreeRepo                    *self,
+               gboolean                       swap_endian,
                const char                    *from,
                const char                    *to,
                GVariant                      *meta_entries,
@@ -608,6 +609,8 @@ show_one_part (OstreeRepo                    *self,
   gint part_fd = -1;
 
   g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects);
+  size = maybe_swap_endian_u64 (swap_endian, size);
+  usize = maybe_swap_endian_u64 (swap_endian, usize);
   *total_size_ref += size;
   *total_usize_ref += usize;
   g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n",
@@ -666,6 +669,45 @@ show_one_part (OstreeRepo                    *self,
   return ret;
 }
 
+OstreeDeltaEndianness
+_ostree_delta_get_endianness (GVariant *superblock)
+{
+  guint8 endianness_char;
+  g_autoptr(GVariant) delta_meta = NULL;
+  g_autoptr(GVariantDict) delta_metadict = NULL;
+
+  delta_meta = g_variant_get_child_value (superblock, 0);
+  delta_metadict = g_variant_dict_new (delta_meta);
+
+  if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
+    {
+      switch (endianness_char)
+        {
+        case 'l':
+          return OSTREE_DELTA_ENDIAN_LITTLE;
+        case 'B':
+          return OSTREE_DELTA_ENDIAN_BIG;
+        default:
+          return OSTREE_DELTA_ENDIAN_INVALID;
+        }
+    }
+  return OSTREE_DELTA_ENDIAN_UNKNOWN;
+}
+
+gboolean
+_ostree_delta_needs_byteswap (GVariant *superblock)
+{
+  switch (_ostree_delta_get_endianness (superblock))
+    {
+    case OSTREE_DELTA_ENDIAN_BIG:
+      return G_BYTE_ORDER == G_LITTLE_ENDIAN;
+    case OSTREE_DELTA_ENDIAN_LITTLE:
+      return G_BYTE_ORDER == G_BIG_ENDIAN;
+    default:
+      return FALSE;
+    }
+}
+
 gboolean
 _ostree_repo_static_delta_dump (OstreeRepo                    *self,
                                 const char                    *delta_id,
@@ -678,11 +720,11 @@ _ostree_repo_static_delta_dump (OstreeRepo                    *self,
   g_autofree char *superblock_path = NULL;
   glnx_fd_close int superblock_fd = -1;
   g_autoptr(GVariant) delta_superblock = NULL;
-  g_autoptr(GVariant) delta_meta = NULL;
-  g_autoptr(GVariantDict) delta_metadict = NULL;
   guint64 total_size = 0, total_usize = 0;
   guint64 total_fallback_size = 0, total_fallback_usize = 0;
   guint i;
+  OstreeDeltaEndianness endianness;
+  gboolean swap_endian = FALSE;
 
   _ostree_parse_delta_name (delta_id, &from, &to);
   superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
@@ -692,30 +734,34 @@ _ostree_repo_static_delta_dump (OstreeRepo                    *self,
                                TRUE, &delta_superblock, error))
     goto out;
 
-  delta_meta = g_variant_get_child_value (delta_superblock, 0);
+  g_print ("%s\n", g_variant_print (delta_superblock, 1));
-  delta_metadict = g_variant_dict_new (delta_meta);
 
   g_print ("Delta: %s\n", delta_id);
-  { guint8 endianness_char;
+  { const char *endianness_description;
-    const char *endianness_description;
 
-    if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
+    endianness = _ostree_delta_get_endianness (delta_superblock);
+
+    switch (endianness)
       {
-        switch (endianness_char)
+      case OSTREE_DELTA_ENDIAN_BIG:
-          {
+        endianness_description = "big";
-          case 'l':
+        if (G_BYTE_ORDER == G_LITTLE_ENDIAN)
-            endianness_description = "little";
+          swap_endian = TRUE;
-            break;
+        break;
-          case 'B':
+      case OSTREE_DELTA_ENDIAN_LITTLE:
-            endianness_description = "big";
+        endianness_description = "little";
-            break;
+        if (G_BYTE_ORDER == G_BIG_ENDIAN)
-          default:
+          swap_endian = TRUE;
-            endianness_description = "invalid";
+        break;
-            break;
+      case OSTREE_DELTA_ENDIAN_UNKNOWN:
-          }
+        endianness_description = "unknown";
+        break;
+      case OSTREE_DELTA_ENDIAN_INVALID:
+        endianness_description = "invalid";
+        break;
+      default:
+        g_assert_not_reached ();
       }
-    else
-      endianness_description = "unknown";
     
     g_print ("Endianness: %s\n", endianness_description);
   }
@@ -739,6 +785,8 @@ _ostree_repo_static_delta_dump (OstreeRepo                    *self,
       {
         guint64 size, usize;
         g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize);
+        size = maybe_swap_endian_u64 (swap_endian, size);
+        usize = maybe_swap_endian_u64 (swap_endian, usize);
         total_fallback_size += size;
         total_fallback_usize += usize;
       }
@@ -757,7 +805,7 @@ _ostree_repo_static_delta_dump (OstreeRepo                    *self,
 
     for (i = 0; i < n_parts; i++)
       {
-        if (!show_one_part (self, from, to, meta_entries, i,
+        if (!show_one_part (self, swap_endian, from, to, meta_entries, i,
                             &total_size, &total_usize,
                             cancellable, error))
           goto out;

src/libostree/ostree-repo-static-delta-private.h

@@ -46,10 +46,10 @@ G_BEGIN_DECLS
 /**
  * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
  *
- *   u: version
+ *   u: version     (non-canonical endian)
  *   ay checksum
- *   guint64 size:   Total size of delta (sum of parts)
+ *   guint64 size:   Total size of delta (sum of parts) (non-canonical endian)
- *   guint64 usize:   Uncompressed size of resulting objects on disk
+ *   guint64 usize:   Uncompressed size of resulting objects on disk (non-canonical endian)
  *   ARRAY[(guint8 objtype, csum object)]
  *
  * The checksum is of the delta payload, and each entry in the array
@@ -64,8 +64,8 @@ G_BEGIN_DECLS
  *
  * y: objtype
  * ay: checksum
- * t: compressed size
+ * t: compressed size (non-canonical endian)
- * t: uncompressed size
+ * t: uncompressed size (non-canonical endian)
  *
  * Object to fetch invididually; includes compressed/uncompressed size.
  */
@@ -79,7 +79,7 @@ G_BEGIN_DECLS
  *
  * delta-descriptor:
  *   metadata: a{sv}
- *   t: timestamp
+ *   t: timestamp (big endian)
  *   from: ay checksum
  *   to: ay checksum
  *   commit: new commit object
@@ -196,4 +196,38 @@ _ostree_repo_static_delta_dump (OstreeRepo                 *repo,
                                 GCancellable               *cancellable,
                                 GError                    **error);
 
+/* Used for static deltas which due to a historical mistake are
+ * inconsistent endian.
+ *
+ * https://bugzilla.gnome.org/show_bug.cgi?id=762515
+ */
+static inline guint32
+maybe_swap_endian_u32 (gboolean swap,
+                       guint32 v)
+{
+  if (!swap)
+    return v;
+  return GUINT32_SWAP_LE_BE (v);
+}
+
+static inline guint64
+maybe_swap_endian_u64 (gboolean swap,
+                       guint64 v)
+{
+  if (!swap)
+    return v;
+  return GUINT64_SWAP_LE_BE (v);
+}
+
+typedef enum {
+  OSTREE_DELTA_ENDIAN_BIG,
+  OSTREE_DELTA_ENDIAN_LITTLE,
+  OSTREE_DELTA_ENDIAN_UNKNOWN,
+  OSTREE_DELTA_ENDIAN_INVALID
+} OstreeDeltaEndianness;
+
+OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock);
+
+gboolean _ostree_delta_needs_byteswap (GVariant *superblock);
+
 G_END_DECLS

src/ostree/ot-builtin-static-delta.c

@@ -32,7 +32,9 @@ static char *opt_to_rev;
 static char *opt_min_fallback_size;
 static char *opt_max_bsdiff_size;
 static char *opt_max_chunk_size;
+static char *opt_endianness;
 static gboolean opt_empty;
+static gboolean opt_swap_endianness;
 static gboolean opt_inline;
 static gboolean opt_disable_bsdiff;
 
@@ -60,6 +62,8 @@ static GOptionEntry generate_options[] = {
   { "inline", 0, 0, G_OPTION_ARG_NONE, &opt_inline, "Inline delta parts into main delta", NULL },
   { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" },
   { "disable-bsdiff", 0, 0, G_OPTION_ARG_NONE, &opt_disable_bsdiff, "Disable use of bsdiff", NULL },
+  { "set-endianness", 0, 0, G_OPTION_ARG_STRING, &opt_endianness, "Choose metadata endianness ('l' or 'B')", "ENDIAN" },
+  { "swap-endianness", 0, 0, G_OPTION_ARG_NONE, &opt_swap_endianness, "Swap metadata endianness from host order", NULL },
   { "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in megabytes for individual HTTP request", NULL},
   { "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL},
   { "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL},
@@ -194,6 +198,7 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
       g_autofree char *to_resolved = NULL;
       g_autofree char *from_parent_str = NULL;
       g_autoptr(GVariantBuilder) parambuilder = NULL;
+      int endianness;
 
       g_assert (opt_to_rev);
 
@@ -224,6 +229,37 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
         }
       if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error))
         goto out;
+      
+      if (opt_endianness)
+        {
+          if (strcmp (opt_endianness, "l") == 0)
+            endianness = G_LITTLE_ENDIAN;
+          else if (strcmp (opt_endianness, "B") == 0)
+            endianness = G_BIG_ENDIAN;
+          else
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Invalid endianness '%s'", opt_endianness);
+              goto out;
+            }
+        }
+      else
+        endianness = G_BYTE_ORDER;
+          
+      if (opt_swap_endianness)
+        {
+          switch (endianness)
+            {
+            case G_LITTLE_ENDIAN:
+              endianness = G_BIG_ENDIAN;
+              break;
+            case G_BIG_ENDIAN:
+              endianness = G_LITTLE_ENDIAN;
+              break;
+            default:
+              g_assert_not_reached ();
+            }
+        }
 
       parambuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
       if (opt_min_fallback_size)
@@ -243,6 +279,8 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
                                "inline-parts", g_variant_new_boolean (TRUE));
 
       g_variant_builder_add (parambuilder, "{sv}", "verbose", g_variant_new_boolean (TRUE));
+      if (opt_endianness || opt_swap_endianness)
+        g_variant_builder_add (parambuilder, "{sv}", "endianness", g_variant_new_uint32 (endianness));
 
       g_print ("Generating static delta:\n");
       g_print ("  From: %s\n", from_resolved ? from_resolved : "empty");

tests/pull-test.sh

@@ -113,8 +113,8 @@ ostree --repo=ostree-srv/gnomerepo summary -u
 cd ${test_tmpdir}
 repo_init
 ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
-${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt
+${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >dry-run-pull.txt
-assert_file_has_content out.txt 'Delta update: 0/1 parts'
+assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts'
 rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
 assert_streq "${prev_rev}" "${rev}"
 ${CMD_PREFIX} ostree --repo=repo fsck
@@ -142,6 +142,21 @@ assert_not_has_file baz/saucer
 
 echo "ok static delta"
 
+cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --swap-endianness main
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
+
+repo_init
+${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
+${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas --dry-run origin main >byteswapped-dry-run-pull.txt
+${CMD_PREFIX} ostree --repo=repo fsck
+
+if ! diff -u dry-run-pull.txt byteswapped-dry-run-pull.txt; then
+    assert_not_reached "byteswapped delta differs in size"
+fi
+
+echo "ok pull byteswapped delta"
+
 cd ${test_tmpdir}
 rm ostree-srv/gnomerepo/deltas -rf
 ostree --repo=ostree-srv/gnomerepo summary -u

tests/test-delta.sh

@@ -119,6 +119,15 @@ assert_file_has_content show.txt 'Endianness: \(little\|big\)'
 
 echo 'ok show'
 
+${CMD_PREFIX} ostree --repo=repo static-delta generate --swap-endianness --from=${origrev} --to=${newrev}
+${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show-swapped.txt
+totalsize_orig=$(grep 'Total Size:' show.txt)
+totalsize_swapped=$(grep 'Total Size:' show-swapped.txt)
+assert_not_streq "${totalsize_orig}" ""
+assert_streq "${totalsize_orig}" "${totalsize_swapped}"
+
+echo 'ok generate + show endian swapped'
+
 mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2
 ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev}
 ${CMD_PREFIX} ostree --repo=repo2 fsck