deltas: Use base64 for csums, add version to parts

src/libostree/ostree-core.c

@@ -1103,6 +1103,35 @@ ostree_object_name_deserialize (GVariant         *variant,
   *out_objtype = (OstreeObjectType)objtype_u32;
 }
 
+/**
+ * ostree_checksum_b64_inplace_to_bytes: (skip)
+ * @checksum: (array fixed-size=32): An binary checksum of length 32
+ * @buf: Output location, must be at least 45 bytes in length
+ *
+ * Overwrite the contents of @buf with stringified version of @csum.
+ */
+void
+ostree_checksum_b64_inplace_to_bytes (const char *checksum,
+                                      guchar     *buf)
+{
+  int state = 0;
+  guint save = 0;
+  char tmpbuf[44];
+  int i;
+
+  for (i = 0; i < 43; i++)
+    {
+      char c = checksum[i];
+      if (c == '_')
+        tmpbuf[i] = '/';
+      else
+        tmpbuf[i] = c;
+    }
+  tmpbuf[43] = '=';
+
+  g_base64_decode_step (tmpbuf, sizeof (tmpbuf), (guchar *) buf, &state, &save);
+}
+
 /**
  * ostree_checksum_inplace_to_bytes:
  * @checksum: a SHA256 string
@@ -1186,6 +1215,48 @@ ostree_checksum_inplace_from_bytes (const guchar *csum,
   buf[j] = '\0';
 }
 
+/**
+ * ostree_checksum_b64_inplace_from_bytes: (skip)
+ * @csum: (array fixed-size=32): An binary checksum of length 32
+ * @buf: Output location, must be at least 44 bytes in length
+ *
+ * Overwrite the contents of @buf with modified base64 encoding of @csum.
+ * The "modified" term refers to the fact that instead of '/', the '_'
+ * character is used.
+ */
+void
+ostree_checksum_b64_inplace_from_bytes (const guchar *csum,
+                                        char         *buf)
+{
+  char tmpbuf[44];
+  int save = 0;
+  int state = 0;
+  gsize outlen;
+  int i;
+
+  /* At some point, we can optimize this, but for now it's
+   * a lot easier to reuse GLib's base64 encoder and postprocess it
+   * to replace the '/' with '_'.
+   */
+  outlen = g_base64_encode_step (csum, 32, FALSE, tmpbuf, &state, &save);
+  outlen += g_base64_encode_close (FALSE, tmpbuf+outlen, &state, &save);
+  g_assert (outlen == 44);
+
+  for (i = 0; i < sizeof (tmpbuf); i++)
+    {
+      char c = tmpbuf[i];
+      if (c == '=')
+        {
+          g_assert (i == 43);
+          buf[i] = '\0';
+        }
+      else if (c == '/')
+        buf[i] = '_';
+      else
+        buf[i] = c;
+    }
+}
+
 /**
  * ostree_checksum_from_bytes:
  * @csum: (array fixed-size=32): An binary checksum of length 32
@@ -1360,21 +1431,35 @@ get_delta_path (const char *from,
                 const char *target)
 {
   char prefix[3];
+  guint8 csum_to[32];
+  char to_b64[44];
+  guint8 csum_to_copy[32];
+
+  ostree_checksum_inplace_to_bytes (to, csum_to);
+  ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64);
+  ostree_checksum_b64_inplace_to_bytes (to_b64, csum_to_copy);
+
+  g_assert (memcmp (csum_to, csum_to_copy, 32) == 0);
+
   if (from == NULL)
     {
-      prefix[0] = to[0];
+      prefix[0] = to_b64[0];
-      prefix[1] = to[1];
+      prefix[1] = to_b64[1];
       prefix[2] = '\0';
-      to += 2;
+      return g_strconcat ("deltas/", prefix, "/", ((char*)to_b64)+2, "/", target, NULL);
-      return g_strconcat ("deltas/", prefix, "/", to, "/", target, NULL);
     }
   else
     {
-      prefix[0] = from[0];
+      guint8 csum_from[32];
-      prefix[1] = from[1];
+      char from_b64[44];
+
+      ostree_checksum_inplace_to_bytes (from, csum_from);
+      ostree_checksum_b64_inplace_from_bytes (csum_from, from_b64);
+
+      prefix[0] = from_b64[0];
+      prefix[1] = from_b64[1];
       prefix[2] = '\0';
-      from += 2;
+      return g_strconcat ("deltas/", prefix, "/", ((char*)from_b64)+2, "-", to_b64, "/", target, NULL);
-      return g_strconcat ("deltas/", prefix, "/", from, "-", to, "/", target, NULL);
     }
 }
 

src/libostree/ostree-core.h

@@ -164,12 +164,16 @@ gboolean ostree_validate_checksum_string (const char *sha256,
 
 guchar *ostree_checksum_to_bytes (const char *checksum);
 GVariant *ostree_checksum_to_bytes_v (const char *checksum);
+void ostree_checksum_b64_inplace_to_bytes (const char *checksum,
+                                           guint8     *buf);
 
 char * ostree_checksum_from_bytes (const guchar *csum);
 char * ostree_checksum_from_bytes_v (GVariant *csum_v);
 
 void ostree_checksum_inplace_from_bytes (const guchar *csum,
                                          char         *buf);
+void ostree_checksum_b64_inplace_from_bytes (const guchar *csum,
+                                             char         *buf);
 
 void ostree_checksum_inplace_to_bytes (const char *checksum,
                                        guchar     *buf);

src/libostree/ostree-repo-private.h

@@ -24,6 +24,8 @@
 
 G_BEGIN_DECLS
 
+#define OSTREE_DELTAPART_VERSION (0)
+
 #define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay"
 
 /**

src/libostree/ostree-repo-pull.c

@@ -1513,9 +1513,17 @@ process_one_static_delta (OtPullData   *pull_data,
       gs_unref_variant GVariant *csum_v = NULL;
       gs_unref_variant GVariant *objects = NULL;
       guint64 size, usize;
+      guint32 version;
 
       header = g_variant_get_child_value (headers, i);
-      g_variant_get (header, "(@aytt@ay)", &csum_v, &size, &usize, &objects);
+      g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
+
+      if (version > OSTREE_DELTAPART_VERSION)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Delta part has too new version %u", version);
+          goto out;
+        }
 
       csum = ostree_checksum_bytes_peek_validate (csum_v, error);
       if (!csum)

src/libostree/ostree-repo-static-delta-compilation.c

@@ -548,7 +548,8 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
 
       checksum_bytes = g_bytes_new (part_checksum, 32);
       objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
-      delta_part_header = g_variant_new ("(@aytt@ay)",
+      delta_part_header = g_variant_new ("(u@aytt@ay)",
+                                         OSTREE_DELTAPART_VERSION,
                                          ot_gvariant_new_ay_bytes (checksum_bytes),
                                          g_variant_get_size (delta_part),
                                          part_builder->uncompressed_size,
@@ -586,13 +587,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
     metadata_source = metadata;
   else
     {
-      GVariantBuilder tmpbuilder;
+      metadata_source = ot_gvariant_new_empty_string_dict ();
-      g_variant_builder_init (&tmpbuilder, G_VARIANT_TYPE ("(a(ss)a(say))"));
-      g_variant_builder_add (&tmpbuilder, "a(ss)", NULL);
-      g_variant_builder_add (&tmpbuilder, "a(say)", NULL);
-      tmp_metadata = g_variant_builder_end (&tmpbuilder);
-      g_variant_ref_sink (tmp_metadata);
-      metadata_source = tmp_metadata;
     }
 
   if (!get_fallback_headers (self, &builder, &fallback_headers,
@@ -607,7 +602,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
     /* floating */ GVariant *to_csum_v =
       ostree_checksum_to_bytes_v (to);
 
-    delta_descriptor = g_variant_new ("(@(a(ss)a(say))t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay"
+    delta_descriptor = g_variant_new ("(@a{sv}t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay"
                                       "a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT
                                       "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")",
                                       metadata_source,

src/libostree/ostree-repo-static-delta-core.c

@@ -127,7 +127,24 @@ ostree_repo_list_static_delta_names (OstreeRepo                  *self,
 
                 if (g_file_query_exists (meta_path, NULL))
                   {
-                    g_ptr_array_add (ret_deltas, g_strconcat (name1, name2, NULL));
+                    gs_free char *buf = g_strconcat (name1, name2, NULL);
+                    GString *out = g_string_new ("");
+                    char checksum[65];
+                    guchar csum[32];
+                    const char *dash = strchr (buf, '-');
+
+                    ostree_checksum_b64_inplace_to_bytes (buf, csum);
+                    ostree_checksum_inplace_from_bytes (csum, checksum);
+                    g_string_append (out, checksum);
+                    if (dash)
+                      {
+                        g_string_append_c (out, '-');
+                        ostree_checksum_b64_inplace_to_bytes (dash+1, csum);
+                        ostree_checksum_inplace_from_bytes (csum, checksum);
+                        g_string_append (out, checksum);
+                      }
+
+                    g_ptr_array_add (ret_deltas, g_string_free (out, FALSE));
                   }
               }
             }

src/libostree/ostree-repo-static-delta-private.h

@@ -42,6 +42,7 @@ G_BEGIN_DECLS
 /**
  * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
  *
+ *   u: version
  *   ay checksum
  *   guint64 size:   Total size of delta (sum of parts)
  *   guint64 usize:   Uncompressed size of resulting objects on disk
@@ -51,7 +52,7 @@ G_BEGIN_DECLS
  * represents an OSTree object which will be created by the deltapart.
  */
 
-#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(ayttay)"
+#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(uayttay)"
 
 
 /**
@@ -131,9 +132,7 @@ gboolean _ostree_static_delta_part_execute_finish (OstreeRepo      *repo,
 typedef enum {
   OSTREE_STATIC_DELTA_OP_WRITE = 1,
   OSTREE_STATIC_DELTA_OP_GUNZIP = 2,
-  OSTREE_STATIC_DELTA_OP_CLOSE = 3,
+  OSTREE_STATIC_DELTA_OP_CLOSE = 3
-  OSTREE_STATIC_DELTA_OP_READOBJECT = 4,
-  OSTREE_STATIC_DELTA_OP_READPAYLOAD = 5
 } OstreeStaticDeltaOpCode;
 
 gboolean

tests/test-delta.sh

@@ -72,21 +72,6 @@ if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to
     assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded"
 fi
 
-origstart=$(echo ${origrev} | dd bs=1 count=2 2>/dev/null)
-origend=$(echo ${origrev} | dd bs=1 skip=2 2>/dev/null)
-assert_has_dir repo/deltas/${origstart}/${origend}-${newrev}
-assert_has_dir repo/deltas/${origstart}/${origend}
-
 mkdir repo2
 ostree --repo=repo2 init --mode=archive-z2
 ostree --repo=repo2 pull-local repo ${origrev}
-
-ostree --repo=repo2 static-delta apply-offline repo/deltas/${origstart}/${origend}-${newrev}
-ostree --repo=repo2 fsck
-ostree --repo=repo2 show ${newrev}
-
-mkdir repo3
-ostree --repo=repo3 init --mode=archive-z2
-ostree --repo=repo3 static-delta apply-offline repo/deltas/${origstart}/${origend}
-ostree --repo=repo3 fsck
-ostree --repo=repo3 show ${origrev}