Static deltas support

https://bugzilla.gnome.org/show_bug.cgi?id=721799

tests/pull-test.sh

@@ -19,21 +19,31 @@
 
 set -e
 
-cd ${test_tmpdir}
-mkdir repo
-${CMD_PREFIX} ostree --repo=repo init
-${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+function repo_init() {
+    cd ${test_tmpdir}
+    rm repo -rf
+    mkdir repo
+    ${CMD_PREFIX} ostree --repo=repo init
+    ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+}
+
+function verify_initial_contents() {
+    rm checkout-origin-main -rf
+    $OSTREE checkout origin/main checkout-origin-main
+    cd checkout-origin-main
+    assert_file_has_content firstfile '^first$'
+    assert_file_has_content baz/cow '^moo$'
+}
+
 # Try both syntaxes
+repo_init
 ${CMD_PREFIX} ostree --repo=repo pull origin main
 ${CMD_PREFIX} ostree --repo=repo pull origin:main
 ${CMD_PREFIX} ostree --repo=repo fsck
 echo "ok pull"
 
 cd ${test_tmpdir}
-$OSTREE checkout origin/main checkout-origin-main
-cd checkout-origin-main
-assert_file_has_content firstfile '^first$'
-assert_file_has_content baz/cow '^moo$'
+verify_initial_contents
 echo "ok pull contents"
 
 cd ${test_tmpdir}
@@ -52,3 +62,63 @@ ${CMD_PREFIX} ostree --repo=repo fsck
 $OSTREE show --print-detached-metadata-key=SIGNATURE main > main-meta
 assert_file_has_content main-meta "HANCOCK"
 echo "ok pull detached metadata"
+
+cd ${test_tmpdir}
+repo_init
+${CMD_PREFIX} ostree --repo=repo pull origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+# Generate a delta from old to current, even though we aren't going to
+# use it.
+ostree --repo=ostree-srv/gnomerepo static-delta generate main
+
+rm main-files -rf
+ostree --repo=ostree-srv/gnomerepo checkout main main-files
+cd main-files
+echo "an added file for static deltas" > added-file
+echo "modified file for static deltas" > baz/cow
+rm baz/saucer
+ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s 'static delta test'
+cd ..
+rm main-files -rf
+# Generate delta that we'll use
+ostree --repo=ostree-srv/gnomerepo static-delta generate main
+
+cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=repo pull origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+
+rm checkout-origin-main -rf
+$OSTREE checkout origin:main checkout-origin-main
+cd checkout-origin-main
+assert_file_has_content firstfile '^first$'
+assert_file_has_content baz/cow "modified file for static deltas"
+assert_not_has_file baz/saucer
+
+echo "ok static delta"
+
+cd ${test_tmpdir}
+rm main-files -rf
+ostree --repo=ostree-srv/gnomerepo checkout main main-files
+cd main-files
+# Make a file larger than 16M for testing
+dd if=/dev/zero of=test-bigfile count=1 seek=42678
+echo "further modified file for static deltas" > baz/cow
+ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s '2nd static delta test'
+cd ..
+rm main-files -rf
+ostree --repo=ostree-srv/gnomerepo static-delta generate main
+
+cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=repo pull origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+
+rm checkout-origin-main -rf
+$OSTREE checkout origin:main checkout-origin-main
+cd checkout-origin-main
+assert_has_file test-bigfile
+stat --format=%s test-bigfile > bigfile-size
+assert_file_has_content bigfile-size 21851648
+assert_file_has_content baz/cow "further modified file for static deltas"
+assert_not_has_file baz/saucer
+
+echo "ok static delta 2"

tests/test-delta.sh

@@ -57,18 +57,20 @@ function permuteDirectory() {
 
 permuteDirectory 1 files
 ostree --repo=repo commit -b test -s test --tree=dir=files
-ostree static-delta --repo=repo
+ostree static-delta --repo=repo list
 
 origrev=$(ostree --repo=repo rev-parse test^)
 newrev=$(ostree --repo=repo rev-parse test)
-ostree static-delta --repo=repo --from=${origrev} --to=${newrev}
+ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev}
 
-assert_has_dir repo/deltas/${origrev}-${newrev}
+origstart=$(echo ${origrev} | dd bs=1 count=2 2>/dev/null)
+origend=$(echo ${origrev} | dd bs=1 skip=2 2>/dev/null)
+assert_has_dir repo/deltas/${origstart}/${origend}-${newrev}
 
 mkdir repo2
 ostree --repo=repo2 init --mode=archive-z2
 ostree --repo=repo2 pull-local repo ${origrev}
 
-ostree --repo=repo2 static-delta --apply=repo/deltas/${origrev}-${newrev}
+ostree --repo=repo2 static-delta apply-offline repo/deltas/${origstart}/${origend}-${newrev}
 ostree --repo=repo2 fsck
 ostree --repo=repo2 show ${newrev}

tests/test-rollsum.c

@@ -25,7 +25,49 @@
 #include "bupsplit.h"
 
 #define BLOB_MAX (8192*4)
-#define BLOB_READ_SIZE (1024*1024)
+
+static GPtrArray *
+rollsum_checksums_for_data (GBytes     *bytes)
+{
+  const guint8 *start;
+  gsize len;
+  GPtrArray *ret = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+
+  start = g_bytes_get_data (bytes, &len);
+  while (TRUE)
+    {
+      int offset, bits;
+      offset = bupsplit_find_ofs (start, MIN(G_MAXINT32, len), &bits); 
+      if (offset == 0)
+        break;
+      if (offset > BLOB_MAX)
+        offset = BLOB_MAX;
+      {
+        gs_free char *blobcsum =
+          g_compute_checksum_for_data (G_CHECKSUM_SHA256,
+                                       start, offset);
+        g_ptr_array_add (ret, g_variant_ref_sink (g_variant_new ("(st)",
+                                                                 blobcsum, (guint64)offset)));
+      }
+      start += offset;
+      len -= offset;
+    }
+  return ret;
+}
+
+static void
+print_rollsums (GPtrArray  *rollsums)
+{
+  guint i;
+  for (i = 0; i < rollsums->len; i++)
+    {
+      GVariant *sum = rollsums->pdata[i];
+      const char *csum;
+      guint64 val;
+      g_variant_get (sum, "(&st)", &csum, &val);
+      g_print ("chunk %s %" G_GUINT64_FORMAT "\n", csum, val);
+    }
+}
 
 int
 main (int argc, char **argv)
@@ -38,29 +80,57 @@ main (int argc, char **argv)
 
   g_setenv ("GIO_USE_VFS", "local", TRUE);
 
-  if (argc > 1)
+  if (argc == 2)
     {
-      const guint8 *start;
-      gsize len;
+      gs_unref_ptrarray GPtrArray *rollsums = NULL;
 
       path = g_file_new_for_path (argv[1]);
       bytes = gs_file_map_readonly (path, cancellable, error);
       if (!bytes)
 	goto out;
 
-      start = g_bytes_get_data (bytes, &len);
-      while (TRUE)
-	{
-	  int offset, bits;
-	  offset = bupsplit_find_ofs (start, MIN(G_MAXINT32, len), &bits); 
-	  if (offset == 0)
-	    break;
-	  if (offset > BLOB_MAX)
-	    offset = BLOB_MAX;
-          g_print ("%" G_GUINT64_FORMAT "\n", (guint64)offset);
-	  start += offset;
-          len -= offset;
+      rollsums = rollsum_checksums_for_data (bytes);
+      print_rollsums (rollsums);
+    }
+  else if (argc > 2)
+    {
+      guint i;
+      gs_unref_hashtable GHashTable *sums = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+      guint64 input_size = 0;
+      guint64 rollsum_size = 0;
+
+      for (i = 1; i < argc; i++)
+        {
+          guint j;
+          gs_unref_ptrarray GPtrArray *rollsums = NULL;
+
+          path = g_file_new_for_path (argv[i]);
+          bytes = gs_file_map_readonly (path, cancellable, error);
+          if (!bytes)
+            goto out;
+
+          input_size += g_bytes_get_size (bytes);
+          
+          g_print ("input: %s size: %" G_GUINT64_FORMAT "\n", argv[i], g_bytes_get_size (bytes));
+
+          rollsums = rollsum_checksums_for_data (bytes);
+          print_rollsums (rollsums);
+          for (j = 0; j < rollsums->len; j++)
+            {
+              GVariant *sum = rollsums->pdata[j];
+              const char *csum;
+              guint64 ofs;
+              g_variant_get (sum, "(&st)", &csum, &ofs);
+              if (!g_hash_table_contains (sums, csum))
+                {
+                  g_hash_table_add (sums, g_strdup (csum));
+                  rollsum_size += ofs;
+                }
+            }
         }
+      g_print ("rollsums:%u input:%" G_GUINT64_FORMAT " output: %" G_GUINT64_FORMAT " speedup:%f\n",
+               g_hash_table_size (sums), input_size, rollsum_size,
+               (((double)(input_size+1)) / ((double) rollsum_size + 1)));
     }
   else
     {