lib/commit: Include object type in sizes metadata

Append a byte encoding the OSTree object type for each object in the
metadata. This allows the commit metadata to be fetched and then for the
program to see which objects it already has for an accurate calculation
of which objects need to be downloaded.

This slightly breaks the `ostree.sizes` `ay` metadata entries. However,
it's unlikely anyone was asserting the length of the entries since the
array currently ends in 2 variable length integers. As far as I know,
the only users of the sizes metadata are the ostree test suite and
Endless' eos-updater[1]. The former is updated here and the latter
already expects this format.

1. https://github.com/endlessm/eos-updater/

src/libostree/ostree-repo-commit.c

@@ -322,16 +322,19 @@ commit_loose_regfile_object (OstreeRepo        *self,
 /* This is used by OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES */
 typedef struct
 {
+  OstreeObjectType objtype;
   goffset unpacked;
   goffset archived;
 } OstreeContentSizeCacheEntry;
 
 static OstreeContentSizeCacheEntry *
-content_size_cache_entry_new (goffset unpacked,
+content_size_cache_entry_new (OstreeObjectType objtype,
-                              goffset archived)
+                              goffset          unpacked,
+                              goffset          archived)
 {
   OstreeContentSizeCacheEntry *entry = g_slice_new0 (OstreeContentSizeCacheEntry);
 
+  entry->objtype = objtype;
   entry->unpacked = unpacked;
   entry->archived = archived;
 
@@ -399,7 +402,7 @@ repo_store_size_entry (OstreeRepo       *self,
   repo_ensure_size_entries (self);
   g_hash_table_replace (self->object_sizes,
                         g_strdup (checksum),
-                        content_size_cache_entry_new (unpacked, archived));
+                        content_size_cache_entry_new (objtype, unpacked, archived));
 }
 
 static int
@@ -450,6 +453,7 @@ add_size_index_to_metadata (OstreeRepo        *self,
             g_hash_table_lookup (self->object_sizes, e_checksum);
           _ostree_write_varuint64 (buffer, e_size->archived);
           _ostree_write_varuint64 (buffer, e_size->unpacked);
+          g_string_append_c (buffer, (gchar) e_size->objtype);
 
           g_variant_builder_add (&index_builder, "@ay",
                                  ot_gvariant_new_bytearray ((guint8*)buffer->str, buffer->len));

tests/test-sizes.js

@@ -67,15 +67,15 @@ function unpackByteArray(variant) {
     return array;
 }
 
-function validateSizes(repo, commit, expectedFiles) {
+function validateSizes(repo, commit, expectedObjects) {
     let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit);
     let metadata = commitVariant.get_child_value(0);
     let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay'));
-    let nSizes = sizes.n_children();
+    let nObjects = sizes.n_children();
-    let expectedNSizes = Object.keys(expectedFiles).length
+    let expectedNObjects = Object.keys(expectedObjects).length
-    assertEquals(nSizes, expectedNSizes);
+    assertEquals(nObjects, expectedNObjects);
 
-    for (let i = 0; i < nSizes; i++) {
+    for (let i = 0; i < nObjects; i++) {
         let sizeEntry = sizes.get_child_value(i);
         assertGreaterEquals(sizeEntry.n_children(), 34);
         let entryBytes = unpackByteArray(sizeEntry);
@@ -94,15 +94,20 @@ function validateSizes(repo, commit, expectedFiles) {
         assertGreaterEquals(remainingBytes.length, 1);
         [uncompressedSize, varintRead] = readVarint(remainingBytes);
         remainingBytes = remainingBytes.slice(varintRead);
-        assertEquals(remainingBytes.length, 0);
+        assertEquals(remainingBytes.length, 1);
+        let objectType = remainingBytes[0];
+        let objectTypeString = OSTree.object_type_to_string(objectType);
         print("compressed = " + compressedSize);
         print("uncompressed = " + uncompressedSize);
+        print("objtype = " + objectTypeString + " (" + objectType + ")");
+        let objectName = OSTree.object_to_string(checksumString, objectType);
+        print("object = " + objectName);
 
-        if (!(checksumString in expectedFiles)) {
+        if (!(objectName in expectedObjects)) {
-            throw new Error("Checksum " + checksumString + " not in " +
+            throw new Error("Object " + objectName + " not in " +
-                            JSON.stringify(expectedFiles));
+                            JSON.stringify(expectedObjects));
         }
-        let expectedSizes = expectedFiles[checksumString];
+        let expectedSizes = expectedObjects[objectName];
         let expectedCompressedSize = expectedSizes[0];
         let expectedUncompressedSize = expectedSizes[1];
         if (compressedSize != expectedCompressedSize) {
@@ -152,15 +157,26 @@ print("commit => " + commit);
 
 repo.commit_transaction(null);
 
-// Test the sizes metadata
+// Test the sizes metadata. The key is the object and the value is an
-let expectedFiles = {
+// array of compressed size and uncompressed size.
-    'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad': [54, 18],
+let expectedObjects = {
-    'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b': [48, 12],
+    'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file': [
-    '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0': [43, 9],
+        54, 18
-    '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51': [185, 185],
+    ],
-    '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488': [12, 12],
+    'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b.file': [
+        48, 12
+    ],
+    '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0.file': [
+        43, 9
+    ],
+    '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree': [
+        185, 185
+    ],
+    '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488.dirmeta': [
+        12, 12
+    ],
 };
-validateSizes(repo, commit, expectedFiles);
+validateSizes(repo, commit, expectedObjects);
 
 print("ok test-sizes");
 
@@ -168,9 +184,11 @@ print("ok test-sizes");
 // previous commit. Remove that file from the expected metadata and
 // replace the dirtree object.
 testDataDir.get_child('another-file').delete(null);
-delete expectedFiles['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad'];
+delete expectedObjects['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file'];
-delete expectedFiles['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51'];
+delete expectedObjects['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree'];
-expectedFiles['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539'] = [138, 138];
+expectedObjects['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539.dirtree'] = [
+    138, 138
+];
 
 repo.prepare_transaction(null);
 mtree = OSTree.MutableTree.new();
@@ -180,7 +198,7 @@ repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null);
 print("commit => " + commit);
 repo.commit_transaction(null);
 
-validateSizes(repo, commit, expectedFiles);
+validateSizes(repo, commit, expectedObjects);
 
 print("ok test-sizes file deleted");
 
@@ -194,6 +212,6 @@ repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null);
 print("commit => " + commit);
 repo.commit_transaction(null);
 
-validateSizes(repo, commit, expectedFiles);
+validateSizes(repo, commit, expectedObjects);
 
 print("ok test-sizes repeated");