From 37045b4b468460785926d73cc2bc4d288caaa9b3 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 13:05:48 -0600 Subject: [PATCH 01/12] lib/commit: Only set generate_sizes for archive repos Rather than checking throughout the code, only set the boolean when appropriate. --- src/libostree/ostree-repo-commit.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8c5d9411..8f059e11 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -345,6 +345,19 @@ content_size_cache_entry_free (gpointer entry) g_slice_free (OstreeContentSizeCacheEntry, entry); } +static void +repo_setup_generate_sizes (OstreeRepo *self, + OstreeRepoCommitModifier *modifier) +{ + if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) + { + if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE) + self->generate_sizes = TRUE; + else + g_debug ("Not generating sizes for non-archive repo"); + } +} + static void repo_store_size_entry (OstreeRepo *self, const gchar *checksum, @@ -956,7 +969,6 @@ write_content_object (OstreeRepo *self, g_auto(OtCleanupUnlinkat) tmp_unlinker = { commit_tmp_dfd (self), NULL }; g_auto(GLnxTmpfile) tmpf = { 0, }; goffset unpacked_size = 0; - gboolean indexable = FALSE; /* Is it a symlink physically? */ if (phys_object_is_symlink) { @@ -982,9 +994,6 @@ write_content_object (OstreeRepo *self, g_assert (repo_mode == OSTREE_REPO_MODE_ARCHIVE); - if (self->generate_sizes) - indexable = TRUE; - if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_WRONLY|O_CLOEXEC, &tmpf, error)) return FALSE; @@ -1108,7 +1117,7 @@ write_content_object (OstreeRepo *self, else { /* Update size metadata if configured */ - if (indexable && object_file_type == G_FILE_TYPE_REGULAR) + if (self->generate_sizes && object_file_type == G_FILE_TYPE_REGULAR) { struct stat stbuf; @@ -3848,8 +3857,7 @@ ostree_repo_write_directory_to_mtree (OstreeRepo *self, } else { - if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) - self->generate_sizes = TRUE; + repo_setup_generate_sizes (self, modifier); g_autoptr(GPtrArray) path = g_ptr_array_new (); if (!write_directory_to_mtree_internal (self, dir, mtree, modifier, path, @@ -3883,8 +3891,7 @@ ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error) { - if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) - self->generate_sizes = TRUE; + repo_setup_generate_sizes (self, modifier); g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error)) From 694b741a366f4abb523f6e4ffbec3f56c5934d1a Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 22 Oct 2019 14:59:19 -0600 Subject: [PATCH 02/12] tests/sizes: Improve metadata validation Ensure all 3 of the checksum, compressed size and uncompressed size are correct. For repeatable objects, skip xattrs and use canonical permissions for the commit. For the sizes, read a varint rather than assuming they will be a single byte. To work around bugs in gjs with byte array unpacking, manually build the array byte by byte. Split out some helper functions to use in subsequent tests. --- tests/test-sizes.js | 126 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 27 deletions(-) diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 73b179c5..622c2d1b 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -28,6 +28,96 @@ function assertEquals(a, b) { throw new Error("assertion failed " + JSON.stringify(a) + " == " + JSON.stringify(b)); } +function assertGreater(a, b) { + if (a <= b) + throw new Error("assertion failed " + JSON.stringify(a) + " > " + JSON.stringify(b)); +} + +function assertGreaterEquals(a, b) { + if (a < b) + throw new Error("assertion failed " + JSON.stringify(a) + " >= " + JSON.stringify(b)); +} + +// Adapted from _ostree_read_varuint64() +function readVarint(buffer) { + let result = 0; + let count = 0; + let len = buffer.length; + let cur; + + do { + assertGreater(len, 0); + cur = buffer[count]; + result = result | ((cur & 0x7F) << (7 * count)); + count++; + len--; + } while (cur & 0x80); + + return [result, count]; +} + +// There have been various bugs with byte array unpacking in GJS, so +// just do it manually. +function unpackByteArray(variant) { + let array = []; + let nBytes = variant.n_children(); + for (let i = 0; i < nBytes; i++) { + array.push(variant.get_child_value(i).get_byte()); + } + return array; +} + +function validateSizes(repo, commit, expectedFiles) { + let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit); + let metadata = commitVariant.get_child_value(0); + let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay')); + let nSizes = sizes.n_children(); + let expectedNSizes = Object.keys(expectedFiles).length + assertEquals(nSizes, expectedNSizes); + + for (let i = 0; i < nSizes; i++) { + let sizeEntry = sizes.get_child_value(i); + assertGreaterEquals(sizeEntry.n_children(), 34); + let entryBytes = unpackByteArray(sizeEntry); + let checksumBytes = entryBytes.slice(0, 32); + let checksumString = OSTree.checksum_from_bytes(checksumBytes); + print("checksum = " + checksumString); + + // Read the sizes from the next 2 varints + let remainingBytes = entryBytes.slice(32); + assertGreaterEquals(remainingBytes.length, 2); + let varintRead; + let compressedSize; + let uncompressedSize; + [compressedSize, varintRead] = readVarint(remainingBytes); + remainingBytes = remainingBytes.slice(varintRead); + assertGreaterEquals(remainingBytes.length, 1); + [uncompressedSize, varintRead] = readVarint(remainingBytes); + remainingBytes = remainingBytes.slice(varintRead); + assertEquals(remainingBytes.length, 0); + print("compressed = " + compressedSize); + print("uncompressed = " + uncompressedSize); + + if (!(checksumString in expectedFiles)) { + throw new Error("Checksum " + checksumString + " not in " + + JSON.stringify(expectedFiles)); + } + let expectedSizes = expectedFiles[checksumString]; + let expectedCompressedSize = expectedSizes[0]; + let expectedUncompressedSize = expectedSizes[1]; + if (compressedSize != expectedCompressedSize) { + throw new Error("Compressed size " + compressedSize + + " for checksum " + checksumString + + " does not match expected " + expectedCompressedSize); + } + if (uncompressedSize != expectedUncompressedSize) { + throw new Error("Uncompressed size " + uncompressedSize + + " for checksum " + checksumString + + " does not match expected " + expectedUncompressedSize); + } + } +} + print('1..1') let testDataDir = Gio.File.new_for_path('test-data'); @@ -41,7 +131,10 @@ repo.create(OSTree.RepoMode.ARCHIVE_Z2, null); repo.open(null); -let commitModifier = OSTree.RepoCommitModifier.new(OSTree.RepoCommitModifierFlags.GENERATE_SIZES, null); +let commitModifierFlags = (OSTree.RepoCommitModifierFlags.GENERATE_SIZES | + OSTree.RepoCommitModifierFlags.SKIP_XATTRS | + OSTree.RepoCommitModifierFlags.CANONICAL_PERMISSIONS); +let commitModifier = OSTree.RepoCommitModifier.new(commitModifierFlags, null); assertEquals(repo.get_mode(), OSTree.RepoMode.ARCHIVE_Z2); @@ -56,31 +149,10 @@ print("commit => " + commit); repo.commit_transaction(null); // Test the sizes metadata -let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit); -let metadata = commitVariant.get_child_value(0); -let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay')); -let nSizes = sizes.n_children(); -assertEquals(nSizes, 2); -let expectedUncompressedSizes = [12, 18]; -let foundExpectedUncompressedSizes = 0; -for (let i = 0; i < nSizes; i++) { - let sizeEntry = sizes.get_child_value(i); - assertEquals(sizeEntry.n_children(), 34); - let compressedSize = sizeEntry.get_child_value(32).get_byte(); - let uncompressedSize = sizeEntry.get_child_value(33).get_byte(); - print("compressed = " + compressedSize); - print("uncompressed = " + uncompressedSize); - for (let j = 0; j < expectedUncompressedSizes.length; j++) { - let expected = expectedUncompressedSizes[j]; - if (expected == uncompressedSize) { - print("Matched expected uncompressed size " + expected); - expectedUncompressedSizes.splice(j, 1); - break; - } - } -} -if (expectedUncompressedSizes.length > 0) { - throw new Error("Failed to match expectedUncompressedSizes: " + JSON.stringify(expectedUncompressedSizes)); -} +let expectedFiles = { + 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad': [54, 18], + 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b': [48, 12], +}; +validateSizes(repo, commit, expectedFiles); print("ok test-sizes"); From 8ec7d6322fe004a93a1b391c279020c5411996c0 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 09:12:08 -0600 Subject: [PATCH 03/12] lib/commit: Fix object sizes metadata for multiple commits The object sizes hash table was only being cleared when the repo was finalized. That means that performing multiple commits while the repo was open would reuse all the object sizes metadata for each commit. Clear the hash table when the sizes metadata is setup and when it's added to a commit. This still does not fix the issue all the way since it does nothing to prevent the program from constructing multiple commits simultaneously. To handle that, the object sizes hash table should be attached to the MutableTree since that has the commit state. However, the MutableTree is gone when the commit is actually created. The hash table would have to be transferred to the root file when writing the MutableTree. That would be an awkward addition to OstreeRepoFile, though. Add a FIXME to capture that. --- src/libostree/ostree-repo-commit.c | 11 ++++++++++- src/libostree/ostree-repo-private.h | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8f059e11..752a01be 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -352,7 +352,13 @@ repo_setup_generate_sizes (OstreeRepo *self, if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) { if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE) - self->generate_sizes = TRUE; + { + self->generate_sizes = TRUE; + + /* Clear any stale data in the object sizes hash table */ + if (self->object_sizes != NULL) + g_hash_table_remove_all (self->object_sizes); + } else g_debug ("Not generating sizes for non-archive repo"); } @@ -428,6 +434,9 @@ add_size_index_to_metadata (OstreeRepo *self, g_variant_builder_add (builder, "{sv}", "ostree.sizes", g_variant_builder_end (&index_builder)); + + /* Clear the object sizes hash table for a subsequent commit. */ + g_hash_table_remove_all (self->object_sizes); } return g_variant_ref_sink (g_variant_builder_end (builder)); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index b57ad799..bc2325e5 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -143,6 +143,14 @@ struct OstreeRepo { guint zlib_compression_level; GHashTable *loose_object_devino_hash; GHashTable *updated_uncompressed_dirs; + + /* FIXME: The object sizes hash table is really per-commit state, not repo + * state. Using a single table for the repo means that commits cannot be + * built simultaneously if they're adding size information. This data should + * probably be in OstreeMutableTree, but that's gone by the time the actual + * commit is constructed. At that point the only commit state is in the root + * OstreeRepoFile. + */ GHashTable *object_sizes; /* Cache the repo's device/inode to use for comparisons elsewhere */ From 44fb5e72a1496d42ff500926a99dbe4c6cb44da6 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 09:28:11 -0600 Subject: [PATCH 04/12] lib/commit: Make size entries for existing objects If the object was already in the repo then the sizes metadata entry was skipped. Move the sizes entry creation after the data has been computed but before the early return for an existing object. --- src/libostree/ostree-repo-commit.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 752a01be..d995686b 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1060,6 +1060,19 @@ write_content_object (OstreeRepo *self, g_assert (actual_checksum != NULL); /* Pacify static analysis */ + /* Update size metadata if configured and entry missing */ + if (self->generate_sizes && object_file_type == G_FILE_TYPE_REGULAR && + (self->object_sizes == NULL || + g_hash_table_lookup (self->object_sizes, actual_checksum) == NULL)) + { + struct stat stbuf; + + if (!glnx_fstat (tmpf.fd, &stbuf, error)) + return FALSE; + + repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); + } + /* See whether or not we have the object, now that we know the * checksum. */ @@ -1125,17 +1138,6 @@ write_content_object (OstreeRepo *self, } else { - /* Update size metadata if configured */ - if (self->generate_sizes && object_file_type == G_FILE_TYPE_REGULAR) - { - struct stat stbuf; - - if (!glnx_fstat (tmpf.fd, &stbuf, error)) - return FALSE; - - repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); - } - /* Check if a file with the same payload is present in the repository, and in case try to reflink it */ if (actual_payload_checksum && !_try_clone_from_payload_link (self, self, actual_payload_checksum, file_info, &tmpf, cancellable, error)) @@ -2638,8 +2640,11 @@ ostree_repo_write_content (OstreeRepo *self, { /* First, if we have an expected checksum, see if we already have this * object. This mirrors the same logic in ostree_repo_write_metadata(). + * + * If size metadata is needed, fall through to write_content_object() + * where the entries are made. */ - if (expected_checksum) + if (expected_checksum && !self->generate_sizes) { gboolean have_obj; if (!_ostree_repo_has_loose_object (self, expected_checksum, From 1ea719b76bfa94181eb5d228c5c6be10e14b64f1 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 22 Oct 2019 15:14:58 -0600 Subject: [PATCH 05/12] tests/sizes: Test sizes metadata with existing objects Repeat the commit to make sure that the files are enumerated again for the size metadata. --- tests/test-sizes.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 622c2d1b..a3928fe8 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -118,7 +118,7 @@ function validateSizes(repo, commit, expectedFiles) { } } -print('1..1') +print('1..2') let testDataDir = Gio.File.new_for_path('test-data'); testDataDir.make_directory(null); @@ -156,3 +156,17 @@ let expectedFiles = { validateSizes(repo, commit, expectedFiles); print("ok test-sizes"); + +// Repeat the commit now that all the objects are cached and ensure the +// metadata is still correct +repo.prepare_transaction(null); +mtree = OSTree.MutableTree.new(); +repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); +[,dirTree] = repo.write_mtree(mtree, null); +[,commit] = repo.write_commit(null, 'Another subject', 'Another body', null, dirTree, null); +print("commit => " + commit); +repo.commit_transaction(null); + +validateSizes(repo, commit, expectedFiles); + +print("ok test-sizes repeated"); From 4f1b991246dbc67ab7176842a61f3022bed5aad5 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 09:10:06 -0600 Subject: [PATCH 06/12] tests/sizes: Test that sizes metadata is not reused Ensure that the object sizes hash table is cleared after a commit and not only when the repo is closed. --- tests/test-sizes.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test-sizes.js b/tests/test-sizes.js index a3928fe8..46848905 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -118,7 +118,7 @@ function validateSizes(repo, commit, expectedFiles) { } } -print('1..2') +print('1..3') let testDataDir = Gio.File.new_for_path('test-data'); testDataDir.make_directory(null); @@ -157,6 +157,23 @@ validateSizes(repo, commit, expectedFiles); print("ok test-sizes"); +// Remove a file to make sure that metadata is not reused from the +// previous commit +testDataDir.get_child('another-file').delete(null); +delete expectedFiles['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad']; + +repo.prepare_transaction(null); +mtree = OSTree.MutableTree.new(); +repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); +[,dirTree] = repo.write_mtree(mtree, null); +[,commit] = repo.write_commit(null, 'Some subject', 'Some body', null, dirTree, null); +print("commit => " + commit); +repo.commit_transaction(null); + +validateSizes(repo, commit, expectedFiles); + +print("ok test-sizes file deleted"); + // Repeat the commit now that all the objects are cached and ensure the // metadata is still correct repo.prepare_transaction(null); From a4592678aa5b71051daed24d08bcc84d8844781c Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 09:43:10 -0600 Subject: [PATCH 07/12] tests/sizes: Check duplicate file doesn't add sizes entry A duplicate file will resolve to the same object, so it shouldn't add any entries to the sizes metadata. --- src/libostree/ostree-repo-commit.c | 64 ++++++++++++++++++++++++++---- tests/test-sizes.js | 12 +++++- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index d995686b..f88e2d78 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -365,15 +365,38 @@ repo_setup_generate_sizes (OstreeRepo *self, } static void -repo_store_size_entry (OstreeRepo *self, - const gchar *checksum, - goffset unpacked, - goffset archived) +repo_ensure_size_entries (OstreeRepo *self) { if (G_UNLIKELY (self->object_sizes == NULL)) self->object_sizes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, content_size_cache_entry_free); +} +static gboolean +repo_has_size_entry (OstreeRepo *self, + OstreeObjectType objtype, + const gchar *checksum) +{ + /* Only file, dirtree and dirmeta objects appropriate for size metadata */ + if (objtype > OSTREE_OBJECT_TYPE_DIR_META) + return TRUE; + + repo_ensure_size_entries (self); + return (g_hash_table_lookup (self->object_sizes, checksum) != NULL); +} + +static void +repo_store_size_entry (OstreeRepo *self, + OstreeObjectType objtype, + const gchar *checksum, + goffset unpacked, + goffset archived) +{ + /* Only file, dirtree and dirmeta objects appropriate for size metadata */ + if (objtype > OSTREE_OBJECT_TYPE_DIR_META) + return; + + repo_ensure_size_entries (self); g_hash_table_replace (self->object_sizes, g_strdup (checksum), content_size_cache_entry_new (unpacked, archived)); @@ -1031,6 +1054,11 @@ write_content_object (OstreeRepo *self, unpacked_size = g_file_info_get_size (file_info); } + else + { + /* For a symlink, the size is the length of the target */ + unpacked_size = strlen (g_file_info_get_symlink_target (file_info)); + } if (!g_output_stream_flush (temp_out, cancellable, error)) return FALSE; @@ -1061,16 +1089,16 @@ write_content_object (OstreeRepo *self, g_assert (actual_checksum != NULL); /* Pacify static analysis */ /* Update size metadata if configured and entry missing */ - if (self->generate_sizes && object_file_type == G_FILE_TYPE_REGULAR && - (self->object_sizes == NULL || - g_hash_table_lookup (self->object_sizes, actual_checksum) == NULL)) + if (self->generate_sizes && + !repo_has_size_entry (self, OSTREE_OBJECT_TYPE_FILE, actual_checksum)) { struct stat stbuf; if (!glnx_fstat (tmpf.fd, &stbuf, error)) return FALSE; - repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); + repo_store_size_entry (self, OSTREE_OBJECT_TYPE_FILE, actual_checksum, + unpacked_size, stbuf.st_size); } /* See whether or not we have the object, now that we know the @@ -1329,6 +1357,11 @@ write_metadata_object (OstreeRepo *self, */ if (have_obj) { + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, actual_checksum)) + repo_store_size_entry (self, objtype, actual_checksum, len, len); + g_mutex_lock (&self->txn_lock); self->txn.stats.metadata_objects_total++; g_mutex_unlock (&self->txn_lock); @@ -1350,6 +1383,11 @@ write_metadata_object (OstreeRepo *self, gsize len; const guint8 *bufp = g_bytes_get_data (buf, &len); + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, actual_checksum)) + repo_store_size_entry (self, objtype, actual_checksum, len, len); + /* Write the metadata to a temporary file */ g_auto(GLnxTmpfile) tmpf = { 0, }; if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_WRONLY|O_CLOEXEC, @@ -2365,6 +2403,16 @@ ostree_repo_write_metadata (OstreeRepo *self, return FALSE; if (have_obj) { + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, expected_checksum)) + { + /* Make sure we have a fully serialized object */ + g_autoptr(GVariant) trusted = g_variant_get_normal_form (object); + gsize size = g_variant_get_size (trusted); + repo_store_size_entry (self, objtype, expected_checksum, size, size); + } + if (out_csum) *out_csum = ostree_checksum_to_bytes (expected_checksum); return TRUE; diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 46848905..685fe3fe 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -123,6 +123,10 @@ print('1..3') let testDataDir = Gio.File.new_for_path('test-data'); testDataDir.make_directory(null); testDataDir.get_child('some-file').replace_contents("hello world!", null, false, 0, null); +testDataDir.get_child('some-file').copy(testDataDir.get_child('duplicate-file'), + Gio.FileCopyFlags.OVERWRITE, + null, null); +testDataDir.get_child('link-file').make_symbolic_link('some-file', null); testDataDir.get_child('another-file').replace_contents("hello world again!", null, false, 0, null); let repoPath = Gio.File.new_for_path('repo'); @@ -152,15 +156,21 @@ repo.commit_transaction(null); let expectedFiles = { 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad': [54, 18], 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b': [48, 12], + '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0': [43, 9], + '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51': [185, 185], + '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488': [12, 12], }; validateSizes(repo, commit, expectedFiles); print("ok test-sizes"); // Remove a file to make sure that metadata is not reused from the -// previous commit +// previous commit. Remove that file from the expected metadata and +// replace the dirtree object. testDataDir.get_child('another-file').delete(null); delete expectedFiles['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad']; +delete expectedFiles['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51']; +expectedFiles['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539'] = [138, 138]; repo.prepare_transaction(null); mtree = OSTree.MutableTree.new(); From 291e9da258dc2c5b7cd5191f1c29e9b5f9d79b3b Mon Sep 17 00:00:00 2001 From: John Hiesey Date: Thu, 24 Oct 2019 09:10:57 -0600 Subject: [PATCH 08/12] lib/commit: Include object type in sizes metadata Append a byte encoding the OSTree object type for each object in the metadata. This allows the commit metadata to be fetched and then for the program to see which objects it already has for an accurate calculation of which objects need to be downloaded. This slightly breaks the `ostree.sizes` `ay` metadata entries. However, it's unlikely anyone was asserting the length of the entries since the array currently ends in 2 variable length integers. As far as I know, the only users of the sizes metadata are the ostree test suite and Endless' eos-updater[1]. The former is updated here and the latter already expects this format. 1. https://github.com/endlessm/eos-updater/ --- src/libostree/ostree-repo-commit.c | 10 +++-- tests/test-sizes.js | 64 +++++++++++++++++++----------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index f88e2d78..2294c846 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -322,16 +322,19 @@ commit_loose_regfile_object (OstreeRepo *self, /* This is used by OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES */ typedef struct { + OstreeObjectType objtype; goffset unpacked; goffset archived; } OstreeContentSizeCacheEntry; static OstreeContentSizeCacheEntry * -content_size_cache_entry_new (goffset unpacked, - goffset archived) +content_size_cache_entry_new (OstreeObjectType objtype, + goffset unpacked, + goffset archived) { OstreeContentSizeCacheEntry *entry = g_slice_new0 (OstreeContentSizeCacheEntry); + entry->objtype = objtype; entry->unpacked = unpacked; entry->archived = archived; @@ -399,7 +402,7 @@ repo_store_size_entry (OstreeRepo *self, repo_ensure_size_entries (self); g_hash_table_replace (self->object_sizes, g_strdup (checksum), - content_size_cache_entry_new (unpacked, archived)); + content_size_cache_entry_new (objtype, unpacked, archived)); } static int @@ -450,6 +453,7 @@ add_size_index_to_metadata (OstreeRepo *self, g_hash_table_lookup (self->object_sizes, e_checksum); _ostree_write_varuint64 (buffer, e_size->archived); _ostree_write_varuint64 (buffer, e_size->unpacked); + g_string_append_c (buffer, (gchar) e_size->objtype); g_variant_builder_add (&index_builder, "@ay", ot_gvariant_new_bytearray ((guint8*)buffer->str, buffer->len)); diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 685fe3fe..a2246536 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -67,15 +67,15 @@ function unpackByteArray(variant) { return array; } -function validateSizes(repo, commit, expectedFiles) { +function validateSizes(repo, commit, expectedObjects) { let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit); let metadata = commitVariant.get_child_value(0); let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay')); - let nSizes = sizes.n_children(); - let expectedNSizes = Object.keys(expectedFiles).length - assertEquals(nSizes, expectedNSizes); + let nObjects = sizes.n_children(); + let expectedNObjects = Object.keys(expectedObjects).length + assertEquals(nObjects, expectedNObjects); - for (let i = 0; i < nSizes; i++) { + for (let i = 0; i < nObjects; i++) { let sizeEntry = sizes.get_child_value(i); assertGreaterEquals(sizeEntry.n_children(), 34); let entryBytes = unpackByteArray(sizeEntry); @@ -94,15 +94,20 @@ function validateSizes(repo, commit, expectedFiles) { assertGreaterEquals(remainingBytes.length, 1); [uncompressedSize, varintRead] = readVarint(remainingBytes); remainingBytes = remainingBytes.slice(varintRead); - assertEquals(remainingBytes.length, 0); + assertEquals(remainingBytes.length, 1); + let objectType = remainingBytes[0]; + let objectTypeString = OSTree.object_type_to_string(objectType); print("compressed = " + compressedSize); print("uncompressed = " + uncompressedSize); + print("objtype = " + objectTypeString + " (" + objectType + ")"); + let objectName = OSTree.object_to_string(checksumString, objectType); + print("object = " + objectName); - if (!(checksumString in expectedFiles)) { - throw new Error("Checksum " + checksumString + " not in " + - JSON.stringify(expectedFiles)); + if (!(objectName in expectedObjects)) { + throw new Error("Object " + objectName + " not in " + + JSON.stringify(expectedObjects)); } - let expectedSizes = expectedFiles[checksumString]; + let expectedSizes = expectedObjects[objectName]; let expectedCompressedSize = expectedSizes[0]; let expectedUncompressedSize = expectedSizes[1]; if (compressedSize != expectedCompressedSize) { @@ -152,15 +157,26 @@ print("commit => " + commit); repo.commit_transaction(null); -// Test the sizes metadata -let expectedFiles = { - 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad': [54, 18], - 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b': [48, 12], - '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0': [43, 9], - '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51': [185, 185], - '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488': [12, 12], +// Test the sizes metadata. The key is the object and the value is an +// array of compressed size and uncompressed size. +let expectedObjects = { + 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file': [ + 54, 18 + ], + 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b.file': [ + 48, 12 + ], + '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0.file': [ + 43, 9 + ], + '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree': [ + 185, 185 + ], + '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488.dirmeta': [ + 12, 12 + ], }; -validateSizes(repo, commit, expectedFiles); +validateSizes(repo, commit, expectedObjects); print("ok test-sizes"); @@ -168,9 +184,11 @@ print("ok test-sizes"); // previous commit. Remove that file from the expected metadata and // replace the dirtree object. testDataDir.get_child('another-file').delete(null); -delete expectedFiles['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad']; -delete expectedFiles['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51']; -expectedFiles['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539'] = [138, 138]; +delete expectedObjects['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file']; +delete expectedObjects['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree']; +expectedObjects['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539.dirtree'] = [ + 138, 138 +]; repo.prepare_transaction(null); mtree = OSTree.MutableTree.new(); @@ -180,7 +198,7 @@ repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); print("commit => " + commit); repo.commit_transaction(null); -validateSizes(repo, commit, expectedFiles); +validateSizes(repo, commit, expectedObjects); print("ok test-sizes file deleted"); @@ -194,6 +212,6 @@ repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); print("commit => " + commit); repo.commit_transaction(null); -validateSizes(repo, commit, expectedFiles); +validateSizes(repo, commit, expectedObjects); print("ok test-sizes repeated"); From 1bbe674d91a23e733e1bb58fc6c28c4ab86f94ae Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 24 Oct 2019 17:00:33 -0600 Subject: [PATCH 09/12] libarchive: Support commit sizes metadata Call the helper to set the generate_sizes boolean so that object size data is stored while writing the mtree. --- src/libostree/ostree-repo-commit.c | 10 +++++----- src/libostree/ostree-repo-libarchive.c | 2 ++ src/libostree/ostree-repo-private.h | 4 ++++ tests/test-libarchive.sh | 16 +++++++++++++++- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 2294c846..87b585fd 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -348,9 +348,9 @@ content_size_cache_entry_free (gpointer entry) g_slice_free (OstreeContentSizeCacheEntry, entry); } -static void -repo_setup_generate_sizes (OstreeRepo *self, - OstreeRepoCommitModifier *modifier) +void +_ostree_repo_setup_generate_sizes (OstreeRepo *self, + OstreeRepoCommitModifier *modifier) { if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) { @@ -3923,7 +3923,7 @@ ostree_repo_write_directory_to_mtree (OstreeRepo *self, } else { - repo_setup_generate_sizes (self, modifier); + _ostree_repo_setup_generate_sizes (self, modifier); g_autoptr(GPtrArray) path = g_ptr_array_new (); if (!write_directory_to_mtree_internal (self, dir, mtree, modifier, path, @@ -3957,7 +3957,7 @@ ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error) { - repo_setup_generate_sizes (self, modifier); + _ostree_repo_setup_generate_sizes (self, modifier); g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error)) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 1850f99f..d55459f4 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -844,6 +844,8 @@ ostree_repo_import_archive_to_mtree (OstreeRepo *self, .modifier = modifier }; + _ostree_repo_setup_generate_sizes (self, modifier); + while (TRUE) { int r = archive_read_next_header (a, &aictx.entry); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index bc2325e5..2864d81e 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -337,6 +337,10 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self, GFileInfo *file_info, GFileInfo **out_modified_info); +void +_ostree_repo_setup_generate_sizes (OstreeRepo *self, + OstreeRepoCommitModifier *modifier); + gboolean _ostree_repo_remote_name_is_file (const char *remote_name); diff --git a/tests/test-libarchive.sh b/tests/test-libarchive.sh index 24de55b2..174be800 100755 --- a/tests/test-libarchive.sh +++ b/tests/test-libarchive.sh @@ -28,7 +28,7 @@ fi . $(dirname $0)/libtest.sh -echo "1..17" +echo "1..18" setup_test_repository "bare" @@ -234,3 +234,17 @@ for filter in '^usr/bin/,usr/sbin/' '/bin/,/sbin/'; do assert_file_has_content usr/lib/libfoo.so 'a library' echo "ok tar pathname filter modification: ${filter}" done + +# Test sizes metadata. This needs an archive repo, so a separate repo is used. +cd ${test_tmpdir} +rm -rf repo2 +ostree_repo_init repo2 --mode=archive +${CMD_PREFIX} ostree --repo=repo2 commit \ + -s "from tar" -b test-tar \ + --generate-sizes \ + --tree=tar=foo.tar.gz +${CMD_PREFIX} ostree --repo=repo2 show --print-sizes test-tar > sizes.txt +assert_file_has_content sizes.txt 'Compressed size (needed/total): 0[  ]bytes/1.1[  ]kB' +assert_file_has_content sizes.txt 'Unpacked size (needed/total): 0[  ]bytes/900[  ]bytes' +assert_file_has_content sizes.txt 'Number of objects (needed/total): 0/12' +echo "ok tar sizes metadata" From fcbb453443c4f22e8621c8593877ab775e5a5884 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 19 Dec 2019 12:50:46 -0700 Subject: [PATCH 10/12] core: Add OstreeCommitSizesEntry type This will be used when reading out entries in the `ostree.sizes` metadata. Each entry corresponds to an object in the metadata array. --- apidoc/ostree-sections.txt | 6 +++ src/libostree/libostree-devel.sym | 5 +++ src/libostree/ostree-autocleanups.h | 1 + src/libostree/ostree-core.c | 70 +++++++++++++++++++++++++++++ src/libostree/ostree-core.h | 32 +++++++++++++ 5 files changed, 114 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 1ef4bbf6..33035ca7 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -151,7 +151,13 @@ ostree_validate_structureof_dirmeta ostree_commit_get_parent ostree_commit_get_timestamp ostree_commit_get_content_checksum +OstreeCommitSizesEntry +ostree_commit_sizes_entry_new +ostree_commit_sizes_entry_copy +ostree_commit_sizes_entry_free ostree_check_version + +ostree_commit_sizes_entry_get_type
diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index d1666176..8e7473c5 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,6 +19,11 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2019.7 { +global: + ostree_commit_sizes_entry_copy; + ostree_commit_sizes_entry_free; + ostree_commit_sizes_entry_get_type; + ostree_commit_sizes_entry_new; ostree_sysroot_initialize; ostree_sysroot_is_booted; ostree_sysroot_set_mount_namespace_in_use; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index c07f88a8..c9692ebe 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -49,6 +49,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoDevInoCache, ostree_repo_devino_cache_u G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeAsyncProgress, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeBootconfigParser, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCommitSizesEntry, ostree_commit_sizes_entry_free) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeDeployment, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeGpgVerifyResult, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeKernelArgs, ostree_kernel_args_free) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 3d16757e..160f954f 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -2430,6 +2430,76 @@ ostree_commit_get_content_checksum (GVariant *commit_variant) return g_strdup (hexdigest); } +G_DEFINE_BOXED_TYPE (OstreeCommitSizesEntry, ostree_commit_sizes_entry, + ostree_commit_sizes_entry_copy, ostree_commit_sizes_entry_free) + +/** + * ostree_commit_sizes_entry_new: + * @checksum: (not nullable): object checksum + * @objtype: object type + * @unpacked: unpacked object size + * @archived: compressed object size + * + * Create a new #OstreeCommitSizesEntry for representing an object in a + * commit's "ostree.sizes" metadata. + * + * Returns: (transfer full) (nullable): a new #OstreeCommitSizesEntry + * Since: 2019.7 + */ +OstreeCommitSizesEntry * +ostree_commit_sizes_entry_new (const gchar *checksum, + OstreeObjectType objtype, + guint64 unpacked, + guint64 archived) +{ + g_return_val_if_fail (checksum == NULL || ostree_validate_checksum_string (checksum, NULL), NULL); + + g_autoptr(OstreeCommitSizesEntry) entry = g_new0 (OstreeCommitSizesEntry, 1); + entry->checksum = g_strdup (checksum); + entry->objtype = objtype; + entry->unpacked = unpacked; + entry->archived = archived; + + return g_steal_pointer (&entry); +} + +/** + * ostree_commit_sizes_entry_copy: + * @entry: (not nullable): an #OstreeCommitSizesEntry + * + * Create a copy of the given @entry. + * + * Returns: (transfer full) (nullable): a new copy of @entry + * Since: 2019.7 + */ +OstreeCommitSizesEntry * +ostree_commit_sizes_entry_copy (const OstreeCommitSizesEntry *entry) +{ + g_return_val_if_fail (entry != NULL, NULL); + + return ostree_commit_sizes_entry_new (entry->checksum, + entry->objtype, + entry->unpacked, + entry->archived); +} + +/** + * ostree_commit_sizes_entry_free: + * @entry: (transfer full): an #OstreeCommitSizesEntry + * + * Free given @entry. + * + * Since: 2019.7 + */ +void +ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry) +{ + g_return_if_fail (entry != NULL); + + g_free (entry->checksum); + g_free (entry); +} + /* Used in pull/deploy to validate we're not being downgraded */ gboolean _ostree_compare_timestamps (const char *current_rev, diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 69477a75..a61ae06c 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -521,6 +521,38 @@ guint64 ostree_commit_get_timestamp (GVariant *commit_variant); _OSTREE_PUBLIC gchar * ostree_commit_get_content_checksum (GVariant *commit_variant); +/** + * OstreeCommitSizesEntry: + * @checksum: (not nullable): object checksum + * @objtype: object type + * @unpacked: unpacked object size + * @archived: compressed object size + * + * Structure representing an entry in the "ostree.sizes" commit metadata. Each + * entry corresponds to an object in the associated commit. + * + * Since: 2019.5 + */ +typedef struct { + gchar *checksum; + OstreeObjectType objtype; + guint64 unpacked; + guint64 archived; +} OstreeCommitSizesEntry; + +_OSTREE_PUBLIC +GType ostree_commit_sizes_entry_get_type (void); + +_OSTREE_PUBLIC +OstreeCommitSizesEntry *ostree_commit_sizes_entry_new (const gchar *checksum, + OstreeObjectType objtype, + guint64 unpacked, + guint64 archived); +_OSTREE_PUBLIC +OstreeCommitSizesEntry *ostree_commit_sizes_entry_copy (const OstreeCommitSizesEntry *entry); +_OSTREE_PUBLIC +void ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry); + _OSTREE_PUBLIC gboolean ostree_check_version (guint required_year, guint required_release); From 260bcd11938be8fbe49846eff913206e5df4168a Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 20 Jan 2020 19:54:00 -0700 Subject: [PATCH 11/12] core: Add ostree_commit_get_object_sizes API This function parses the object listing in the `ostree.sizes` metadata and returns an array of `OstreeCommitSizesEntry` structures. Unfortunately, for reasons I don't understand, the linker wants to resolve `_ostree_read_varuint64` from `ostree-core.c` even though it's not used by `test-checksum.c` at all. --- Makefile-tests.am | 5 +- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-core-private.h | 3 + src/libostree/ostree-core.c | 112 ++++++++++++++++++++++++++++ src/libostree/ostree-core.h | 5 ++ src/libostree/ostree-repo-private.h | 2 - 7 files changed, 126 insertions(+), 3 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index fc2f2d91..a4acb8e0 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -347,7 +347,10 @@ tests_test_varint_LDADD = $(TESTS_LDADD) tests_test_bsdiff_CFLAGS = $(TESTS_CFLAGS) tests_test_bsdiff_LDADD = libbsdiff.la $(TESTS_LDADD) -tests_test_checksum_SOURCES = src/libostree/ostree-core.c tests/test-checksum.c +tests_test_checksum_SOURCES = \ + src/libostree/ostree-core.c \ + src/libostree/ostree-varint.c \ + tests/test-checksum.c tests_test_checksum_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) tests_test_checksum_LDADD = $(TESTS_LDADD) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 33035ca7..32cf5228 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -151,6 +151,7 @@ ostree_validate_structureof_dirmeta ostree_commit_get_parent ostree_commit_get_timestamp ostree_commit_get_content_checksum +ostree_commit_get_object_sizes OstreeCommitSizesEntry ostree_commit_sizes_entry_new ostree_commit_sizes_entry_copy diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 8e7473c5..ff5f52c4 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -20,6 +20,7 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2019.7 { global: + ostree_commit_get_object_sizes; ostree_commit_sizes_entry_copy; ostree_commit_sizes_entry_free; ostree_commit_sizes_entry_get_type; diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 43cf22c4..c1a82386 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -102,6 +102,9 @@ _ostree_checksum_inplace_from_bytes_v (GVariant *csum_v, char *buf) */ #define _OSTREE_LOOSE_PATH_MAX (256) +/* GVariant format for ostree.sizes metadata entries. */ +#define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" + char * _ostree_get_relative_object_path (const char *checksum, OstreeObjectType type, diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 160f954f..4667dd8f 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -32,6 +32,7 @@ #include "ostree.h" #include "ostree-core-private.h" #include "ostree-chain-input-stream.h" +#include "ostree-varint.h" #include "otutil.h" /* Generic ABI checks */ @@ -2500,6 +2501,117 @@ ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry) g_free (entry); } +static gboolean +read_sizes_entry (GVariant *entry, + OstreeCommitSizesEntry **out_sizes, + GError **error) +{ + gsize entry_size = g_variant_get_size (entry); + g_return_val_if_fail (entry_size >= OSTREE_SHA256_DIGEST_LEN + 2, FALSE); + + const guchar *buffer = g_variant_get_data (entry); + if (buffer == NULL) + return glnx_throw (error, "Could not read ostree.sizes metadata entry"); + + char checksum[OSTREE_SHA256_STRING_LEN + 1]; + ostree_checksum_inplace_from_bytes (buffer, checksum); + buffer += OSTREE_SHA256_DIGEST_LEN; + entry_size -= OSTREE_SHA256_DIGEST_LEN; + + gsize bytes_read = 0; + guint64 archived = 0; + if (!_ostree_read_varuint64 (buffer, entry_size, &archived, &bytes_read)) + return glnx_throw (error, "Unexpected EOF reading ostree.sizes varint"); + buffer += bytes_read; + entry_size -= bytes_read; + + guint64 unpacked = 0; + if (!_ostree_read_varuint64 (buffer, entry_size, &unpacked, &bytes_read)) + return glnx_throw (error, "Unexpected EOF reading ostree.sizes varint"); + buffer += bytes_read; + entry_size -= bytes_read; + + /* On newer commits, an additional byte is used for the object type. */ + OstreeObjectType objtype; + if (entry_size > 0) + { + objtype = *buffer; + if (objtype < OSTREE_OBJECT_TYPE_FILE || objtype > OSTREE_OBJECT_TYPE_LAST) + return glnx_throw (error, "Unexpected ostree.sizes object type %u", + objtype); + buffer++; + entry_size--; + } + else + { + /* Assume the object is a file. */ + objtype = OSTREE_OBJECT_TYPE_FILE; + } + + g_autoptr(OstreeCommitSizesEntry) sizes = ostree_commit_sizes_entry_new (checksum, + objtype, + unpacked, + archived); + + if (out_sizes != NULL) + *out_sizes = g_steal_pointer (&sizes); + + return TRUE; +} + +/** + * ostree_commit_get_object_sizes: + * @commit_variant: (not nullable): variant of type %OSTREE_OBJECT_TYPE_COMMIT + * @out_sizes_entries: (out) (element-type OstreeCommitSizesEntry) (transfer container) (optional): + * return location for an array of object size entries + * @error: Error + * + * Reads a commit's "ostree.sizes" metadata and returns an array of + * #OstreeCommitSizesEntry in @out_sizes_entries. Each element + * represents an object in the commit. If the commit does not contain + * the "ostree.sizes" metadata, a %G_IO_ERROR_NOT_FOUND error will be + * returned. + * + * Since: 2019.7 + */ +gboolean +ostree_commit_get_object_sizes (GVariant *commit_variant, + GPtrArray **out_sizes_entries, + GError **error) +{ + g_return_val_if_fail (commit_variant != NULL, FALSE); + + g_autoptr(GVariant) metadata = g_variant_get_child_value (commit_variant, 0); + g_autoptr(GVariant) sizes_variant = + g_variant_lookup_value (metadata, "ostree.sizes", + G_VARIANT_TYPE ("a" _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE)); + if (sizes_variant == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "No metadata key ostree.sizes in commit"); + return FALSE; + } + + g_autoptr(GPtrArray) sizes_entries = + g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_commit_sizes_entry_free); + g_autoptr(GVariant) entry = NULL; + GVariantIter entry_iter; + g_variant_iter_init (&entry_iter, sizes_variant); + while ((entry = g_variant_iter_next_value (&entry_iter))) + { + OstreeCommitSizesEntry *sizes_entry = NULL; + if (!read_sizes_entry (entry, &sizes_entry, error)) + return FALSE; + g_clear_pointer (&entry, g_variant_unref); + g_ptr_array_add (sizes_entries, sizes_entry); + } + + if (out_sizes_entries != NULL) + *out_sizes_entries = g_steal_pointer (&sizes_entries); + + return TRUE; +} + /* Used in pull/deploy to validate we're not being downgraded */ gboolean _ostree_compare_timestamps (const char *current_rev, diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index a61ae06c..10601123 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -553,6 +553,11 @@ OstreeCommitSizesEntry *ostree_commit_sizes_entry_copy (const OstreeCommitSizesE _OSTREE_PUBLIC void ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry); +_OSTREE_PUBLIC +gboolean ostree_commit_get_object_sizes (GVariant *commit_variant, + GPtrArray **out_sizes_entries, + GError **error); + _OSTREE_PUBLIC gboolean ostree_check_version (guint required_year, guint required_release); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 2864d81e..0465327c 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -31,8 +31,6 @@ G_BEGIN_DECLS #define OSTREE_DELTAPART_VERSION (0) -#define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" - #define _OSTREE_SUMMARY_CACHE_DIR "summaries" #define _OSTREE_CACHE_DIR "cache" From 97c831dd5fe509483939cdd40703a0ce8e0e9bfd Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 24 Oct 2019 15:21:49 -0600 Subject: [PATCH 12/12] bin/show: Add --print-sizes option to show sizes metadata Use the new `ostree_commit_get_object_sizes()` API to read the `ostree.sizes` commit metadata and print a summary. --- Makefile-tests.am | 1 + bash/ostree | 1 + man/ostree-show.xml | 11 ++++++ src/ostree/ot-builtin-show.c | 69 ++++++++++++++++++++++++++++++++++++ tests/test-pull-sizes.sh | 58 ++++++++++++++++++++++++++++++ 5 files changed, 140 insertions(+) create mode 100755 tests/test-pull-sizes.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index a4acb8e0..615b8480 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -86,6 +86,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-resume.sh \ tests/test-pull-basicauth.sh \ tests/test-pull-repeated.sh \ + tests/test-pull-sizes.sh \ tests/test-pull-untrusted.sh \ tests/test-pull-override-url.sh \ tests/test-pull-localcache.sh \ diff --git a/bash/ostree b/bash/ostree index fc429983..4aec588b 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1445,6 +1445,7 @@ _ostree_show() { local boolean_options=" $main_boolean_options --print-related + --print-sizes --raw " diff --git a/man/ostree-show.xml b/man/ostree-show.xml index a3d9aa4a..a28f704c 100644 --- a/man/ostree-show.xml +++ b/man/ostree-show.xml @@ -99,6 +99,17 @@ Boston, MA 02111-1307, USA. + + + + + Show the commit size metadata. This in only supported for + commits that contain ostree.sizes + metadata. This can be included when creating commits with + ostree commit --generate-sizes. + + + diff --git a/src/ostree/ot-builtin-show.c b/src/ostree/ot-builtin-show.c index 5091a93c..96e2d4c6 100644 --- a/src/ostree/ot-builtin-show.c +++ b/src/ostree/ot-builtin-show.c @@ -33,6 +33,7 @@ static gboolean opt_print_related; static char* opt_print_variant_type; static char* opt_print_metadata_key; static char* opt_print_detached_metadata_key; +static gboolean opt_print_sizes; static gboolean opt_raw; static gboolean opt_no_byteswap; static char *opt_gpg_homedir; @@ -48,6 +49,7 @@ static GOptionEntry options[] = { { "print-variant-type", 0, 0, G_OPTION_ARG_STRING, &opt_print_variant_type, "Memory map OBJECT (in this case a filename) to the GVariant type string", "TYPE" }, { "print-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_metadata_key, "Print string value of metadata key", "KEY" }, { "print-detached-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_detached_metadata_key, "Print string value of detached metadata key", "KEY" }, + { "print-sizes", 0, 0, G_OPTION_ARG_NONE, &opt_print_sizes, "Show the commit size metadata", NULL }, { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "Show raw variant data" }, { "no-byteswap", 'B', 0, G_OPTION_ARG_NONE, &opt_no_byteswap, "Do not automatically convert variant data from big endian" }, { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, @@ -146,6 +148,65 @@ do_print_metadata_key (OstreeRepo *repo, return TRUE; } +static gboolean +do_print_sizes (OstreeRepo *repo, + const char *rev, + GError **error) +{ + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, rev, + &commit, error)) + { + g_prefix_error (error, "Failed to read commit: "); + return FALSE; + } + + g_autoptr(GPtrArray) sizes = NULL; + if (!ostree_commit_get_object_sizes (commit, &sizes, error)) + return FALSE; + + gint64 new_archived = 0; + gint64 new_unpacked = 0; + gsize new_objects = 0; + gint64 archived = 0; + gint64 unpacked = 0; + gsize objects = 0; + for (guint i = 0; i < sizes->len; i++) + { + OstreeCommitSizesEntry *entry = sizes->pdata[i]; + + archived += entry->archived; + unpacked += entry->unpacked; + objects++; + + gboolean exists; + if (!ostree_repo_has_object (repo, entry->objtype, entry->checksum, + &exists, NULL, error)) + return FALSE; + + if (!exists) + { + /* Object not in local repo */ + new_archived += entry->archived; + new_unpacked += entry->unpacked; + new_objects++; + } + } + + g_autofree char *new_archived_str = g_format_size (new_archived); + g_autofree char *archived_str = g_format_size (archived); + g_autofree char *new_unpacked_str = g_format_size (new_unpacked); + g_autofree char *unpacked_str = g_format_size (unpacked); + g_print ("Compressed size (needed/total): %s/%s\n" + "Unpacked size (needed/total): %s/%s\n" + "Number of objects (needed/total): %" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT "\n", + new_archived_str, archived_str, + new_unpacked_str, unpacked_str, + new_objects, objects); + + return TRUE; +} + static gboolean print_object (OstreeRepo *repo, OstreeObjectType objtype, @@ -279,6 +340,14 @@ ostree_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, if (!do_print_variant_generic (G_VARIANT_TYPE (opt_print_variant_type), rev, error)) return FALSE; } + else if (opt_print_sizes) + { + if (!ostree_repo_resolve_rev (repo, rev, FALSE, &resolved_rev, error)) + return FALSE; + + if (!do_print_sizes (repo, resolved_rev, error)) + return FALSE; + } else { gboolean found = FALSE; diff --git a/tests/test-pull-sizes.sh b/tests/test-pull-sizes.sh new file mode 100755 index 00000000..8ee07cc8 --- /dev/null +++ b/tests/test-pull-sizes.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright (C) 2019 Endless Mobile, Inc. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive" "--generate-sizes" + +echo '1..3' + +cd ${test_tmpdir} +mkdir repo +ostree_repo_init repo +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo + +# Pull commit metadata only. All size and objects will be needed. +${CMD_PREFIX} ostree --repo=repo pull --commit-metadata-only origin main +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 637[  ]bytes/637[  ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 457[  ]bytes/457[  ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 10/10' +echo "ok sizes commit metadata only" + +# Pull the parent commit so we get most of the objects +parent=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main^) +${CMD_PREFIX} ostree --repo=repo pull origin ${parent} +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 501[  ]bytes/637[  ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 429[  ]bytes/457[  ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 6/10' +echo "ok sizes commit partial" + +# Finish pulling the commit and check that no objects needed +${CMD_PREFIX} ostree --repo=repo pull origin main +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 0[  ]bytes/637[  ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 0[  ]bytes/457[  ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 0/10' +echo "ok sizes commit full"