packaging/ostree
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1957 from dbnicholson/commit-sizes

Browse Source 
Upstream Endless sizes metadata changes
This commit is contained in:
OpenShift Merge Robot authored and GitHub committed 2020-01-27 08:25:59 -08:00
2 parents 08e292bc14 97c831dd5f
commit 8a9a496501
16 files changed
+669 -60
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/test-libarchive.sh
+15 -1
View File
@@ -28,7 +28,7 @@ fi
. $(dirname $0)/libtest.sh
echo "1..17"
echo "1..18"
setup_test_repository "bare"
@@ -234,3 +234,17 @@ for filter in '^usr/bin/,usr/sbin/' '/bin/,/sbin/'; do
assert_file_has_content usr/lib/libfoo.so 'a library'
echo "ok tar pathname filter modification: ${filter}"
done
# Test sizes metadata. This needs an archive repo, so a separate repo is used.
cd ${test_tmpdir}
rm -rf repo2
ostree_repo_init repo2 --mode=archive
${CMD_PREFIX} ostree --repo=repo2 commit \
-s "from tar" -b test-tar \
--generate-sizes \
--tree=tar=foo.tar.gz
${CMD_PREFIX} ostree --repo=repo2 show --print-sizes test-tar > sizes.txt
assert_file_has_content sizes.txt 'Compressed size (needed/total): 0[  ]bytes/1.1[  ]kB'
assert_file_has_content sizes.txt 'Unpacked size (needed/total): 0[  ]bytes/900[  ]bytes'
assert_file_has_content sizes.txt 'Number of objects (needed/total): 0/12'
echo "ok tar sizes metadata"
tests/test-pull-sizes.sh
Executable
+58
View File
@@ -0,0 +1,58 @@
#!/bin/bash
#
# Copyright (C) 2019 Endless Mobile, Inc.
#
# SPDX-License-Identifier: LGPL-2.0+
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
set -euo pipefail
. $(dirname $0)/libtest.sh
setup_fake_remote_repo1 "archive" "--generate-sizes"
echo '1..3'
cd ${test_tmpdir}
mkdir repo
ostree_repo_init repo
${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
# Pull commit metadata only. All size and objects will be needed.
${CMD_PREFIX} ostree --repo=repo pull --commit-metadata-only origin main
${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt
assert_file_has_content show.txt 'Compressed size (needed/total): 637[  ]bytes/637[  ]bytes'
assert_file_has_content show.txt 'Unpacked size (needed/total): 457[  ]bytes/457[  ]bytes'
assert_file_has_content show.txt 'Number of objects (needed/total): 10/10'
echo "ok sizes commit metadata only"
# Pull the parent commit so we get most of the objects
parent=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main^)
${CMD_PREFIX} ostree --repo=repo pull origin ${parent}
${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt
assert_file_has_content show.txt 'Compressed size (needed/total): 501[  ]bytes/637[  ]bytes'
assert_file_has_content show.txt 'Unpacked size (needed/total): 429[  ]bytes/457[  ]bytes'
assert_file_has_content show.txt 'Number of objects (needed/total): 6/10'
echo "ok sizes commit partial"
# Finish pulling the commit and check that no objects needed
${CMD_PREFIX} ostree --repo=repo pull origin main
${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt
assert_file_has_content show.txt 'Compressed size (needed/total): 0[  ]bytes/637[  ]bytes'
assert_file_has_content show.txt 'Unpacked size (needed/total): 0[  ]bytes/457[  ]bytes'
assert_file_has_content show.txt 'Number of objects (needed/total): 0/10'
echo "ok sizes commit full"
tests/test-sizes.js
+160 -29
View File
@@ -28,11 +28,110 @@ function assertEquals(a, b) {
throw new Error("assertion failed " + JSON.stringify(a) + " == " + JSON.stringify(b));
}
print('1..1')
function assertGreater(a, b) {
if (a <= b)
throw new Error("assertion failed " + JSON.stringify(a) + " > " + JSON.stringify(b));
}
function assertGreaterEquals(a, b) {
if (a < b)
throw new Error("assertion failed " + JSON.stringify(a) + " >= " + JSON.stringify(b));
}
// Adapted from _ostree_read_varuint64()
function readVarint(buffer) {
let result = 0;
let count = 0;
let len = buffer.length;
let cur;
do {
assertGreater(len, 0);
cur = buffer[count];
result = result | ((cur & 0x7F) << (7 * count));
count++;
len--;
} while (cur & 0x80);
return [result, count];
}
// There have been various bugs with byte array unpacking in GJS, so
// just do it manually.
function unpackByteArray(variant) {
let array = [];
let nBytes = variant.n_children();
for (let i = 0; i < nBytes; i++) {
array.push(variant.get_child_value(i).get_byte());
}
return array;
}
function validateSizes(repo, commit, expectedObjects) {
let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit);
let metadata = commitVariant.get_child_value(0);
let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay'));
let nObjects = sizes.n_children();
let expectedNObjects = Object.keys(expectedObjects).length
assertEquals(nObjects, expectedNObjects);
for (let i = 0; i < nObjects; i++) {
let sizeEntry = sizes.get_child_value(i);
assertGreaterEquals(sizeEntry.n_children(), 34);
let entryBytes = unpackByteArray(sizeEntry);
let checksumBytes = entryBytes.slice(0, 32);
let checksumString = OSTree.checksum_from_bytes(checksumBytes);
print("checksum = " + checksumString);
// Read the sizes from the next 2 varints
let remainingBytes = entryBytes.slice(32);
assertGreaterEquals(remainingBytes.length, 2);
let varintRead;
let compressedSize;
let uncompressedSize;
[compressedSize, varintRead] = readVarint(remainingBytes);
remainingBytes = remainingBytes.slice(varintRead);
assertGreaterEquals(remainingBytes.length, 1);
[uncompressedSize, varintRead] = readVarint(remainingBytes);
remainingBytes = remainingBytes.slice(varintRead);
assertEquals(remainingBytes.length, 1);
let objectType = remainingBytes[0];
let objectTypeString = OSTree.object_type_to_string(objectType);
print("compressed = " + compressedSize);
print("uncompressed = " + uncompressedSize);
print("objtype = " + objectTypeString + " (" + objectType + ")");
let objectName = OSTree.object_to_string(checksumString, objectType);
print("object = " + objectName);
if (!(objectName in expectedObjects)) {
throw new Error("Object " + objectName + " not in " +
JSON.stringify(expectedObjects));
}
let expectedSizes = expectedObjects[objectName];
let expectedCompressedSize = expectedSizes[0];
let expectedUncompressedSize = expectedSizes[1];
if (compressedSize != expectedCompressedSize) {
throw new Error("Compressed size " + compressedSize +
" for checksum " + checksumString +
" does not match expected " + expectedCompressedSize);
}
if (uncompressedSize != expectedUncompressedSize) {
throw new Error("Uncompressed size " + uncompressedSize +
" for checksum " + checksumString +
" does not match expected " + expectedUncompressedSize);
}
}
}
print('1..3')
let testDataDir = Gio.File.new_for_path('test-data');
testDataDir.make_directory(null);
testDataDir.get_child('some-file').replace_contents("hello world!", null, false, 0, null);
testDataDir.get_child('some-file').copy(testDataDir.get_child('duplicate-file'),
Gio.FileCopyFlags.OVERWRITE,
null, null);
testDataDir.get_child('link-file').make_symbolic_link('some-file', null);
testDataDir.get_child('another-file').replace_contents("hello world again!", null, false, 0, null);
let repoPath = Gio.File.new_for_path('repo');
@@ -41,7 +140,10 @@ repo.create(OSTree.RepoMode.ARCHIVE_Z2, null);
repo.open(null);
let commitModifier = OSTree.RepoCommitModifier.new(OSTree.RepoCommitModifierFlags.GENERATE_SIZES, null);
let commitModifierFlags = (OSTree.RepoCommitModifierFlags.GENERATE_SIZES |
OSTree.RepoCommitModifierFlags.SKIP_XATTRS |
OSTree.RepoCommitModifierFlags.CANONICAL_PERMISSIONS);
let commitModifier = OSTree.RepoCommitModifier.new(commitModifierFlags, null);
assertEquals(repo.get_mode(), OSTree.RepoMode.ARCHIVE_Z2);
@@ -55,32 +157,61 @@ print("commit => " + commit);
repo.commit_transaction(null);
// Test the sizes metadata
let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit);
let metadata = commitVariant.get_child_value(0);
let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay'));
let nSizes = sizes.n_children();
assertEquals(nSizes, 2);
let expectedUncompressedSizes = [12, 18];
let foundExpectedUncompressedSizes = 0;
for (let i = 0; i < nSizes; i++) {
let sizeEntry = sizes.get_child_value(i);
assertEquals(sizeEntry.n_children(), 34);
let compressedSize = sizeEntry.get_child_value(32).get_byte();
let uncompressedSize = sizeEntry.get_child_value(33).get_byte();
print("compressed = " + compressedSize);
print("uncompressed = " + uncompressedSize);
for (let j = 0; j < expectedUncompressedSizes.length; j++) {
let expected = expectedUncompressedSizes[j];
if (expected == uncompressedSize) {
print("Matched expected uncompressed size " + expected);
expectedUncompressedSizes.splice(j, 1);
break;
}
}
}
if (expectedUncompressedSizes.length > 0) {
throw new Error("Failed to match expectedUncompressedSizes: " + JSON.stringify(expectedUncompressedSizes));
}
// Test the sizes metadata. The key is the object and the value is an
// array of compressed size and uncompressed size.
let expectedObjects = {
'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file': [
54, 18
],
'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b.file': [
48, 12
],
'8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0.file': [
43, 9
],
'1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree': [
185, 185
],
'446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488.dirmeta': [
12, 12
],
};
validateSizes(repo, commit, expectedObjects);
print("ok test-sizes");
// Remove a file to make sure that metadata is not reused from the
// previous commit. Remove that file from the expected metadata and
// replace the dirtree object.
testDataDir.get_child('another-file').delete(null);
delete expectedObjects['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file'];
delete expectedObjects['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree'];
expectedObjects['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539.dirtree'] = [
138, 138
];
repo.prepare_transaction(null);
mtree = OSTree.MutableTree.new();
repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null);
[,dirTree] = repo.write_mtree(mtree, null);
[,commit] = repo.write_commit(null, 'Some subject', 'Some body', null, dirTree, null);
print("commit => " + commit);
repo.commit_transaction(null);
validateSizes(repo, commit, expectedObjects);
print("ok test-sizes file deleted");
// Repeat the commit now that all the objects are cached and ensure the
// metadata is still correct
repo.prepare_transaction(null);
mtree = OSTree.MutableTree.new();
repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null);
[,dirTree] = repo.write_mtree(mtree, null);
[,commit] = repo.write_commit(null, 'Another subject', 'Another body', null, dirTree, null);
print("commit => " + commit);
repo.commit_transaction(null);
validateSizes(repo, commit, expectedObjects);
print("ok test-sizes repeated");