From 87e9a29f1101c3d10515ce9ac07223933111c390 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Mar 2012 19:58:07 -0500 Subject: [PATCH] ostbuild: Further work on regenerating a snapshot from a compose --- Makefile-ostbuild.am | 1 + gnomeos/3.4/manifest.json | 6 +- src/ostbuild/pyostbuild/builtin_build.py | 48 ++++++------- src/ostbuild/pyostbuild/builtin_checkout.py | 12 ++-- .../pyostbuild/builtin_chroot_compile_one.py | 11 +-- .../pyostbuild/builtin_gen_snapshot.py | 68 +++++++++++++++++++ src/ostbuild/pyostbuild/builtin_resolve.py | 7 +- src/ostbuild/pyostbuild/main.py | 1 + 8 files changed, 112 insertions(+), 42 deletions(-) create mode 100755 src/ostbuild/pyostbuild/builtin_gen_snapshot.py diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index b64aa07e..f45eb0a2 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -27,6 +27,7 @@ pyostbuild_PYTHON = \ src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \ src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py \ src/ostbuild/pyostbuild/builtin_compile_one.py \ + src/ostbuild/pyostbuild/builtin_gen_snapshot.py \ src/ostbuild/pyostbuild/builtin_resolve.py \ src/ostbuild/pyostbuild/builtin_status.py \ src/ostbuild/pyostbuild/builtins.py \ diff --git a/gnomeos/3.4/manifest.json b/gnomeos/3.4/manifest.json index 4a3e8fb5..214acc32 100644 --- a/gnomeos/3.4/manifest.json +++ b/gnomeos/3.4/manifest.json @@ -52,9 +52,6 @@ {"src": "gnome:linux-user-chroot"}, - {"src": "gnome:ostree", - "config-opts": ["--disable-documentation"]}, - {"src": "gnome:libxml2"}, {"src": "gnome:libxslt"}, @@ -576,6 +573,9 @@ {"src": "gnome:libsoup", "config-args": ["--disable-tls-check"]}, + {"src": "gnome:ostree", + "config-opts": ["--disable-documentation"]}, + {"src": "fd-gstreamer:gstreamer", "config-opts": ["--disable-tests", "--disable-fatal-warnings"]}, diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index 7f1ca8d2..aa7f2721 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -100,12 +100,9 @@ class OstbuildBuild(builtins.Builtin): buildname = buildutil.manifest_buildname(self.manifest, meta) buildroot_name = buildutil.manifest_buildroot_name(self.manifest, meta) - (keytype, uri) = buildutil.parse_src_key(meta['src']) - - mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri) - checkoutdir = os.path.join(self.workdir, 'src', name) - component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, branch, - overwrite=not self.args.debug_shell) + checkoutdir = os.path.join(self.workdir, 'src') + component_src = os.path.join(checkoutdir, name) + run_sync(['ostbuild', 'checkout', '--overwrite', '--manifest=' + self.manifest_path, name], cwd=checkoutdir) current_vcs_version = meta['revision'] @@ -116,10 +113,13 @@ class OstbuildBuild(builtins.Builtin): if previous_build_version is not None: log("Previous build of '%s' is %s" % (buildname, previous_build_version)) - previous_vcs_version = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'show', '--print-metadata-key=ostbuild-revision', - previous_build_version]) - previous_vcs_version = previous_vcs_version.strip() + previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'cat', previous_build_version, + '/_ostbuild-meta.json'], + log_initiation=True) + previous_meta = json.loads(previous_metadata_text) + + previous_vcs_version = previous_meta['revision'] vcs_version_matches = False if previous_vcs_version == current_vcs_version: @@ -139,8 +139,6 @@ class OstbuildBuild(builtins.Builtin): json.dump(artifact_meta, f, indent=4, sort_keys=True) f.close() - run_sync(['ostbuild', 'checkout', '--manifest=' + self.manifest_path, name], cwd=checkoutdir) - logdir = os.path.join(self.workdir, 'logs', 'compile', name) old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name) if not os.path.isdir(logdir): @@ -166,7 +164,6 @@ class OstbuildBuild(builtins.Builtin): args = ['ostree', '--repo=' + self.repo, 'commit', '-b', buildname, '-s', 'Build', - '--add-metadata-string=ostbuild-revision=' + artifact_meta['revision'], '--owner-uid=0', '--owner-gid=0', '--no-xattrs', '--skip-if-unchanged'] @@ -188,33 +185,35 @@ class OstbuildBuild(builtins.Builtin): return True def _compose(self, components): - base_ref = self.manifest['base'] - + base = self.manifest['base'] + base_branch = base['branch'] + base_revision = base['revision'] # HACK manifest_build_name = self.manifest['name'] is_runtime = manifest_build_name.endswith('-runtime') branch_to_rev = {} - branches = [base_ref] + branch_to_subtrees = {} + + component_branches = [] for component in components: branch = buildutil.manifest_buildname(self.manifest, component) - branches.append(branch) + component_branches.append(branch) args = ['ostree', '--repo=' + self.repo, 'rev-parse'] - args.extend(branches) + args.extend(component_branches) branch_revs_text = run_sync_get_output(args) branch_revs = branch_revs_text.split('\n') - for (branch, rev) in zip(branches, branch_revs): + for (branch, rev) in zip(component_branches, branch_revs): branch_to_rev[branch] = rev - branch_to_subtrees = {} - branch_to_subtrees[base_ref] = ['/'] - contents = [base_ref] + contents = [base_branch] + branch_to_subtrees[base_branch] = ['/'] + branch_to_rev[base_branch] = base_revision - for component in components: - branch = buildutil.manifest_buildname(self.manifest, component) + for branch in component_branches: contents.append(branch) subtrees = ['/runtime'] branch_to_subtrees[branch] = subtrees @@ -230,6 +229,7 @@ class OstbuildBuild(builtins.Builtin): metadata_contents = [] metadata = {'source': 'ostbuild compose v0', + 'base': base, 'contents': metadata_contents} for branch in contents: branch_rev = branch_to_rev[branch] diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py index e77589e5..d45a742e 100755 --- a/src/ostbuild/pyostbuild/builtin_checkout.py +++ b/src/ostbuild/pyostbuild/builtin_checkout.py @@ -39,6 +39,7 @@ class OstbuildCheckout(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) parser.add_argument('--manifest', required=True) + parser.add_argument('--overwrite', action='store_true') parser.add_argument('components', nargs='*') args = parser.parse_args(argv) @@ -64,23 +65,22 @@ class OstbuildCheckout(builtins.Builtin): component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, component['revision'], - overwrite=False) + overwrite=args.overwrite) patches = component.get('patches') if patches is not None: - patches_meta = self.manifest['patches'] - (patches_keytype, patches_uri) = buildutil.parse_src_key(patches_meta['src']) + (patches_keytype, patches_uri) = buildutil.parse_src_key(patches['src']) patches_mirror = buildutil.get_mirrordir(self.mirrordir, patches_keytype, patches_uri) vcs.get_vcs_checkout(self.mirrordir, patches_keytype, patches_uri, - self.patchdir, patches_meta['branch'], + self.patchdir, patches['branch'], overwrite=True) - patch_prefix = patches_meta.get('prefix', None) + patch_prefix = patches.get('prefix', None) if patch_prefix is not None: patchdir = os.path.join(self.patchdir, patch_prefix) else: patchdir = self.patchdir - for patch in patches: + for patch in patches['files']: patch_path = os.path.join(patchdir, patch) run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir) diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py index fd0a35fc..6569a39d 100755 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py @@ -35,7 +35,8 @@ class OstbuildChrootCompileOne(builtins.Builtin): dependencies = components[:index] base = self.manifest['base'] - checkout_trees = [(base, '/')] + base_revision = base['revision'] + checkout_trees = [(base_revision, '/')] for dep in dependencies: buildname = buildutil.manifest_buildname(self.manifest, dep) checkout_trees.append((buildname, '/runtime')) @@ -146,15 +147,9 @@ class OstbuildChrootCompileOne(builtins.Builtin): env_copy['PWD'] = chroot_sourcedir run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell) - recorded_meta = dict(self.metadata) - del recorded_meta['revision'] - patches_recorded_meta = recorded_meta.get('patches') - if patches_recorded_meta is not None: - del patches_recorded_meta['revision'] - recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json') recorded_meta_f = open(recorded_meta_path, 'w') - json.dump(recorded_meta, recorded_meta_f, indent=4, sort_keys=True) + json.dump(self.metadata, recorded_meta_f, indent=4, sort_keys=True) recorded_meta_f.close() builtins.register(OstbuildChrootCompileOne) diff --git a/src/ostbuild/pyostbuild/builtin_gen_snapshot.py b/src/ostbuild/pyostbuild/builtin_gen_snapshot.py new file mode 100755 index 00000000..c70b8063 --- /dev/null +++ b/src/ostbuild/pyostbuild/builtin_gen_snapshot.py @@ -0,0 +1,68 @@ +# Copyright (C) 2011,2012 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +# ostbuild-compile-one-make wraps systems that implement the GNOME build API: +# http://people.gnome.org/~walters/docs/build-api.txt + +import os,sys,stat,subprocess,tempfile,re,shutil +import argparse +from StringIO import StringIO +import json + +from . import builtins +from .ostbuildlog import log, fatal +from .subprocess_helpers import run_sync, run_sync_get_output +from . import buildutil + +class OstbuildGenSnapshot(builtins.Builtin): + name = "gen-snapshot" + short_description = "Generate a snapshot description from a tree" + + def __init__(self): + builtins.Builtin.__init__(self) + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('--branch', required=True) + + args = parser.parse_args(argv) + self.args = args + self.parse_config() + + contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'cat', args.branch, 'contents.json']) + + contents = json.loads(contents_json_text) + contents_list = contents['contents'] + + base = contents_list[0] + artifacts = contents_list[1:] + + components = [] + snapshot = {'name': args.branch, + 'base': contents['base'], + 'components': components} + + for artifact in artifacts: + component_meta_text = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'cat', artifact['rev'], '/_ostbuild-meta.json']) + component_meta = json.loads(component_meta_text) + components.append(component_meta) + + json.dump(snapshot, sys.stdout) + +builtins.register(OstbuildGenSnapshot) diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py index 5018b49e..67fbd2c2 100755 --- a/src/ostbuild/pyostbuild/builtin_resolve.py +++ b/src/ostbuild/pyostbuild/builtin_resolve.py @@ -240,7 +240,12 @@ class OstbuildResolve(builtins.Builtin): del snapshot['base-prefix'] snapshot['name'] = '%s-%s-%s' % (name_prefix, architecture, component_type) - snapshot['base'] = '%s-%s-%s' % (base_prefix, architecture, component_type) + + base_ref = '%s-%s-%s' % (base_prefix, architecture, component_type) + base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'rev-parse', base_ref]) + snapshot['base'] = {'branch': base_ref, + 'revision': base_revision} out_snapshot = os.path.join(self.workdir, snapshot['name'] + '.snapshot') f = open(out_snapshot, 'w') json.dump(snapshot, f, indent=4, sort_keys=True) diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py index 81b029c5..b34d2e6e 100755 --- a/src/ostbuild/pyostbuild/main.py +++ b/src/ostbuild/pyostbuild/main.py @@ -27,6 +27,7 @@ from . import builtin_checkout from . import builtin_chroot_compile_one from . import builtin_chroot_run_triggers from . import builtin_compile_one +from . import builtin_gen_snapshot from . import builtin_resolve from . import builtin_status