From 40aedeb7a4aa592be59fae966659858515c48819 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 24 Feb 2012 21:36:24 -0500 Subject: [PATCH] ostbuild: Move more intelligence into chroot-compile-one Let's have chroot-compile-one be the core logic for composing a buildroot, etc. This allows us to use it as a developer tool by checking out a tree, then doing a chroot build. --- src/ostbuild/pyostbuild/buildutil.py | 31 ++++++ src/ostbuild/pyostbuild/builtin_build.py | 96 +++++-------------- .../pyostbuild/builtin_chroot_compile_one.py | 78 ++++++++------- .../pyostbuild/builtin_compile_one.py | 33 +++---- 4 files changed, 108 insertions(+), 130 deletions(-) diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py index f6a00c33..2e3b4a13 100755 --- a/src/ostbuild/pyostbuild/buildutil.py +++ b/src/ostbuild/pyostbuild/buildutil.py @@ -18,6 +18,7 @@ import os import re import urlparse +import tempfile from .subprocess_helpers import run_sync_get_output @@ -90,3 +91,33 @@ def manifest_buildroot_name(manifest, component, architecture): return 'buildroots/%s/%s/%s' % (manifest_target (manifest, architecture), component['name'], component['branch']) + +def find_component_in_manifest(manifest, component_name): + for component in manifest['components']: + if component['name'] == component_name: + return component + return None + +def compose(repo, target, artifacts): + child_args = ['ostree', '--repo=' + repo, 'compose', + '-b', target, '-s', 'Compose'] + (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') + f = os.fdopen(fd, 'w') + for artifact in artifacts: + f.write(artifact) + f.write('\n') + f.close() + child_args.extend(['-F', path]) + revision = run_sync_get_output(child_args, log_initiation=True).strip() + os.unlink(path) + return revision + +def compose_buildroot(manifest, repo, buildroot_name, component, dependencies, architecture): + base = manifest_base(manifest, 'devel', architecture) + buildroot_contents = [base + ':/'] + for dep in dependencies: + dep_buildname = manifest_buildname(manifest, dep, architecture) + buildroot_contents.append(dep_buildname + ':/runtime') + buildroot_contents.append(dep_buildname + ':/devel') + + return compose(repo, buildroot_name, buildroot_contents) diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index 21c7f4e7..124d3c11 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -48,8 +48,7 @@ class OstbuildBuild(builtins.Builtin): args = ['setarch', architecture] else: args = [] - args.extend(['ostbuild', 'chroot-compile-one', - '--repo=' + self.repo]) + args.extend(['ostbuild', 'chroot-compile-one']) return args def _launch_debug_shell(self, architecture, buildroot, cwd=None): @@ -92,17 +91,7 @@ class OstbuildBuild(builtins.Builtin): return result - def _compose_buildroot(self, buildroot_name, component, dependencies, architecture): - base = buildutil.manifest_base(self.manifest, 'devel', architecture) - buildroot_contents = [base + ':/'] - for dep in dependencies: - dep_buildname = buildutil.manifest_buildname(self.manifest, dep, architecture) - buildroot_contents.append(dep_buildname + ':/runtime') - buildroot_contents.append(dep_buildname + ':/devel') - - return self._compose(buildroot_name, buildroot_contents) - - def _build_one_component(self, meta, dependencies, architecture): + def _build_one_component(self, meta, architecture): name = meta['name'] branch = meta['branch'] @@ -117,10 +106,6 @@ class OstbuildBuild(builtins.Builtin): component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, branch, overwrite=not self.args.debug_shell) - if self.args.debug_shell: - buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture) - self._launch_debug_shell(architecture, buildroot_name, cwd=component_src) - current_vcs_version = meta['revision'] previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo, @@ -146,39 +131,19 @@ class OstbuildBuild(builtins.Builtin): else: log("No previous build for '%s' found" % (buildname, )) - buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture) + artifact_meta = dict(meta) - if meta.get('rm-configure', False): - configure_path = os.path.join(component_src, 'configure') - if os.path.exists(configure_path): - os.unlink(configure_path) - - artifact_meta = {'buildroot': buildroot_name, - 'buildroot-version': buildroot_version, - 'name': name, - 'branch': branch, - 'version': current_vcs_version - } - - metadata_dir = os.path.join(self.workdir, 'meta') - if not os.path.isdir(metadata_dir): - os.makedirs(metadata_dir) - metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, )) + metadata_path = os.path.join(component_src, '_ostbuild-meta.json') f = open(metadata_path, 'w') json.dump(artifact_meta, f) f.close() - + patches = meta.get('patches') if patches is not None: for patch in patches: patch_path = os.path.join(self.patchdir, patch) run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src) - component_resultdir = os.path.join(self.workdir, 'results', name) - if os.path.isdir(component_resultdir): - shutil.rmtree(component_resultdir) - os.makedirs(component_resultdir) - logdir = os.path.join(self.workdir, 'logs', 'compile', name) old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name) if not os.path.isdir(logdir): @@ -194,16 +159,7 @@ class OstbuildBuild(builtins.Builtin): log("Logging to %s" % (log_path, )) f = open(log_path, 'w') chroot_args = self._get_ostbuild_chroot_args(architecture) - chroot_args.extend(['--buildroot=' + buildroot_name, - '--workdir=' + self.workdir, - '--resultdir=' + component_resultdir, - '--meta=' + metadata_path]) - global_config_opts = self.manifest.get('config-opts') - if global_config_opts is not None: - chroot_args.extend(global_config_opts) - component_config_opts = meta.get('config-opts') - if component_config_opts is not None: - chroot_args.extend(component_config_opts) + chroot_args.extend(['--meta=' + metadata_path]) if self.buildopts.shell_on_failure: ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False) if ecode != 0: @@ -211,14 +167,19 @@ class OstbuildBuild(builtins.Builtin): else: run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src) + # Reread metadata to get buildroot version + f = open(metadata_path) + artifact_meta = json.load(f) + f.close() + args = ['ostree', '--repo=' + self.repo, - 'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['version'], - '--add-metadata-string=ostbuild-buildroot-version=' + buildroot_version, - '--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['version'], + 'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['revision'], + '--add-metadata-string=ostbuild-buildroot-version=' + artifact_meta['buildroot-version'], + '--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['revision'], '--owner-uid=0', '--owner-gid=0', '--no-xattrs', '--skip-if-unchanged'] - setuid_files = meta.get('setuid', []) + setuid_files = artifact_meta.get('setuid', []) statoverride_path = None if len(setuid_files) > 0: (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-') @@ -227,26 +188,14 @@ class OstbuildBuild(builtins.Builtin): f.write('+2048 ' + path) f.close() args.append('--statoverride=' + statoverride_path) + + component_resultdir = os.path.join(self.workdir, 'results', name) run_sync(args, cwd=component_resultdir) if statoverride_path is not None: os.unlink(statoverride_path) return True - def _compose(self, target, artifacts): - child_args = ['ostree', '--repo=' + self.repo, 'compose', - '-b', target, '-s', 'Compose'] - (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') - f = os.fdopen(fd, 'w') - for artifact in artifacts: - f.write(artifact) - f.write('\n') - f.close() - child_args.extend(['-F', path]) - revision = run_sync_get_output(child_args, log_initiation=True).strip() - os.unlink(path) - return revision - def _compose_arch(self, architecture, components): runtime_base = buildutil.manifest_base(self.manifest, 'runtime', architecture) devel_base = buildutil.manifest_base(self.manifest, 'devel', architecture) @@ -261,10 +210,10 @@ class OstbuildBuild(builtins.Builtin): devel_contents.append(branch + ':/doc') devel_contents.append(branch + ':/devel') - self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'), - runtime_contents) - self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'devel'), - devel_contents) + buildutil.compose(self.repo, '%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'), + runtime_contents) + buildutil.compose(self.repo, '%s-%s-%s' % (self.manifest['name'], architecture, 'devel'), + devel_contents) def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) @@ -321,9 +270,8 @@ class OstbuildBuild(builtins.Builtin): for component in build_components[start_at_index:]: index = components.index(component) - dependencies = components[:index] for architecture in self.manifest['architectures']: - self._build_one_component(component, dependencies, architecture) + self._build_one_component(component, architecture) for architecture in self.manifest['architectures']: self._compose_arch(architecture, components) diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py index cfc18369..ce699dc2 100755 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py @@ -31,33 +31,48 @@ class OstbuildChrootCompileOne(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--workdir') - parser.add_argument('--repo', required=True) - parser.add_argument('--resultdir') - parser.add_argument('--buildroot', required=True) parser.add_argument('--meta') parser.add_argument('--debug-shell', action='store_true') - (args, rest_args) = parser.parse_known_args(argv) + args = parser.parse_args(argv) - if args.meta is None: - output = run_sync_get_output(['ostbuild', 'autodiscover-meta']) - self.metadata = json.loads(output) - else: + self.parse_config() + + component_name = os.path.basename(os.getcwd()) + build_manifest_path = os.path.join(self.workdir, 'manifest.json') + self.manifest = json.load(open(build_manifest_path)) + + if args.meta is not None: f = open(args.meta) self.metadata = json.load(f) f.close() - for k in ['name']: - if k not in self.metadata: - sys.stderr.write('Missing required key "%s" in metadata' % (k, )) - sys.exit(1) - - workdir_is_tmp = (args.workdir is None) - if workdir_is_tmp: - workdir = tempfile.mkdtemp(prefix='ostbuild-chroot-compile-') + component = buildutil.find_component_in_manifest(self.manifest, self.metadata['name']) else: - workdir = args.workdir + component = buildutil.find_component_in_manifest(self.manifest, component_name) + if component is None: + fatal("Couldn't find component '%s' in manifest" % (component_name, )) + + self.metadata = component + self.metadata['src'] = 'dirty:worktree' + self.metadata['revision'] = 'dirty-worktree' + + components = self.manifest['components'] + index = components.index(component) + dependencies = components[:index] + + architecture = os.uname()[4] + buildroot_name = buildutil.manifest_buildroot_name(self.manifest, self.metadata, architecture) + buildroot_version = buildutil.compose_buildroot(self.manifest, self.repo, buildroot_name, + self.metadata, dependencies, architecture) + self.metadata['buildroot-name'] = buildroot_name + self.metadata['buildroot-version'] = buildroot_version + + if 'name' not in self.metadata: + sys.stderr.write('Missing required key "%s" in metadata' % (k, )) + sys.exit(1) + + workdir = self.workdir log("Using working directory: %s" % (workdir, )) @@ -66,24 +81,23 @@ class OstbuildChrootCompileOne(builtins.Builtin): log("Cleaning up previous tmpdir: %r" % (child_tmpdir, )) shutil.rmtree(child_tmpdir) os.mkdir(child_tmpdir) - - rev = run_sync_get_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.buildroot]) - rev = rev.strip() - - self.metadata['buildroot'] = args.buildroot - self.metadata['buildroot-version'] = rev + + resultdir = os.path.join(self.workdir, 'results', component['name']) + if os.path.isdir(resultdir): + shutil.rmtree(resultdir) + os.makedirs(resultdir) rootdir_prefix = os.path.join(workdir, 'roots') if not os.path.isdir(rootdir_prefix): os.makedirs(rootdir_prefix) - rootdir = os.path.join(rootdir_prefix, rev) + rootdir = os.path.join(rootdir_prefix, buildroot_version) rootdir_tmp = rootdir + '.tmp' builddir = os.path.join(rootdir, 'ostbuild'); if not os.path.isdir(rootdir): if os.path.isdir(rootdir_tmp): shutil.rmtree(rootdir_tmp) - child_args = ['ostree', '--repo=' + args.repo, 'checkout', '-U', rev, rootdir_tmp] + child_args = ['ostree', '--repo=' + self.repo, 'checkout', '-U', buildroot_version, rootdir_tmp] run_sync(child_args) child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp] run_sync(child_args) @@ -100,7 +114,7 @@ class OstbuildChrootCompileOne(builtins.Builtin): if not os.path.isdir(sourcedir): os.mkdir(sourcedir) - output_metadata = open('_ostbuild-meta', 'w') + output_metadata = open('_ostbuild-meta.json', 'w') json.dump(self.metadata, output_metadata) output_metadata.close() @@ -114,22 +128,18 @@ class OstbuildChrootCompileOne(builtins.Builtin): '--mount-bind', '/dev', '/dev', '--mount-bind', child_tmpdir, '/tmp', '--mount-bind', os.getcwd(), chroot_sourcedir, + '--mount-bind', resultdir, '/ostbuild/results', '--chdir', chroot_sourcedir] - if args.resultdir: - child_args.extend(['--mount-bind', args.resultdir, '/ostbuild/results']) if args.debug_shell: child_args.extend([rootdir, '/bin/sh']) else: child_args.extend([rootdir, '/usr/bin/ostbuild', 'compile-one', '--ostbuild-resultdir=/ostbuild/results', - '--ostbuild-meta=_ostbuild-meta']) - child_args.extend(rest_args) + '--ostbuild-meta=_ostbuild-meta.json']) + child_args.extend(self.metadata.get('config-opts', [])) env_copy = dict(buildutil.BUILD_ENV) env_copy['PWD'] = chroot_sourcedir run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell) - if workdir_is_tmp: - shutil.rmtree(workdir) - builtins.register(OstbuildChrootCompileOne) diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py index d7fbd943..03805ca3 100755 --- a/src/ostbuild/pyostbuild/builtin_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_compile_one.py @@ -96,21 +96,22 @@ class OstbuildCompileOne(builtins.Builtin): if self.ostbuild_resultdir is None: fatal("Must specify --ostbuild-resultdir=") - - self.metadata = {} - if self.ostbuild_meta is None: - output = run_sync_get_output(['ostbuild', 'autodiscover-meta']) - self.metadata = json.loads(output) - else: - f = open(self.ostbuild_meta) - self.metadata = json.load(f) - f.close() + fatal("Must specify --ostbuild-meta=") - for k in ['name', 'version']: + f = open(self.ostbuild_meta) + self.metadata = json.load(f) + f.close() + + for k in ['name', 'revision']: if k not in self.metadata: fatal('Missing required key "%s" in metadata' % (k, )) + if self.metadata.get('rm-configure', False): + configure_path = 'configure' + if os.path.exists(configure_path): + os.unlink(configure_path) + autogen_script = None if not os.path.exists('configure'): log("No 'configure' script found, looking for autogen/bootstrap") @@ -181,18 +182,6 @@ class OstbuildCompileOne(builtins.Builtin): name = self.metadata['name'] assert ',' not in name - branch = self.metadata['branch'] - assert ',' not in name - version = self.metadata['version'] - assert ',' not in version - - root_name = self.metadata.get('buildroot', None) - # TODO - pick up current sysroot version from ostree - if root_name is None: - root_name = 'unknown-' + self.build_target - root_version = 'UNKNOWN' - else: - root_version = self.metadata.get('buildroot-version') tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,)) self.tempfiles.append(tempdir)