From b44afdef233c3742b859e862015c9c71504edbd9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 12 Jan 2012 10:48:11 -0500 Subject: [PATCH] ostbuild: Lots of stuff * Only create one build commit which contains multiple artifact trees, rather than one per artifact. This is atomic. We can use the new compose syntax like foo:/devel to slice out the /devel tree. * Create the minimal buildroot for each component by composing the previous components in the build order, instead of continually updating one big tree. * Ensure the artifact builder gets empty directories in /etc --- src/ostbuild/pyostbuild/buildutil.py | 18 +- src/ostbuild/pyostbuild/builtin_build.py | 252 ++++++++++-------- .../pyostbuild/builtin_compile_one.py | 98 ++++--- 3 files changed, 210 insertions(+), 158 deletions(-) diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py index fb3e634f..67d4130b 100755 --- a/src/ostbuild/pyostbuild/buildutil.py +++ b/src/ostbuild/pyostbuild/buildutil.py @@ -21,22 +21,10 @@ from .subprocess_helpers import run_sync_get_output ARTIFACT_RE = re.compile(r'^artifact-([^,]+),([^,]+),([^,]+),([^,]+),(.+)-((?:runtime)|(?:devel))\.tar$') -def parse_artifact_name(artifact_basename): - match = ARTIFACT_RE.match(artifact_basename) - if match is None: - raise ValueError("Invalid artifact basename %s" % (artifact_basename)) - return {'buildroot': match.group(1), - 'buildroot-version': match.group(2), - 'name': match.group(3), - 'branch': match.group(4), - 'version': match.group(5), - 'type': match.group(6)} - def branch_name_for_artifact(a): - return 'artifacts/%s/%s/%s/%s' % (a['buildroot'], - a['name'], - a['branch'], - a['type']) + return 'artifacts/%s/%s/%s' % (a['buildroot'], + a['name'], + a['branch']) def get_git_version_describe(dirpath): version = run_sync_get_output(['git', 'describe', '--long', '--abbrev=42', '--always'], diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index 3ac05c9d..2a07451b 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -18,6 +18,7 @@ import os,sys,subprocess,tempfile,re,shutil import argparse import json +from StringIO import StringIO from . import builtins from .ostbuildlog import log, fatal @@ -25,6 +26,7 @@ from .subprocess_helpers import run_sync, run_sync_get_output from . import ostbuildrc from . import buildutil from . import kvfile +from . import odict class BuildOptions(object): pass @@ -57,7 +59,7 @@ class OstbuildBuild(builtins.Builtin): shutil.rmtree(dest) if os.path.isdir(tmp_dest): shutil.rmtree(tmp_dest) - subprocess.check_call(['git', 'clone', '-q', mirrordir, tmp_dest]) + subprocess.check_call(['git', 'clone', '-q', '--recursive', mirrordir, tmp_dest]) subprocess.check_call(['git', 'checkout', '-q', branch], cwd=tmp_dest) subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=tmp_dest) os.rename(tmp_dest, dest) @@ -117,84 +119,94 @@ class OstbuildBuild(builtins.Builtin): name = name[:-4] name = name.replace('/', '-') result['name'] = name + + if 'branch' not in result: + result['branch'] = 'master' + return result - def _build_one_component(self, meta, architecture): + def _get_target(self, architecture): + return '%s-%s-devel' % (self.manifest['name'], architecture) + + def _get_base(self, roottype, architecture): + return 'bases/%s-%s-%s' % (self.manifest['base'], + architecture, roottype) + + def _get_buildname(self, component, architecture): + return 'artifacts/%s/%s/%s' % (self._get_target (architecture), + component['name'], + component['branch']) + + def _get_buildroot_name(self, component, architecture): + return 'buildroots/%s/%s/%s' % (self._get_target (architecture), + component['name'], + component['branch']) + + def _compose_buildroot(self, buildroot_name, component, dependencies, architecture): + base = self._get_base('devel', architecture) + buildroot_contents = [base + ':/'] + for dep in dependencies: + dep_buildname = self._get_buildname(dep, architecture) + buildroot_contents.append(dep_buildname + ':/runtime') + buildroot_contents.append(dep_buildname + ':/devel') + + return self._compose(buildroot_name, buildroot_contents) + + def _build_one_component(self, meta, dependencies, architecture): name = meta['name'] + branch = meta['branch'] + + target = self._get_target(architecture) + buildname = self._get_buildname(meta, architecture) + buildroot_name = self._get_buildroot_name(meta, architecture) (keytype, uri) = self._parse_src_key(meta['src']) - branch = meta.get('branch', 'master') - - buildroot = '%s-%s-devel' % (self.manifest['name'], architecture) - runtime_branchname = 'artifacts/%s/%s/%s/runtime' % (buildroot, name, branch) - current_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', buildroot]) - current_buildroot_version = current_buildroot_version.strip() - - artifact_base = {'buildroot': buildroot, - 'buildroot-version': current_buildroot_version, - 'name': name, - 'branch': branch, - } component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch) component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch) current_vcs_version = buildutil.get_git_version_describe(component_src) - artifact_base['version'] = current_vcs_version + + previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'rev-parse', buildname], + stderr=open('/dev/null', 'w'), + none_on_error=True) + if previous_build_version is not None: + log("Previous build of '%s' is %s" % (buildname, previous_build_version)) + + previous_vcs_version = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'show', '--print-metadata-key=ostbuild-artifact-version', + previous_build_version]) + previous_vcs_version = previous_vcs_version.strip() + + vcs_version_matches = False + if previous_vcs_version == current_vcs_version: + vcs_version_matches = True + log("VCS version is unchanged from '%s'" % (previous_vcs_version, )) + if self.buildopts.skip_built: + return False + else: + log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version)) + else: + log("No previous build for '%s' found" % (buildname, )) + + buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture) + + artifact_meta = {'buildroot': buildroot_name, + 'buildroot-version': buildroot_version, + 'name': name, + 'branch': branch, + 'version': current_vcs_version + } metadata_dir = os.path.join(self.workdir, 'meta') if not os.path.isdir(metadata_dir): os.makedirs(metadata_dir) metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, )) f = open(metadata_path, 'w') - json.dump(artifact_base, f) + json.dump(artifact_meta, f) f.close() - previous_commit_version = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', runtime_branchname], - stderr=open('/dev/null', 'w'), - none_on_error=True) - if previous_commit_version is not None: - log("Previous build of '%s' is %s" % (runtime_branchname, previous_commit_version)) - - previous_artifact_version = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'show', '--print-metadata-key=ostbuild-artifact-version', previous_commit_version]) - previous_artifact_version = previous_artifact_version.strip() - previous_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'show', '--print-metadata-key=ostbuild-buildroot-version', previous_commit_version]) - previous_buildroot_version = previous_buildroot_version.strip() - - previous_artifact_base = dict(artifact_base) - previous_artifact_base['version'] = previous_artifact_version - previous_artifact_base['buildroot-version'] = previous_buildroot_version - - previous_artifact_runtime = dict(previous_artifact_base) - previous_artifact_runtime['type'] = 'runtime' - previous_artifact_devel = dict(previous_artifact_base) - previous_artifact_devel['type'] = 'devel' - previous_artifacts = [previous_artifact_runtime, - previous_artifact_devel] - - vcs_version_matches = False - if previous_artifact_version == current_vcs_version: - vcs_version_matches = True - log("VCS version is unchanged from '%s'" % (previous_artifact_version, )) - if self.buildopts.skip_built: - return previous_artifacts - else: - log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_artifact_version)) - buildroot_version_matches = False - if vcs_version_matches: - buildroot_version_matches = (current_buildroot_version == previous_buildroot_version) - if buildroot_version_matches: - log("Already have build '%s' of src commit '%s' for '%s' in buildroot '%s'" % (previous_commit_version, previous_artifact_version, runtime_branchname, buildroot)) - return previous_artifacts - else: - log("Buildroot is now '%s'" % (current_buildroot_version, )) - else: - log("No previous build for '%s' found" % (runtime_branchname, )) - patches = meta.get('patches') if patches is not None: for patch in patches: @@ -206,48 +218,68 @@ class OstbuildBuild(builtins.Builtin): shutil.rmtree(component_resultdir) os.makedirs(component_resultdir) - chroot_args = self._get_ostbuild_chroot_args(architecture) - chroot_args.extend(['--buildroot=' + buildroot, - '--workdir=' + self.workdir, - '--resultdir=' + component_resultdir, - '--meta=' + metadata_path]) - global_config_opts = self.manifest.get('config-opts') - if global_config_opts is not None: - chroot_args.extend(global_config_opts) - component_config_opts = meta.get('config-opts') - if component_config_opts is not None: - chroot_args.extend(component_config_opts) - if self.buildopts.shell_on_failure: - ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False) - if ecode != 0: - self._launch_debug_shell(architecture, buildroot, cwd=component_src) + if self.args.debug_shell: + self._launch_debug_shell(architecture, buildroot_name, cwd=component_src) else: - run_sync(chroot_args, cwd=component_src, fatal_on_error=True) + chroot_args = self._get_ostbuild_chroot_args(architecture) + chroot_args.extend(['--buildroot=' + buildroot_name, + '--workdir=' + self.workdir, + '--resultdir=' + component_resultdir, + '--meta=' + metadata_path]) + global_config_opts = self.manifest.get('config-opts') + if global_config_opts is not None: + chroot_args.extend(global_config_opts) + component_config_opts = meta.get('config-opts') + if component_config_opts is not None: + chroot_args.extend(component_config_opts) + if self.buildopts.shell_on_failure: + ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False) + if ecode != 0: + self._launch_debug_shell(architecture, buildroot_name, cwd=component_src) + else: + run_sync(chroot_args, cwd=component_src, fatal_on_error=True) - artifacts = [] - for artifact_type in ['runtime', 'devel']: - artifact = dict(artifact_base) - artifacts.append(artifact) - artifact['type'] = artifact_type + run_sync(['ostree', '--repo=' + self.repo, + 'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['version'], + '--add-metadata-string=ostbuild-buildroot-version=' + buildroot_version, + '--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['version'], + '--owner-uid=0', '--owner-gid=0', '--no-xattrs', + '--skip-if-unchanged'], + cwd=component_resultdir) + return True - artifact_branch = buildutil.branch_name_for_artifact(artifact) + def _compose(self, target, artifacts): + child_args = ['ostree', '--repo=' + self.repo, 'compose', + '-b', target, '-s', 'Compose'] + (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') + f = os.fdopen(fd, 'w') + for artifact in artifacts: + f.write(artifact) + f.write('\n') + f.close() + child_args.extend(['-F', path]) + revision = run_sync_get_output(child_args, log_initiation=True).strip() + os.unlink(path) + return revision - artifact_resultdir = os.path.join(component_resultdir, artifact_branch) - - run_sync(['ostree', '--repo=' + self.repo, - 'commit', '-b', artifact_branch, '-s', 'Build ' + artifact_base['version'], - '--add-metadata-string=ostbuild-buildroot-version=' + current_buildroot_version, - '--add-metadata-string=ostbuild-artifact-version=' + artifact_base['version'], - '--owner-uid=0', '--owner-gid=0', '--no-xattrs', - '--skip-if-unchanged'], - cwd=artifact_resultdir) - return artifacts + def _compose_arch(self, architecture, components): + runtime_base = self._get_base('runtime', architecture) + devel_base = self._get_base('runtime', architecture) + runtime_contents = [runtime_base + ':/'] + devel_contents = [devel_base + ':/'] - def _compose(self, suffix, artifacts): - child_args = ['ostree', '--repo=' + self.repo, 'compose', '--recompose', - '-b', self.manifest['name'] + '-' + suffix, '-s', 'Compose'] - child_args.extend(artifacts) - run_sync(child_args) + for component in components: + branch = self._get_buildname(component, architecture) + runtime_contents.append(branch + ':/runtime') + devel_contents.append(branch + ':/runtime') + # For now just hardcode docs going in devel + devel_contents.append(branch + ':/doc') + devel_contents.append(branch + ':/devel') + + self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'), + runtime_contents) + self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'devel'), + devel_contents) def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) @@ -259,6 +291,7 @@ class OstbuildBuild(builtins.Builtin): parser.add_argument('components', nargs='*') args = parser.parse_args(argv) + self.args = args self.parse_config() @@ -268,11 +301,6 @@ class OstbuildBuild(builtins.Builtin): self.manifest = json.load(open(args.manifest)) - if args.debug_shell: - debug_shell_arch = self.manifest['architectures'][0] - debug_shell_buildroot = '%s-%s-devel' % (self.manifest['name'], debug_shell_arch) - self._launch_debug_shell(debug_shell_arch, debug_shell_buildroot) - self.manifestdir = os.path.dirname(args.manifest) self.resolved_components = map(self._resolve_component_meta, self.manifest['components']) @@ -293,6 +321,8 @@ class OstbuildBuild(builtins.Builtin): start_at_index = -1 if args.start_at is not None: + if build_components != self.resolved_components: + fatal("Can't specify --start-at with component list") for i,component in enumerate(build_components): if component['name'] == args.start_at: start_at_index = i @@ -301,16 +331,14 @@ class OstbuildBuild(builtins.Builtin): fatal("Unknown component %r specified for --start-at" % (args.start_at, )) else: start_at_index = 0 - + for component in build_components[start_at_index:]: + index = self.resolved_components.index(component) + dependencies = self.resolved_components[:index] for architecture in self.manifest['architectures']: - (runtime_artifact,devel_artifact) = self._build_one_component(component, architecture) - runtime_branch = buildutil.branch_name_for_artifact(runtime_artifact) - devel_branch = buildutil.branch_name_for_artifact(devel_artifact) - - target_component = component.get('component') - if target_component != 'devel': - self._compose(architecture + '-runtime', [runtime_branch]) - self._compose(architecture + '-devel', [runtime_branch, devel_branch]) + self._build_one_component(component, dependencies, architecture) + + for architecture in self.manifest['architectures']: + self._compose_arch(architecture, self.resolved_components) builtins.register(OstbuildBuild) diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py index 5b05e9ac..87d4bcfb 100755 --- a/src/ostbuild/pyostbuild/builtin_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_compile_one.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011 Colin Walters +# Copyright (C) 2011,2012 Colin Walters # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -34,14 +34,20 @@ _BLACKLIST_REGEXPS = map(re.compile, [r'.*\.la$', ]) +_RUNTIME_DIRS = ['/etc'] + +_DOC_DIRS = ['/usr/share/doc', + '/usr/share/gtk-doc', + '/usr/share/man', + '/usr/share/info'] + +_DEVEL_DIRS = ['/usr/include', + '/usr/share/aclocal', + '/usr/share/pkgconfig', + '/usr/lib/pkgconfig'] + _DEVEL_REGEXPS = map(re.compile, - [r'/usr/include/', - r'/usr/share/pkgconfig/', - r'/usr/share/aclocal/', - r'/(?:usr/)lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc$', - r'/(?:usr/)lib(?:|(?:32)|(?:64))/[^/]+\.so$' - r'/(?:usr/)lib(?:|(?:32)|(?:64))/[^/]+\.a$' - ]) + [r'/(?:usr/)lib/[^/]+\.(?:so|a)$']) class OstbuildCompileOne(builtins.Builtin): name = "compile-one" @@ -66,15 +72,9 @@ class OstbuildCompileOne(builtins.Builtin): machine=uname[4] self.build_target='%s-%s' % (machine, kernel) - # libdir detection - if os.path.isdir('/lib64'): - libdir=os.path.join(PREFIX, 'lib64') - else: - libdir=os.path.join(PREFIX, 'lib') - self.configargs = ['--build=' + self.build_target, '--prefix=' + PREFIX, - '--libdir=' + libdir, + '--libdir=' + os.path.join(PREFIX, 'lib'), '--sysconfdir=/etc', '--localstatedir=/var', '--bindir=' + os.path.join(PREFIX, 'bin'), @@ -199,20 +199,45 @@ class OstbuildCompileOne(builtins.Builtin): else: root_version = self.metadata.get('buildroot-version') - artifact_prefix=os.path.join('artifacts', root_name, name, branch) - tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,)) self.tempfiles.append(tempdir) args = ['make', 'install', 'DESTDIR=' + tempdir] run_sync(args, cwd=builddir) devel_files = set() - dbg_files = set() + doc_files = set() runtime_files = set() oldpwd=os.getcwd() os.chdir(tempdir) for root, dirs, files in os.walk('.'): + deleted_dirs = set() + for dirname in dirs: + path = os.path.join(root, dirname) + subpath = path[1:] + matched = False + for runtime_name in _RUNTIME_DIRS: + if subpath.startswith(runtime_name): + runtime_files.add(path) + matched = True + break + if not matched: + for devel_name in _DEVEL_DIRS: + if subpath.startswith(devel_name): + devel_files.add(path) + matched = True + break + if not matched: + for doc_name in _DOC_DIRS: + if subpath.startswith(doc_name): + doc_files.add(path) + matched = True + break + if matched: + deleted_dirs.add(dirname) + for dirname in deleted_dirs: + dirs.remove(dirname) + for filename in files: path = os.path.join(root, filename) @@ -224,7 +249,7 @@ class OstbuildCompileOne(builtins.Builtin): if blacklisted: continue - + matched = False for r in _DEVEL_REGEXPS: if not r.match(path[1:]): @@ -236,8 +261,9 @@ class OstbuildCompileOne(builtins.Builtin): runtime_files.add(path) os.chdir(oldpwd) - self.make_artifact(artifact_prefix, 'devel', devel_files, tempdir=tempdir) - self.make_artifact(artifact_prefix, 'runtime', runtime_files, tempdir=tempdir) + self.make_artifact('devel', devel_files, tempdir=tempdir) + self.make_artifact('doc', doc_files, tempdir=tempdir) + self.make_artifact('runtime', runtime_files, tempdir=tempdir) for tmpname in self.tempfiles: assert os.path.isabs(tmpname) @@ -252,17 +278,27 @@ class OstbuildCompileOne(builtins.Builtin): def _rename_or_copy(self, src, dest): statsrc = os.lstat(src) statdest = os.lstat(os.path.dirname(dest)) - try: - os.rename(src, dest) - except OSError, e: - if stat.S_ISLNK(statsrc.st_mode): - linkto = os.readlink(src) - os.symlink(linkto, dest) - else: - shutil.copy2(src, dest) + + if stat.S_ISDIR(statsrc.st_mode): + if not os.path.isdir(dest): + os.mkdir(dest) + for filename in os.listdir(src): + src_child = os.path.join(src, filename) + dest_child = os.path.join(dest, filename) + + self._rename_or_copy(src_child, dest_child) + else: + try: + os.rename(src, dest) + except OSError, e: + if stat.S_ISLNK(statsrc.st_mode): + linkto = os.readlink(src) + os.symlink(linkto, dest) + else: + shutil.copy2(src, dest) - def make_artifact(self, prefix, dirtype, from_files, tempdir): - resultdir = os.path.join(self.ostbuild_resultdir, prefix, dirtype) + def make_artifact(self, dirtype, from_files, tempdir): + resultdir = os.path.join(self.ostbuild_resultdir, dirtype) if os.path.isdir(resultdir): shutil.rmtree(resultdir) os.makedirs(resultdir)