packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ostbuild: Lots of stuff 

* Only create one build commit which contains multiple artifact trees,
  rather than one per artifact.  This is atomic.  We can use the new
  compose syntax like foo:/devel to slice out the /devel tree.
* Create the minimal buildroot for each component by composing the
  previous components in the build order, instead of continually
  updating one big tree.
* Ensure the artifact builder gets empty directories in /etc
This commit is contained in:
Colin Walters 2012-01-12 10:48:11 -05:00
parent 82aae2e57f
commit b44afdef23
3 changed files with 210 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/ostbuild/pyostbuild/buildutil.py
View File

@ -21,22 +21,10 @@ from .subprocess_helpers import run_sync_get_output
ARTIFACT_RE = re.compile(r'^artifact-([^,]+),([^,]+),([^,]+),([^,]+),(.+)-((?:runtime)|(?:devel))\.tar$')
def parse_artifact_name(artifact_basename):
match = ARTIFACT_RE.match(artifact_basename)
if match is None:
raise ValueError("Invalid artifact basename %s" % (artifact_basename))
return {'buildroot': match.group(1),
'buildroot-version': match.group(2),
'name': match.group(3),
'branch': match.group(4),
'version': match.group(5),
'type': match.group(6)}
def branch_name_for_artifact(a):
return 'artifacts/%s/%s/%s/%s' % (a['buildroot'],
a['name'],
a['branch'],
a['type'])
return 'artifacts/%s/%s/%s' % (a['buildroot'],
a['name'],
a['branch'])
def get_git_version_describe(dirpath):
version = run_sync_get_output(['git', 'describe', '--long', '--abbrev=42', '--always'],

252
src/ostbuild/pyostbuild/builtin_build.py
View File

@ -18,6 +18,7 @@
import os,sys,subprocess,tempfile,re,shutil
import argparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
@ -25,6 +26,7 @@ from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
from . import buildutil
from . import kvfile
from . import odict
class BuildOptions(object):
pass
@ -57,7 +59,7 @@ class OstbuildBuild(builtins.Builtin):
shutil.rmtree(dest)
if os.path.isdir(tmp_dest):
shutil.rmtree(tmp_dest)
subprocess.check_call(['git', 'clone', '-q', mirrordir, tmp_dest])
subprocess.check_call(['git', 'clone', '-q', '--recursive', mirrordir, tmp_dest])
subprocess.check_call(['git', 'checkout', '-q', branch], cwd=tmp_dest)
subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=tmp_dest)
os.rename(tmp_dest, dest)
@ -117,84 +119,94 @@ class OstbuildBuild(builtins.Builtin):
name = name[:-4]
name = name.replace('/', '-')
result['name'] = name
if 'branch' not in result:
result['branch'] = 'master'
return result
def _build_one_component(self, meta, architecture):
def _get_target(self, architecture):
return '%s-%s-devel' % (self.manifest['name'], architecture)
def _get_base(self, roottype, architecture):
return 'bases/%s-%s-%s' % (self.manifest['base'],
architecture, roottype)
def _get_buildname(self, component, architecture):
return 'artifacts/%s/%s/%s' % (self._get_target (architecture),
component['name'],
component['branch'])
def _get_buildroot_name(self, component, architecture):
return 'buildroots/%s/%s/%s' % (self._get_target (architecture),
component['name'],
component['branch'])
def _compose_buildroot(self, buildroot_name, component, dependencies, architecture):
base = self._get_base('devel', architecture)
buildroot_contents = [base + ':/']
for dep in dependencies:
dep_buildname = self._get_buildname(dep, architecture)
buildroot_contents.append(dep_buildname + ':/runtime')
buildroot_contents.append(dep_buildname + ':/devel')
return self._compose(buildroot_name, buildroot_contents)
def _build_one_component(self, meta, dependencies, architecture):
name = meta['name']
branch = meta['branch']
target = self._get_target(architecture)
buildname = self._get_buildname(meta, architecture)
buildroot_name = self._get_buildroot_name(meta, architecture)
(keytype, uri) = self._parse_src_key(meta['src'])
branch = meta.get('branch', 'master')
buildroot = '%s-%s-devel' % (self.manifest['name'], architecture)
runtime_branchname = 'artifacts/%s/%s/%s/runtime' % (buildroot, name, branch)
current_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', buildroot])
current_buildroot_version = current_buildroot_version.strip()
artifact_base = {'buildroot': buildroot,
'buildroot-version': current_buildroot_version,
'name': name,
'branch': branch,
}
component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch)
component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch)
current_vcs_version = buildutil.get_git_version_describe(component_src)
artifact_base['version'] = current_vcs_version
previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', buildname],
stderr=open('/dev/null', 'w'),
none_on_error=True)
if previous_build_version is not None:
log("Previous build of '%s' is %s" % (buildname, previous_build_version))
previous_vcs_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'show', '--print-metadata-key=ostbuild-artifact-version',
previous_build_version])
previous_vcs_version = previous_vcs_version.strip()
vcs_version_matches = False
if previous_vcs_version == current_vcs_version:
vcs_version_matches = True
log("VCS version is unchanged from '%s'" % (previous_vcs_version, ))
if self.buildopts.skip_built:
return False
else:
log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
else:
log("No previous build for '%s' found" % (buildname, ))
buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
artifact_meta = {'buildroot': buildroot_name,
'buildroot-version': buildroot_version,
'name': name,
'branch': branch,
'version': current_vcs_version
}
metadata_dir = os.path.join(self.workdir, 'meta')
if not os.path.isdir(metadata_dir):
os.makedirs(metadata_dir)
metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, ))
f = open(metadata_path, 'w')
json.dump(artifact_base, f)
json.dump(artifact_meta, f)
f.close()
previous_commit_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', runtime_branchname],
stderr=open('/dev/null', 'w'),
none_on_error=True)
if previous_commit_version is not None:
log("Previous build of '%s' is %s" % (runtime_branchname, previous_commit_version))
previous_artifact_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'show', '--print-metadata-key=ostbuild-artifact-version', previous_commit_version])
previous_artifact_version = previous_artifact_version.strip()
previous_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'show', '--print-metadata-key=ostbuild-buildroot-version', previous_commit_version])
previous_buildroot_version = previous_buildroot_version.strip()
previous_artifact_base = dict(artifact_base)
previous_artifact_base['version'] = previous_artifact_version
previous_artifact_base['buildroot-version'] = previous_buildroot_version
previous_artifact_runtime = dict(previous_artifact_base)
previous_artifact_runtime['type'] = 'runtime'
previous_artifact_devel = dict(previous_artifact_base)
previous_artifact_devel['type'] = 'devel'
previous_artifacts = [previous_artifact_runtime,
previous_artifact_devel]
vcs_version_matches = False
if previous_artifact_version == current_vcs_version:
vcs_version_matches = True
log("VCS version is unchanged from '%s'" % (previous_artifact_version, ))
if self.buildopts.skip_built:
return previous_artifacts
else:
log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_artifact_version))
buildroot_version_matches = False
if vcs_version_matches:
buildroot_version_matches = (current_buildroot_version == previous_buildroot_version)
if buildroot_version_matches:
log("Already have build '%s' of src commit '%s' for '%s' in buildroot '%s'" % (previous_commit_version, previous_artifact_version, runtime_branchname, buildroot))
return previous_artifacts
else:
log("Buildroot is now '%s'" % (current_buildroot_version, ))
else:
log("No previous build for '%s' found" % (runtime_branchname, ))
patches = meta.get('patches')
if patches is not None:
for patch in patches:
@ -206,48 +218,68 @@ class OstbuildBuild(builtins.Builtin):
shutil.rmtree(component_resultdir)
os.makedirs(component_resultdir)
chroot_args = self._get_ostbuild_chroot_args(architecture)
chroot_args.extend(['--buildroot=' + buildroot,
'--workdir=' + self.workdir,
'--resultdir=' + component_resultdir,
'--meta=' + metadata_path])
global_config_opts = self.manifest.get('config-opts')
if global_config_opts is not None:
chroot_args.extend(global_config_opts)
component_config_opts = meta.get('config-opts')
if component_config_opts is not None:
chroot_args.extend(component_config_opts)
if self.buildopts.shell_on_failure:
ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False)
if ecode != 0:
self._launch_debug_shell(architecture, buildroot, cwd=component_src)
if self.args.debug_shell:
self._launch_debug_shell(architecture, buildroot_name, cwd=component_src)
else:
run_sync(chroot_args, cwd=component_src, fatal_on_error=True)
chroot_args = self._get_ostbuild_chroot_args(architecture)
chroot_args.extend(['--buildroot=' + buildroot_name,
'--workdir=' + self.workdir,
'--resultdir=' + component_resultdir,
'--meta=' + metadata_path])
global_config_opts = self.manifest.get('config-opts')
if global_config_opts is not None:
chroot_args.extend(global_config_opts)
component_config_opts = meta.get('config-opts')
if component_config_opts is not None:
chroot_args.extend(component_config_opts)
if self.buildopts.shell_on_failure:
ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False)
if ecode != 0:
self._launch_debug_shell(architecture, buildroot_name, cwd=component_src)
else:
run_sync(chroot_args, cwd=component_src, fatal_on_error=True)
artifacts = []
for artifact_type in ['runtime', 'devel']:
artifact = dict(artifact_base)
artifacts.append(artifact)
artifact['type'] = artifact_type
run_sync(['ostree', '--repo=' + self.repo,
'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['version'],
'--add-metadata-string=ostbuild-buildroot-version=' + buildroot_version,
'--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['version'],
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged'],
cwd=component_resultdir)
return True
artifact_branch = buildutil.branch_name_for_artifact(artifact)
def _compose(self, target, artifacts):
child_args = ['ostree', '--repo=' + self.repo, 'compose',
'-b', target, '-s', 'Compose']
(fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
f = os.fdopen(fd, 'w')
for artifact in artifacts:
f.write(artifact)
f.write('\n')
f.close()
child_args.extend(['-F', path])
revision = run_sync_get_output(child_args, log_initiation=True).strip()
os.unlink(path)
return revision
artifact_resultdir = os.path.join(component_resultdir, artifact_branch)
run_sync(['ostree', '--repo=' + self.repo,
'commit', '-b', artifact_branch, '-s', 'Build ' + artifact_base['version'],
'--add-metadata-string=ostbuild-buildroot-version=' + current_buildroot_version,
'--add-metadata-string=ostbuild-artifact-version=' + artifact_base['version'],
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged'],
cwd=artifact_resultdir)
return artifacts
def _compose_arch(self, architecture, components):
runtime_base = self._get_base('runtime', architecture)
devel_base = self._get_base('runtime', architecture)
runtime_contents = [runtime_base + ':/']
devel_contents = [devel_base + ':/']
def _compose(self, suffix, artifacts):
child_args = ['ostree', '--repo=' + self.repo, 'compose', '--recompose',
'-b', self.manifest['name'] + '-' + suffix, '-s', 'Compose']
child_args.extend(artifacts)
run_sync(child_args)
for component in components:
branch = self._get_buildname(component, architecture)
runtime_contents.append(branch + ':/runtime')
devel_contents.append(branch + ':/runtime')
# For now just hardcode docs going in devel
devel_contents.append(branch + ':/doc')
devel_contents.append(branch + ':/devel')
self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'),
runtime_contents)
self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'devel'),
devel_contents)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
@ -259,6 +291,7 @@ class OstbuildBuild(builtins.Builtin):
parser.add_argument('components', nargs='*')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
@ -268,11 +301,6 @@ class OstbuildBuild(builtins.Builtin):
self.manifest = json.load(open(args.manifest))
if args.debug_shell:
debug_shell_arch = self.manifest['architectures'][0]
debug_shell_buildroot = '%s-%s-devel' % (self.manifest['name'], debug_shell_arch)
self._launch_debug_shell(debug_shell_arch, debug_shell_buildroot)
self.manifestdir = os.path.dirname(args.manifest)
self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
@ -293,6 +321,8 @@ class OstbuildBuild(builtins.Builtin):
start_at_index = -1
if args.start_at is not None:
if build_components != self.resolved_components:
fatal("Can't specify --start-at with component list")
for i,component in enumerate(build_components):
if component['name'] == args.start_at:
start_at_index = i
@ -301,16 +331,14 @@ class OstbuildBuild(builtins.Builtin):
fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
else:
start_at_index = 0
for component in build_components[start_at_index:]:
index = self.resolved_components.index(component)
dependencies = self.resolved_components[:index]
for architecture in self.manifest['architectures']:
(runtime_artifact,devel_artifact) = self._build_one_component(component, architecture)
runtime_branch = buildutil.branch_name_for_artifact(runtime_artifact)
devel_branch = buildutil.branch_name_for_artifact(devel_artifact)
target_component = component.get('component')
if target_component != 'devel':
self._compose(architecture + '-runtime', [runtime_branch])
self._compose(architecture + '-devel', [runtime_branch, devel_branch])
self._build_one_component(component, dependencies, architecture)
for architecture in self.manifest['architectures']:
self._compose_arch(architecture, self.resolved_components)
builtins.register(OstbuildBuild)

98
src/ostbuild/pyostbuild/builtin_compile_one.py
View File

@ -1,4 +1,4 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@ -34,14 +34,20 @@ _BLACKLIST_REGEXPS = map(re.compile,
[r'.*\.la$',
])
_RUNTIME_DIRS = ['/etc']
_DOC_DIRS = ['/usr/share/doc',
'/usr/share/gtk-doc',
'/usr/share/man',
'/usr/share/info']
_DEVEL_DIRS = ['/usr/include',
'/usr/share/aclocal',
'/usr/share/pkgconfig',
'/usr/lib/pkgconfig']
_DEVEL_REGEXPS = map(re.compile,
[r'/usr/include/',
r'/usr/share/pkgconfig/',
r'/usr/share/aclocal/',
r'/(?:usr/)lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc$',
r'/(?:usr/)lib(?:|(?:32)|(?:64))/[^/]+\.so$'
r'/(?:usr/)lib(?:|(?:32)|(?:64))/[^/]+\.a$'
])
[r'/(?:usr/)lib/[^/]+\.(?:so|a)$'])
class OstbuildCompileOne(builtins.Builtin):
name = "compile-one"
@ -66,15 +72,9 @@ class OstbuildCompileOne(builtins.Builtin):
machine=uname[4]
self.build_target='%s-%s' % (machine, kernel)
# libdir detection
if os.path.isdir('/lib64'):
libdir=os.path.join(PREFIX, 'lib64')
else:
libdir=os.path.join(PREFIX, 'lib')
self.configargs = ['--build=' + self.build_target,
'--prefix=' + PREFIX,
'--libdir=' + libdir,
'--libdir=' + os.path.join(PREFIX, 'lib'),
'--sysconfdir=/etc',
'--localstatedir=/var',
'--bindir=' + os.path.join(PREFIX, 'bin'),
@ -199,20 +199,45 @@ class OstbuildCompileOne(builtins.Builtin):
else:
root_version = self.metadata.get('buildroot-version')
artifact_prefix=os.path.join('artifacts', root_name, name, branch)
tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,))
self.tempfiles.append(tempdir)
args = ['make', 'install', 'DESTDIR=' + tempdir]
run_sync(args, cwd=builddir)
devel_files = set()
dbg_files = set()
doc_files = set()
runtime_files = set()
oldpwd=os.getcwd()
os.chdir(tempdir)
for root, dirs, files in os.walk('.'):
deleted_dirs = set()
for dirname in dirs:
path = os.path.join(root, dirname)
subpath = path[1:]
matched = False
for runtime_name in _RUNTIME_DIRS:
if subpath.startswith(runtime_name):
runtime_files.add(path)
matched = True
break
if not matched:
for devel_name in _DEVEL_DIRS:
if subpath.startswith(devel_name):
devel_files.add(path)
matched = True
break
if not matched:
for doc_name in _DOC_DIRS:
if subpath.startswith(doc_name):
doc_files.add(path)
matched = True
break
if matched:
deleted_dirs.add(dirname)
for dirname in deleted_dirs:
dirs.remove(dirname)
for filename in files:
path = os.path.join(root, filename)
@ -224,7 +249,7 @@ class OstbuildCompileOne(builtins.Builtin):
if blacklisted:
continue
matched = False
for r in _DEVEL_REGEXPS:
if not r.match(path[1:]):
@ -236,8 +261,9 @@ class OstbuildCompileOne(builtins.Builtin):
runtime_files.add(path)
os.chdir(oldpwd)
self.make_artifact(artifact_prefix, 'devel', devel_files, tempdir=tempdir)
self.make_artifact(artifact_prefix, 'runtime', runtime_files, tempdir=tempdir)
self.make_artifact('devel', devel_files, tempdir=tempdir)
self.make_artifact('doc', doc_files, tempdir=tempdir)
self.make_artifact('runtime', runtime_files, tempdir=tempdir)
for tmpname in self.tempfiles:
assert os.path.isabs(tmpname)
@ -252,17 +278,27 @@ class OstbuildCompileOne(builtins.Builtin):
def _rename_or_copy(self, src, dest):
statsrc = os.lstat(src)
statdest = os.lstat(os.path.dirname(dest))
try:
os.rename(src, dest)
except OSError, e:
if stat.S_ISLNK(statsrc.st_mode):
linkto = os.readlink(src)
os.symlink(linkto, dest)
else:
shutil.copy2(src, dest)
if stat.S_ISDIR(statsrc.st_mode):
if not os.path.isdir(dest):
os.mkdir(dest)
for filename in os.listdir(src):
src_child = os.path.join(src, filename)
dest_child = os.path.join(dest, filename)
self._rename_or_copy(src_child, dest_child)
else:
try:
os.rename(src, dest)
except OSError, e:
if stat.S_ISLNK(statsrc.st_mode):
linkto = os.readlink(src)
os.symlink(linkto, dest)
else:
shutil.copy2(src, dest)
def make_artifact(self, prefix, dirtype, from_files, tempdir):
resultdir = os.path.join(self.ostbuild_resultdir, prefix, dirtype)
def make_artifact(self, dirtype, from_files, tempdir):
resultdir = os.path.join(self.ostbuild_resultdir, dirtype)
if os.path.isdir(resultdir):
shutil.rmtree(resultdir)
os.makedirs(resultdir)