packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ostbuild: Move more intelligence into chroot-compile-one 

Let's have chroot-compile-one be the core logic for composing a
buildroot, etc.  This allows us to use it as a developer tool by
checking out a tree, then doing a chroot build.
This commit is contained in:
Colin Walters 2012-02-24 21:36:24 -05:00
parent db93c43255
commit 40aedeb7a4
4 changed files with 108 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/ostbuild/pyostbuild/buildutil.py
View File

@ -18,6 +18,7 @@
import os
import re
import urlparse
import tempfile
from .subprocess_helpers import run_sync_get_output
@ -90,3 +91,33 @@ def manifest_buildroot_name(manifest, component, architecture):
return 'buildroots/%s/%s/%s' % (manifest_target (manifest, architecture),
component['name'],
component['branch'])
def find_component_in_manifest(manifest, component_name):
for component in manifest['components']:
if component['name'] == component_name:
return component
return None
def compose(repo, target, artifacts):
child_args = ['ostree', '--repo=' + repo, 'compose',
'-b', target, '-s', 'Compose']
(fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
f = os.fdopen(fd, 'w')
for artifact in artifacts:
f.write(artifact)
f.write('\n')
f.close()
child_args.extend(['-F', path])
revision = run_sync_get_output(child_args, log_initiation=True).strip()
os.unlink(path)
return revision
def compose_buildroot(manifest, repo, buildroot_name, component, dependencies, architecture):
base = manifest_base(manifest, 'devel', architecture)
buildroot_contents = [base + ':/']
for dep in dependencies:
dep_buildname = manifest_buildname(manifest, dep, architecture)
buildroot_contents.append(dep_buildname + ':/runtime')
buildroot_contents.append(dep_buildname + ':/devel')
return compose(repo, buildroot_name, buildroot_contents)

96
src/ostbuild/pyostbuild/builtin_build.py
View File

@ -48,8 +48,7 @@ class OstbuildBuild(builtins.Builtin):
args = ['setarch', architecture]
else:
args = []
args.extend(['ostbuild', 'chroot-compile-one',
'--repo=' + self.repo])
args.extend(['ostbuild', 'chroot-compile-one'])
return args
def _launch_debug_shell(self, architecture, buildroot, cwd=None):
@ -92,17 +91,7 @@ class OstbuildBuild(builtins.Builtin):
return result
def _compose_buildroot(self, buildroot_name, component, dependencies, architecture):
base = buildutil.manifest_base(self.manifest, 'devel', architecture)
buildroot_contents = [base + ':/']
for dep in dependencies:
dep_buildname = buildutil.manifest_buildname(self.manifest, dep, architecture)
buildroot_contents.append(dep_buildname + ':/runtime')
buildroot_contents.append(dep_buildname + ':/devel')
return self._compose(buildroot_name, buildroot_contents)
def _build_one_component(self, meta, dependencies, architecture):
def _build_one_component(self, meta, architecture):
name = meta['name']
branch = meta['branch']
@ -117,10 +106,6 @@ class OstbuildBuild(builtins.Builtin):
component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, branch,
overwrite=not self.args.debug_shell)
if self.args.debug_shell:
buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
self._launch_debug_shell(architecture, buildroot_name, cwd=component_src)
current_vcs_version = meta['revision']
previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
@ -146,39 +131,19 @@ class OstbuildBuild(builtins.Builtin):
else:
log("No previous build for '%s' found" % (buildname, ))
buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
artifact_meta = dict(meta)
if meta.get('rm-configure', False):
configure_path = os.path.join(component_src, 'configure')
if os.path.exists(configure_path):
os.unlink(configure_path)
artifact_meta = {'buildroot': buildroot_name,
'buildroot-version': buildroot_version,
'name': name,
'branch': branch,
'version': current_vcs_version
}
metadata_dir = os.path.join(self.workdir, 'meta')
if not os.path.isdir(metadata_dir):
os.makedirs(metadata_dir)
metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, ))
metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
f = open(metadata_path, 'w')
json.dump(artifact_meta, f)
f.close()
patches = meta.get('patches')
if patches is not None:
for patch in patches:
patch_path = os.path.join(self.patchdir, patch)
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src)
component_resultdir = os.path.join(self.workdir, 'results', name)
if os.path.isdir(component_resultdir):
shutil.rmtree(component_resultdir)
os.makedirs(component_resultdir)
logdir = os.path.join(self.workdir, 'logs', 'compile', name)
old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name)
if not os.path.isdir(logdir):
@ -194,16 +159,7 @@ class OstbuildBuild(builtins.Builtin):
log("Logging to %s" % (log_path, ))
f = open(log_path, 'w')
chroot_args = self._get_ostbuild_chroot_args(architecture)
chroot_args.extend(['--buildroot=' + buildroot_name,
'--workdir=' + self.workdir,
'--resultdir=' + component_resultdir,
'--meta=' + metadata_path])
global_config_opts = self.manifest.get('config-opts')
if global_config_opts is not None:
chroot_args.extend(global_config_opts)
component_config_opts = meta.get('config-opts')
if component_config_opts is not None:
chroot_args.extend(component_config_opts)
chroot_args.extend(['--meta=' + metadata_path])
if self.buildopts.shell_on_failure:
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
if ecode != 0:
@ -211,14 +167,19 @@ class OstbuildBuild(builtins.Builtin):
else:
run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
# Reread metadata to get buildroot version
f = open(metadata_path)
artifact_meta = json.load(f)
f.close()
args = ['ostree', '--repo=' + self.repo,
'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['version'],
'--add-metadata-string=ostbuild-buildroot-version=' + buildroot_version,
'--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['version'],
'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['revision'],
'--add-metadata-string=ostbuild-buildroot-version=' + artifact_meta['buildroot-version'],
'--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['revision'],
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged']
setuid_files = meta.get('setuid', [])
setuid_files = artifact_meta.get('setuid', [])
statoverride_path = None
if len(setuid_files) > 0:
(fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
@ -227,26 +188,14 @@ class OstbuildBuild(builtins.Builtin):
f.write('+2048 ' + path)
f.close()
args.append('--statoverride=' + statoverride_path)
component_resultdir = os.path.join(self.workdir, 'results', name)
run_sync(args, cwd=component_resultdir)
if statoverride_path is not None:
os.unlink(statoverride_path)
return True
def _compose(self, target, artifacts):
child_args = ['ostree', '--repo=' + self.repo, 'compose',
'-b', target, '-s', 'Compose']
(fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
f = os.fdopen(fd, 'w')
for artifact in artifacts:
f.write(artifact)
f.write('\n')
f.close()
child_args.extend(['-F', path])
revision = run_sync_get_output(child_args, log_initiation=True).strip()
os.unlink(path)
return revision
def _compose_arch(self, architecture, components):
runtime_base = buildutil.manifest_base(self.manifest, 'runtime', architecture)
devel_base = buildutil.manifest_base(self.manifest, 'devel', architecture)
@ -261,10 +210,10 @@ class OstbuildBuild(builtins.Builtin):
devel_contents.append(branch + ':/doc')
devel_contents.append(branch + ':/devel')
self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'),
runtime_contents)
self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'devel'),
devel_contents)
buildutil.compose(self.repo, '%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'),
runtime_contents)
buildutil.compose(self.repo, '%s-%s-%s' % (self.manifest['name'], architecture, 'devel'),
devel_contents)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
@ -321,9 +270,8 @@ class OstbuildBuild(builtins.Builtin):
for component in build_components[start_at_index:]:
index = components.index(component)
dependencies = components[:index]
for architecture in self.manifest['architectures']:
self._build_one_component(component, dependencies, architecture)
self._build_one_component(component, architecture)
for architecture in self.manifest['architectures']:
self._compose_arch(architecture, components)

78
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
View File

@ -31,33 +31,48 @@ class OstbuildChrootCompileOne(builtins.Builtin):
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--workdir')
parser.add_argument('--repo', required=True)
parser.add_argument('--resultdir')
parser.add_argument('--buildroot', required=True)
parser.add_argument('--meta')
parser.add_argument('--debug-shell', action='store_true')
(args, rest_args) = parser.parse_known_args(argv)
args = parser.parse_args(argv)
if args.meta is None:
output = run_sync_get_output(['ostbuild', 'autodiscover-meta'])
self.metadata = json.loads(output)
else:
self.parse_config()
component_name = os.path.basename(os.getcwd())
build_manifest_path = os.path.join(self.workdir, 'manifest.json')
self.manifest = json.load(open(build_manifest_path))
if args.meta is not None:
f = open(args.meta)
self.metadata = json.load(f)
f.close()
for k in ['name']:
if k not in self.metadata:
sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
sys.exit(1)
workdir_is_tmp = (args.workdir is None)
if workdir_is_tmp:
workdir = tempfile.mkdtemp(prefix='ostbuild-chroot-compile-')
component = buildutil.find_component_in_manifest(self.manifest, self.metadata['name'])
else:
workdir = args.workdir
component = buildutil.find_component_in_manifest(self.manifest, component_name)
if component is None:
fatal("Couldn't find component '%s' in manifest" % (component_name, ))
self.metadata = component
self.metadata['src'] = 'dirty:worktree'
self.metadata['revision'] = 'dirty-worktree'
components = self.manifest['components']
index = components.index(component)
dependencies = components[:index]
architecture = os.uname()[4]
buildroot_name = buildutil.manifest_buildroot_name(self.manifest, self.metadata, architecture)
buildroot_version = buildutil.compose_buildroot(self.manifest, self.repo, buildroot_name,
self.metadata, dependencies, architecture)
self.metadata['buildroot-name'] = buildroot_name
self.metadata['buildroot-version'] = buildroot_version
if 'name' not in self.metadata:
sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
sys.exit(1)
workdir = self.workdir
log("Using working directory: %s" % (workdir, ))
@ -66,24 +81,23 @@ class OstbuildChrootCompileOne(builtins.Builtin):
log("Cleaning up previous tmpdir: %r" % (child_tmpdir, ))
shutil.rmtree(child_tmpdir)
os.mkdir(child_tmpdir)
rev = run_sync_get_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.buildroot])
rev = rev.strip()
self.metadata['buildroot'] = args.buildroot
self.metadata['buildroot-version'] = rev
resultdir = os.path.join(self.workdir, 'results', component['name'])
if os.path.isdir(resultdir):
shutil.rmtree(resultdir)
os.makedirs(resultdir)
rootdir_prefix = os.path.join(workdir, 'roots')
if not os.path.isdir(rootdir_prefix):
os.makedirs(rootdir_prefix)
rootdir = os.path.join(rootdir_prefix, rev)
rootdir = os.path.join(rootdir_prefix, buildroot_version)
rootdir_tmp = rootdir + '.tmp'
builddir = os.path.join(rootdir, 'ostbuild');
if not os.path.isdir(rootdir):
if os.path.isdir(rootdir_tmp):
shutil.rmtree(rootdir_tmp)
child_args = ['ostree', '--repo=' + args.repo, 'checkout', '-U', rev, rootdir_tmp]
child_args = ['ostree', '--repo=' + self.repo, 'checkout', '-U', buildroot_version, rootdir_tmp]
run_sync(child_args)
child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
run_sync(child_args)
@ -100,7 +114,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
if not os.path.isdir(sourcedir):
os.mkdir(sourcedir)
output_metadata = open('_ostbuild-meta', 'w')
output_metadata = open('_ostbuild-meta.json', 'w')
json.dump(self.metadata, output_metadata)
output_metadata.close()
@ -114,22 +128,18 @@ class OstbuildChrootCompileOne(builtins.Builtin):
'--mount-bind', '/dev', '/dev',
'--mount-bind', child_tmpdir, '/tmp',
'--mount-bind', os.getcwd(), chroot_sourcedir,
'--mount-bind', resultdir, '/ostbuild/results',
'--chdir', chroot_sourcedir]
if args.resultdir:
child_args.extend(['--mount-bind', args.resultdir, '/ostbuild/results'])
if args.debug_shell:
child_args.extend([rootdir, '/bin/sh'])
else:
child_args.extend([rootdir, '/usr/bin/ostbuild',
'compile-one',
'--ostbuild-resultdir=/ostbuild/results',
'--ostbuild-meta=_ostbuild-meta'])
child_args.extend(rest_args)
'--ostbuild-meta=_ostbuild-meta.json'])
child_args.extend(self.metadata.get('config-opts', []))
env_copy = dict(buildutil.BUILD_ENV)
env_copy['PWD'] = chroot_sourcedir
run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
if workdir_is_tmp:
shutil.rmtree(workdir)
builtins.register(OstbuildChrootCompileOne)

33
src/ostbuild/pyostbuild/builtin_compile_one.py
View File

@ -96,21 +96,22 @@ class OstbuildCompileOne(builtins.Builtin):
if self.ostbuild_resultdir is None:
fatal("Must specify --ostbuild-resultdir=")
self.metadata = {}
if self.ostbuild_meta is None:
output = run_sync_get_output(['ostbuild', 'autodiscover-meta'])
self.metadata = json.loads(output)
else:
f = open(self.ostbuild_meta)
self.metadata = json.load(f)
f.close()
fatal("Must specify --ostbuild-meta=")
for k in ['name', 'version']:
f = open(self.ostbuild_meta)
self.metadata = json.load(f)
f.close()
for k in ['name', 'revision']:
if k not in self.metadata:
fatal('Missing required key "%s" in metadata' % (k, ))
if self.metadata.get('rm-configure', False):
configure_path = 'configure'
if os.path.exists(configure_path):
os.unlink(configure_path)
autogen_script = None
if not os.path.exists('configure'):
log("No 'configure' script found, looking for autogen/bootstrap")
@ -181,18 +182,6 @@ class OstbuildCompileOne(builtins.Builtin):
name = self.metadata['name']
assert ',' not in name
branch = self.metadata['branch']
assert ',' not in name
version = self.metadata['version']
assert ',' not in version
root_name = self.metadata.get('buildroot', None)
# TODO - pick up current sysroot version from ostree
if root_name is None:
root_name = 'unknown-' + self.build_target
root_version = 'UNKNOWN'
else:
root_version = self.metadata.get('buildroot-version')
tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,))
self.tempfiles.append(tempdir)