346 lines
14 KiB
Python
Executable File
346 lines
14 KiB
Python
Executable File
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
|
#
|
|
# This library is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
# License as published by the Free Software Foundation; either
|
|
# version 2 of the License, or (at your option) any later version.
|
|
#
|
|
# This library is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
# License along with this library; if not, write to the
|
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
# Boston, MA 02111-1307, USA.
|
|
|
|
import os,sys,subprocess,tempfile,re,shutil
|
|
import argparse
|
|
import time
|
|
import urlparse
|
|
import json
|
|
from StringIO import StringIO
|
|
|
|
from . import builtins
|
|
from .ostbuildlog import log, fatal
|
|
from .subprocess_helpers import run_sync, run_sync_get_output
|
|
from .subprocess_helpers import run_sync_monitor_log_file
|
|
from . import ostbuildrc
|
|
from . import buildutil
|
|
from . import kvfile
|
|
from . import odict
|
|
from . import vcs
|
|
|
|
class BuildOptions(object):
|
|
pass
|
|
|
|
class OstbuildBuild(builtins.Builtin):
|
|
name = "build"
|
|
short_description = "Rebuild all artifacts from the given manifest"
|
|
|
|
def __init__(self):
|
|
builtins.Builtin.__init__(self)
|
|
|
|
def _get_ostbuild_chroot_args(self, architecture):
|
|
current_machine = os.uname()[4]
|
|
if current_machine != architecture:
|
|
args = ['setarch', architecture]
|
|
else:
|
|
args = []
|
|
args.extend(['ostbuild', 'chroot-compile-one',
|
|
'--repo=' + self.repo])
|
|
return args
|
|
|
|
def _launch_debug_shell(self, architecture, buildroot, cwd=None):
|
|
args = self._get_ostbuild_chroot_args(architecture)
|
|
args.extend(['--buildroot=' + buildroot,
|
|
'--workdir=' + self.workdir,
|
|
'--debug-shell'])
|
|
run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
|
|
fatal("Exiting after debug shell")
|
|
|
|
def _resolve_component_meta(self, component_meta):
|
|
result = dict(component_meta)
|
|
orig_src = component_meta['src']
|
|
|
|
did_expand = False
|
|
for (vcsprefix, expansion) in self.manifest['vcsconfig'].iteritems():
|
|
prefix = vcsprefix + ':'
|
|
if orig_src.startswith(prefix):
|
|
result['src'] = expansion + orig_src[len(prefix):]
|
|
did_expand = True
|
|
break
|
|
|
|
name = component_meta.get('name')
|
|
if name is None:
|
|
if did_expand:
|
|
src = orig_src
|
|
idx = src.rindex(':')
|
|
name = src[idx+1:]
|
|
else:
|
|
src = result['src']
|
|
idx = src.rindex('/')
|
|
name = src[idx+1:]
|
|
if name.endswith('.git'):
|
|
name = name[:-4]
|
|
name = name.replace('/', '-')
|
|
result['name'] = name
|
|
|
|
if 'branch' not in result:
|
|
result['branch'] = 'master'
|
|
|
|
return result
|
|
|
|
def _get_target(self, architecture):
|
|
return '%s-%s-devel' % (self.manifest['name'], architecture)
|
|
|
|
def _get_base(self, roottype, architecture):
|
|
return 'bases/%s-%s-%s' % (self.manifest['base'],
|
|
architecture, roottype)
|
|
|
|
def _get_buildname(self, component, architecture):
|
|
return 'artifacts/%s/%s/%s' % (self._get_target (architecture),
|
|
component['name'],
|
|
component['branch'])
|
|
|
|
def _get_buildroot_name(self, component, architecture):
|
|
return 'buildroots/%s/%s/%s' % (self._get_target (architecture),
|
|
component['name'],
|
|
component['branch'])
|
|
|
|
def _compose_buildroot(self, buildroot_name, component, dependencies, architecture):
|
|
base = self._get_base('devel', architecture)
|
|
buildroot_contents = [base + ':/']
|
|
for dep in dependencies:
|
|
dep_buildname = self._get_buildname(dep, architecture)
|
|
buildroot_contents.append(dep_buildname + ':/runtime')
|
|
buildroot_contents.append(dep_buildname + ':/devel')
|
|
|
|
return self._compose(buildroot_name, buildroot_contents)
|
|
|
|
def _build_one_component(self, meta, dependencies, architecture):
|
|
name = meta['name']
|
|
branch = meta['branch']
|
|
|
|
target = self._get_target(architecture)
|
|
buildname = self._get_buildname(meta, architecture)
|
|
buildroot_name = self._get_buildroot_name(meta, architecture)
|
|
|
|
(keytype, uri) = buildutil.parse_src_key(meta['src'])
|
|
|
|
mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
|
|
checkoutdir = os.path.join(self.workdir, 'src', name)
|
|
component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, branch,
|
|
overwrite=not self.args.debug_shell)
|
|
|
|
if self.args.debug_shell:
|
|
buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
|
|
self._launch_debug_shell(architecture, buildroot_name, cwd=component_src)
|
|
|
|
current_vcs_version = meta['revision']
|
|
|
|
previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
|
|
'rev-parse', buildname],
|
|
stderr=open('/dev/null', 'w'),
|
|
none_on_error=True)
|
|
if previous_build_version is not None:
|
|
log("Previous build of '%s' is %s" % (buildname, previous_build_version))
|
|
|
|
previous_vcs_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
|
|
'show', '--print-metadata-key=ostbuild-artifact-version',
|
|
previous_build_version])
|
|
previous_vcs_version = previous_vcs_version.strip()
|
|
|
|
vcs_version_matches = False
|
|
if previous_vcs_version == current_vcs_version:
|
|
vcs_version_matches = True
|
|
log("VCS version is unchanged from '%s'" % (previous_vcs_version, ))
|
|
if self.buildopts.skip_built:
|
|
return False
|
|
else:
|
|
log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
|
|
else:
|
|
log("No previous build for '%s' found" % (buildname, ))
|
|
|
|
buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
|
|
|
|
if meta.get('rm-configure', False):
|
|
configure_path = os.path.join(component_src, 'configure')
|
|
if os.path.exists(configure_path):
|
|
os.unlink(configure_path)
|
|
|
|
artifact_meta = {'buildroot': buildroot_name,
|
|
'buildroot-version': buildroot_version,
|
|
'name': name,
|
|
'branch': branch,
|
|
'version': current_vcs_version
|
|
}
|
|
|
|
metadata_dir = os.path.join(self.workdir, 'meta')
|
|
if not os.path.isdir(metadata_dir):
|
|
os.makedirs(metadata_dir)
|
|
metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, ))
|
|
f = open(metadata_path, 'w')
|
|
json.dump(artifact_meta, f)
|
|
f.close()
|
|
|
|
patches = meta.get('patches')
|
|
if patches is not None:
|
|
for patch in patches:
|
|
patch_path = os.path.join(self.patchdir, patch)
|
|
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src)
|
|
|
|
component_resultdir = os.path.join(self.workdir, 'results', name)
|
|
if os.path.isdir(component_resultdir):
|
|
shutil.rmtree(component_resultdir)
|
|
os.makedirs(component_resultdir)
|
|
|
|
logdir = os.path.join(self.workdir, 'logs', 'compile', name)
|
|
old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name)
|
|
if not os.path.isdir(logdir):
|
|
os.makedirs(logdir)
|
|
if not os.path.isdir(old_logdir):
|
|
os.makedirs(old_logdir)
|
|
log_path = os.path.join(logdir, '%s.log' % (name, ))
|
|
if os.path.isfile(log_path):
|
|
curtime = int(time.time())
|
|
saved_name = '%s-%d.log' % (name, int(time.time()),)
|
|
os.rename(log_path, os.path.join(old_logdir, saved_name))
|
|
|
|
log("Logging to %s" % (log_path, ))
|
|
f = open(log_path, 'w')
|
|
chroot_args = self._get_ostbuild_chroot_args(architecture)
|
|
chroot_args.extend(['--buildroot=' + buildroot_name,
|
|
'--workdir=' + self.workdir,
|
|
'--resultdir=' + component_resultdir,
|
|
'--meta=' + metadata_path])
|
|
global_config_opts = self.manifest.get('config-opts')
|
|
if global_config_opts is not None:
|
|
chroot_args.extend(global_config_opts)
|
|
component_config_opts = meta.get('config-opts')
|
|
if component_config_opts is not None:
|
|
chroot_args.extend(component_config_opts)
|
|
if self.buildopts.shell_on_failure:
|
|
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
|
|
if ecode != 0:
|
|
self._launch_debug_shell(architecture, buildroot_name, cwd=component_src)
|
|
else:
|
|
run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
|
|
|
|
args = ['ostree', '--repo=' + self.repo,
|
|
'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['version'],
|
|
'--add-metadata-string=ostbuild-buildroot-version=' + buildroot_version,
|
|
'--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['version'],
|
|
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
|
|
'--skip-if-unchanged']
|
|
|
|
setuid_files = meta.get('setuid', [])
|
|
statoverride_path = None
|
|
if len(setuid_files) > 0:
|
|
(fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
|
|
f = os.fdopen(fd, 'w')
|
|
for path in setuid_files:
|
|
f.write('+2048 ' + path)
|
|
f.close()
|
|
args.append('--statoverride=' + statoverride_path)
|
|
|
|
run_sync(args, cwd=component_resultdir)
|
|
if statoverride_path is not None:
|
|
os.unlink(statoverride_path)
|
|
return True
|
|
|
|
def _compose(self, target, artifacts):
|
|
child_args = ['ostree', '--repo=' + self.repo, 'compose',
|
|
'-b', target, '-s', 'Compose']
|
|
(fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
|
|
f = os.fdopen(fd, 'w')
|
|
for artifact in artifacts:
|
|
f.write(artifact)
|
|
f.write('\n')
|
|
f.close()
|
|
child_args.extend(['-F', path])
|
|
revision = run_sync_get_output(child_args, log_initiation=True).strip()
|
|
os.unlink(path)
|
|
return revision
|
|
|
|
def _compose_arch(self, architecture, components):
|
|
runtime_base = self._get_base('runtime', architecture)
|
|
devel_base = self._get_base('devel', architecture)
|
|
runtime_contents = [runtime_base + ':/']
|
|
devel_contents = [devel_base + ':/']
|
|
|
|
for component in components:
|
|
branch = self._get_buildname(component, architecture)
|
|
runtime_contents.append(branch + ':/runtime')
|
|
devel_contents.append(branch + ':/runtime')
|
|
# For now just hardcode docs going in devel
|
|
devel_contents.append(branch + ':/doc')
|
|
devel_contents.append(branch + ':/devel')
|
|
|
|
self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'),
|
|
runtime_contents)
|
|
self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'devel'),
|
|
devel_contents)
|
|
|
|
def execute(self, argv):
|
|
parser = argparse.ArgumentParser(description=self.short_description)
|
|
parser.add_argument('--skip-built', action='store_true')
|
|
parser.add_argument('--start-at')
|
|
parser.add_argument('--shell-on-failure', action='store_true')
|
|
parser.add_argument('--debug-shell', action='store_true')
|
|
parser.add_argument('components', nargs='*')
|
|
|
|
args = parser.parse_args(argv)
|
|
self.args = args
|
|
|
|
self.parse_config()
|
|
|
|
self.buildopts = BuildOptions()
|
|
self.buildopts.shell_on_failure = args.shell_on_failure
|
|
self.buildopts.skip_built = args.skip_built
|
|
|
|
build_manifest_path = os.path.join(self.workdir, 'manifest.json')
|
|
self.manifest = json.load(open(build_manifest_path))
|
|
|
|
self.patchdir = os.path.join(self.workdir, 'patches')
|
|
|
|
components = self.manifest['components']
|
|
if len(args.components) == 0:
|
|
build_components = components
|
|
else:
|
|
build_components = []
|
|
for name in args.components:
|
|
found = False
|
|
for child in components:
|
|
if child['name'] == name:
|
|
found = True
|
|
build_components.append(child)
|
|
break
|
|
if not found:
|
|
fatal("Unknown component %r" % (name, ))
|
|
|
|
start_at_index = -1
|
|
if args.start_at is not None:
|
|
if build_components != components:
|
|
fatal("Can't specify --start-at with component list")
|
|
for i,component in enumerate(build_components):
|
|
if component['name'] == args.start_at:
|
|
start_at_index = i
|
|
break
|
|
if start_at_index == -1:
|
|
fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
|
|
else:
|
|
start_at_index = 0
|
|
|
|
for component in build_components[start_at_index:]:
|
|
index = components.index(component)
|
|
dependencies = components[:index]
|
|
for architecture in self.manifest['architectures']:
|
|
self._build_one_component(component, dependencies, architecture)
|
|
|
|
for architecture in self.manifest['architectures']:
|
|
self._compose_arch(architecture, components)
|
|
|
|
builtins.register(OstbuildBuild)
|