diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py index 36e5954b..fb3e634f 100755 --- a/src/ostbuild/pyostbuild/buildutil.py +++ b/src/ostbuild/pyostbuild/buildutil.py @@ -26,7 +26,7 @@ def parse_artifact_name(artifact_basename): if match is None: raise ValueError("Invalid artifact basename %s" % (artifact_basename)) return {'buildroot': match.group(1), - 'buildroot_version': match.group(2), + 'buildroot-version': match.group(2), 'name': match.group(3), 'branch': match.group(4), 'version': match.group(5), diff --git a/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py b/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py index fe69be09..9e7c6ffd 100755 --- a/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py +++ b/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py @@ -19,6 +19,7 @@ import os,sys,re,subprocess,tempfile,shutil import argparse +import json from . import builtins from . import buildutil @@ -42,15 +43,13 @@ class OstbuildAutodiscoverMeta(builtins.Builtin): AUTODISCOVERED_KEYS[key] = [] AUTODISCOVERED_KEYS[key].append(func) - _register_discover_func('NAME', self._discover_name_from_cwd) - _register_discover_func('VERSION', self._discover_version_from_git) - _register_discover_func('BRANCH', self._discover_branch_from_git) + _register_discover_func('name', self._discover_name_from_cwd) + _register_discover_func('version', self._discover_version_from_git) + _register_discover_func('branch', self._discover_branch_from_git) if args.meta: f = open(args.meta) - for line in f.readlines(): - (k,v) = line.split('=', 1) - KEYS[k.strip()] = v.strip() + KEYS = json.load(f) f.close() for (key,hooks) in AUTODISCOVERED_KEYS.iteritems(): @@ -65,8 +64,7 @@ class OstbuildAutodiscoverMeta(builtins.Builtin): KEYS[key] = value break - for (key,value) in KEYS.iteritems(): - print "%s=%s" % (key, value) + json.dump(KEYS, sys.stdout, indent=2) def _discover_name_from_cwd(self): return os.path.basename(os.getcwd()) @@ -79,8 +77,11 @@ class OstbuildAutodiscoverMeta(builtins.Builtin): def _discover_branch_from_git(self): if os.path.isdir('.git'): try: - ref = subprocess.check_output(['git', 'symbolic-ref', 'HEAD']) - return ref.replace('refs/heads/', '').strip() + try: + ref = subprocess.check_output(['git', 'symbolic-ref', 'HEAD'], stderr=open('/dev/null', 'w')) + return ref.replace('refs/heads/', '').strip() + except subprocess.CalledProcessError, e: + return subprocess.check_output(['git', 'describe', '--tags', '--exact-match', 'HEAD']).strip() except subprocess.CalledProcessError, e: return None return None diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index 9c9ef898..7118a6fd 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -68,16 +68,10 @@ class OstbuildBuild(builtins.Builtin): if idx < 0: raise ValueError("Invalid SRC uri=%s" % (srckey, )) keytype = srckey[:idx] - if keytype not in ('git'): + if keytype not in ['git']: raise ValueError("Unsupported SRC uri=%s" % (srckey, )) uri = srckey[idx+1:] - idx = uri.rfind('#') - if idx < 0: - branch = "master" - else: - branch = uri[idx+1:] - uri = uri[0:idx] - return (keytype, uri, branch) + return (keytype, uri) def _get_ostbuild_chroot_args(self, architecture): current_machine = os.uname()[4] @@ -128,7 +122,8 @@ class OstbuildBuild(builtins.Builtin): def _build_one_component(self, meta, architecture): name = meta['name'] - (keytype, uri, branch) = self._parse_src_key(meta['src']) + (keytype, uri) = self._parse_src_key(meta['src']) + branch = meta.get('branch', 'master') buildroot = '%s-%s-devel' % (self.manifest['name'], architecture) runtime_branchname = 'artifacts/%s/%s/%s/runtime' % (buildroot, name, branch) @@ -137,7 +132,7 @@ class OstbuildBuild(builtins.Builtin): current_buildroot_version = current_buildroot_version.strip() artifact_base = {'buildroot': buildroot, - 'buildroot_version': current_buildroot_version, + 'buildroot-version': current_buildroot_version, 'name': name, 'branch': branch, } @@ -148,6 +143,14 @@ class OstbuildBuild(builtins.Builtin): current_vcs_version = buildutil.get_git_version_describe(component_src) artifact_base['version'] = current_vcs_version + metadata_dir = os.path.join(self.workdir, 'meta') + if not os.path.isdir(metadata_dir): + os.makedirs(metadata_dir) + metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, )) + f = open(metadata_path, 'w') + json.dump(artifact_base, f) + f.close() + previous_commit_version = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', runtime_branchname], stderr=open('/dev/null', 'w'), @@ -164,7 +167,7 @@ class OstbuildBuild(builtins.Builtin): previous_artifact_base = dict(artifact_base) previous_artifact_base['version'] = previous_artifact_version - previous_artifact_base['buildroot_version'] = previous_buildroot_version + previous_artifact_base['buildroot-version'] = previous_buildroot_version previous_artifact_runtime = dict(previous_artifact_base) previous_artifact_runtime['type'] = 'runtime' @@ -198,7 +201,7 @@ class OstbuildBuild(builtins.Builtin): patch_path = os.path.join(self.manifestdir, patch) run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src) - component_resultdir = os.path.join(self.workdir, name, 'results') + component_resultdir = os.path.join(self.workdir, 'results', name) if os.path.isdir(component_resultdir): shutil.rmtree(component_resultdir) os.makedirs(component_resultdir) @@ -206,10 +209,14 @@ class OstbuildBuild(builtins.Builtin): chroot_args = self._get_ostbuild_chroot_args(architecture) chroot_args.extend(['--buildroot=' + buildroot, '--workdir=' + self.workdir, - '--resultdir=' + component_resultdir]) + '--resultdir=' + component_resultdir, + '--meta=' + metadata_path]) global_config_opts = self.manifest.get('config-opts') if global_config_opts is not None: chroot_args.extend(global_config_opts) + component_config_opts = meta.get('config-opts') + if component_config_opts is not None: + chroot_args.extend(component_config_opts) if self.buildopts.shell_on_failure: ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False) if ecode != 0: @@ -246,6 +253,7 @@ class OstbuildBuild(builtins.Builtin): parser = argparse.ArgumentParser(description=self.short_description) parser.add_argument('--manifest', required=True) parser.add_argument('--skip-built', action='store_true') + parser.add_argument('--start-at') parser.add_argument('--shell-on-failure', action='store_true') parser.add_argument('--debug-shell', action='store_true') @@ -265,16 +273,27 @@ class OstbuildBuild(builtins.Builtin): self._launch_debug_shell(debug_shell_arch, debug_shell_buildroot) self.manifestdir = os.path.dirname(args.manifest) + + self.resolved_components = map(self._resolve_component_meta, self.manifest['components']) + + start_at_index = -1 + if args.start_at is not None: + for i,component in enumerate(self.resolved_components): + if component['name'] == args.start_at: + start_at_index = i + break + if start_at_index == -1: + fatal("Unknown component %r specified for --start-at" % (args.start_at, )) + else: + start_at_index = 0 - for component in self.manifest['components']: + for component in self.resolved_components[start_at_index:]: for architecture in self.manifest['architectures']: - component_meta = self._resolve_component_meta(component) - - (runtime_artifact,devel_artifact) = self._build_one_component(component_meta, architecture) + (runtime_artifact,devel_artifact) = self._build_one_component(component, architecture) runtime_branch = buildutil.branch_name_for_artifact(runtime_artifact) devel_branch = buildutil.branch_name_for_artifact(devel_artifact) - target_component = component_meta.get('component') + target_component = component.get('component') if target_component != 'devel': self._compose(architecture + '-runtime', [runtime_branch]) self._compose(architecture + '-devel', [runtime_branch, devel_branch]) diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py index 182ad7ec..34a8d15d 100755 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py @@ -18,6 +18,7 @@ import os,sys,re,subprocess,tempfile,shutil from StringIO import StringIO import argparse +import json from . import builtins from .ostbuildlog import log, fatal @@ -51,17 +52,14 @@ class OstbuildChrootCompileOne(builtins.Builtin): if args.meta is None: output = subprocess.check_output(['ostbuild', 'autodiscover-meta']) - ostbuild_meta_f = StringIO(output) + self.metadata = json.loads(output) else: - ostbuild_meta_f = open(args.meta) + f = open(args.meta) + self.metadata = json.load(f) + f.close() - metadata = {} - for line in ostbuild_meta_f: - (k,v) = line.split('=', 1) - metadata[k.strip()] = v.strip() - - for k in ['NAME']: - if k not in metadata: + for k in ['name']: + if k not in self.metadata: sys.stderr.write('Missing required key "%s" in metadata' % (k, )) sys.exit(1) @@ -82,10 +80,14 @@ class OstbuildChrootCompileOne(builtins.Builtin): rev = subprocess.check_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.buildroot]) rev=rev.strip() - metadata['BUILDROOT'] = args.buildroot - metadata['BUILDROOT_VERSION'] = rev + self.metadata['buildroot'] = args.buildroot + self.metadata['buildroot-version'] = rev + + rootdir_prefix = os.path.join(workdir, 'roots') + if not os.path.isdir(rootdir_prefix): + os.makedirs(rootdir_prefix) + rootdir = os.path.join(rootdir_prefix, rev) - rootdir = os.path.join(workdir, 'root-' + rev) rootdir_tmp = rootdir + '.tmp' builddir = os.path.join(rootdir, 'ostbuild'); if not os.path.isdir(rootdir): @@ -102,16 +104,15 @@ class OstbuildChrootCompileOne(builtins.Builtin): else: log("Using existing root: %s" % (rootdir, )) - sourcedir=os.path.join(builddir, 'source', metadata['NAME']) + sourcedir=os.path.join(builddir, 'source', self.metadata['name']) if not os.path.isdir(sourcedir): os.mkdir(sourcedir) output_metadata = open('_ostbuild-meta', 'w') - for (k,v) in metadata.iteritems(): - output_metadata.write('%s=%s\n' % (k, v)) + json.dump(self.metadata, output_metadata) output_metadata.close() - chroot_sourcedir = os.path.join('/ostbuild', 'source', metadata['NAME']) + chroot_sourcedir = os.path.join('/ostbuild', 'source', self.metadata['name']) # We need to search PATH here manually so we correctly pick up an # ostree install in e.g. ~/bin even though we're going to set PATH diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py index 4a3d78c3..e2f3ca02 100755 --- a/src/ostbuild/pyostbuild/builtin_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_compile_one.py @@ -20,6 +20,7 @@ import os,sys,stat,subprocess,tempfile,re,shutil from StringIO import StringIO +import json from multiprocessing import cpu_count import select,time @@ -109,19 +110,13 @@ class OstbuildCompileOne(builtins.Builtin): if self.ostbuild_meta is None: output = subprocess.check_output(['ostbuild', 'autodiscover-meta']) - ostbuild_meta_f = StringIO(output) + self.metadata = json.loads(output) else: - ostbuild_meta_f = open(self.ostbuild_meta) + f = open(self.ostbuild_meta) + self.metadata = json.load(f) + f.close() - for line in ostbuild_meta_f: - if line == '': - continue - (k,v) = line.split('=', 1) - self.metadata[k.strip()] = v.strip() - - ostbuild_meta_f.close() - - for k in ['NAME', 'VERSION']: + for k in ['name', 'version']: if k not in self.metadata: fatal('Missing required key "%s" in metadata' % (k, )) @@ -192,20 +187,20 @@ class OstbuildCompileOne(builtins.Builtin): run_sync(args, cwd=builddir) - name = self.metadata['NAME'] + name = self.metadata['name'] assert ',' not in name - branch = self.metadata['BRANCH'] + branch = self.metadata['branch'] assert ',' not in name - version = self.metadata['VERSION'] + version = self.metadata['version'] assert ',' not in version - root_name = self.metadata.get('BUILDROOT', None) + root_name = self.metadata.get('buildroot', None) # TODO - pick up current sysroot version from ostree if root_name is None: root_name = 'unknown-' + self.build_target root_version = 'UNKNOWN' else: - root_version = self.metadata.get('BUILDROOT_VERSION') + root_version = self.metadata.get('buildroot-version') artifact_prefix=os.path.join('artifacts', root_name, name, branch)