diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index 41e34d0c..7cc2ac8b 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -27,6 +27,7 @@ pyostbuild_PYTHON = \ src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \ src/ostbuild/pyostbuild/builtin_commit_artifacts.py \ src/ostbuild/pyostbuild/builtin_compile_one.py \ + src/ostbuild/pyostbuild/builtin_resolve.py \ src/ostbuild/pyostbuild/builtins.py \ src/ostbuild/pyostbuild/__init__.py \ src/ostbuild/pyostbuild/kvfile.py \ diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py index 67d4130b..b120fd88 100755 --- a/src/ostbuild/pyostbuild/buildutil.py +++ b/src/ostbuild/pyostbuild/buildutil.py @@ -26,7 +26,9 @@ def branch_name_for_artifact(a): a['name'], a['branch']) -def get_git_version_describe(dirpath): - version = run_sync_get_output(['git', 'describe', '--long', '--abbrev=42', '--always'], - cwd=dirpath) +def get_git_version_describe(dirpath, commit=None): + args = ['git', 'describe', '--long', '--abbrev=42', '--always'] + if commit is not None: + args.append(commit) + version = run_sync_get_output(args, cwd=dirpath) return version.strip() diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index 2a07451b..d3deafd1 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -162,10 +162,7 @@ class OstbuildBuild(builtins.Builtin): (keytype, uri) = self._parse_src_key(meta['src']) - component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch) - component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch) - - current_vcs_version = buildutil.get_git_version_describe(component_src) + current_vcs_version = meta['revision'] previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', buildname], @@ -190,6 +187,9 @@ class OstbuildBuild(builtins.Builtin): else: log("No previous build for '%s' found" % (buildname, )) + mirror = os.path.join(self.mirrordir, name) + component_src = self._get_vcs_checkout(name, keytype, mirror, branch) + buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture) artifact_meta = {'buildroot': buildroot_name, @@ -210,7 +210,7 @@ class OstbuildBuild(builtins.Builtin): patches = meta.get('patches') if patches is not None: for patch in patches: - patch_path = os.path.join(self.manifestdir, patch) + patch_path = os.path.join(self.patchdir, patch) run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src) component_resultdir = os.path.join(self.workdir, 'results', name) @@ -283,7 +283,6 @@ class OstbuildBuild(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--manifest', required=True) parser.add_argument('--skip-built', action='store_true') parser.add_argument('--start-at') parser.add_argument('--shell-on-failure', action='store_true') @@ -299,19 +298,19 @@ class OstbuildBuild(builtins.Builtin): self.buildopts.shell_on_failure = args.shell_on_failure self.buildopts.skip_built = args.skip_built - self.manifest = json.load(open(args.manifest)) + build_manifest_path = os.path.join(self.workdir, 'manifest.json') + self.manifest = json.load(open(build_manifest_path)) - self.manifestdir = os.path.dirname(args.manifest) - - self.resolved_components = map(self._resolve_component_meta, self.manifest['components']) + self.patchdir = os.path.join(self.workdir, 'patches') + components = self.manifest['components'] if len(args.components) == 0: - build_components = self.resolved_components + build_components = components else: build_components = [] for name in args.components: found = False - for child in self.resolved_components: + for child in components: if child['name'] == name: found = True build_components.append(child) @@ -321,7 +320,7 @@ class OstbuildBuild(builtins.Builtin): start_at_index = -1 if args.start_at is not None: - if build_components != self.resolved_components: + if build_components != components: fatal("Can't specify --start-at with component list") for i,component in enumerate(build_components): if component['name'] == args.start_at: @@ -333,12 +332,12 @@ class OstbuildBuild(builtins.Builtin): start_at_index = 0 for component in build_components[start_at_index:]: - index = self.resolved_components.index(component) - dependencies = self.resolved_components[:index] + index = components.index(component) + dependencies = components[:index] for architecture in self.manifest['architectures']: self._build_one_component(component, dependencies, architecture) for architecture in self.manifest['architectures']: - self._compose_arch(architecture, self.resolved_components) + self._compose_arch(architecture, components) builtins.register(OstbuildBuild) diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py new file mode 100755 index 00000000..38379e3a --- /dev/null +++ b/src/ostbuild/pyostbuild/builtin_resolve.py @@ -0,0 +1,139 @@ +# Copyright (C) 2011 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import os,sys,subprocess,tempfile,re,shutil +import argparse +import json +from StringIO import StringIO + +from . import builtins +from .ostbuildlog import log, fatal +from .subprocess_helpers import run_sync, run_sync_get_output +from . import ostbuildrc +from . import buildutil +from . import kvfile +from . import odict + +class OstbuildResolve(builtins.Builtin): + name = "resolve" + short_description = "Download the source code for a given manifest" + + def __init__(self): + builtins.Builtin.__init__(self) + + def _ensure_vcs_mirror(self, name, keytype, uri, branch): + assert keytype == 'git' + mirror = os.path.join(self.mirrordir, name) + tmp_mirror = mirror + '.tmp' + if os.path.isdir(tmp_mirror): + shutil.rmtree(tmp_mirror) + if not os.path.isdir(mirror): + run_sync(['git', 'clone', '--mirror', uri, tmp_mirror]) + os.rename(tmp_mirror, mirror) + elif self.args.fetch: + log("Running git fetch for %s" % (name, )) + run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False) + return mirror + + def _parse_src_key(self, srckey): + idx = srckey.find(':') + if idx < 0: + raise ValueError("Invalid SRC uri=%s" % (srckey, )) + keytype = srckey[:idx] + if keytype not in ['git']: + raise ValueError("Unsupported SRC uri=%s" % (srckey, )) + uri = srckey[idx+1:] + return (keytype, uri) + + def _resolve_component_meta(self, component_meta): + result = dict(component_meta) + orig_src = component_meta['src'] + + did_expand = False + for (vcsprefix, expansion) in self.manifest['vcsconfig'].iteritems(): + prefix = vcsprefix + ':' + if orig_src.startswith(prefix): + result['src'] = expansion + orig_src[len(prefix):] + did_expand = True + break + + name = component_meta.get('name') + if name is None: + if did_expand: + src = orig_src + idx = src.rindex(':') + name = src[idx+1:] + else: + src = result['src'] + idx = src.rindex('/') + name = src[idx+1:] + if name.endswith('.git'): + name = name[:-4] + name = name.replace('/', '-') + result['name'] = name + + if 'branch' not in result: + result['branch'] = 'master' + + return result + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('--fetch', action='store_true') + + args = parser.parse_args(argv) + self.args = args + + self.parse_config() + + manifest_path = self.ostbuildrc.get_key('manifest') + self.manifest = json.load(open(manifest_path)) + + self.resolved_components = map(self._resolve_component_meta, self.manifest['components']) + + for component in self.resolved_components: + (keytype, uri) = self._parse_src_key(component['src']) + mirrordir = self._ensure_vcs_mirror(component['name'], + keytype, uri, + component['branch']) + revision = buildutil.get_git_version_describe(mirrordir, + component['branch']) + component['revision'] = revision + + self.manifest['components'] = self.resolved_components + + out_manifest = os.path.join(self.workdir, 'manifest.json') + patchdir = os.path.join(self.workdir, 'patches') + if not os.path.isdir(patchdir): + os.mkdir(patchdir) + all_patches = {} + for component in self.resolved_components: + patches = component.get('patches', []) + for patch in patches: + all_patches[patch] = True + for patch in all_patches: + src = os.path.join(os.path.dirname(manifest_path), + patch) + dest = os.path.join(patchdir, patch) + shutil.copy(src, dest) + + f = open(out_manifest, 'w') + json.dump(self.manifest, f, indent=4) + f.close() + print "Created: %s, %d patches" % (out_manifest, len(all_patches.keys())) + +builtins.register(OstbuildResolve) diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py index 5fd4b331..ddafbcd4 100755 --- a/src/ostbuild/pyostbuild/builtins.py +++ b/src/ostbuild/pyostbuild/builtins.py @@ -31,6 +31,7 @@ class Builtin(object): short_description = None def parse_config(self): + self.ostbuildrc = ostbuildrc self.repo = ostbuildrc.get_key('repo') self.mirrordir = ostbuildrc.get_key('mirrordir') if not os.path.isdir(self.mirrordir): diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py index eb9a659d..558957e5 100755 --- a/src/ostbuild/pyostbuild/main.py +++ b/src/ostbuild/pyostbuild/main.py @@ -27,6 +27,7 @@ from . import builtin_build from . import builtin_chroot_compile_one from . import builtin_commit_artifacts from . import builtin_compile_one +from . import builtin_resolve def usage(ecode): print "Builtins:" diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py index de85f300..f8ff3e56 100755 --- a/src/ostbuild/pyostbuild/subprocess_helpers.py +++ b/src/ostbuild/pyostbuild/subprocess_helpers.py @@ -67,8 +67,9 @@ def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=Fal return None def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False, - log_success=True): - log("running: %s" % (subprocess.list2cmdline(args),)) + log_success=True, log_initiation=True): + if log_initiation: + log("running: %s" % (subprocess.list2cmdline(args),)) # This dance is necessary because we want to keep the PWD # environment variable up to date. Not doing so is a recipie # for triggering edge conditions in pwd lookup.