ostbuild: Make explicit resolve (and fetch steps)
Build should only work on already-downloaded sources.
This commit is contained in:
Colin Walters
parent
70a17244eb
commit
134cddaa96
|
|
@ -27,6 +27,7 @@ pyostbuild_PYTHON = \
|
|||
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
|
||||
src/ostbuild/pyostbuild/builtin_commit_artifacts.py \
|
||||
src/ostbuild/pyostbuild/builtin_compile_one.py \
|
||||
src/ostbuild/pyostbuild/builtin_resolve.py \
|
||||
src/ostbuild/pyostbuild/builtins.py \
|
||||
src/ostbuild/pyostbuild/__init__.py \
|
||||
src/ostbuild/pyostbuild/kvfile.py \
|
||||
|
|
|
|||
|
|
@ -26,7 +26,9 @@ def branch_name_for_artifact(a):
|
|||
a['name'],
|
||||
a['branch'])
|
||||
|
||||
def get_git_version_describe(dirpath):
|
||||
version = run_sync_get_output(['git', 'describe', '--long', '--abbrev=42', '--always'],
|
||||
cwd=dirpath)
|
||||
def get_git_version_describe(dirpath, commit=None):
|
||||
args = ['git', 'describe', '--long', '--abbrev=42', '--always']
|
||||
if commit is not None:
|
||||
args.append(commit)
|
||||
version = run_sync_get_output(args, cwd=dirpath)
|
||||
return version.strip()
|
||||
|
|
|
|||
|
|
@ -162,10 +162,7 @@ class OstbuildBuild(builtins.Builtin):
|
|||
|
||||
(keytype, uri) = self._parse_src_key(meta['src'])
|
||||
|
||||
component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch)
|
||||
component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch)
|
||||
|
||||
current_vcs_version = buildutil.get_git_version_describe(component_src)
|
||||
current_vcs_version = meta['revision']
|
||||
|
||||
previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
|
||||
'rev-parse', buildname],
|
||||
|
|
@ -190,6 +187,9 @@ class OstbuildBuild(builtins.Builtin):
|
|||
else:
|
||||
log("No previous build for '%s' found" % (buildname, ))
|
||||
|
||||
mirror = os.path.join(self.mirrordir, name)
|
||||
component_src = self._get_vcs_checkout(name, keytype, mirror, branch)
|
||||
|
||||
buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
|
||||
|
||||
artifact_meta = {'buildroot': buildroot_name,
|
||||
|
|
@ -210,7 +210,7 @@ class OstbuildBuild(builtins.Builtin):
|
|||
patches = meta.get('patches')
|
||||
if patches is not None:
|
||||
for patch in patches:
|
||||
patch_path = os.path.join(self.manifestdir, patch)
|
||||
patch_path = os.path.join(self.patchdir, patch)
|
||||
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src)
|
||||
|
||||
component_resultdir = os.path.join(self.workdir, 'results', name)
|
||||
|
|
@ -283,7 +283,6 @@ class OstbuildBuild(builtins.Builtin):
|
|||
|
||||
def execute(self, argv):
|
||||
parser = argparse.ArgumentParser(description=self.short_description)
|
||||
parser.add_argument('--manifest', required=True)
|
||||
parser.add_argument('--skip-built', action='store_true')
|
||||
parser.add_argument('--start-at')
|
||||
parser.add_argument('--shell-on-failure', action='store_true')
|
||||
|
|
@ -299,19 +298,19 @@ class OstbuildBuild(builtins.Builtin):
|
|||
self.buildopts.shell_on_failure = args.shell_on_failure
|
||||
self.buildopts.skip_built = args.skip_built
|
||||
|
||||
self.manifest = json.load(open(args.manifest))
|
||||
build_manifest_path = os.path.join(self.workdir, 'manifest.json')
|
||||
self.manifest = json.load(open(build_manifest_path))
|
||||
|
||||
self.manifestdir = os.path.dirname(args.manifest)
|
||||
|
||||
self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
|
||||
self.patchdir = os.path.join(self.workdir, 'patches')
|
||||
|
||||
components = self.manifest['components']
|
||||
if len(args.components) == 0:
|
||||
build_components = self.resolved_components
|
||||
build_components = components
|
||||
else:
|
||||
build_components = []
|
||||
for name in args.components:
|
||||
found = False
|
||||
for child in self.resolved_components:
|
||||
for child in components:
|
||||
if child['name'] == name:
|
||||
found = True
|
||||
build_components.append(child)
|
||||
|
|
@ -321,7 +320,7 @@ class OstbuildBuild(builtins.Builtin):
|
|||
|
||||
start_at_index = -1
|
||||
if args.start_at is not None:
|
||||
if build_components != self.resolved_components:
|
||||
if build_components != components:
|
||||
fatal("Can't specify --start-at with component list")
|
||||
for i,component in enumerate(build_components):
|
||||
if component['name'] == args.start_at:
|
||||
|
|
@ -333,12 +332,12 @@ class OstbuildBuild(builtins.Builtin):
|
|||
start_at_index = 0
|
||||
|
||||
for component in build_components[start_at_index:]:
|
||||
index = self.resolved_components.index(component)
|
||||
dependencies = self.resolved_components[:index]
|
||||
index = components.index(component)
|
||||
dependencies = components[:index]
|
||||
for architecture in self.manifest['architectures']:
|
||||
self._build_one_component(component, dependencies, architecture)
|
||||
|
||||
for architecture in self.manifest['architectures']:
|
||||
self._compose_arch(architecture, self.resolved_components)
|
||||
self._compose_arch(architecture, components)
|
||||
|
||||
builtins.register(OstbuildBuild)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,139 @@
|
|||
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
||||
#
|
||||
# This library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2 of the License, or (at your option) any later version.
|
||||
#
|
||||
# This library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this library; if not, write to the
|
||||
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||
# Boston, MA 02111-1307, USA.
|
||||
|
||||
import os,sys,subprocess,tempfile,re,shutil
|
||||
import argparse
|
||||
import json
|
||||
from StringIO import StringIO
|
||||
|
||||
from . import builtins
|
||||
from .ostbuildlog import log, fatal
|
||||
from .subprocess_helpers import run_sync, run_sync_get_output
|
||||
from . import ostbuildrc
|
||||
from . import buildutil
|
||||
from . import kvfile
|
||||
from . import odict
|
||||
|
||||
class OstbuildResolve(builtins.Builtin):
|
||||
name = "resolve"
|
||||
short_description = "Download the source code for a given manifest"
|
||||
|
||||
def __init__(self):
|
||||
builtins.Builtin.__init__(self)
|
||||
|
||||
def _ensure_vcs_mirror(self, name, keytype, uri, branch):
|
||||
assert keytype == 'git'
|
||||
mirror = os.path.join(self.mirrordir, name)
|
||||
tmp_mirror = mirror + '.tmp'
|
||||
if os.path.isdir(tmp_mirror):
|
||||
shutil.rmtree(tmp_mirror)
|
||||
if not os.path.isdir(mirror):
|
||||
run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
|
||||
os.rename(tmp_mirror, mirror)
|
||||
elif self.args.fetch:
|
||||
log("Running git fetch for %s" % (name, ))
|
||||
run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False)
|
||||
return mirror
|
||||
|
||||
def _parse_src_key(self, srckey):
|
||||
idx = srckey.find(':')
|
||||
if idx < 0:
|
||||
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
|
||||
keytype = srckey[:idx]
|
||||
if keytype not in ['git']:
|
||||
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
|
||||
uri = srckey[idx+1:]
|
||||
return (keytype, uri)
|
||||
|
||||
def _resolve_component_meta(self, component_meta):
|
||||
result = dict(component_meta)
|
||||
orig_src = component_meta['src']
|
||||
|
||||
did_expand = False
|
||||
for (vcsprefix, expansion) in self.manifest['vcsconfig'].iteritems():
|
||||
prefix = vcsprefix + ':'
|
||||
if orig_src.startswith(prefix):
|
||||
result['src'] = expansion + orig_src[len(prefix):]
|
||||
did_expand = True
|
||||
break
|
||||
|
||||
name = component_meta.get('name')
|
||||
if name is None:
|
||||
if did_expand:
|
||||
src = orig_src
|
||||
idx = src.rindex(':')
|
||||
name = src[idx+1:]
|
||||
else:
|
||||
src = result['src']
|
||||
idx = src.rindex('/')
|
||||
name = src[idx+1:]
|
||||
if name.endswith('.git'):
|
||||
name = name[:-4]
|
||||
name = name.replace('/', '-')
|
||||
result['name'] = name
|
||||
|
||||
if 'branch' not in result:
|
||||
result['branch'] = 'master'
|
||||
|
||||
return result
|
||||
|
||||
def execute(self, argv):
|
||||
parser = argparse.ArgumentParser(description=self.short_description)
|
||||
parser.add_argument('--fetch', action='store_true')
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
self.args = args
|
||||
|
||||
self.parse_config()
|
||||
|
||||
manifest_path = self.ostbuildrc.get_key('manifest')
|
||||
self.manifest = json.load(open(manifest_path))
|
||||
|
||||
self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
|
||||
|
||||
for component in self.resolved_components:
|
||||
(keytype, uri) = self._parse_src_key(component['src'])
|
||||
mirrordir = self._ensure_vcs_mirror(component['name'],
|
||||
keytype, uri,
|
||||
component['branch'])
|
||||
revision = buildutil.get_git_version_describe(mirrordir,
|
||||
component['branch'])
|
||||
component['revision'] = revision
|
||||
|
||||
self.manifest['components'] = self.resolved_components
|
||||
|
||||
out_manifest = os.path.join(self.workdir, 'manifest.json')
|
||||
patchdir = os.path.join(self.workdir, 'patches')
|
||||
if not os.path.isdir(patchdir):
|
||||
os.mkdir(patchdir)
|
||||
all_patches = {}
|
||||
for component in self.resolved_components:
|
||||
patches = component.get('patches', [])
|
||||
for patch in patches:
|
||||
all_patches[patch] = True
|
||||
for patch in all_patches:
|
||||
src = os.path.join(os.path.dirname(manifest_path),
|
||||
patch)
|
||||
dest = os.path.join(patchdir, patch)
|
||||
shutil.copy(src, dest)
|
||||
|
||||
f = open(out_manifest, 'w')
|
||||
json.dump(self.manifest, f, indent=4)
|
||||
f.close()
|
||||
print "Created: %s, %d patches" % (out_manifest, len(all_patches.keys()))
|
||||
|
||||
builtins.register(OstbuildResolve)
|
||||
|
|
@ -31,6 +31,7 @@ class Builtin(object):
|
|||
short_description = None
|
||||
|
||||
def parse_config(self):
|
||||
self.ostbuildrc = ostbuildrc
|
||||
self.repo = ostbuildrc.get_key('repo')
|
||||
self.mirrordir = ostbuildrc.get_key('mirrordir')
|
||||
if not os.path.isdir(self.mirrordir):
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from . import builtin_build
|
|||
from . import builtin_chroot_compile_one
|
||||
from . import builtin_commit_artifacts
|
||||
from . import builtin_compile_one
|
||||
from . import builtin_resolve
|
||||
|
||||
def usage(ecode):
|
||||
print "Builtins:"
|
||||
|
|
|
|||
|
|
@ -67,7 +67,8 @@ def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=Fal
|
|||
return None
|
||||
|
||||
def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
|
||||
log_success=True):
|
||||
log_success=True, log_initiation=True):
|
||||
if log_initiation:
|
||||
log("running: %s" % (subprocess.list2cmdline(args),))
|
||||
# This dance is necessary because we want to keep the PWD
|
||||
# environment variable up to date. Not doing so is a recipie
|
||||
|
|
|
|||
Loading…
Reference in New Issue