packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ostbuild: Switch to JSON metadata, various other fixes

This commit is contained in:
Colin Walters 2012-01-06 20:02:46 -05:00
parent f0ddff68f8
commit a2fdbcbe63
5 changed files with 77 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/ostbuild/pyostbuild/buildutil.py
View File

@ -26,7 +26,7 @@ def parse_artifact_name(artifact_basename):
if match is None:
raise ValueError("Invalid artifact basename %s" % (artifact_basename))
return {'buildroot': match.group(1),
'buildroot_version': match.group(2),
'buildroot-version': match.group(2),
'name': match.group(3),
'branch': match.group(4),
'version': match.group(5),

21
src/ostbuild/pyostbuild/builtin_autodiscover_meta.py
View File

@ -19,6 +19,7 @@
import os,sys,re,subprocess,tempfile,shutil
import argparse
import json
from . import builtins
from . import buildutil
@ -42,15 +43,13 @@ class OstbuildAutodiscoverMeta(builtins.Builtin):
AUTODISCOVERED_KEYS[key] = []
AUTODISCOVERED_KEYS[key].append(func)
_register_discover_func('NAME', self._discover_name_from_cwd)
_register_discover_func('VERSION', self._discover_version_from_git)
_register_discover_func('BRANCH', self._discover_branch_from_git)
_register_discover_func('name', self._discover_name_from_cwd)
_register_discover_func('version', self._discover_version_from_git)
_register_discover_func('branch', self._discover_branch_from_git)
if args.meta:
f = open(args.meta)
for line in f.readlines():
(k,v) = line.split('=', 1)
KEYS[k.strip()] = v.strip()
KEYS = json.load(f)
f.close()
for (key,hooks) in AUTODISCOVERED_KEYS.iteritems():
@ -65,8 +64,7 @@ class OstbuildAutodiscoverMeta(builtins.Builtin):
KEYS[key] = value
break
for (key,value) in KEYS.iteritems():
print "%s=%s" % (key, value)
json.dump(KEYS, sys.stdout, indent=2)
def _discover_name_from_cwd(self):
return os.path.basename(os.getcwd())
@ -79,8 +77,11 @@ class OstbuildAutodiscoverMeta(builtins.Builtin):
def _discover_branch_from_git(self):
if os.path.isdir('.git'):
try:
ref = subprocess.check_output(['git', 'symbolic-ref', 'HEAD'])
return ref.replace('refs/heads/', '').strip()
try:
ref = subprocess.check_output(['git', 'symbolic-ref', 'HEAD'], stderr=open('/dev/null', 'w'))
return ref.replace('refs/heads/', '').strip()
except subprocess.CalledProcessError, e:
return subprocess.check_output(['git', 'describe', '--tags', '--exact-match', 'HEAD']).strip()
except subprocess.CalledProcessError, e:
return None
return None

55
src/ostbuild/pyostbuild/builtin_build.py
View File

@ -68,16 +68,10 @@ class OstbuildBuild(builtins.Builtin):
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
if keytype not in ('git'):
if keytype not in ['git']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
idx = uri.rfind('#')
if idx < 0:
branch = "master"
else:
branch = uri[idx+1:]
uri = uri[0:idx]
return (keytype, uri, branch)
return (keytype, uri)
def _get_ostbuild_chroot_args(self, architecture):
current_machine = os.uname()[4]
@ -128,7 +122,8 @@ class OstbuildBuild(builtins.Builtin):
def _build_one_component(self, meta, architecture):
name = meta['name']
(keytype, uri, branch) = self._parse_src_key(meta['src'])
(keytype, uri) = self._parse_src_key(meta['src'])
branch = meta.get('branch', 'master')
buildroot = '%s-%s-devel' % (self.manifest['name'], architecture)
runtime_branchname = 'artifacts/%s/%s/%s/runtime' % (buildroot, name, branch)
@ -137,7 +132,7 @@ class OstbuildBuild(builtins.Builtin):
current_buildroot_version = current_buildroot_version.strip()
artifact_base = {'buildroot': buildroot,
'buildroot_version': current_buildroot_version,
'buildroot-version': current_buildroot_version,
'name': name,
'branch': branch,
}
@ -148,6 +143,14 @@ class OstbuildBuild(builtins.Builtin):
current_vcs_version = buildutil.get_git_version_describe(component_src)
artifact_base['version'] = current_vcs_version
metadata_dir = os.path.join(self.workdir, 'meta')
if not os.path.isdir(metadata_dir):
os.makedirs(metadata_dir)
metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, ))
f = open(metadata_path, 'w')
json.dump(artifact_base, f)
f.close()
previous_commit_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', runtime_branchname],
stderr=open('/dev/null', 'w'),
@ -164,7 +167,7 @@ class OstbuildBuild(builtins.Builtin):
previous_artifact_base = dict(artifact_base)
previous_artifact_base['version'] = previous_artifact_version
previous_artifact_base['buildroot_version'] = previous_buildroot_version
previous_artifact_base['buildroot-version'] = previous_buildroot_version
previous_artifact_runtime = dict(previous_artifact_base)
previous_artifact_runtime['type'] = 'runtime'
@ -198,7 +201,7 @@ class OstbuildBuild(builtins.Builtin):
patch_path = os.path.join(self.manifestdir, patch)
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src)
component_resultdir = os.path.join(self.workdir, name, 'results')
component_resultdir = os.path.join(self.workdir, 'results', name)
if os.path.isdir(component_resultdir):
shutil.rmtree(component_resultdir)
os.makedirs(component_resultdir)
@ -206,10 +209,14 @@ class OstbuildBuild(builtins.Builtin):
chroot_args = self._get_ostbuild_chroot_args(architecture)
chroot_args.extend(['--buildroot=' + buildroot,
'--workdir=' + self.workdir,
'--resultdir=' + component_resultdir])
'--resultdir=' + component_resultdir,
'--meta=' + metadata_path])
global_config_opts = self.manifest.get('config-opts')
if global_config_opts is not None:
chroot_args.extend(global_config_opts)
component_config_opts = meta.get('config-opts')
if component_config_opts is not None:
chroot_args.extend(component_config_opts)
if self.buildopts.shell_on_failure:
ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False)
if ecode != 0:
@ -246,6 +253,7 @@ class OstbuildBuild(builtins.Builtin):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--manifest', required=True)
parser.add_argument('--skip-built', action='store_true')
parser.add_argument('--start-at')
parser.add_argument('--shell-on-failure', action='store_true')
parser.add_argument('--debug-shell', action='store_true')
@ -265,16 +273,27 @@ class OstbuildBuild(builtins.Builtin):
self._launch_debug_shell(debug_shell_arch, debug_shell_buildroot)
self.manifestdir = os.path.dirname(args.manifest)
self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
start_at_index = -1
if args.start_at is not None:
for i,component in enumerate(self.resolved_components):
if component['name'] == args.start_at:
start_at_index = i
break
if start_at_index == -1:
fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
else:
start_at_index = 0
for component in self.manifest['components']:
for component in self.resolved_components[start_at_index:]:
for architecture in self.manifest['architectures']:
component_meta = self._resolve_component_meta(component)
(runtime_artifact,devel_artifact) = self._build_one_component(component_meta, architecture)
(runtime_artifact,devel_artifact) = self._build_one_component(component, architecture)
runtime_branch = buildutil.branch_name_for_artifact(runtime_artifact)
devel_branch = buildutil.branch_name_for_artifact(devel_artifact)
target_component = component_meta.get('component')
target_component = component.get('component')
if target_component != 'devel':
self._compose(architecture + '-runtime', [runtime_branch])
self._compose(architecture + '-devel', [runtime_branch, devel_branch])

33
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
View File

@ -18,6 +18,7 @@
import os,sys,re,subprocess,tempfile,shutil
from StringIO import StringIO
import argparse
import json
from . import builtins
from .ostbuildlog import log, fatal
@ -51,17 +52,14 @@ class OstbuildChrootCompileOne(builtins.Builtin):
if args.meta is None:
output = subprocess.check_output(['ostbuild', 'autodiscover-meta'])
ostbuild_meta_f = StringIO(output)
self.metadata = json.loads(output)
else:
ostbuild_meta_f = open(args.meta)
f = open(args.meta)
self.metadata = json.load(f)
f.close()
metadata = {}
for line in ostbuild_meta_f:
(k,v) = line.split('=', 1)
metadata[k.strip()] = v.strip()
for k in ['NAME']:
if k not in metadata:
for k in ['name']:
if k not in self.metadata:
sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
sys.exit(1)
@ -82,10 +80,14 @@ class OstbuildChrootCompileOne(builtins.Builtin):
rev = subprocess.check_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.buildroot])
rev=rev.strip()
metadata['BUILDROOT'] = args.buildroot
metadata['BUILDROOT_VERSION'] = rev
self.metadata['buildroot'] = args.buildroot
self.metadata['buildroot-version'] = rev
rootdir_prefix = os.path.join(workdir, 'roots')
if not os.path.isdir(rootdir_prefix):
os.makedirs(rootdir_prefix)
rootdir = os.path.join(rootdir_prefix, rev)
rootdir = os.path.join(workdir, 'root-' + rev)
rootdir_tmp = rootdir + '.tmp'
builddir = os.path.join(rootdir, 'ostbuild');
if not os.path.isdir(rootdir):
@ -102,16 +104,15 @@ class OstbuildChrootCompileOne(builtins.Builtin):
else:
log("Using existing root: %s" % (rootdir, ))
sourcedir=os.path.join(builddir, 'source', metadata['NAME'])
sourcedir=os.path.join(builddir, 'source', self.metadata['name'])
if not os.path.isdir(sourcedir):
os.mkdir(sourcedir)
output_metadata = open('_ostbuild-meta', 'w')
for (k,v) in metadata.iteritems():
output_metadata.write('%s=%s\n' % (k, v))
json.dump(self.metadata, output_metadata)
output_metadata.close()
chroot_sourcedir = os.path.join('/ostbuild', 'source', metadata['NAME'])
chroot_sourcedir = os.path.join('/ostbuild', 'source', self.metadata['name'])
# We need to search PATH here manually so we correctly pick up an
# ostree install in e.g. ~/bin even though we're going to set PATH

27
src/ostbuild/pyostbuild/builtin_compile_one.py
View File

@ -20,6 +20,7 @@
import os,sys,stat,subprocess,tempfile,re,shutil
from StringIO import StringIO
import json
from multiprocessing import cpu_count
import select,time
@ -109,19 +110,13 @@ class OstbuildCompileOne(builtins.Builtin):
if self.ostbuild_meta is None:
output = subprocess.check_output(['ostbuild', 'autodiscover-meta'])
ostbuild_meta_f = StringIO(output)
self.metadata = json.loads(output)
else:
ostbuild_meta_f = open(self.ostbuild_meta)
f = open(self.ostbuild_meta)
self.metadata = json.load(f)
f.close()
for line in ostbuild_meta_f:
if line == '':
continue
(k,v) = line.split('=', 1)
self.metadata[k.strip()] = v.strip()
ostbuild_meta_f.close()
for k in ['NAME', 'VERSION']:
for k in ['name', 'version']:
if k not in self.metadata:
fatal('Missing required key "%s" in metadata' % (k, ))
@ -192,20 +187,20 @@ class OstbuildCompileOne(builtins.Builtin):
run_sync(args, cwd=builddir)
name = self.metadata['NAME']
name = self.metadata['name']
assert ',' not in name
branch = self.metadata['BRANCH']
branch = self.metadata['branch']
assert ',' not in name
version = self.metadata['VERSION']
version = self.metadata['version']
assert ',' not in version
root_name = self.metadata.get('BUILDROOT', None)
root_name = self.metadata.get('buildroot', None)
# TODO - pick up current sysroot version from ostree
if root_name is None:
root_name = 'unknown-' + self.build_target
root_version = 'UNKNOWN'
else:
root_version = self.metadata.get('BUILDROOT_VERSION')
root_version = self.metadata.get('buildroot-version')
artifact_prefix=os.path.join('artifacts', root_name, name, branch)