packaging/ostree
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ostbuild: Major rework

Browse Source
This commit is contained in:
Colin Walters committed 2012-05-04 16:16:16 -04:00
1 parent b052758652
commit 717cec46c3
27 files changed
+1205 -353
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ostbuild/pyostbuild/buildutil.py
+1 -1
View File
@@ -40,7 +40,7 @@ def parse_src_key(srckey):
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
if keytype not in ['git']:
if keytype not in ['git', 'dirty-git']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
return (keytype, uri)
src/ostbuild/pyostbuild/builtin_bin_to_src.py
Executable
+76
View File
@@ -0,0 +1,76 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildBinToSrc(builtins.Builtin):
name = "bin-to-src"
short_description = "Turn a binary snapshot into a source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
def bin_snapshot_to_src(self, bin_snapshot):
del bin_snapshot['00ostree-bin-snapshot-version']
src_snapshot = dict(bin_snapshot)
src_snapshot['00ostree-src-snapshot-version'] = 0
all_architectures = src_snapshot['architecture-buildroots'].keys()
# Arbitrarily take first architecture
first_arch = all_architectures[0]
bin_components = src_snapshot['components']
src_components = {}
src_snapshot['components'] = src_components
for archname,rev in bin_components.iteritems():
(name, arch) = archname.rsplit('/', 1)
if arch != first_arch:
continue
meta = dict(self.get_component_meta_from_revision(rev))
del meta['name']
src_components[name] = meta
for target in src_snapshot['targets']:
del target['base']['ostree-revision']
return src_snapshot
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--bin-snapshot')
args = parser.parse_args(argv)
self.parse_config()
self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
snapshot = self.bin_snapshot_to_src(self.bin_snapshot)
json.dump(snapshot, sys.stdout, indent=4, sort_keys=True)
builtins.register(OstbuildBinToSrc)
src/ostbuild/pyostbuild/builtin_query_content.py → src/ostbuild/pyostbuild/builtin_branch_prefix.py
+29 -25
View File
@@ -28,40 +28,44 @@ from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildQueryContent(builtins.Builtin):
name = "query-content"
short_description = "Output metadata from a component"
class OstbuildBranchPrefix(builtins.Builtin):
name = "prefix-branch"
short_description = "Copy current source snapshot to new prefix"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--branch', required=True)
parser.add_argument('--component')
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
parser.add_argument('newprefix')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', args.branch, 'contents.json'])
if args.newprefix == self.prefix:
fatal("Specified prefix %r matches active prefix" % (args.newprefix, ))
db = self.create_db('src-snapshot', prefix=args.newprefix)
log("Branching from source snapshot %r" % (self.snapshot_path, ))
orig_prefix = self.prefix
forked_snapshot = dict(self.snapshot)
forked_snapshot['prefix'] = args.newprefix
for target in forked_snapshot['targets']:
name = target['name']
if not name.startswith(orig_prefix):
fatal("Mismatched name %r in snapshot" % (name, ))
target['name'] = name.replace(orig_prefix, args.newprefix)
if args.component is None:
sys.stdout.write(contents_json_text)
else:
contents = json.loads(contents_json_text)
contents_list = contents['contents']
found = False
for content in contents_list:
if content['name'] != args.component:
found = True
break
if not found:
fatal("Unknown component '%s'" % (args.component, ))
ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', content['ostree-revision'],
'/_ostbuild-meta.json'])
sys.stdout.write(ostbuildmeta_json)
db.store(forked_snapshot)
builtins.register(OstbuildQueryContent)
run_sync(['ostbuild', 'prefix', args.newprefix],
log_initiation=False, log_success=False)
builtins.register(OstbuildBranchPrefix)
src/ostbuild/pyostbuild/builtin_build.py → src/ostbuild/pyostbuild/builtin_build_components.py
+70 -80
View File
@@ -36,9 +36,9 @@ from . import vcs
class BuildOptions(object):
pass
class OstbuildBuild(builtins.Builtin):
name = "build"
short_description = "Rebuild all artifacts from the given manifest"
class OstbuildBuildComponents(builtins.Builtin):
name = "build-components"
short_description = "Build multiple components from given source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
@@ -49,7 +49,8 @@ class OstbuildBuild(builtins.Builtin):
args = ['setarch', architecture]
else:
args = []
args.extend(['ostbuild', 'chroot-compile-one'])
args.extend(['ostbuild', 'chroot-compile-one',
'--snapshot=' + self.snapshot_path])
return args
def _launch_debug_shell(self, architecture, buildroot, cwd=None):
@@ -60,10 +61,10 @@ class OstbuildBuild(builtins.Builtin):
run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
fatal("Exiting after debug shell")
def _build_one_component(self, name, component):
def _build_one_component(self, basename, component, architecture):
branch = component['branch']
architecture = component['architecture']
name = '%s/%s' % (basename, architecture)
buildname = 'components/%s' % (name, )
current_vcs_version = component['revision']
@@ -73,7 +74,7 @@ class OstbuildBuild(builtins.Builtin):
stderr=open('/dev/null', 'w'),
none_on_error=True)
if previous_build_version is not None:
log("Previous build of '%s' is %s" % (buildname, previous_build_version))
log("Previous build of '%s' is %s" % (name, previous_build_version))
previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', previous_build_version,
@@ -92,19 +93,15 @@ class OstbuildBuild(builtins.Builtin):
else:
log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
else:
log("No previous build for '%s' found" % (buildname, ))
log("No previous build for '%s' found" % (name, ))
checkoutdir = os.path.join(self.workdir, 'src')
component_src = os.path.join(checkoutdir, name)
run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir)
component_src = os.path.join(checkoutdir, basename)
run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
'--clean', '--overwrite', basename], cwd=checkoutdir)
artifact_meta = dict(component)
metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
f = open(metadata_path, 'w')
json.dump(artifact_meta, f, indent=4, sort_keys=True)
f.close()
logdir = os.path.join(self.workdir, 'logs', name)
fileutil.ensure_dir(logdir)
log_path = os.path.join(logdir, 'compile.log')
@@ -116,7 +113,7 @@ class OstbuildBuild(builtins.Builtin):
log("Logging to %s" % (log_path, ))
f = open(log_path, 'w')
chroot_args = self._get_ostbuild_chroot_args(architecture)
chroot_args.extend(['--pristine', '--name=' + name])
chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
if self.buildopts.shell_on_failure:
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
if ecode != 0:
@@ -146,69 +143,48 @@ class OstbuildBuild(builtins.Builtin):
os.unlink(statoverride_path)
return True
def _compose(self, target):
base_name = 'bases/%s' % (target['base']['name'], )
branch_to_rev = {}
branch_to_subtrees = {}
contents = [base_name]
branch_to_subtrees[base_name] = ['/']
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', base_name])
branch_to_rev[base_name] = base_revision
def _resolve_refs(self, refs):
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
for component in target['contents']:
name = component['name']
contents.append(name)
args.append('components/%s' % (name, ))
branch_to_subtrees[name] = component['trees']
branch_revs_text = run_sync_get_output(args)
branch_revs = branch_revs_text.split('\n')
args.extend(refs)
output = run_sync_get_output(args)
return output.split('\n')
for (content, rev) in zip(target['contents'], branch_revs):
name = content['name']
branch_to_rev[name] = rev
compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
if os.path.isdir(compose_rootdir):
shutil.rmtree(compose_rootdir)
os.mkdir(compose_rootdir)
def _save_bin_snapshot(self, components, component_architectures):
bin_snapshot = dict(self.snapshot)
resolved_base = dict(target['base'])
resolved_base['ostree-revision'] = base_revision
resolved_contents = list(target['contents'])
for component in resolved_contents:
component['ostree-revision'] = branch_to_rev[component['name']]
metadata = {'source': 'ostbuild compose v0',
'base': resolved_base,
'contents': resolved_contents}
del bin_snapshot['00ostree-src-snapshot-version']
bin_snapshot['00ostree-bin-snapshot-version'] = 0
for branch in contents:
branch_rev = branch_to_rev[branch]
subtrees = branch_to_subtrees[branch]
for subtree in subtrees:
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--user-mode',
'--union', '--subpath=' + subtree,
branch_rev, compose_rootdir])
for target in bin_snapshot['targets']:
base = target['base']
base_name = 'bases/%s' % (base['name'], )
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', base_name])
base['ostree-revision'] = base_revision
contents_path = os.path.join(compose_rootdir, 'contents.json')
f = open(contents_path, 'w')
json.dump(metadata, f, indent=4, sort_keys=True)
f.close()
component_refs = []
for name in components.iterkeys():
for architecture in component_architectures[name]:
component_refs.append('components/%s/%s' % (name, architecture))
run_sync(['ostree', '--repo=' + self.repo,
'commit', '-b', target['name'], '-s', 'Compose',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged'], cwd=compose_rootdir)
new_components = {}
resolved_refs = self._resolve_refs(component_refs)
for name,rev in zip(components.iterkeys(), resolved_refs):
for architecture in component_architectures[name]:
archname = '%s/%s' % (name, architecture)
new_components[archname] = rev
bin_snapshot['components'] = new_components
path = self.get_bin_snapshot_db().store(bin_snapshot)
log("Binary snapshot: %s" % (path, ))
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--skip-built', action='store_true')
parser.add_argument('--recompose', action='store_true')
parser.add_argument('--skip-compose', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
parser.add_argument('--compose', action='store_true')
parser.add_argument('--start-at')
parser.add_argument('--shell-on-failure', action='store_true')
parser.add_argument('--debug-shell', action='store_true')
@@ -218,17 +194,28 @@ class OstbuildBuild(builtins.Builtin):
self.args = args
self.parse_config()
self.parse_snapshot()
self.parse_snapshot(args.prefix, args.src_snapshot)
log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
self.buildopts = BuildOptions()
self.buildopts.shell_on_failure = args.shell_on_failure
self.buildopts.skip_built = args.skip_built
required_components = {}
component_architectures = {}
for target in self.snapshot['targets']:
for tree_content in target['contents']:
(name, arch) = tree_content['name'].rsplit('/', 1)
required_components[name] = self.snapshot['components'][name]
if name not in component_architectures:
component_architectures[name] = set([arch])
else:
component_architectures[name].add(arch)
build_component_order = []
if args.recompose:
pass
elif len(args.components) == 0:
tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends')
if len(args.components) == 0:
tsorted = buildutil.tsort_components(required_components, 'build-depends')
tsorted.reverse()
build_component_order = tsorted
else:
@@ -253,11 +240,14 @@ class OstbuildBuild(builtins.Builtin):
start_at_index = 0
for component_name in build_component_order[start_at_index:]:
component = self.snapshot['components'].get(component_name)
self._build_one_component(component_name, component)
component = required_components[component_name]
architectures = component_architectures[component_name]
for architecture in architectures:
self._build_one_component(component_name, component, architecture)
if not args.skip_compose:
for target in self.snapshot['targets']:
self._compose(target)
self._save_bin_snapshot(required_components, component_architectures)
if args.compose:
run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix])
builtins.register(OstbuildBuild)
builtins.register(OstbuildBuildComponents)
src/ostbuild/pyostbuild/builtin_checkout.py
+33 -12
View File
@@ -40,6 +40,9 @@ class OstbuildCheckout(builtins.Builtin):
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--overwrite', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--snapshot')
parser.add_argument('-a', '--active-tree', action='store_true')
parser.add_argument('--clean', action='store_true')
parser.add_argument('components', nargs='*')
@@ -47,28 +50,41 @@ class OstbuildCheckout(builtins.Builtin):
self.args = args
self.parse_config()
self.parse_snapshot()
if len(args.components) > 0:
checkout_components = args.components
else:
checkout_components = [os.path.basename(os.getcwd())]
if args.active_tree:
self.parse_active_branch()
else:
self.parse_snapshot(args.prefix, args.snapshot)
for component_name in checkout_components:
found = False
component = self.snapshot['components'].get(component_name)
if component is None:
fatal("Unknown component %r" % (component_name, ))
component = self.get_component_meta(component_name)
(keytype, uri) = buildutil.parse_src_key(component['src'])
checkoutdir = os.path.join(os.getcwd(), component_name)
fileutil.ensure_parent_dir(checkoutdir)
component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
component['revision'],
overwrite=args.overwrite)
is_dirty = (keytype == 'dirty-git')
if is_dirty:
# Kind of a hack, but...
if os.path.lexists(checkoutdir):
os.unlink(checkoutdir)
os.symlink(uri, checkoutdir)
else:
vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
component['revision'],
overwrite=args.overwrite)
if args.clean:
vcs.clean(keytype, checkoutdir)
if is_dirty:
log("note: ignoring --clean argument due to \"dirty-git:\" specification")
else:
vcs.clean(keytype, checkoutdir)
patches = component.get('patches')
if patches is not None:
@@ -78,15 +94,20 @@ class OstbuildCheckout(builtins.Builtin):
self.patchdir, patches['branch'],
overwrite=True)
patch_prefix = patches.get('prefix', None)
if patch_prefix is not None:
patchdir = os.path.join(self.patchdir, patch_prefix)
patch_subdir = patches.get('subdir', None)
if patch_subdir is not None:
patchdir = os.path.join(self.patchdir, patch_subdir)
else:
patchdir = self.patchdir
for patch in patches['files']:
patch_path = os.path.join(patchdir, patch)
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json')
f = open(metadata_path, 'w')
json.dump(component, f, indent=4, sort_keys=True)
f.close()
print "Checked out: %r" % (component_src, )
log("Checked out: %r" % (checkoutdir, ))
builtins.register(OstbuildCheckout)
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+106 -40
View File
@@ -18,7 +18,9 @@
import os,sys,re,subprocess,tempfile,shutil
from StringIO import StringIO
import argparse
import time
import json
import hashlib
from . import builtins
from . import buildutil
@@ -31,47 +33,133 @@ class OstbuildChrootCompileOne(builtins.Builtin):
name = "chroot-compile-one"
short_description = "Build artifacts from the current source directory in a chroot"
def _compose_buildroot(self, component_name, dirpath):
def _resolve_refs(self, refs):
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
args.extend(refs)
output = run_sync_get_output(args)
return output.split('\n')
def _compose_buildroot(self, component_name, architecture):
starttime = time.time()
rootdir_prefix = os.path.join(self.workdir, 'roots')
rootdir = os.path.join(rootdir_prefix, component_name)
fileutil.ensure_parent_dir(rootdir)
# Clean up any leftover root dir
rootdir_tmp = rootdir + '.tmp'
if os.path.isdir(rootdir_tmp):
shutil.rmtree(rootdir_tmp)
components = self.snapshot['components']
dependencies = buildutil.build_depends(component_name, components)
component = components.get(component_name)
base_devel_name = 'bases/%s-%s-%s' % (self.snapshot['base-prefix'],
component['architecture'],
'devel')
buildroots = self.snapshot['architecture-buildroots']
base_devel_name = 'bases/' + buildroots[architecture]
refs_to_resolve = [base_devel_name]
checkout_trees = [(base_devel_name, '/')]
for dependency_name in dependencies:
buildname = 'components/%s' % (dependency_name, )
buildname = 'components/%s/%s' % (dependency_name, architecture)
refs_to_resolve.append(buildname)
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
resolved_refs = self._resolve_refs(refs_to_resolve)
ref_to_rev = {}
for ref,rev in zip(refs_to_resolve, resolved_refs):
ref_to_rev[ref] = rev
link_cache_dir = os.path.join(self.workdir, 'link-cache')
fileutil.ensure_dir(link_cache_dir)
for (branch, rootpath) in checkout_trees:
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--user-mode', '--link-cache=' + link_cache_dir,
'--union', '--subpath=' + rootpath,
branch, dirpath])
sha = hashlib.sha256()
(fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
f = os.fdopen(fd, 'w')
for (branch, subpath) in checkout_trees:
f.write(ref_to_rev[branch])
f.write('\0')
f.write(subpath)
f.write('\0')
f.close()
f = open(tmppath)
buf = f.read(8192)
while buf != '':
sha.update(buf)
buf = f.read(8192)
f.close()
new_root_cacheid = sha.hexdigest()
rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid')
if os.path.isdir(rootdir):
if os.path.isfile(rootdir_cache_path):
f = open(rootdir_cache_path)
prev_cache_id = f.read().strip()
f.close()
if prev_cache_id == new_root_cacheid:
log("Reusing previous buildroot")
os.unlink(tmppath)
return rootdir
else:
log("New buildroot differs from previous")
shutil.rmtree(rootdir)
os.mkdir(rootdir_tmp)
if len(checkout_trees) > 0:
log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees),
checkout_trees[-1][0]))
link_cache_dir = os.path.join(self.workdir, 'link-cache')
fileutil.ensure_dir(link_cache_dir)
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--link-cache=' + link_cache_dir,
'--user-mode', '--union', '--from-stdin', rootdir_tmp],
stdin=open(tmppath))
os.unlink(tmppath);
builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
os.mkdir(builddir_tmp)
os.mkdir(os.path.join(builddir_tmp, 'source'))
os.mkdir(os.path.join(builddir_tmp, 'results'))
os.rename(rootdir_tmp, rootdir)
f = open(rootdir_cache_path, 'w')
f.write(new_root_cacheid)
f.write('\n')
f.close()
endtime = time.time()
log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
return rootdir
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--pristine', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--snapshot', required=True)
parser.add_argument('--name')
parser.add_argument('--arch', required=True)
parser.add_argument('--debug-shell', action='store_true')
args = parser.parse_args(argv)
self.parse_config()
self.parse_snapshot()
self.parse_snapshot(args.prefix, args.snapshot)
if args.name:
component_name = args.name
else:
cwd = os.getcwd()
parent = os.path.dirname(cwd)
parentparent = os.path.dirname(parent)
component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
component_name = self.get_component_from_cwd()
components = self.snapshot['components']
component = components.get(component_name)
@@ -93,37 +181,16 @@ class OstbuildChrootCompileOne(builtins.Builtin):
shutil.rmtree(child_tmpdir)
fileutil.ensure_dir(child_tmpdir)
resultdir = os.path.join(self.workdir, 'results', component_name)
resultdir = os.path.join(self.workdir, 'results', component_name, args.arch)
if os.path.isdir(resultdir):
shutil.rmtree(resultdir)
fileutil.ensure_dir(resultdir)
rootdir_prefix = os.path.join(workdir, 'roots')
fileutil.ensure_dir(rootdir_prefix)
rootdir = os.path.join(rootdir_prefix, component_name)
fileutil.ensure_parent_dir(rootdir)
if os.path.isdir(rootdir):
shutil.rmtree(rootdir)
rootdir_tmp = rootdir + '.tmp'
builddir = os.path.join(rootdir, 'ostbuild');
if os.path.isdir(rootdir_tmp):
shutil.rmtree(rootdir_tmp)
os.mkdir(rootdir_tmp)
self._compose_buildroot(component_name, rootdir_tmp)
rootdir = self._compose_buildroot(component_name, args.arch)
child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
run_sync(child_args)
builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
os.mkdir(builddir_tmp)
os.mkdir(os.path.join(builddir_tmp, 'source'))
os.mkdir(os.path.join(builddir_tmp, 'results'))
os.rename(rootdir_tmp, rootdir)
log("Checked out buildroot: %s" % (rootdir, ))
sourcedir=os.path.join(builddir, 'source', component_name)
sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name)
fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
@@ -148,7 +215,6 @@ class OstbuildChrootCompileOne(builtins.Builtin):
'compile-one',
'--ostbuild-resultdir=/ostbuild/results',
'--ostbuild-meta=_ostbuild-meta.json'])
child_args.extend(self.metadata.get('config-opts', []))
env_copy = dict(buildutil.BUILD_ENV)
env_copy['PWD'] = chroot_sourcedir
run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py
-47
View File
@@ -1,47 +0,0 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,re,subprocess,tempfile,shutil
from StringIO import StringIO
import argparse
import json
from . import builtins
from . import buildutil
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync
class OstbuildChrootRunTriggers(builtins.Builtin):
name = "chroot-run-triggers"
short_description = "Run ostree-run-triggers inside a chroot"
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('root')
args = parser.parse_args(argv)
child_args = buildutil.get_base_user_chroot_args()
child_args.extend(['--mount-proc', '/proc',
'--mount-bind', '/dev', '/dev',
args.root,
'/usr/bin/ostree-run-triggers'])
env_copy = dict(buildutil.BUILD_ENV)
env_copy['PWD'] = '/'
run_sync(child_args, env=env_copy)
builtins.register(OstbuildChrootRunTriggers)
src/ostbuild/pyostbuild/builtin_compile_one.py
+14 -11
View File
@@ -57,6 +57,8 @@ class OstbuildCompileOne(builtins.Builtin):
def execute(self, args):
self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
starttime = time.time()
uname=os.uname()
kernel=uname[0].lower()
@@ -77,32 +79,28 @@ class OstbuildCompileOne(builtins.Builtin):
'--infodir=' + os.path.join(PREFIX, 'share', 'info')]
self.makeargs = ['make']
self.ostbuild_resultdir=None
self.ostbuild_meta=None
self.ostbuild_resultdir='_ostbuild-results'
self.ostbuild_meta_path='_ostbuild-meta.json'
chdir = None
opt_install = False
for arg in args:
if arg.startswith('--ostbuild-resultdir='):
self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
elif arg.startswith('--ostbuild-meta='):
self.ostbuild_meta=arg[len('--ostbuild-meta='):]
self.ostbuild_meta_path=arg[len('--ostbuild-meta='):]
elif arg.startswith('--chdir='):
os.chdir(arg[len('--chdir='):])
elif arg.startswith('--'):
self.configargs.append(arg)
else:
self.makeargs.append(arg)
if self.ostbuild_resultdir is None:
fatal("Must specify --ostbuild-resultdir=")
if self.ostbuild_meta is None:
fatal("Must specify --ostbuild-meta=")
f = open(self.ostbuild_meta)
f = open(self.ostbuild_meta_path)
self.metadata = json.load(f)
f.close()
self.configargs.extend(self.metadata.get('config-opts', []))
if self.metadata.get('rm-configure', False):
configure_path = 'configure'
if os.path.exists(configure_path):
@@ -235,6 +233,11 @@ class OstbuildCompileOne(builtins.Builtin):
except OSError, e:
pass
endtime = time.time()
log("Compliation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
log("Results placed in %s" % (self.ostbuild_resultdir, ))
def _install_and_unlink(self, src, dest):
statsrc = os.lstat(src)
dirname = os.path.dirname(dest)
src/ostbuild/pyostbuild/builtin_compose.py
Executable
+110
View File
@@ -0,0 +1,110 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from .subprocess_helpers import run_sync_monitor_log_file
from . import ostbuildrc
from . import buildutil
from . import fileutil
from . import kvfile
from . import odict
from . import vcs
class OstbuildCompose(builtins.Builtin):
name = "compose"
short_description = "Build complete trees from components"
def __init__(self):
builtins.Builtin.__init__(self)
def _compose_one_target(self, bin_snapshot, target):
components = bin_snapshot['components']
base = target['base']
base_name = 'bases/%s' % (base['name'], )
base_revision = target['base']['ostree-revision']
compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
if os.path.isdir(compose_rootdir):
shutil.rmtree(compose_rootdir)
os.mkdir(compose_rootdir)
compose_contents = [(base_revision, '/')]
for tree_content in target['contents']:
name = tree_content['name']
rev = components[name]
subtrees = tree_content['trees']
for subpath in subtrees:
compose_contents.append((rev, subpath))
(fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
f = os.fdopen(fd, 'w')
for (branch, subpath) in compose_contents:
f.write(branch)
f.write('\0')
f.write(subpath)
f.write('\0')
f.close()
link_cache_dir = os.path.join(self.workdir, 'link-cache')
fileutil.ensure_dir(link_cache_dir)
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--link-cache=' + link_cache_dir,
'--user-mode', '--no-triggers',
'--union', '--from-stdin', compose_rootdir],
stdin=open(tmppath))
os.unlink(tmppath)
contents_path = os.path.join(compose_rootdir, 'contents.json')
f = open(contents_path, 'w')
json.dump(bin_snapshot, f, indent=4, sort_keys=True)
f.close()
run_sync(['ostree', '--repo=' + self.repo,
'commit', '-b', target['name'], '-s', 'Compose',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged'], cwd=compose_rootdir)
shutil.rmtree(compose_rootdir)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--bin-snapshot')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), ))
for target in self.bin_snapshot['targets']:
log("Composing target %r from %u components" % (target['name'],
len(target['contents'])))
self._compose_one_target(self.bin_snapshot, target)
builtins.register(OstbuildCompose)
src/ostbuild/pyostbuild/builtin_modify_snapshot.py
Executable
+63
View File
@@ -0,0 +1,63 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,stat,subprocess,tempfile,re,shutil
from StringIO import StringIO
import json
import select,time
import argparse
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
class OstbuildModifySnapshot(builtins.Builtin):
name = "modify-snapshot"
short_description = "Change the current source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
args = parser.parse_args(argv)
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
component_name = self.get_component_from_cwd()
current_meta = self.get_component_meta(component_name)
new_snapshot = dict(self.snapshot)
new_meta = dict(current_meta)
if 'patches' in new_meta:
del new_meta['patches']
new_meta['src'] = "dirty-git:%s" % (os.getcwd(), )
new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD'])
new_snapshot['components'][component_name] = new_meta
db = self.get_src_snapshot_db()
path = db.store(new_snapshot)
log("Replaced %s with %s %s" % (component_name, new_meta['src'],
new_meta['revision']))
log("New source snapshot: %s" % (path, ))
builtins.register(OstbuildModifySnapshot)
src/ostbuild/pyostbuild/builtin_prefix.py
Executable
+73
View File
@@ -0,0 +1,73 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildPrefix(builtins.Builtin):
name = "prefix"
short_description = "Display or modify \"prefix\" (build target)"
def __init__(self):
builtins.Builtin.__init__(self)
def _set_prefix(self, prefix):
f = open(self.path, 'w')
f.write(prefix)
f.write('\n')
f.close()
log("Prefix is now %r" % (prefix, ))
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('-a', '--active', action='store_true')
parser.add_argument('prefix', nargs='?', default=None)
args = parser.parse_args(argv)
self.path = os.path.expanduser('~/.config/ostbuild-prefix')
if args.prefix is None and not args.active:
if os.path.exists(self.path):
f = open(self.path)
print "%s" % (f.read().strip(), )
f.close()
else:
log("No currently active prefix")
elif args.prefix is not None and args.active:
fatal("Can't specify -a with prefix")
elif args.prefix is not None:
self._set_prefix(args.prefix)
else:
assert args.active
self.parse_active_branch()
active_prefix = self.active_branch_contents['prefix']
self._set_prefix(active_prefix)
builtins.register(OstbuildPrefix)
src/ostbuild/pyostbuild/builtin_pull_components.py
Executable
+70
View File
@@ -0,0 +1,70 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import copy
import argparse
import json
import time
import urlparse
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from . import ostbuildrc
from . import buildutil
from .subprocess_helpers import run_sync, run_sync_get_output
from . import kvfile
from . import odict
class OstbuildPullComponents(builtins.Builtin):
name = "pull-components"
short_description = "Download the component data for active branch"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('targets', nargs='*')
args = parser.parse_args(argv)
self.parse_active_branch()
if len(args.targets) == 0:
targets = [self.active_branch]
else:
targets = args.targets
tree_contents_list = []
for target in targets:
tree_contents_path = os.path.join(self.ostree_dir, target, 'contents.json')
tree_contents = json.load(open(tree_contents_path))
tree_contents_list.append(tree_contents)
revisions = set()
for tree_contents in tree_contents_list:
for content_item in tree_contents['contents']:
revisions.add(content_item['ostree-revision'])
args = ['ostree-pull', '--repo=' + self.repo]
# FIXME FIXME - don't hardcode origin here
args.append('gnome')
for revision in revisions:
args.append(revision)
run_sync(args)
builtins.register(OstbuildPullComponents)
src/ostbuild/pyostbuild/builtin_resolve.py
+55 -48
View File
@@ -19,6 +19,7 @@ import os,sys,subprocess,tempfile,re,shutil
import copy
import argparse
import json
import time
import urlparse
from StringIO import StringIO
@@ -26,6 +27,7 @@ from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
from . import jsondb
from . import buildutil
from . import kvfile
from . import odict
@@ -38,6 +40,7 @@ class OstbuildResolve(builtins.Builtin):
builtins.Builtin.__init__(self)
def _ensure_vcs_mirror(self, name, keytype, uri, branch):
# FIXME - remove "name" from parameter list here - hash uri?
mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
tmp_mirror = mirror + '.tmp'
if os.path.isdir(tmp_mirror):
@@ -135,6 +138,7 @@ class OstbuildResolve(builtins.Builtin):
self.args = args
self.parse_config()
self.repo = ostbuildrc.get_key('repo')
manifest_path = self.ostbuildrc.get_key('manifest')
self.manifest = json.load(open(manifest_path))
@@ -192,52 +196,52 @@ class OstbuildResolve(builtins.Builtin):
component['patches'] = copy.deepcopy(global_patches_meta)
component['patches']['files'] = patch_files
name_prefix = snapshot['name-prefix']
base_prefix = snapshot['base-prefix']
manifest_architectures = snapshot['architectures']
ostree_prefix = snapshot['prefix']
base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix)
snapshot['architecture-buildroots'] = {}
for architecture in manifest_architectures:
snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
components_by_name = {}
component_ordering = []
build_prev_component_by_arch = {}
runtime_prev_component_by_arch = {}
runtime_components_by_arch = {}
devel_components_by_arch = {}
for architecture in manifest_architectures:
runtime_components_by_arch[architecture] = []
devel_components_by_arch[architecture] = []
build_prev_component = None
runtime_prev_component = None
runtime_components = []
devel_components = []
builds = {}
for component in component_source_list:
component_architectures = component.get('architectures', manifest_architectures)
for architecture in component_architectures:
component_binary = copy.deepcopy(component)
source_name = component['name']
binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name)
component_binary['name'] = binary_name
component_binary['architecture'] = architecture
base_name = component['name']
name = '%s/%s' % (ostree_prefix, base_name)
component['name'] = name
components_by_name[binary_name] = component_binary
components_by_name[name] = component
prev_component = build_prev_component_by_arch.get(architecture)
if prev_component is not None:
component_binary['build-depends'] = [prev_component['name']]
build_prev_component_by_arch[architecture] = component_binary
if build_prev_component is not None:
component['build-depends'] = [build_prev_component['name']]
build_prev_component = component
is_runtime = component.get('component', 'runtime') == 'runtime'
is_runtime = component.get('component', 'runtime') == 'runtime'
prev_component = runtime_prev_component_by_arch.get(architecture)
if prev_component is not None:
component_binary['runtime-depends'] = [prev_component['name']]
if runtime_prev_component is not None:
component['runtime-depends'] = [runtime_prev_component['name']]
if is_runtime:
runtime_prev_component_by_arch[architecture] = component_binary
if is_runtime:
runtime_prev_component = component
runtime_components.append(component)
devel_components.append(component)
if is_runtime:
runtime_components_by_arch[architecture].append(component_binary)
devel_components_by_arch[architecture].append(component_binary)
if 'architectures' in component_binary:
del component_binary['architectures']
is_noarch = component.get('noarch', False)
if is_noarch:
# Just use the first specified architecture
component_arches = [manifest_architectures[0]]
else:
component_arches = component.get('architectures', manifest_architectures)
builds[name] = component_arches
# We expanded these keys
del snapshot['config-opts']
@@ -247,26 +251,27 @@ class OstbuildResolve(builtins.Builtin):
targets_list = []
snapshot['targets'] = targets_list
for architecture in manifest_architectures:
for target_component_type in ['runtime', 'devel']:
for target_component_type in ['runtime', 'devel']:
for architecture in manifest_architectures:
target = {}
targets_list.append(target)
target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type)
target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type)
base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', 'bases/%s' % (base_ref, )])
target['base'] = {'name': base_ref}
if target_component_type == 'runtime':
target_components = runtime_components_by_arch[architecture]
target_components = runtime_components
else:
target_components = devel_components_by_arch[architecture]
target_components = devel_components
contents = []
for component in target_components:
name = component['name']
component_ref = {'name': name}
builds_for_component = builds[component['name']]
if architecture not in builds_for_component:
continue
binary_name = '%s/%s' % (component['name'], architecture)
component_ref = {'name': binary_name}
if target_component_type == 'runtime':
component_ref['trees'] = ['/runtime']
else:
@@ -278,10 +283,12 @@ class OstbuildResolve(builtins.Builtin):
del component['name']
snapshot['components'] = components_by_name
out_snapshot = os.path.join(self.workdir, '%s-snapshot.json' % (name_prefix, ))
f = open(out_snapshot, 'w')
json.dump(snapshot, f, indent=4, sort_keys=True)
f.close()
print "Created: %s" % (out_snapshot, )
snapshot['00ostree-src-snapshot-version'] = 0
current_time = time.time()
src_db = self.get_src_snapshot_db()
path = src_db.store(snapshot)
log("Source snapshot: %s" % (path, ))
builtins.register(OstbuildResolve)
src/ostbuild/pyostbuild/builtins.py
+155 -10
View File
@@ -19,11 +19,15 @@
import os
import sys
import stat
import argparse
import json
from . import ostbuildrc
from . import fileutil
from . import jsondb
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
_all_builtins = {}
@@ -31,26 +35,167 @@ class Builtin(object):
name = None
short_description = None
def __init__(self):
self._meta_cache = {}
self.prefix = None
self.manifest = None
self.snapshot = None
self.bin_snapshot = None
self.repo = None
self.ostree_dir = self._find_ostree_dir()
(self.active_branch, self.active_branch_checksum) = self._find_active_branch()
self._src_snapshots = None
self._bin_snapshots = None
def _find_ostree_dir(self):
for path in ['/ostree', '/sysroot/ostree']:
if os.path.isdir(path):
return path
return None
def _find_active_branch(self):
if self.ostree_dir is None:
return (None, None)
current_path = os.path.join(self.ostree_dir, 'current')
while True:
try:
target = os.path.join(self.ostree_dir, current_path)
stbuf = os.lstat(target)
except OSError, e:
current_path = None
break
if not stat.S_ISLNK(stbuf.st_mode):
break
current_path = os.readlink(target)
if current_path is not None:
basename = os.path.basename(current_path)
return basename.rsplit('-', 1)
else:
return (None, None)
def get_component_from_cwd(self):
cwd = os.getcwd()
parent = os.path.dirname(cwd)
parentparent = os.path.dirname(parent)
return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
def parse_config(self):
self.ostbuildrc = ostbuildrc
self.repo = ostbuildrc.get_key('repo')
self.mirrordir = ostbuildrc.get_key('mirrordir')
if not os.path.isdir(self.mirrordir):
fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
self.workdir = ostbuildrc.get_key('workdir')
if not os.path.isdir(self.workdir):
fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
self.patchdir = os.path.join(self.workdir, 'patches')
def parse_manifest(self):
self.manifest_path = ostbuildrc.get_key('manifest')
self.manifest = json.load(open(self.manifest_path))
self.name_prefix = self.manifest['name-prefix']
def parse_active_branch(self):
if self.ostree_dir is None:
fatal("/ostree directory not found")
repo_path = os.path.join(self.ostree_dir, 'repo')
if not os.path.isdir(repo_path):
fatal("Repository '%s' doesn't exist" % (repo_path, ))
self.repo = repo_path
if self.active_branch is None:
fatal("No \"current\" link found")
branch_path = os.path.join(self.ostree_dir, self.active_branch)
contents_path = os.path.join(branch_path, 'contents.json')
f = open(contents_path)
self.active_branch_contents = json.load(f)
f.close()
def parse_snapshot(self):
self.parse_manifest()
snapshot_path = os.path.join(self.workdir, '%s-snapshot.json' % (self.name_prefix, ))
self.snapshot = json.load(open(snapshot_path))
def get_component_snapshot(self, name):
found = False
for content in self.active_branch_contents['contents']:
if content['name'] == name:
found = True
break
if not found:
fatal("Unknown component '%s'" % (name, ))
return content
def get_component_meta_from_revision(self, revision):
text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', revision,
'/_ostbuild-meta.json'])
return json.loads(text)
def get_component_meta(self, name):
assert self.repo is not None
if self.snapshot is not None:
return self.snapshot['components'][name]
meta = self._meta_cache.get(name)
if meta is None:
content = self.get_component_snapshot(name)
meta = self.get_component_meta_from_revision(content['ostree-revision'])
self._meta_cache[name] = meta
return meta
def get_prefix(self):
if self.prefix is None:
path = os.path.expanduser('~/.config/ostbuild-prefix')
if not os.path.exists(path):
fatal("No prefix set; use \"ostbuild prefix\" to set one")
f = open(path)
self.prefix = f.read().strip()
f.close()
return self.prefix
def create_db(self, dbsuffix, prefix=None):
if prefix is None:
target_prefix = self.get_prefix()
else:
target_prefix = prefix
name = '%s-%s' % (target_prefix, dbsuffix)
fileutil.ensure_dir(self.snapshot_dir)
return jsondb.JsonDB(self.snapshot_dir, prefix=name)
def get_src_snapshot_db(self):
if self._src_snapshots is None:
self._src_snapshots = self.create_db('src-snapshot')
return self._src_snapshots
def get_bin_snapshot_db(self):
if self._bin_snapshots is None:
self._bin_snapshots = self.create_db('bin-snapshot')
return self._bin_snapshots
def parse_snapshot(self, prefix, path):
if prefix is not None:
self.prefix = prefix
self.repo = ostbuildrc.get_key('repo')
if path is None:
latest_path = self.get_src_snapshot_db().get_latest_path()
if latest_path is None:
raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
self.snapshot_path = latest_path
else:
self.snapshot_path = path
self.snapshot = json.load(open(self.snapshot_path))
src_ver = self.snapshot['00ostree-src-snapshot-version']
if src_ver != 0:
fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0", src_ver)
def parse_bin_snapshot(self, prefix, path):
if prefix is not None:
self.prefix = prefix
self.repo = ostbuildrc.get_key('repo')
if path is None:
latest_path = self.get_bin_snapshot_db().get_latest_path()
if latest_path is None:
raise Exception("No binary snapshot found for prefix %r" % (self.prefix, ))
self.bin_snapshot_path = latest_path
else:
self.bin_snapshot_path = path
self.bin_snapshot = json.load(open(self.bin_snapshot_path))
bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version']
if bin_ver != 0:
fatal("Unhandled 00ostree-bin-snapshot-version \"%d\", expected 0", bin_ver)
def execute(self, args):
raise NotImplementedError()
@@ -65,4 +210,4 @@ def get(name):
return None
def get_all():
return _all_builtins.itervalues()
return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name))
src/ostbuild/pyostbuild/jsondb.py
+110
View File
@@ -0,0 +1,110 @@
#
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import stat
import time
import tempfile
import re
import shutil
import hashlib
import json
class JsonDB(object):
def __init__(self, dirpath, prefix):
self._dirpath = dirpath
self._prefix = prefix
self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$')
def _cmp_match_by_version(self, a, b):
# Note this is a reversed comparison; bigger is earlier
a_major = a[0]
a_minor = a[1]
b_major = b[0]
b_minor = b[1]
c = cmp(b_major, a_major)
if c == 0:
return cmp(b_minor, a_minor)
return 0
def _get_all(self):
result = []
for fname in os.listdir(self._dirpath):
if not (fname.startswith(self._prefix) and fname.endswith('.json')):
continue
path = os.path.join(self._dirpath, fname)
match = self._version_csum_re.search(fname)
if not match:
raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum",
path)
result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname))
result.sort(self._cmp_match_by_version)
return result
def get_latest(self):
path = self.get_latest_path()
if path is None:
return None
return json.load(open(path))
def get_latest_path(self):
files = self._get_all()
if len(files) == 0:
return None
return os.path.join(self._dirpath, files[0][3])
def store(self, obj):
files = self._get_all()
if len(files) == 0:
latest = None
else:
latest = files[0]
current_time = time.gmtime()
(fd, tmppath) = tempfile.mkstemp(suffix='.tmp', prefix='tmp-jsondb-')
os.close(fd)
f = open(tmppath, 'w')
json.dump(obj, f, indent=4, sort_keys=True)
f.close()
csum = hashlib.sha256()
f = open(tmppath)
buf = f.read(8192)
while buf != '':
csum.update(buf)
buf = f.read(8192)
f.close()
digest = csum.hexdigest()
if latest is not None:
if digest == latest[2]:
return latest[3]
latest_version = (latest[0], latest[1])
else:
latest_version = (current_time.tm_year, 0)
target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year,
latest_version[1] + 1, digest)
target_path = os.path.join(self._dirpath, target_name)
os.rename(tmppath, target_path)
return target_path
src/ostbuild/pyostbuild/main.py
+7 -3
View File
@@ -22,13 +22,17 @@ import sys
import argparse
from . import builtins
from . import builtin_build
from . import builtin_bin_to_src
from . import builtin_build_components
from . import builtin_branch_prefix
from . import builtin_checkout
from . import builtin_chroot_compile_one
from . import builtin_chroot_run_triggers
from . import builtin_compose
from . import builtin_compile_one
from . import builtin_query_content
from . import builtin_pull_components
from . import builtin_prefix
from . import builtin_resolve
from . import builtin_modify_snapshot
from . import builtin_status
def usage(ecode):
src/ostbuild/pyostbuild/ostbuildlog.py
+7 -3
View File
@@ -20,12 +20,16 @@
import os
import sys
def log(msg):
fullmsg = '%s: %s\n' % (os.path.basename(sys.argv[0]), msg)
def log(msg, prefix=None):
if prefix is None:
prefix_target = ''
else:
prefix_target = prefix
fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg)
sys.stdout.write(fullmsg)
sys.stdout.flush()
def fatal(msg):
log(msg)
log(msg, prefix="FATAL: ")
sys.exit(1)
src/ostbuild/pyostbuild/subprocess_helpers.py
+4 -2
View File
@@ -70,14 +70,16 @@ def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none
return None
def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
log_success=True, log_initiation=True, stdout=None,
log_success=True, log_initiation=True, stdin=None, stdout=None,
stderr=None):
if log_initiation:
log("running: %s" % (subprocess.list2cmdline(args),))
env_copy = _get_env_for_cwd(cwd, env)
if keep_stdin:
if stdin is not None:
stdin_target = stdin
elif keep_stdin:
stdin_target = sys.stdin
else:
stdin_target = open('/dev/null', 'r')
src/ostbuild/pyostbuild/vcs.py
+1 -1
View File
@@ -71,5 +71,5 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
return dest
def clean(keytype, checkoutdir):
assert keytype == 'git'
assert keytype in ('git', 'dirty-git')
run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)