ostbuild: Rework again, split into components.json and targets.json

The "resolve" builtin now does a lot more heavy lifting; we expand the
manifest.json, and "build" consequently is less intelligent now, more
of a low-level implementation.

Makefile-ostbuild.am

@@ -32,6 +32,7 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/builtin_status.py	\
 	src/ostbuild/pyostbuild/builtins.py		\
 	src/ostbuild/pyostbuild/filemonitor.py		\
+	src/ostbuild/pyostbuild/fileutil.py		\
 	src/ostbuild/pyostbuild/__init__.py		\
 	src/ostbuild/pyostbuild/kvfile.py		\
 	src/ostbuild/pyostbuild/main.py			\

gnomeos/3.4/manifest.json

@@ -1,7 +1,7 @@
 {
   "name-prefix": "gnomeos-3.4",
   "architectures": ["i686"],
-  "base-prefix": "bases/yocto/gnomeos-3.4",
+  "base-prefix": "yocto/gnomeos-3.4",
 
   "config-opts": ["--disable-static", "--disable-silent-rules"],
 

src/ostbuild/pyostbuild/buildutil.py

@@ -19,6 +19,7 @@ import os
 import re
 import urlparse
 import tempfile
+import StringIO
 
 from .subprocess_helpers import run_sync_get_output
 
@@ -75,23 +76,51 @@ def get_git_version_describe(dirpath, commit=None):
     version = run_sync_get_output(args, cwd=dirpath)
     return version.strip()
 
-def manifest_target(manifest):
+def ref_to_unix_name(ref):
-    name = manifest['name']
+    return ref.replace('/', '.')
-    is_runtime = name.endswith('-runtime')
-    # HACK - we should really name builds just like e.g. gnomeos-3.4-i686 
-    if is_runtime:
-        return name[:-len('-runtime')] + '-devel'
-    return name
 
-def manifest_buildname(manifest, component):
+def tsort_components(components, key):
-    return 'artifacts/%s/%s/%s' % (manifest_target(manifest),
+    (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-tsort-')
-                                   component['name'],
+    f = os.fdopen(fd, 'w')
-                                   component['branch'])
+    for name,component in components.iteritems():
+        build_prev = component.get(key)
+        if (build_prev is not None and len(build_prev) > 0):
+            for dep_name in build_prev:
+                f.write('%s %s\n' % (name, dep_name))
+    f.close()
+    
+    output = run_sync_get_output(['tsort', path])
+    os.unlink(path)
+    output_stream = StringIO.StringIO(output)
+    result = []
+    for line in output_stream:
+        result.append(line.strip())
+    return result
 
-def manifest_buildroot_name(manifest, component):
+def _recurse_depends(depkey, component_name, components, dep_names):
-    return 'buildroots/%s/%s/%s' % (manifest_target (manifest),
+    component = components[component_name]
-                                    component['name'],
+    depends = component.get(depkey)
-                                    component['branch'])
+    if (depends is None or len(depends) == 0):
+        return
+    for depname in depends:
+        dep_names.add(depname)
+        _recurse_depends(depkey, depname, components, dep_names)
+
+def _sorted_depends(deptype, component_name, components):
+    dep_names = set()
+    _recurse_depends(deptype, component_name, components, dep_names)
+    dep_components = {}
+    for component_name in dep_names:
+        dep_components[component_name] = components[component_name]
+    result = tsort_components(dep_components, deptype)
+    result.reverse()
+    return result
+    
+def build_depends(component_name, components):
+    return _sorted_depends('build-depends', component_name, components)
+
+def runtime_depends(component_name, components):
+    return _sorted_depends('runtime-depends', component_name, components)
 
 def find_component_in_manifest(manifest, component_name):
     for component in manifest['components']:

src/ostbuild/pyostbuild/builtin_build.py

@@ -28,6 +28,7 @@ from .subprocess_helpers import run_sync, run_sync_get_output
 from .subprocess_helpers import run_sync_monitor_log_file
 from . import ostbuildrc
 from . import buildutil
+from . import fileutil
 from . import kvfile
 from . import odict
 from . import vcs
@@ -91,16 +92,13 @@ class OstbuildBuild(builtins.Builtin):
 
         return result
 
-    def _build_one_component(self, meta):
+    def _build_one_component(self, name, component):
-        name = meta['name']
+        branch = component['branch']
-        branch = meta['branch']
+        architecture = component['architecture']
-        architecture = meta['arch']
 
-        target = buildutil.manifest_target(self.manifest)
+        buildname = 'components/%s' % (name, )
-        buildname = buildutil.manifest_buildname(self.manifest, meta)
-        buildroot_name = buildutil.manifest_buildroot_name(self.manifest, meta)
 
-        current_vcs_version = meta['revision']
+        current_vcs_version = component['revision']
 
         previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                       'rev-parse', buildname],
@@ -130,31 +128,27 @@ class OstbuildBuild(builtins.Builtin):
 
         checkoutdir = os.path.join(self.workdir, 'src')
         component_src = os.path.join(checkoutdir, name)
-        run_sync(['ostbuild', 'checkout', '--overwrite', '--manifest=' + self.manifest_path, name], cwd=checkoutdir)
+        run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir)
 
-        artifact_meta = dict(meta)
+        artifact_meta = dict(component)
 
         metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
         f = open(metadata_path, 'w')
         json.dump(artifact_meta, f, indent=4, sort_keys=True)
         f.close()
 
-        logdir = os.path.join(self.workdir, 'logs', 'compile', name)
+        logdir = os.path.join(self.workdir, 'logs', name)
-        old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name)
+        fileutil.ensure_dir(logdir)
-        if not os.path.isdir(logdir):
+        log_path = os.path.join(logdir, 'compile.log')
-            os.makedirs(logdir)
-        if not os.path.isdir(old_logdir):
-            os.makedirs(old_logdir)
-        log_path = os.path.join(logdir, '%s.log' % (name, ))
         if os.path.isfile(log_path):
             curtime = int(time.time())
-            saved_name = '%s-%d.log' % (name, int(time.time()),)
+            saved_name = os.path.join(logdir, 'compile-prev.log')
-            os.rename(log_path, os.path.join(old_logdir, saved_name))
+            os.rename(log_path, saved_name)
 
         log("Logging to %s" % (log_path, ))
         f = open(log_path, 'w')
         chroot_args = self._get_ostbuild_chroot_args(architecture)
-        chroot_args.extend(['--pristine', '--manifest=' + self.manifest_path])
+        chroot_args.extend(['--pristine', '--name=' + name])
         if self.buildopts.shell_on_failure:
             ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
             if ecode != 0:
@@ -184,53 +178,45 @@ class OstbuildBuild(builtins.Builtin):
             os.unlink(statoverride_path)
         return True
 
-    def _compose(self, components):
+    def _compose(self, target):
-        base = self.manifest['base']
+        base_name = 'bases/%s' % (target['base']['name'], )
-        base_branch = base['branch']
-        base_revision = base['revision']
-        # HACK
-        manifest_build_name = self.manifest['name']
-        is_runtime = manifest_build_name.endswith('-runtime')
-
         branch_to_rev = {}
         branch_to_subtrees = {}
 
-        component_branches = []
+        contents = [base_name]
-        for component in components:
+        branch_to_subtrees[base_name] = ['/']
-            branch = buildutil.manifest_buildname(self.manifest, component)
+        base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
-            component_branches.append(branch)
+                                             'rev-parse', base_name])
 
-        args = ['ostree', '--repo=' + self.repo,
+        branch_to_rev[base_name] = base_revision
-                'rev-parse']
+
-        args.extend(component_branches)
+        args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+        for component in target['contents']:
+            name = component['name']
+            contents.append(name)
+            args.append('components/%s' % (name, ))
+            branch_to_subtrees[name] = component['trees']
         branch_revs_text = run_sync_get_output(args)
         branch_revs = branch_revs_text.split('\n')
 
-        for (branch, rev) in zip(component_branches, branch_revs):
+        for (content, rev) in zip(target['contents'], branch_revs):
-            branch_to_rev[branch] = rev
+            name = content['name']
-
+            branch_to_rev[name] = rev
-        contents = [base_branch]
-        branch_to_subtrees[base_branch] = ['/']
-        branch_to_rev[base_branch] = base_revision
         
-        for branch in component_branches:
+        compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
-            contents.append(branch)
-            subtrees = ['/runtime']
-            branch_to_subtrees[branch] = subtrees
-            if not is_runtime:
-                # For now just hardcode docs going in devel
-                subtrees.append('/doc')
-                subtrees.append('/devel')
-
-        compose_rootdir = os.path.join(self.workdir, 'roots', self.manifest['name'])
         if os.path.isdir(compose_rootdir):
             shutil.rmtree(compose_rootdir)
         os.mkdir(compose_rootdir)
 
-        metadata_contents = []
+        resolved_base = dict(target['base'])
+        resolved_base['ostree-revision'] = base_revision
+        resolved_contents = list(target['contents'])
+        for component in resolved_contents:
+            component['ostree-revision'] = branch_to_rev[component['name']]
         metadata = {'source': 'ostbuild compose v0',
-                    'base': base,
+                    'base': resolved_base, 
-                    'contents': metadata_contents}
+                    'contents': resolved_contents}
+
         for branch in contents:
             branch_rev = branch_to_rev[branch]
             subtrees = branch_to_subtrees[branch]
@@ -239,18 +225,14 @@ class OstbuildBuild(builtins.Builtin):
                           'checkout', '--user-mode',
                           '--union', '--subpath=' + subtree,
                           branch_rev, compose_rootdir])
-            branch_meta = {'name': branch,
-                           'rev': branch_rev,
-                           'subtrees': subtrees}
-            metadata_contents.append(branch_meta)
 
-        contents_path = os.path.join(compose_rootdir, 'contents.json')
+        contents_path = os.path.join(compose_rootdir, 'manifest.json')
         f = open(contents_path, 'w')
         json.dump(metadata, f, indent=4, sort_keys=True)
         f.close()
 
         run_sync(['ostree', '--repo=' + self.repo,
-                  'commit', '-b', self.manifest['name'], '-s', 'Compose',
+                  'commit', '-b', target['name'], '-s', 'Compose',
                   '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
                   '--skip-if-unchanged'], cwd=compose_rootdir)
 
@@ -259,7 +241,6 @@ class OstbuildBuild(builtins.Builtin):
         parser.add_argument('--skip-built', action='store_true')
         parser.add_argument('--recompose', action='store_true')
         parser.add_argument('--start-at')
-        parser.add_argument('--manifest', required=True)
         parser.add_argument('--shell-on-failure', action='store_true')
         parser.add_argument('--debug-shell', action='store_true')
         parser.add_argument('components', nargs='*')
@@ -268,37 +249,33 @@ class OstbuildBuild(builtins.Builtin):
         self.args = args
         
         self.parse_config()
+        self.parse_components_and_targets()
 
         self.buildopts = BuildOptions()
         self.buildopts.shell_on_failure = args.shell_on_failure
         self.buildopts.skip_built = args.skip_built
 
-        self.manifest_path = args.manifest
+        build_component_order = []
-        self.manifest = json.load(open(args.manifest))
-
-        components = self.manifest['components']
         if args.recompose:
-            build_components = []
+            pass
         elif len(args.components) == 0:
-            build_components = components
+            tsorted = buildutil.tsort_components(self.components, 'build-depends')
+            tsorted.reverse()
+            build_component_order = tsorted
         else:
-            build_components = []
+            if args.start_at is not None:
+                fatal("Can't specify --start-at with component list")
             for name in args.components:
                 found = False
-                for child in components:
+                component = self.components.get(name)
-                    if child['name'] == name:
+                if component is None:
-                        found = True
-                        build_components.append(child)
-                        break
-                if not found:
                     fatal("Unknown component %r" % (name, ))
+                build_component_order.append(name)
 
         start_at_index = -1
         if args.start_at is not None:
-            if build_components != components:
+            for i,component_name in enumerate(build_component_order):
-                fatal("Can't specify --start-at with component list")
+                if component_name == args.start_at:
-            for i,component in enumerate(build_components):
-                if component['name'] == args.start_at:
                     start_at_index = i
                     break
             if start_at_index == -1:
@@ -306,10 +283,11 @@ class OstbuildBuild(builtins.Builtin):
         else:
             start_at_index = 0
 
-        for component in build_components[start_at_index:]:
+        for component_name in build_component_order[start_at_index:]:
-            index = components.index(component)
+            component = self.components.get(component_name)
-            self._build_one_component(component)
+            self._build_one_component(component_name, component)
 
-        self._compose(components)
+        for target in self.targets['targets']:
+            self._compose(target)
         
 builtins.register(OstbuildBuild)

src/ostbuild/pyostbuild/builtin_checkout.py

@@ -26,6 +26,7 @@ from .ostbuildlog import log, fatal
 from .subprocess_helpers import run_sync, run_sync_get_output
 from . import ostbuildrc
 from . import buildutil
+from . import fileutil
 from . import odict
 from . import vcs
 
@@ -38,16 +39,15 @@ class OstbuildCheckout(builtins.Builtin):
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--manifest', required=True)
         parser.add_argument('--overwrite', action='store_true')
+        parser.add_argument('--clean', action='store_true')
         parser.add_argument('components', nargs='*')
 
         args = parser.parse_args(argv)
         self.args = args
         
         self.parse_config()
-
+        self.parse_components_and_targets()
-        self.manifest = json.load(open(args.manifest))
 
         if len(args.components) > 0:
             checkout_components = args.components
@@ -56,17 +56,20 @@ class OstbuildCheckout(builtins.Builtin):
 
         for component_name in checkout_components:
             found = False
-            component = buildutil.find_component_in_manifest(self.manifest,
+            component = self.components.get(component_name)
-                                                             component_name)
             if component is None:
                 fatal("Unknown component %r" % (component_name, ))
             (keytype, uri) = buildutil.parse_src_key(component['src'])
-            checkoutdir = os.path.join(os.getcwd(), component['name'])
+            checkoutdir = os.path.join(os.getcwd(), component_name)
+            fileutil.ensure_parent_dir(checkoutdir)
 
             component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
                                                  component['revision'],
                                                  overwrite=args.overwrite)
 
+            if args.clean:
+                vcs.clean(keytype, checkoutdir)
+
             patches = component.get('patches')
             if patches is not None:
                 (patches_keytype, patches_uri) = buildutil.parse_src_key(patches['src'])

src/ostbuild/pyostbuild/builtin_chroot_compile_one.py

@@ -22,6 +22,7 @@ import json
 
 from . import builtins
 from . import buildutil
+from . import fileutil
 from .ostbuildlog import log, fatal
 from .subprocess_helpers import run_sync, run_sync_get_output
 
@@ -29,16 +30,16 @@ class OstbuildChrootCompileOne(builtins.Builtin):
     name = "chroot-compile-one"
     short_description = "Build artifacts from the current source directory in a chroot"
 
-    def _compose_buildroot(self, component, dirpath):
+    def _compose_buildroot(self, component_name, dirpath):
-        components = self.manifest['components']
+        dependencies = buildutil.build_depends(component_name, self.components)
-        index = components.index(component)
+        component = self.components.get(component_name)
-        dependencies = components[:index]
 
-        base = self.manifest['base']
+        base_devel_name = 'bases/%s-%s-%s' % (self.manifest['base-prefix'],
-        base_revision = base['revision']
+                                              component['architecture'],
-        checkout_trees = [(base_revision, '/')]
+                                              'devel')
-        for dep in dependencies:
+        checkout_trees = [(base_devel_name, '/')]
-            buildname = buildutil.manifest_buildname(self.manifest, dep)
+        for dependency_name in dependencies:
+            buildname = 'components/%s' % (dependency_name, )
             checkout_trees.append((buildname, '/runtime'))
             checkout_trees.append((buildname, '/devel'))
 
@@ -50,31 +51,32 @@ class OstbuildChrootCompileOne(builtins.Builtin):
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--manifest', required=True)
         parser.add_argument('--pristine', action='store_true')
+        parser.add_argument('--name')
         parser.add_argument('--debug-shell', action='store_true')
         
         args = parser.parse_args(argv)
 
         self.parse_config()
+        self.parse_components_and_targets()
 
-        component_name = os.path.basename(os.getcwd())
+        if args.name:
-        self.manifest = json.load(open(args.manifest))
+            component_name = args.name
+        else:
+            cwd = os.getcwd()
+            parent = os.path.dirname(cwd)
+            parentparent = os.path.dirname(parent)
+            component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
 
-        component = buildutil.find_component_in_manifest(self.manifest, component_name)
+        component = self.components.get(component_name)
-        self.metadata = component
         if component is None:
             fatal("Couldn't find component '%s' in manifest" % (component_name, ))
+        self.metadata = dict(component)
+        self.metadata['name'] = component_name
         if not args.pristine:
             self.metadata['src'] = 'dirty:worktree'
             self.metadata['revision'] = 'dirty-worktree'
 
-        architecture = os.uname()[4]
-
-        if 'name' not in self.metadata:
-            sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
-            sys.exit(1)
-
         workdir = self.workdir
             
         log("Using working directory: %s" % (workdir, ))
@@ -83,17 +85,17 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         if os.path.isdir(child_tmpdir):
             log("Cleaning up previous tmpdir: %r" % (child_tmpdir, ))
             shutil.rmtree(child_tmpdir)
-        os.mkdir(child_tmpdir)
+        fileutil.ensure_dir(child_tmpdir)
 
-        resultdir = os.path.join(self.workdir, 'results', component['name'])
+        resultdir = os.path.join(self.workdir, 'results', component_name)
         if os.path.isdir(resultdir):
             shutil.rmtree(resultdir)
-        os.makedirs(resultdir)
+        fileutil.ensure_dir(resultdir)
         
         rootdir_prefix = os.path.join(workdir, 'roots')
-        if not os.path.isdir(rootdir_prefix):
+        fileutil.ensure_dir(rootdir_prefix)
-            os.makedirs(rootdir_prefix)
+        rootdir = os.path.join(rootdir_prefix, component_name)
-        rootdir = os.path.join(rootdir_prefix, component['name'])
+        fileutil.ensure_parent_dir(rootdir)
         if os.path.isdir(rootdir):
             shutil.rmtree(rootdir)
         
@@ -103,7 +105,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
             shutil.rmtree(rootdir_tmp)
         os.mkdir(rootdir_tmp)
             
-        self._compose_buildroot(component, rootdir_tmp)
+        self._compose_buildroot(component_name, rootdir_tmp)
 
         child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
         run_sync(child_args)
@@ -115,15 +117,14 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         os.rename(rootdir_tmp, rootdir)
         log("Checked out buildroot: %s" % (rootdir, ))
         
-        sourcedir=os.path.join(builddir, 'source', self.metadata['name'])
+        sourcedir=os.path.join(builddir, 'source', component_name)
-        if not os.path.isdir(sourcedir):
+        fileutil.ensure_dir(sourcedir)
-            os.mkdir(sourcedir)
         
         output_metadata = open('_ostbuild-meta.json', 'w')
         json.dump(self.metadata, output_metadata, indent=4, sort_keys=True)
         output_metadata.close()
         
-        chroot_sourcedir = os.path.join('/ostbuild', 'source', self.metadata['name'])
+        chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name)
 
         ostbuild_user_chroot_path = buildutil.find_user_chroot_path()
         

src/ostbuild/pyostbuild/builtin_compile_one.py

@@ -103,10 +103,6 @@ class OstbuildCompileOne(builtins.Builtin):
         self.metadata = json.load(f)
         f.close()
 
-        for k in ['name', 'revision']:
-            if k not in self.metadata:
-                fatal('Missing required key "%s" in metadata' % (k, ))
-
         if self.metadata.get('rm-configure', False):
             configure_path = 'configure'
             if os.path.exists(configure_path):
@@ -180,10 +176,7 @@ class OstbuildCompileOne(builtins.Builtin):
     
         run_sync(args, cwd=builddir)
 
-        name = self.metadata['name']
+        tempdir = tempfile.mkdtemp(prefix='ostbuild-destdir-%s' % (self.metadata['name'].replace('/', '_'), ))
-        assert ',' not in name
-    
-        tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,))
         self.tempfiles.append(tempdir)
         args = ['make', 'install', 'DESTDIR=' + tempdir]
         run_sync(args, cwd=builddir)

src/ostbuild/pyostbuild/builtin_resolve.py

@@ -16,6 +16,7 @@
 # Boston, MA 02111-1307, USA.
 
 import os,sys,subprocess,tempfile,re,shutil
+import copy
 import argparse
 import json
 import urlparse
@@ -138,16 +139,18 @@ class OstbuildResolve(builtins.Builtin):
         manifest_path = self.ostbuildrc.get_key('manifest')
         self.manifest = json.load(open(manifest_path))
 
-        self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
+        snapshot = copy.deepcopy(self.manifest)
+        component_source_list = map(self._resolve_component_meta, self.manifest['components'])
+        del snapshot['components']
 
         if args.fetch:
             if len(args.components) == 0:
-                fetch_components = map(lambda x: x['name'], self.resolved_components)
+                fetch_components = map(lambda x: x['name'], component_source_list)
             else:
                 fetch_components = args.components
             for component_name in fetch_components:
                 found = False
-                for component in self.resolved_components:
+                for component in component_source_list:
                     if component['name'] == component_name:
                         found = True
                         break
@@ -159,76 +162,132 @@ class OstbuildResolve(builtins.Builtin):
                 run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
         else:
             fetch_components = []
-            
+
         global_patches_meta = self._resolve_component_meta(self.manifest['patches'])
         (keytype, uri) = self._parse_src_key(global_patches_meta['src'])
         mirrordir = self._ensure_vcs_mirror(global_patches_meta['name'], keytype, uri, global_patches_meta['branch'])
         revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch'])
         global_patches_meta['revision'] = revision
 
-        for component in self.resolved_components:
+        unique_component_names = set()
+        for component in component_source_list:
             (keytype, uri) = self._parse_src_key(component['src'])
             name = component['name']
+
+            if name in unique_component_names:
+                fatal("Duplicate component name '%s'" % (name, ))
+            unique_component_names.add(name)
+
             mirrordir = self._ensure_vcs_mirror(name, keytype, uri, component['branch'])
             revision = buildutil.get_git_version_describe(mirrordir,
                                                           component['branch'])
             component['revision'] = revision
 
-            if 'component' not in component:
-                component['component'] = 'runtime'
-
             config_opts = list(self.manifest['config-opts'])
             config_opts.extend(component.get('config-opts', []))
             component['config-opts'] = config_opts
 
             patch_files = component.get('patches')
             if patch_files is not None:
-                component['patches'] = dict(global_patches_meta)
+                component['patches'] = copy.deepcopy(global_patches_meta)
                 component['patches']['files'] = patch_files
 
-        self.manifest['components'] = self.resolved_components
+        name_prefix = snapshot['name-prefix']
+        del snapshot['name-prefix']
+        base_prefix = snapshot['base-prefix']
+        del snapshot['base-prefix']
+
+        manifest_architectures = snapshot['architectures']
+
+        components_by_name = {}
+        component_ordering = []
+        build_prev_component_by_arch = {}
+        runtime_prev_component_by_arch = {}
+        runtime_components_by_arch = {}
+        devel_components_by_arch = {}
+        for architecture in manifest_architectures:
+            runtime_components_by_arch[architecture] = []
+            devel_components_by_arch[architecture] = []
+
+        for component in component_source_list:
+            component_architectures = component.get('architectures', manifest_architectures)
+            for architecture in component_architectures:
+                component_binary = copy.deepcopy(component)
+                source_name = component['name']
+                binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name)
+                component_binary['name'] = binary_name
+                component_binary['architecture'] = architecture
+
+                components_by_name[binary_name] = component_binary
+
+                prev_component = build_prev_component_by_arch.get(architecture)
+                if prev_component is not None:
+                    component_binary['build-depends'] = [prev_component['name']]
+                build_prev_component_by_arch[architecture] = component_binary
+
+                is_runtime = component.get('component', 'runtime') == 'runtime'
+
+                prev_component = runtime_prev_component_by_arch.get(architecture)
+                if prev_component is not None:
+                    component_binary['runtime-depends'] = [prev_component['name']]
+
+                if is_runtime:
+                    runtime_prev_component_by_arch[architecture] = component_binary
+
+                if is_runtime:
+                    runtime_components_by_arch[architecture].append(component_binary)
+                devel_components_by_arch[architecture].append(component_binary)
+
+                if 'architectures' in component_binary:
+                    del component_binary['architectures']
 
         # We expanded these keys
-        del self.manifest['config-opts']
+        del snapshot['config-opts']
-        del self.manifest['vcsconfig']
+        del snapshot['vcsconfig']
-        del self.manifest['patches']
+        del snapshot['patches']
+        del snapshot['architectures']
 
-        manifest_architectures = self.manifest['architectures']
+        targets_json = {}
-        del self.manifest['architectures']
+        targets_list = []
+        targets_json['targets'] = targets_list
         for architecture in manifest_architectures:
-            arch_manifest = dict(self.manifest)
+            for target_component_type in ['runtime', 'devel']:
-            for component in arch_manifest['components']:
+                target = {}
-                component['arch'] = architecture
+                targets_list.append(target)
+                target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type)
 
-            runtime_components = filter(lambda x: x.get('component', 'runtime') == 'runtime', arch_manifest['components'])
+                base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
-            devel_components = arch_manifest['components']
-            for component in arch_manifest['components']:
-                if 'component' in component:
-                    del component['component']
-
-            for component_type in ['runtime', 'devel']:
-                snapshot = dict(arch_manifest)
-                if component_type == 'runtime':
-                    snapshot['components'] = runtime_components
-                else:
-                    snapshot['components'] = devel_components
-
-                name_prefix = snapshot['name-prefix']
-                del snapshot['name-prefix']
-                base_prefix = snapshot['base-prefix']
-                del snapshot['base-prefix']
-
-                snapshot['name'] = '%s-%s-%s' % (name_prefix, architecture, component_type)
-
-                base_ref = '%s-%s-%s' % (base_prefix, architecture, component_type)
                 base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                     'rev-parse', base_ref])
+                                                     'rev-parse', 'bases/%s' % (base_ref, )])
-                snapshot['base'] = {'branch': base_ref,
+                target['base'] = {'name': base_ref}
-                                    'revision': base_revision}
+
-                out_snapshot = os.path.join(self.workdir, snapshot['name'] + '.snapshot')
+                if target_component_type == 'runtime':
-                f = open(out_snapshot, 'w')
+                    target_components = runtime_components_by_arch[architecture]
-                json.dump(snapshot, f, indent=4, sort_keys=True)
+                else:
-                f.close()
+                    target_components = devel_components_by_arch[architecture]
-                print "Created: %s" % (out_snapshot, )
+                    
+                contents = []
+                for component in target_components:
+                    name = component['name']
+                    component_ref = {'name': name}
+                    if target_component_type == 'runtime':
+                        component_ref['trees'] = ['/runtime']
+                    else:
+                        component_ref['trees'] = ['/runtime', '/devel', '/doc']
+                    contents.append(component_ref)
+                target['contents'] = contents
+        out_targets = os.path.join(self.workdir, '%s-targets.json' % (name_prefix, ))
+        f = open(out_targets, 'w')
+        json.dump(targets_json, f, indent=4, sort_keys=True)
+        f.close()
+        print "Created: %s" % (out_targets, )
+
+        out_components = os.path.join(self.workdir, '%s-components.json' % (name_prefix, ))
+        f = open(out_components, 'w')
+        for component in components_by_name.itervalues():
+            del component['name']
+        json.dump(components_by_name, f, indent=4, sort_keys=True)
+        f.close()
+        print "Created: %s" % (out_components, )
         
 builtins.register(OstbuildResolve)

src/ostbuild/pyostbuild/builtin_status.py

@@ -42,14 +42,14 @@ class OstbuildStatus(builtins.Builtin):
         args = parser.parse_args(argv)
 
         self.parse_config()
-        self.manifest = json.load(open(args.manifest))
+        self.parse_components_and_targets()
 
-        for component in self.manifest['components']:
+        for name,component in self.components.iteritems():
-            branch = buildutil.manifest_buildname(self.manifest, component)
+            buildname = 'components/%s' % (name, )
             build_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                   'show',
                                                   '--print-metadata-key=ostbuild-artifact-version',
-                                                  branch],
+                                                  buildname],
                                                  none_on_error=True)
             if build_revision is None:
                 build_revision = '(not built)'
@@ -57,7 +57,7 @@ class OstbuildStatus(builtins.Builtin):
                 build_status = '(needs build)'
             else:
                 build_status = 'ok'
-            sys.stdout.write('{:<40} {:<95} {:<10}\n'.format(component['name'],
+            sys.stdout.write('{:<40} {:<95} {:<10}\n'.format(name,
                                                              build_revision, build_status))
     
 builtins.register(OstbuildStatus)

src/ostbuild/pyostbuild/builtins.py

@@ -20,6 +20,7 @@
 import os
 import sys
 import argparse
+import json
 
 from . import ostbuildrc
 from .ostbuildlog import log, fatal
@@ -41,6 +42,18 @@ class Builtin(object):
             fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
         self.patchdir = os.path.join(self.workdir, 'patches')
 
+    def parse_manifest(self):
+        self.manifest_path = ostbuildrc.get_key('manifest')
+        self.manifest = json.load(open(self.manifest_path))
+        self.name_prefix = self.manifest['name-prefix']
+
+    def parse_components_and_targets(self):
+        self.parse_manifest()
+        components_path = os.path.join(self.workdir, '%s-components.json' % (self.name_prefix, ))
+        self.components = json.load(open(components_path))
+        targets_path = os.path.join(self.workdir, '%s-targets.json' % (self.name_prefix, ))
+        self.targets = json.load(open(targets_path))
+
     def execute(self, args):
         raise NotImplementedError()
 

src/ostbuild/pyostbuild/fileutil.py

@@ -0,0 +1,26 @@
+#
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+
+def ensure_dir(path):
+    if not os.path.isdir(path):
+        os.makedirs(path)
+
+def ensure_parent_dir(path):
+    ensure_dir(os.path.dirname(path))

src/ostbuild/pyostbuild/vcs.py

@@ -69,3 +69,7 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
     if tmp_dest != dest:
         os.rename(tmp_dest, dest)
     return dest
+
+def clean(keytype, checkoutdir):
+    assert keytype == 'git'
+    run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)