ostbuild: Major rework

Makefile-ostbuild.am

@@ -22,18 +22,23 @@ bin_SCRIPTS += ostbuild
 pyostbuilddir=$(libdir)/ostbuild/pyostbuild
 pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/buildutil.py		\
-	src/ostbuild/pyostbuild/builtin_build.py	\
+	src/ostbuild/pyostbuild/builtin_bin_to_src.py	\
+	src/ostbuild/pyostbuild/builtin_branch_prefix.py	\
+	src/ostbuild/pyostbuild/builtin_build_components.py	\
 	src/ostbuild/pyostbuild/builtin_checkout.py	\
+	src/ostbuild/pyostbuild/builtin_compose.py	\
 	src/ostbuild/pyostbuild/builtin_chroot_compile_one.py	\
-	src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py	\
 	src/ostbuild/pyostbuild/builtin_compile_one.py	\
-	src/ostbuild/pyostbuild/builtin_query_content.py	\
+	src/ostbuild/pyostbuild/builtin_pull_components.py	\
+	src/ostbuild/pyostbuild/builtin_prefix.py	\
 	src/ostbuild/pyostbuild/builtin_resolve.py	\
+	src/ostbuild/pyostbuild/builtin_modify_snapshot.py	\
 	src/ostbuild/pyostbuild/builtin_status.py	\
 	src/ostbuild/pyostbuild/builtins.py		\
 	src/ostbuild/pyostbuild/filemonitor.py		\
 	src/ostbuild/pyostbuild/fileutil.py		\
 	src/ostbuild/pyostbuild/__init__.py		\
+	src/ostbuild/pyostbuild/jsondb.py		\
 	src/ostbuild/pyostbuild/kvfile.py		\
 	src/ostbuild/pyostbuild/main.py			\
 	src/ostbuild/pyostbuild/mainloop.py		\

gnomeos/3.4/gnomeos-3.4-src.json

@@ -1,7 +1,10 @@
 {
-  "name-prefix": "gnomeos-3.4",
+  "prefix": "gnomeos-3.4",
   "architectures": ["i686"],
-  "base-prefix": "yocto/gnomeos-3.4",
+  "base": {
+            "name": "yocto",
+	    "src": "cgwalters:poky"
+          },
 
   "config-opts": ["--disable-static", "--disable-silent-rules"],
 
@@ -25,13 +28,15 @@
 
   "patches": {"src": "git:/src/ostree",
               "branch": "wip/ostbuild-v2",
-              "prefix": "gnomeos/3.4"},
+              "subdir": "gnomeos/3.4"},
 
   "components": [
-   		{"src": "cgwalters:ginitscripts"},
+   		{"src": "cgwalters:ginitscripts",
+		 "noarch": true},
 
    		{"src": "gnome:gtk-doc-stub",
-		 "component": "devel"},
+		 "component": "devel",
+		 "noarch": true},
 
 		{"src": "git:git://github.com/atgreen/libffi.git"},
 

gnomeos/README-build.md

@@ -16,6 +16,9 @@ base "runtime", and one "devel" with all of the development tools like
 gcc.  We then import that into an OSTree branch
 e.g. "bases/yocto/gnomeos-3.4-i686-devel".
 
+At present, it's still (mostly) possible to put this data on an ext4
+filesystem and boot into it.
+
 We also have a Yocto recipe "ostree-native" which generates (as you
 might guess) a native binary of ostree.  That binary is used to import
 into an "archive mode" OSTree repository.  You can see it in
@@ -26,26 +29,59 @@ can use "ostbuild" which uses "linux-user-chroot" to chroot inside,
 run a build on a source tree, and outputs binaries, which we then add
 to the build tree for the next module, and so on.
 
+The final result of all of this is that the OSTree repository gains
+new commits (which can be downloaded by clients), while still
+retaining old build history.
+
+Yocto details
+-------------
+
+I have a branch of Yocto here:
+
+https://github.com/cgwalters/poky
+
+It has a collection of patches on top of the "Edison" release of
+Yocto, some of which are hacky, others upstreamable.  The most
+important part though are the modifications to commit the generated
+root filesystem into OSTree.
+
 ostbuild details
 ----------------
 
 The simple goal of ostbuild is that it only takes as input a
-"manifest" which is basically just a list of components to build.  A
+"manifest" which is basically just a list of components to build.  You
-component is a pure metadata file which includes the git repository
+can see this here:
+
+http://git.gnome.org/browse/ostree/tree/gnomeos/3.4/gnomeos-3.4-src.json
+
+A component is a pure metadata file which includes the git repository
 URL and branch name, as well as ./configure flags (--enable-foo).
 
 There is no support for building from "tarballs" - I want the ability
 to review all of the code that goes in, and to efficiently store
-source code updates.
+source code updates.  It's also just significantly easier from an
+implementation perspective, versus having to maintain a version
+control abstraction layer.
 
 The result of a build of a component is an OSTree branch like
 "artifacts/gnomeos-3.4-i686-devel/libxslt/master".  Then, a "compose"
 process merges together the individual filesystem trees into the final
 branches (e.g. gnomeos-3.4-i686-devel).
 
+Doing local builds
+------------------
+
+This is where you want to modify one (or a few) components on top of
+what comes from the ostree.gnome.org server, and test the result
+locally.  I'm working on this.
+
 Doing a full build on your system
 ---------------------------------
 
+Following this process is equivalent to what we have set up on the
+ostree.gnome.org build server.  It will generate a completely new
+repository.
+
 srcdir=/src
 builddir=/src/build
 

gnomeos/gnomeos-install.sh

@@ -46,17 +46,19 @@ if ! test -d /ostree/repo/objects; then
     mkdir -p /ostree
 
     $SRCDIR/gnomeos-setup.sh /ostree
+
+    cd /ostree
+
+    ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel}
+    ostree-pull --repo=repo gnome
+    for branch in runtime devel; do
+        ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch}
+    done
+    ln -sf ${BRANCH_PREFIX}runtime current
+
+    cd ${WORKDIR}
 fi
 
-cd /ostree
-
-ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel}
-ostree-pull --repo=repo gnome
-for branch in runtime devel; do
-    ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch}
-done
-ln -sf ${BRANCH_PREFIX}runtime current
-
 uname=$(uname -r)
 
 if test -d /etc/grub.d; then

gnomeos/gnomeos-qemu-create.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; -*-
+# Create ostree-qemu.img file in the current directory, suitable
+# for booting via qemu.
+#
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+set -x
+
+SRCDIR=`dirname $0`
+WORKDIR=`pwd`
+
+if test $(id -u) != 0; then
+    cat <<EOF
+This script should be run as root.
+EOF
+    exit 1
+fi
+
+usage () {
+    echo "$0"
+    exit 1
+}
+
+OBJ=ostree-qemu.img
+if ! test -f ${OBJ}; then
+    # Hardcoded 6 gigabyte filesystem size here; 6 gigabytes should be
+    # enough for everybody.
+    qemu-img create $OBJ 6G
+    mkfs.ext4 -q -F $OBJ
+fi
+
+mkdir -p fs
+umount fs || true
+sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
+mount -o loop ostree-qemu.img fs
+
+cd fs
+
+if ! test -d ./ostree/repo/objects; then
+    mkdir -p ./ostree
+    
+    $SRCDIR/gnomeos-setup.sh $(pwd)/ostree
+fi
+
+mkdir -p ./run ./home ./root ./sys
+mkdir -p ./tmp 
+chmod 01777 ./tmp
+
+mkdir -p $(pwd)/ostree/modules
+rsync -a -H -v --delete /ostree/modules/ ./ostree/modules/
+
+cd ..
+umount fs
+
+cat << EOF
+Next, run gnomeos-qemu-pull.sh to copy data.
+EOF

gnomeos/gnomeos-qemu-pull.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; -*-
+# Run built image in QEMU 
+#
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+set -x
+
+SRCDIR=`dirname $0`
+WORKDIR=`pwd`
+
+if test $(id -u) != 0; then
+    cat <<EOF
+This script should be run as root.
+EOF
+    exit 1
+fi
+
+usage () {
+    cat <<EOF 
+usage: $0 SRC_REPO_PATH CURRENT_REF [REFS...]
+EOF
+    exit 1
+}
+
+SRC_REPO_PATH=$1
+test -n "$SRC_REPO_PATH" || usage
+shift
+
+CURRENT_REF=$1
+test -n "$CURRENT_REF" || usage
+shift
+
+if ! test -f ostree-qemu.img; then
+    cat <<EOF
+ostree-qemu.img not found; You must run gnomeos-qemu-create.sh first
+EOF
+fi
+
+mkdir -p fs
+umount fs || true
+sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
+mount -o loop ostree-qemu.img fs
+
+cd fs
+ostree --repo=${SRC_REPO_PATH} local-clone ./ostree/repo ${CURRENT_REF} "$@"
+
+cd ostree
+ostree --repo=./repo checkout --atomic-retarget ${CURRENT_REF}
+ln -sf ${CURRENT_REF} ${CURRENT_REF}.tmplink
+mv -T ${CURRENT_REF}.tmplink current
+
+cd ${WORKDIR}
+umount fs

gnomeos/gnomeos-qemu-run.sh

@@ -33,62 +33,16 @@ EOF
 fi
 
 usage () {
-    echo "$0 OSTREE_REPO_PATH"
+    echo "$0"
     exit 1
 }
 
-ARCH=i686
+if ! test -f ostree-qemu.img; then
-BRANCH_PREFIX="gnomeos-3.4-${ARCH}-"
+    echo "ostree-qemu.img not found; You must run gnomeos-qemu-create.sh first"
-
-OBJ=gnomeos-fs.img
-if ! test -f ${OBJ}; then
-    cat <<EOF
-Create gnomeos-fs.img like this:
-$ qemu-img create gnomeos-fs.img 6G
-$ mkfs.ext4 -q -F gnomeos-fs.img
-EOF
-    exit 1
 fi
 
-if ! test -d ${WORKDIR}/repo/objects; then
-    cat <<EOF
-No ./repo/objects found
-EOF
-    exit 1
-fi
-
-mkdir -p fs
-umount fs || true
-sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
-mount -o loop gnomeos-fs.img fs
-
-cd fs
-
-TOPROOT_BIND_MOUNTS="home root tmp"
-
-for d in $TOPROOT_BIND_MOUNTS; do
-    if ! test -d $d; then
-        mkdir -m 0755 $d
-    fi
-done
-chmod a=rwxt tmp
-
-if ! test -d ostree; then
-    mkdir -p ostree
-
-    $SRCDIR/gnomeos-setup.sh $(pwd)/ostree
-fi
-
-rsync -a -H -v ${WORKDIR}/repo ${WORKDIR}/current ${WORKDIR}/modules ${WORKDIR}/var ${WORKDIR}/gnomeos-3.4-* ./ostree
-
 current_uname=$(uname -r)
 
-cd ${WORKDIR}
-
-sync
-umount fs
-rmdir fs
-
 ARGS="$@"
 if ! echo $ARGS | grep -q 'ostree='; then
     ARGS="ostree=current $ARGS"
@@ -97,4 +51,4 @@ ARGS="rd.plymouth=0 root=/dev/sda $ARGS"
 KERNEL=/boot/vmlinuz-${current_uname}
 INITRD_ARG="-initrd /boot/initramfs-ostree-${current_uname}.img"
 
-exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda gnomeos-fs.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio
+exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda ostree-qemu.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio

gnomeos/gnomeos-setup.sh

@@ -58,6 +58,11 @@ fi
 
 mkdir -p ./var/lib/gdm
 chown 2:2 ./var/lib/gdm
+mkdir -p ./var/log/gdm
+chown 2:2 ./var/log/gdm
+chmod 01770 ./var/log/gdm
+
+mkdir -p ./var/lib/AccountsService
 
 touch ./var/shadow
 chmod 0600 ./var/shadow

src/ostbuild/pyostbuild/buildutil.py

@@ -40,7 +40,7 @@ def parse_src_key(srckey):
     if idx < 0:
         raise ValueError("Invalid SRC uri=%s" % (srckey, ))
     keytype = srckey[:idx]
-    if keytype not in ['git']:
+    if keytype not in ['git', 'dirty-git']:
         raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
     uri = srckey[idx+1:]
     return (keytype, uri)

src/ostbuild/pyostbuild/builtin_bin_to_src.py

@@ -0,0 +1,76 @@
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildBinToSrc(builtins.Builtin):
+    name = "bin-to-src"
+    short_description = "Turn a binary snapshot into a source snapshot"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def bin_snapshot_to_src(self, bin_snapshot):
+        del bin_snapshot['00ostree-bin-snapshot-version']
+
+        src_snapshot = dict(bin_snapshot)
+        src_snapshot['00ostree-src-snapshot-version'] = 0
+
+        all_architectures = src_snapshot['architecture-buildroots'].keys()
+        # Arbitrarily take first architecture
+        first_arch = all_architectures[0]
+
+        bin_components = src_snapshot['components']
+        src_components = {}
+        src_snapshot['components'] = src_components
+        for archname,rev in bin_components.iteritems():
+            (name, arch) = archname.rsplit('/', 1)
+            if arch != first_arch:
+                continue
+            meta = dict(self.get_component_meta_from_revision(rev))
+            del meta['name']
+            src_components[name] = meta
+
+        for target in src_snapshot['targets']:
+            del target['base']['ostree-revision']
+
+        return src_snapshot
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('--prefix')
+        parser.add_argument('--bin-snapshot')
+
+        args = parser.parse_args(argv)
+        self.parse_config()
+        self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
+
+        snapshot = self.bin_snapshot_to_src(self.bin_snapshot)
+        json.dump(snapshot, sys.stdout, indent=4, sort_keys=True)
+
+builtins.register(OstbuildBinToSrc)

src/ostbuild/pyostbuild/builtin_branch_prefix.py

@@ -28,40 +28,44 @@ from .ostbuildlog import log, fatal
 from .subprocess_helpers import run_sync, run_sync_get_output
 from . import buildutil
 
-class OstbuildQueryContent(builtins.Builtin):
+class OstbuildBranchPrefix(builtins.Builtin):
-    name = "query-content"
+    name = "prefix-branch"
-    short_description = "Output metadata from a component"
+    short_description = "Copy current source snapshot to new prefix"
 
     def __init__(self):
         builtins.Builtin.__init__(self)
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--branch', required=True)
+        parser.add_argument('--prefix')
-        parser.add_argument('--component')
+        parser.add_argument('--src-snapshot')
+        parser.add_argument('newprefix')
 
         args = parser.parse_args(argv)
-        self.args = args
         self.parse_config()
+        self.parse_snapshot(args.prefix, args.src_snapshot)
 
-        contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+        if args.newprefix == self.prefix:
-                                                  'cat', args.branch, 'contents.json'])
+            fatal("Specified prefix %r matches active prefix" % (args.newprefix, ))
+
+        db = self.create_db('src-snapshot', prefix=args.newprefix)
+
+        log("Branching from source snapshot %r" % (self.snapshot_path, ))
+
+        orig_prefix = self.prefix
+
+        forked_snapshot = dict(self.snapshot)
+        forked_snapshot['prefix'] = args.newprefix
+
+        for target in forked_snapshot['targets']:
+            name = target['name']
+            if not name.startswith(orig_prefix):
+                fatal("Mismatched name %r in snapshot" % (name, ))
+            target['name'] = name.replace(orig_prefix, args.newprefix)
         
-        if args.component is None:
+        db.store(forked_snapshot)
-            sys.stdout.write(contents_json_text)
-        else:
-            contents = json.loads(contents_json_text)
-            contents_list = contents['contents']
-            found = False
-            for content in contents_list:
-                if content['name'] != args.component:
-                    found = True
-                    break
-            if not found:
-                fatal("Unknown component '%s'" % (args.component, ))
-            ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                     'cat', content['ostree-revision'],
-                                                     '/_ostbuild-meta.json'])
-            sys.stdout.write(ostbuildmeta_json)
 
-builtins.register(OstbuildQueryContent)
+        run_sync(['ostbuild', 'prefix', args.newprefix],
+                 log_initiation=False, log_success=False)
+
+builtins.register(OstbuildBranchPrefix)

src/ostbuild/pyostbuild/builtin_build_components.py

@@ -36,9 +36,9 @@ from . import vcs
 class BuildOptions(object):
     pass
 
-class OstbuildBuild(builtins.Builtin):
+class OstbuildBuildComponents(builtins.Builtin):
-    name = "build"
+    name = "build-components"
-    short_description = "Rebuild all artifacts from the given manifest"
+    short_description = "Build multiple components from given source snapshot"
 
     def __init__(self):
         builtins.Builtin.__init__(self)
@@ -49,7 +49,8 @@ class OstbuildBuild(builtins.Builtin):
             args = ['setarch', architecture]
         else:
             args = []
-        args.extend(['ostbuild', 'chroot-compile-one'])
+        args.extend(['ostbuild', 'chroot-compile-one',
+                     '--snapshot=' + self.snapshot_path])
         return args
 
     def _launch_debug_shell(self, architecture, buildroot, cwd=None):
@@ -60,10 +61,10 @@ class OstbuildBuild(builtins.Builtin):
         run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
         fatal("Exiting after debug shell")
 
-    def _build_one_component(self, name, component):
+    def _build_one_component(self, basename, component, architecture):
         branch = component['branch']
-        architecture = component['architecture']
 
+        name = '%s/%s' % (basename, architecture)
         buildname = 'components/%s' % (name, )
 
         current_vcs_version = component['revision']
@@ -73,7 +74,7 @@ class OstbuildBuild(builtins.Builtin):
                                                      stderr=open('/dev/null', 'w'),
                                                      none_on_error=True)
         if previous_build_version is not None:
-            log("Previous build of '%s' is %s" % (buildname, previous_build_version))
+            log("Previous build of '%s' is %s" % (name, previous_build_version))
 
             previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                           'cat', previous_build_version,
@@ -92,19 +93,15 @@ class OstbuildBuild(builtins.Builtin):
             else:
                 log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
         else:
-            log("No previous build for '%s' found" % (buildname, ))
+            log("No previous build for '%s' found" % (name, ))
 
         checkoutdir = os.path.join(self.workdir, 'src')
-        component_src = os.path.join(checkoutdir, name)
+        component_src = os.path.join(checkoutdir, basename)
-        run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir)
+        run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
+                  '--clean', '--overwrite', basename], cwd=checkoutdir)
 
         artifact_meta = dict(component)
 
-        metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
-        f = open(metadata_path, 'w')
-        json.dump(artifact_meta, f, indent=4, sort_keys=True)
-        f.close()
-
         logdir = os.path.join(self.workdir, 'logs', name)
         fileutil.ensure_dir(logdir)
         log_path = os.path.join(logdir, 'compile.log')
@@ -116,7 +113,7 @@ class OstbuildBuild(builtins.Builtin):
         log("Logging to %s" % (log_path, ))
         f = open(log_path, 'w')
         chroot_args = self._get_ostbuild_chroot_args(architecture)
-        chroot_args.extend(['--pristine', '--name=' + name])
+        chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
         if self.buildopts.shell_on_failure:
             ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
             if ecode != 0:
@@ -146,69 +143,48 @@ class OstbuildBuild(builtins.Builtin):
             os.unlink(statoverride_path)
         return True
 
-    def _compose(self, target):
+    def _resolve_refs(self, refs):
-        base_name = 'bases/%s' % (target['base']['name'], )
-        branch_to_rev = {}
-        branch_to_subtrees = {}
-
-        contents = [base_name]
-        branch_to_subtrees[base_name] = ['/']
-        base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                             'rev-parse', base_name])
-
-        branch_to_rev[base_name] = base_revision
-
         args = ['ostree', '--repo=' + self.repo, 'rev-parse']
-        for component in target['contents']:
+        args.extend(refs)
-            name = component['name']
+        output = run_sync_get_output(args)
-            contents.append(name)
+        return output.split('\n')
-            args.append('components/%s' % (name, ))
-            branch_to_subtrees[name] = component['trees']
-        branch_revs_text = run_sync_get_output(args)
-        branch_revs = branch_revs_text.split('\n')
 
-        for (content, rev) in zip(target['contents'], branch_revs):
+    def _save_bin_snapshot(self, components, component_architectures):
-            name = content['name']
+        bin_snapshot = dict(self.snapshot)
-            branch_to_rev[name] = rev
-        
-        compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
-        if os.path.isdir(compose_rootdir):
-            shutil.rmtree(compose_rootdir)
-        os.mkdir(compose_rootdir)
 
-        resolved_base = dict(target['base'])
+        del bin_snapshot['00ostree-src-snapshot-version']
-        resolved_base['ostree-revision'] = base_revision
+        bin_snapshot['00ostree-bin-snapshot-version'] = 0
-        resolved_contents = list(target['contents'])
-        for component in resolved_contents:
-            component['ostree-revision'] = branch_to_rev[component['name']]
-        metadata = {'source': 'ostbuild compose v0',
-                    'base': resolved_base, 
-                    'contents': resolved_contents}
 
-        for branch in contents:
+        for target in bin_snapshot['targets']:
-            branch_rev = branch_to_rev[branch]
+            base = target['base']
-            subtrees = branch_to_subtrees[branch]
+            base_name = 'bases/%s' % (base['name'], )
-            for subtree in subtrees:
+            base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                run_sync(['ostree', '--repo=' + self.repo,
+                                                 'rev-parse', base_name])
-                          'checkout', '--user-mode',
+            base['ostree-revision'] = base_revision
-                          '--union', '--subpath=' + subtree,
-                          branch_rev, compose_rootdir])
 
-        contents_path = os.path.join(compose_rootdir, 'contents.json')
+        component_refs = []
-        f = open(contents_path, 'w')
+        for name in components.iterkeys():
-        json.dump(metadata, f, indent=4, sort_keys=True)
+            for architecture in component_architectures[name]:
-        f.close()
+                component_refs.append('components/%s/%s' % (name, architecture))
 
-        run_sync(['ostree', '--repo=' + self.repo,
+        new_components = {}
-                  'commit', '-b', target['name'], '-s', 'Compose',
+        resolved_refs = self._resolve_refs(component_refs)
-                  '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
+        for name,rev in zip(components.iterkeys(), resolved_refs):
-                  '--skip-if-unchanged'], cwd=compose_rootdir)
+            for architecture in component_architectures[name]:
+                archname = '%s/%s' % (name, architecture)
+                new_components[archname] = rev
+
+        bin_snapshot['components'] = new_components
+
+        path = self.get_bin_snapshot_db().store(bin_snapshot)
+        log("Binary snapshot: %s" % (path, ))
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--skip-built', action='store_true')
-        parser.add_argument('--recompose', action='store_true')
+        parser.add_argument('--prefix')
-        parser.add_argument('--skip-compose', action='store_true')
+        parser.add_argument('--src-snapshot')
+        parser.add_argument('--compose', action='store_true')
         parser.add_argument('--start-at')
         parser.add_argument('--shell-on-failure', action='store_true')
         parser.add_argument('--debug-shell', action='store_true')
@@ -218,17 +194,28 @@ class OstbuildBuild(builtins.Builtin):
         self.args = args
         
         self.parse_config()
-        self.parse_snapshot()
+        self.parse_snapshot(args.prefix, args.src_snapshot)
+
+        log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
 
         self.buildopts = BuildOptions()
         self.buildopts.shell_on_failure = args.shell_on_failure
         self.buildopts.skip_built = args.skip_built
 
+        required_components = {}
+        component_architectures = {}
+        for target in self.snapshot['targets']:
+            for tree_content in target['contents']:
+                (name, arch) = tree_content['name'].rsplit('/', 1)
+                required_components[name] = self.snapshot['components'][name]
+                if name not in component_architectures:
+                    component_architectures[name] = set([arch])
+                else:
+                    component_architectures[name].add(arch)
+
         build_component_order = []
-        if args.recompose:
+        if len(args.components) == 0:
-            pass
+            tsorted = buildutil.tsort_components(required_components, 'build-depends')
-        elif len(args.components) == 0:
-            tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends')
             tsorted.reverse()
             build_component_order = tsorted
         else:
@@ -253,11 +240,14 @@ class OstbuildBuild(builtins.Builtin):
             start_at_index = 0
 
         for component_name in build_component_order[start_at_index:]:
-            component = self.snapshot['components'].get(component_name)
+            component = required_components[component_name]
-            self._build_one_component(component_name, component)
+            architectures = component_architectures[component_name]
+            for architecture in architectures:
+                self._build_one_component(component_name, component, architecture)
 
-        if not args.skip_compose:
+        self._save_bin_snapshot(required_components, component_architectures)   
-            for target in self.snapshot['targets']:
+
-                self._compose(target)
+        if args.compose:
+            run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix])
         
-builtins.register(OstbuildBuild)
+builtins.register(OstbuildBuildComponents)

src/ostbuild/pyostbuild/builtin_checkout.py

@@ -40,6 +40,9 @@ class OstbuildCheckout(builtins.Builtin):
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--overwrite', action='store_true')
+        parser.add_argument('--prefix')
+        parser.add_argument('--snapshot')
+        parser.add_argument('-a', '--active-tree', action='store_true')
         parser.add_argument('--clean', action='store_true')
         parser.add_argument('components', nargs='*')
 
@@ -47,28 +50,41 @@ class OstbuildCheckout(builtins.Builtin):
         self.args = args
         
         self.parse_config()
-        self.parse_snapshot()
 
         if len(args.components) > 0:
             checkout_components = args.components
         else:
             checkout_components = [os.path.basename(os.getcwd())]
 
+        if args.active_tree:
+            self.parse_active_branch()
+        else:
+            self.parse_snapshot(args.prefix, args.snapshot)
+
         for component_name in checkout_components:
             found = False
-            component = self.snapshot['components'].get(component_name)
+            component = self.get_component_meta(component_name)
-            if component is None:
-                fatal("Unknown component %r" % (component_name, ))
             (keytype, uri) = buildutil.parse_src_key(component['src'])
             checkoutdir = os.path.join(os.getcwd(), component_name)
             fileutil.ensure_parent_dir(checkoutdir)
 
-            component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+            is_dirty = (keytype == 'dirty-git')
-                                                 component['revision'],
-                                                 overwrite=args.overwrite)
 
+            if is_dirty:
+                # Kind of a hack, but...
+                if os.path.lexists(checkoutdir):
+                    os.unlink(checkoutdir)
+                os.symlink(uri, checkoutdir)
+            else:
+                vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+                                     component['revision'],
+                                     overwrite=args.overwrite)
+                
             if args.clean:
-                vcs.clean(keytype, checkoutdir)
+                if is_dirty:
+                    log("note: ignoring --clean argument due to \"dirty-git:\" specification")
+                else:
+                    vcs.clean(keytype, checkoutdir)
 
             patches = component.get('patches')
             if patches is not None:
@@ -78,15 +94,20 @@ class OstbuildCheckout(builtins.Builtin):
                                      self.patchdir, patches['branch'],
                                      overwrite=True)
 
-                patch_prefix = patches.get('prefix', None)
+                patch_subdir = patches.get('subdir', None)
-                if patch_prefix is not None:
+                if patch_subdir is not None:
-                    patchdir = os.path.join(self.patchdir, patch_prefix)
+                    patchdir = os.path.join(self.patchdir, patch_subdir)
                 else:
                     patchdir = self.patchdir
                 for patch in patches['files']:
                     patch_path = os.path.join(patchdir, patch)
                     run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
+
+            metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json')
+            f = open(metadata_path, 'w')
+            json.dump(component, f, indent=4, sort_keys=True)
+            f.close()
         
-            print "Checked out: %r" % (component_src, )
+            log("Checked out: %r" % (checkoutdir, ))
         
 builtins.register(OstbuildCheckout)

src/ostbuild/pyostbuild/builtin_chroot_compile_one.py

@@ -18,7 +18,9 @@
 import os,sys,re,subprocess,tempfile,shutil
 from StringIO import StringIO
 import argparse
+import time
 import json
+import hashlib
 
 from . import builtins
 from . import buildutil
@@ -31,47 +33,133 @@ class OstbuildChrootCompileOne(builtins.Builtin):
     name = "chroot-compile-one"
     short_description = "Build artifacts from the current source directory in a chroot"
 
-    def _compose_buildroot(self, component_name, dirpath):
+    def _resolve_refs(self, refs):
+        args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+        args.extend(refs)
+        output = run_sync_get_output(args)
+        return output.split('\n')
+
+    def _compose_buildroot(self, component_name, architecture):
+        starttime = time.time()
+
+        rootdir_prefix = os.path.join(self.workdir, 'roots')
+        rootdir = os.path.join(rootdir_prefix, component_name)
+        fileutil.ensure_parent_dir(rootdir)
+
+        # Clean up any leftover root dir
+        rootdir_tmp = rootdir + '.tmp'
+        if os.path.isdir(rootdir_tmp):
+            shutil.rmtree(rootdir_tmp)
+
         components = self.snapshot['components']
         dependencies = buildutil.build_depends(component_name, components)
         component = components.get(component_name)
 
-        base_devel_name = 'bases/%s-%s-%s' % (self.snapshot['base-prefix'],
+        buildroots = self.snapshot['architecture-buildroots']
-                                              component['architecture'],
+        base_devel_name = 'bases/' + buildroots[architecture]
-                                              'devel')
+
+        refs_to_resolve = [base_devel_name]
         checkout_trees = [(base_devel_name, '/')]
         for dependency_name in dependencies:
-            buildname = 'components/%s' % (dependency_name, )
+            buildname = 'components/%s/%s' % (dependency_name, architecture)
+            refs_to_resolve.append(buildname)
             checkout_trees.append((buildname, '/runtime'))
             checkout_trees.append((buildname, '/devel'))
 
+        resolved_refs = self._resolve_refs(refs_to_resolve)
+        ref_to_rev = {}
+        for ref,rev in zip(refs_to_resolve, resolved_refs):
+            ref_to_rev[ref] = rev
+
         link_cache_dir = os.path.join(self.workdir, 'link-cache')
         fileutil.ensure_dir(link_cache_dir)
 
-        for (branch, rootpath) in checkout_trees:
+        sha = hashlib.sha256()
-            run_sync(['ostree', '--repo=' + self.repo,
+
-                      'checkout', '--user-mode', '--link-cache=' + link_cache_dir,
+        (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
-                      '--union', '--subpath=' + rootpath,
+        f = os.fdopen(fd, 'w')
-                      branch, dirpath])
+        for (branch, subpath) in checkout_trees:
+            f.write(ref_to_rev[branch])
+            f.write('\0')
+            f.write(subpath)
+            f.write('\0')
+        f.close()
+
+        f = open(tmppath)
+        buf = f.read(8192)
+        while buf != '':
+            sha.update(buf)
+            buf = f.read(8192)
+        f.close()
+
+        new_root_cacheid = sha.hexdigest()
+
+        rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid')
+
+        if os.path.isdir(rootdir):
+            if os.path.isfile(rootdir_cache_path):
+                f = open(rootdir_cache_path)
+                prev_cache_id = f.read().strip()
+                f.close()
+                if prev_cache_id == new_root_cacheid:
+                    log("Reusing previous buildroot")
+                    os.unlink(tmppath)
+                    return rootdir
+                else:
+                    log("New buildroot differs from previous")
+
+            shutil.rmtree(rootdir)
+
+        os.mkdir(rootdir_tmp)
+
+        if len(checkout_trees) > 0:
+            log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees),
+                                                                    checkout_trees[-1][0]))
+
+        link_cache_dir = os.path.join(self.workdir, 'link-cache')
+        fileutil.ensure_dir(link_cache_dir)
+
+        run_sync(['ostree', '--repo=' + self.repo,
+                  'checkout', '--link-cache=' + link_cache_dir,
+                  '--user-mode', '--union', '--from-stdin', rootdir_tmp],
+                 stdin=open(tmppath))
+
+        os.unlink(tmppath);
+
+        builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
+        os.mkdir(builddir_tmp)
+        os.mkdir(os.path.join(builddir_tmp, 'source'))
+        os.mkdir(os.path.join(builddir_tmp, 'results'))
+        os.rename(rootdir_tmp, rootdir)
+
+        f = open(rootdir_cache_path, 'w')
+        f.write(new_root_cacheid)
+        f.write('\n')
+        f.close()
+
+        endtime = time.time()
+        log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
+
+        return rootdir
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--pristine', action='store_true')
+        parser.add_argument('--prefix')
+        parser.add_argument('--snapshot', required=True)
         parser.add_argument('--name')
+        parser.add_argument('--arch', required=True)
         parser.add_argument('--debug-shell', action='store_true')
         
         args = parser.parse_args(argv)
 
         self.parse_config()
-        self.parse_snapshot()
+        self.parse_snapshot(args.prefix, args.snapshot)
 
         if args.name:
             component_name = args.name
         else:
-            cwd = os.getcwd()
+            component_name = self.get_component_from_cwd()
-            parent = os.path.dirname(cwd)
-            parentparent = os.path.dirname(parent)
-            component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
 
         components = self.snapshot['components']
         component = components.get(component_name)
@@ -93,37 +181,16 @@ class OstbuildChrootCompileOne(builtins.Builtin):
             shutil.rmtree(child_tmpdir)
         fileutil.ensure_dir(child_tmpdir)
 
-        resultdir = os.path.join(self.workdir, 'results', component_name)
+        resultdir = os.path.join(self.workdir, 'results', component_name, args.arch)
         if os.path.isdir(resultdir):
             shutil.rmtree(resultdir)
         fileutil.ensure_dir(resultdir)
         
-        rootdir_prefix = os.path.join(workdir, 'roots')
+        rootdir = self._compose_buildroot(component_name, args.arch)
-        fileutil.ensure_dir(rootdir_prefix)
-        rootdir = os.path.join(rootdir_prefix, component_name)
-        fileutil.ensure_parent_dir(rootdir)
-        if os.path.isdir(rootdir):
-            shutil.rmtree(rootdir)
-        
-        rootdir_tmp = rootdir + '.tmp'
-        builddir = os.path.join(rootdir, 'ostbuild');
-        if os.path.isdir(rootdir_tmp):
-            shutil.rmtree(rootdir_tmp)
-        os.mkdir(rootdir_tmp)
-            
-        self._compose_buildroot(component_name, rootdir_tmp)
 
-        child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
-        run_sync(child_args)
-
-        builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
-        os.mkdir(builddir_tmp)
-        os.mkdir(os.path.join(builddir_tmp, 'source'))
-        os.mkdir(os.path.join(builddir_tmp, 'results'))
-        os.rename(rootdir_tmp, rootdir)
         log("Checked out buildroot: %s" % (rootdir, ))
         
-        sourcedir=os.path.join(builddir, 'source', component_name)
+        sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name)
         fileutil.ensure_dir(sourcedir)
         
         output_metadata = open('_ostbuild-meta.json', 'w')
@@ -148,7 +215,6 @@ class OstbuildChrootCompileOne(builtins.Builtin):
                                'compile-one',
                                '--ostbuild-resultdir=/ostbuild/results',
                                '--ostbuild-meta=_ostbuild-meta.json'])
-            child_args.extend(self.metadata.get('config-opts', []))
         env_copy = dict(buildutil.BUILD_ENV)
         env_copy['PWD'] = chroot_sourcedir
         run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)

src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py

@@ -1,47 +0,0 @@
-# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-import os,sys,re,subprocess,tempfile,shutil
-from StringIO import StringIO
-import argparse
-import json
-
-from . import builtins
-from . import buildutil
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync
-
-class OstbuildChrootRunTriggers(builtins.Builtin):
-    name = "chroot-run-triggers"
-    short_description = "Run ostree-run-triggers inside a chroot"
-
-    def execute(self, argv):
-        parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('root')
-        
-        args = parser.parse_args(argv)
-
-        child_args = buildutil.get_base_user_chroot_args()
-        child_args.extend(['--mount-proc', '/proc', 
-                           '--mount-bind', '/dev', '/dev',
-                           args.root,
-                           '/usr/bin/ostree-run-triggers'])
-        env_copy = dict(buildutil.BUILD_ENV)
-        env_copy['PWD'] = '/' 
-        run_sync(child_args, env=env_copy)
-                       
-builtins.register(OstbuildChrootRunTriggers)

src/ostbuild/pyostbuild/builtin_compile_one.py

@@ -57,6 +57,8 @@ class OstbuildCompileOne(builtins.Builtin):
 
     def execute(self, args):
         self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
+
+        starttime = time.time()
         
         uname=os.uname()
         kernel=uname[0].lower()
@@ -77,32 +79,28 @@ class OstbuildCompileOne(builtins.Builtin):
                       '--infodir=' + os.path.join(PREFIX, 'share', 'info')]
         self.makeargs = ['make']
 
-        self.ostbuild_resultdir=None
+        self.ostbuild_resultdir='_ostbuild-results'
-        self.ostbuild_meta=None
+        self.ostbuild_meta_path='_ostbuild-meta.json'
 
         chdir = None
+        opt_install = False
 
         for arg in args:
             if arg.startswith('--ostbuild-resultdir='):
                 self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
             elif arg.startswith('--ostbuild-meta='):
-                self.ostbuild_meta=arg[len('--ostbuild-meta='):]
+                self.ostbuild_meta_path=arg[len('--ostbuild-meta='):]
             elif arg.startswith('--chdir='):
                 os.chdir(arg[len('--chdir='):])
-            elif arg.startswith('--'):
-                self.configargs.append(arg)
             else:
                 self.makeargs.append(arg)
         
-        if self.ostbuild_resultdir is None:
+        f = open(self.ostbuild_meta_path)
-            fatal("Must specify --ostbuild-resultdir=")
-        if self.ostbuild_meta is None:
-            fatal("Must specify --ostbuild-meta=")
-
-        f = open(self.ostbuild_meta)
         self.metadata = json.load(f)
         f.close()
 
+        self.configargs.extend(self.metadata.get('config-opts', []))
+
         if self.metadata.get('rm-configure', False):
             configure_path = 'configure'
             if os.path.exists(configure_path):
@@ -235,6 +233,11 @@ class OstbuildCompileOne(builtins.Builtin):
                 except OSError, e:
                     pass
 
+        endtime = time.time()
+
+        log("Compliation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
+        log("Results placed in %s" % (self.ostbuild_resultdir, ))
+
     def _install_and_unlink(self, src, dest):
         statsrc = os.lstat(src)
         dirname = os.path.dirname(dest)

src/ostbuild/pyostbuild/builtin_compose.py

@@ -0,0 +1,110 @@
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import fileutil
+from . import kvfile
+from . import odict
+from . import vcs
+
+class OstbuildCompose(builtins.Builtin):
+    name = "compose"
+    short_description = "Build complete trees from components"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def _compose_one_target(self, bin_snapshot, target):
+        components = bin_snapshot['components']
+        base = target['base']
+        base_name = 'bases/%s' % (base['name'], )
+        base_revision = target['base']['ostree-revision']
+
+        compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
+        if os.path.isdir(compose_rootdir):
+            shutil.rmtree(compose_rootdir)
+        os.mkdir(compose_rootdir)
+
+        compose_contents = [(base_revision, '/')]
+        for tree_content in target['contents']:
+            name = tree_content['name']
+            rev = components[name]
+            subtrees = tree_content['trees']
+            for subpath in subtrees:
+                compose_contents.append((rev, subpath))
+
+        (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+        f = os.fdopen(fd, 'w')
+        for (branch, subpath) in compose_contents:
+            f.write(branch)
+            f.write('\0')
+            f.write(subpath)
+            f.write('\0')
+        f.close()
+
+        link_cache_dir = os.path.join(self.workdir, 'link-cache')
+        fileutil.ensure_dir(link_cache_dir)
+
+        run_sync(['ostree', '--repo=' + self.repo,
+                  'checkout', '--link-cache=' + link_cache_dir,
+                  '--user-mode', '--no-triggers',
+                  '--union', '--from-stdin', compose_rootdir],
+                 stdin=open(tmppath))
+        os.unlink(tmppath)
+
+        contents_path = os.path.join(compose_rootdir, 'contents.json')
+        f = open(contents_path, 'w')
+        json.dump(bin_snapshot, f, indent=4, sort_keys=True)
+        f.close()
+
+        run_sync(['ostree', '--repo=' + self.repo,
+                  'commit', '-b', target['name'], '-s', 'Compose',
+                  '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
+                  '--skip-if-unchanged'], cwd=compose_rootdir)
+        shutil.rmtree(compose_rootdir)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('--prefix')
+        parser.add_argument('--bin-snapshot')
+
+        args = parser.parse_args(argv)
+        self.args = args
+        
+        self.parse_config()
+        self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
+        
+        log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), ))
+
+        for target in self.bin_snapshot['targets']:
+            log("Composing target %r from %u components" % (target['name'],
+                                                            len(target['contents'])))
+            self._compose_one_target(self.bin_snapshot, target)
+        
+builtins.register(OstbuildCompose)

src/ostbuild/pyostbuild/builtin_modify_snapshot.py

@@ -0,0 +1,63 @@
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+from StringIO import StringIO
+import json
+import select,time
+import argparse
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+
+class OstbuildModifySnapshot(builtins.Builtin):
+    name = "modify-snapshot"
+    short_description = "Change the current source snapshot"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('--prefix')
+        parser.add_argument('--src-snapshot')
+
+        args = parser.parse_args(argv)
+        
+        self.parse_config()
+        self.parse_snapshot(args.prefix, args.src_snapshot)
+
+        component_name = self.get_component_from_cwd()
+        current_meta = self.get_component_meta(component_name)
+       
+        new_snapshot = dict(self.snapshot)
+        new_meta = dict(current_meta)
+        if 'patches' in new_meta:
+            del new_meta['patches']
+        new_meta['src'] = "dirty-git:%s" % (os.getcwd(), )
+        new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD'])
+
+        new_snapshot['components'][component_name] = new_meta
+
+        db = self.get_src_snapshot_db()
+        path = db.store(new_snapshot)
+        log("Replaced %s with %s %s" % (component_name, new_meta['src'],
+                                        new_meta['revision']))
+        log("New source snapshot: %s" % (path, ))
+    
+builtins.register(OstbuildModifySnapshot)

src/ostbuild/pyostbuild/builtin_prefix.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildPrefix(builtins.Builtin):
+    name = "prefix"
+    short_description = "Display or modify \"prefix\" (build target)"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def _set_prefix(self, prefix):
+        f = open(self.path, 'w')
+        f.write(prefix)
+        f.write('\n')
+        f.close()
+        log("Prefix is now %r" % (prefix, ))
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('-a', '--active', action='store_true')
+        parser.add_argument('prefix', nargs='?', default=None)
+
+        args = parser.parse_args(argv)
+
+        self.path = os.path.expanduser('~/.config/ostbuild-prefix')
+        if args.prefix is None and not args.active:
+            if os.path.exists(self.path):
+                f = open(self.path)
+                print "%s" % (f.read().strip(), )
+                f.close()
+            else:
+                log("No currently active prefix")
+        elif args.prefix is not None and args.active:
+            fatal("Can't specify -a with prefix")
+        elif args.prefix is not None:
+            self._set_prefix(args.prefix)
+        else:
+            assert args.active
+
+            self.parse_active_branch()
+
+            active_prefix = self.active_branch_contents['prefix']
+            
+            self._set_prefix(active_prefix)
+
+builtins.register(OstbuildPrefix)

src/ostbuild/pyostbuild/builtin_pull_components.py

@@ -0,0 +1,70 @@
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import copy
+import argparse
+import json
+import time
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import ostbuildrc
+from . import buildutil
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import kvfile
+from . import odict
+
+class OstbuildPullComponents(builtins.Builtin):
+    name = "pull-components"
+    short_description = "Download the component data for active branch"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('targets', nargs='*')
+
+        args = parser.parse_args(argv)
+
+        self.parse_active_branch()
+
+        if len(args.targets) == 0:
+            targets = [self.active_branch]
+        else:
+            targets = args.targets
+
+        tree_contents_list = []
+        for target in targets:
+            tree_contents_path = os.path.join(self.ostree_dir, target, 'contents.json')
+            tree_contents = json.load(open(tree_contents_path))
+            tree_contents_list.append(tree_contents)
+        revisions = set()
+        for tree_contents in tree_contents_list:
+            for content_item in tree_contents['contents']:
+                revisions.add(content_item['ostree-revision'])
+        args = ['ostree-pull', '--repo=' + self.repo]
+        # FIXME FIXME - don't hardcode origin here
+        args.append('gnome')
+        for revision in revisions:
+            args.append(revision)
+        run_sync(args)
+        
+builtins.register(OstbuildPullComponents)

src/ostbuild/pyostbuild/builtin_resolve.py

@@ -19,6 +19,7 @@ import os,sys,subprocess,tempfile,re,shutil
 import copy
 import argparse
 import json
+import time
 import urlparse
 from StringIO import StringIO
 
@@ -26,6 +27,7 @@ from . import builtins
 from .ostbuildlog import log, fatal
 from .subprocess_helpers import run_sync, run_sync_get_output
 from . import ostbuildrc
+from . import jsondb
 from . import buildutil
 from . import kvfile
 from . import odict
@@ -38,6 +40,7 @@ class OstbuildResolve(builtins.Builtin):
         builtins.Builtin.__init__(self)
 
     def _ensure_vcs_mirror(self, name, keytype, uri, branch):
+        # FIXME - remove "name" from parameter list here - hash uri?
         mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
         tmp_mirror = mirror + '.tmp'
         if os.path.isdir(tmp_mirror):
@@ -135,6 +138,7 @@ class OstbuildResolve(builtins.Builtin):
         self.args = args
         
         self.parse_config()
+        self.repo = ostbuildrc.get_key('repo')
 
         manifest_path = self.ostbuildrc.get_key('manifest')
         self.manifest = json.load(open(manifest_path))
@@ -192,52 +196,52 @@ class OstbuildResolve(builtins.Builtin):
                 component['patches'] = copy.deepcopy(global_patches_meta)
                 component['patches']['files'] = patch_files
 
-        name_prefix = snapshot['name-prefix']
-        base_prefix = snapshot['base-prefix']
-
         manifest_architectures = snapshot['architectures']
 
+        ostree_prefix = snapshot['prefix']
+        base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix)
+        
+        snapshot['architecture-buildroots'] = {}
+        for architecture in manifest_architectures:
+            snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
+
         components_by_name = {}
         component_ordering = []
-        build_prev_component_by_arch = {}
+        build_prev_component = None
-        runtime_prev_component_by_arch = {}
+        runtime_prev_component = None
-        runtime_components_by_arch = {}
+        runtime_components = []
-        devel_components_by_arch = {}
+        devel_components = []
-        for architecture in manifest_architectures:
+
-            runtime_components_by_arch[architecture] = []
+        builds = {}
-            devel_components_by_arch[architecture] = []
 
         for component in component_source_list:
-            component_architectures = component.get('architectures', manifest_architectures)
+            base_name = component['name']
-            for architecture in component_architectures:
+            name = '%s/%s' % (ostree_prefix, base_name)
-                component_binary = copy.deepcopy(component)
+            component['name'] = name
-                source_name = component['name']
-                binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name)
-                component_binary['name'] = binary_name
-                component_binary['architecture'] = architecture
 
-                components_by_name[binary_name] = component_binary
+            components_by_name[name] = component
 
-                prev_component = build_prev_component_by_arch.get(architecture)
+            if build_prev_component is not None:
-                if prev_component is not None:
+                component['build-depends'] = [build_prev_component['name']]
-                    component_binary['build-depends'] = [prev_component['name']]
+            build_prev_component = component
-                build_prev_component_by_arch[architecture] = component_binary
 
-                is_runtime = component.get('component', 'runtime') == 'runtime'
+            is_runtime = component.get('component', 'runtime') == 'runtime'
 
-                prev_component = runtime_prev_component_by_arch.get(architecture)
+            if runtime_prev_component is not None:
-                if prev_component is not None:
+                component['runtime-depends'] = [runtime_prev_component['name']]
-                    component_binary['runtime-depends'] = [prev_component['name']]
 
-                if is_runtime:
+            if is_runtime:
-                    runtime_prev_component_by_arch[architecture] = component_binary
+                runtime_prev_component = component
+                runtime_components.append(component)
+            devel_components.append(component)
 
-                if is_runtime:
+            is_noarch = component.get('noarch', False)
-                    runtime_components_by_arch[architecture].append(component_binary)
+            if is_noarch:
-                devel_components_by_arch[architecture].append(component_binary)
+                # Just use the first specified architecture
-
+                component_arches = [manifest_architectures[0]]
-                if 'architectures' in component_binary:
+            else:
-                    del component_binary['architectures']
+                component_arches = component.get('architectures', manifest_architectures)
+            builds[name] = component_arches
 
         # We expanded these keys
         del snapshot['config-opts']
@@ -247,26 +251,27 @@ class OstbuildResolve(builtins.Builtin):
 
         targets_list = []
         snapshot['targets'] = targets_list
-        for architecture in manifest_architectures:
+        for target_component_type in ['runtime', 'devel']:
-            for target_component_type in ['runtime', 'devel']:
+            for architecture in manifest_architectures:
                 target = {}
                 targets_list.append(target)
-                target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type)
+                target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type)
 
                 base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
-                base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                     'rev-parse', 'bases/%s' % (base_ref, )])
                 target['base'] = {'name': base_ref}
 
                 if target_component_type == 'runtime':
-                    target_components = runtime_components_by_arch[architecture]
+                    target_components = runtime_components
                 else:
-                    target_components = devel_components_by_arch[architecture]
+                    target_components = devel_components
                     
                 contents = []
                 for component in target_components:
-                    name = component['name']
+                    builds_for_component = builds[component['name']]
-                    component_ref = {'name': name}
+                    if architecture not in builds_for_component:
+                        continue
+                    binary_name = '%s/%s' % (component['name'], architecture)
+                    component_ref = {'name': binary_name}
                     if target_component_type == 'runtime':
                         component_ref['trees'] = ['/runtime']
                     else:
@@ -278,10 +283,12 @@ class OstbuildResolve(builtins.Builtin):
             del component['name']
         snapshot['components'] = components_by_name
 
-        out_snapshot = os.path.join(self.workdir, '%s-snapshot.json' % (name_prefix, ))
+        snapshot['00ostree-src-snapshot-version'] = 0
-        f = open(out_snapshot, 'w')
+
-        json.dump(snapshot, f, indent=4, sort_keys=True)
+        current_time = time.time()
-        f.close()
+
-        print "Created: %s" % (out_snapshot, )
+        src_db = self.get_src_snapshot_db()
+        path = src_db.store(snapshot)
+        log("Source snapshot: %s" % (path, ))
         
 builtins.register(OstbuildResolve)

src/ostbuild/pyostbuild/builtins.py

@@ -19,11 +19,15 @@
 
 import os
 import sys
+import stat
 import argparse
 import json
 
 from . import ostbuildrc
+from . import fileutil
+from . import jsondb
 from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
 
 _all_builtins = {}
 
@@ -31,26 +35,167 @@ class Builtin(object):
     name = None
     short_description = None
 
+    def __init__(self):
+        self._meta_cache = {}
+        self.prefix = None
+        self.manifest = None
+        self.snapshot = None
+        self.bin_snapshot = None
+        self.repo = None
+        self.ostree_dir = self._find_ostree_dir()
+        (self.active_branch, self.active_branch_checksum) = self._find_active_branch()
+        self._src_snapshots = None
+        self._bin_snapshots = None
+
+    def _find_ostree_dir(self):
+        for path in ['/ostree', '/sysroot/ostree']:
+            if os.path.isdir(path):
+                return path
+        return None
+        
+    def _find_active_branch(self):
+        if self.ostree_dir is None:
+            return (None, None)
+        current_path = os.path.join(self.ostree_dir, 'current')
+        while True:
+            try:
+                target = os.path.join(self.ostree_dir, current_path)
+                stbuf = os.lstat(target)
+            except OSError, e:
+                current_path = None
+                break
+            if not stat.S_ISLNK(stbuf.st_mode):
+                break
+            current_path = os.readlink(target)
+        if current_path is not None:
+            basename = os.path.basename(current_path)
+            return basename.rsplit('-', 1)
+        else:
+            return (None, None)
+
+    def get_component_from_cwd(self):
+        cwd = os.getcwd()
+        parent = os.path.dirname(cwd)
+        parentparent = os.path.dirname(parent)
+        return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
+
     def parse_config(self):
         self.ostbuildrc = ostbuildrc
-        self.repo = ostbuildrc.get_key('repo')
+
         self.mirrordir = ostbuildrc.get_key('mirrordir')
         if not os.path.isdir(self.mirrordir):
             fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
         self.workdir = ostbuildrc.get_key('workdir')
         if not os.path.isdir(self.workdir):
             fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
+
+        self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
         self.patchdir = os.path.join(self.workdir, 'patches')
 
-    def parse_manifest(self):
+    def parse_active_branch(self):
-        self.manifest_path = ostbuildrc.get_key('manifest')
+        if self.ostree_dir is None:
-        self.manifest = json.load(open(self.manifest_path))
+            fatal("/ostree directory not found")
-        self.name_prefix = self.manifest['name-prefix']
+        repo_path = os.path.join(self.ostree_dir, 'repo')
+        if not os.path.isdir(repo_path):
+            fatal("Repository '%s' doesn't exist" % (repo_path, ))
+        self.repo = repo_path
+        if self.active_branch is None:
+            fatal("No \"current\" link found")
+        branch_path = os.path.join(self.ostree_dir, self.active_branch)
+        contents_path = os.path.join(branch_path, 'contents.json')
+        f = open(contents_path)
+        self.active_branch_contents = json.load(f)
+        f.close()
 
-    def parse_snapshot(self):
+    def get_component_snapshot(self, name):
-        self.parse_manifest()
+        found = False
-        snapshot_path = os.path.join(self.workdir, '%s-snapshot.json' % (self.name_prefix, ))
+        for content in self.active_branch_contents['contents']:
-        self.snapshot = json.load(open(snapshot_path))
+            if content['name'] == name:
+                found = True
+                break
+        if not found:
+            fatal("Unknown component '%s'" % (name, ))
+        return content
+
+    def get_component_meta_from_revision(self, revision):
+        text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                    'cat', revision,
+                                    '/_ostbuild-meta.json'])
+        return json.loads(text)
+
+    def get_component_meta(self, name):
+        assert self.repo is not None
+
+        if self.snapshot is not None:
+            return self.snapshot['components'][name]
+
+        meta = self._meta_cache.get(name)
+        if meta is None:
+            content = self.get_component_snapshot(name)
+            meta = self.get_component_meta_from_revision(content['ostree-revision'])
+            self._meta_cache[name] = meta
+        return meta
+
+    def get_prefix(self):
+        if self.prefix is None:
+            path = os.path.expanduser('~/.config/ostbuild-prefix')
+            if not os.path.exists(path):
+                fatal("No prefix set; use \"ostbuild prefix\" to set one")
+            f = open(path)
+            self.prefix = f.read().strip()
+            f.close()
+        return self.prefix
+
+    def create_db(self, dbsuffix, prefix=None):
+        if prefix is None:
+            target_prefix = self.get_prefix()
+        else:
+            target_prefix = prefix
+        name = '%s-%s' % (target_prefix, dbsuffix)
+        fileutil.ensure_dir(self.snapshot_dir)
+        return jsondb.JsonDB(self.snapshot_dir, prefix=name)
+
+    def get_src_snapshot_db(self):
+        if self._src_snapshots is None:
+            self._src_snapshots = self.create_db('src-snapshot')
+        return self._src_snapshots
+
+    def get_bin_snapshot_db(self):
+        if self._bin_snapshots is None:
+            self._bin_snapshots = self.create_db('bin-snapshot')
+        return self._bin_snapshots
+
+    def parse_snapshot(self, prefix, path):
+        if prefix is not None:
+            self.prefix = prefix
+        self.repo = ostbuildrc.get_key('repo')
+        if path is None:
+            latest_path = self.get_src_snapshot_db().get_latest_path()
+            if latest_path is None:
+                raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
+            self.snapshot_path = latest_path
+        else:
+            self.snapshot_path = path
+        self.snapshot = json.load(open(self.snapshot_path))
+        src_ver = self.snapshot['00ostree-src-snapshot-version']
+        if src_ver != 0:
+            fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0", src_ver)
+
+    def parse_bin_snapshot(self, prefix, path):
+        if prefix is not None:
+            self.prefix = prefix
+        self.repo = ostbuildrc.get_key('repo')
+        if path is None:
+            latest_path = self.get_bin_snapshot_db().get_latest_path()
+            if latest_path is None:
+                raise Exception("No binary snapshot found for prefix %r" % (self.prefix, ))
+            self.bin_snapshot_path = latest_path
+        else:
+            self.bin_snapshot_path = path
+        self.bin_snapshot = json.load(open(self.bin_snapshot_path))
+        bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version']
+        if bin_ver != 0:
+            fatal("Unhandled 00ostree-bin-snapshot-version \"%d\", expected 0", bin_ver)
 
     def execute(self, args):
         raise NotImplementedError()
@@ -65,4 +210,4 @@ def get(name):
     return None
 
 def get_all():
-    return _all_builtins.itervalues()
+    return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name))

src/ostbuild/pyostbuild/jsondb.py

@@ -0,0 +1,110 @@
+#
+# Copyright (C) 2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import stat
+import time
+import tempfile
+import re
+import shutil
+import hashlib
+import json
+
+class JsonDB(object):
+    def __init__(self, dirpath, prefix):
+        self._dirpath = dirpath
+        self._prefix = prefix
+        self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$')
+
+    def _cmp_match_by_version(self, a, b):
+        # Note this is a reversed comparison; bigger is earlier
+        a_major = a[0]
+        a_minor = a[1]
+        b_major = b[0]
+        b_minor = b[1]
+
+        c = cmp(b_major, a_major)
+        if c == 0:
+            return cmp(b_minor, a_minor)
+        return 0
+
+    def _get_all(self):
+        result = []
+        for fname in os.listdir(self._dirpath):
+            if not (fname.startswith(self._prefix) and fname.endswith('.json')):
+                continue
+
+            path = os.path.join(self._dirpath, fname)
+            match = self._version_csum_re.search(fname)
+            if not match:
+                raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum",
+                                path)
+            result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname))
+        result.sort(self._cmp_match_by_version)
+        return result
+
+    def get_latest(self):
+        path = self.get_latest_path()
+        if path is None:
+            return None
+        return json.load(open(path))
+
+    def get_latest_path(self):
+        files = self._get_all()
+        if len(files) == 0:
+            return None
+        return os.path.join(self._dirpath, files[0][3])
+
+    def store(self, obj):
+        files = self._get_all()
+        if len(files) == 0:
+            latest = None
+        else:
+            latest = files[0]
+
+        current_time = time.gmtime()
+
+        (fd, tmppath) = tempfile.mkstemp(suffix='.tmp', prefix='tmp-jsondb-')
+        os.close(fd)
+        f = open(tmppath, 'w')
+        json.dump(obj, f, indent=4, sort_keys=True)
+        f.close()
+
+        csum = hashlib.sha256()
+        f = open(tmppath)
+        buf = f.read(8192)
+        while buf != '':
+            csum.update(buf)
+            buf = f.read(8192)
+        f.close()
+        digest = csum.hexdigest()
+        
+        if latest is not None:
+            if digest == latest[2]:
+                return latest[3]
+            latest_version = (latest[0], latest[1])
+        else:
+            latest_version = (current_time.tm_year, 0)
+        target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year,
+                                            latest_version[1] + 1, digest)
+        target_path = os.path.join(self._dirpath, target_name)
+        os.rename(tmppath, target_path)
+        return target_path
+                
+                
+        

src/ostbuild/pyostbuild/main.py

@@ -22,13 +22,17 @@ import sys
 import argparse
 
 from . import builtins
-from . import builtin_build
+from . import builtin_bin_to_src
+from . import builtin_build_components
+from . import builtin_branch_prefix
 from . import builtin_checkout
 from . import builtin_chroot_compile_one
-from . import builtin_chroot_run_triggers
+from . import builtin_compose
 from . import builtin_compile_one
-from . import builtin_query_content
+from . import builtin_pull_components
+from . import builtin_prefix
 from . import builtin_resolve
+from . import builtin_modify_snapshot
 from . import builtin_status
 
 def usage(ecode):

src/ostbuild/pyostbuild/ostbuildlog.py

@@ -20,12 +20,16 @@
 import os
 import sys
 
-def log(msg):
+def log(msg, prefix=None):
-    fullmsg = '%s: %s\n' % (os.path.basename(sys.argv[0]), msg)
+    if prefix is None:
+        prefix_target = ''
+    else:
+        prefix_target = prefix
+    fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg)
     sys.stdout.write(fullmsg)
     sys.stdout.flush()
 
 def fatal(msg):
-    log(msg)
+    log(msg, prefix="FATAL: ")
     sys.exit(1)
 

src/ostbuild/pyostbuild/subprocess_helpers.py

@@ -70,14 +70,16 @@ def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none
     return None
 
 def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
-             log_success=True, log_initiation=True, stdout=None,
+             log_success=True, log_initiation=True, stdin=None, stdout=None,
              stderr=None):
     if log_initiation:
         log("running: %s" % (subprocess.list2cmdline(args),))
 
     env_copy = _get_env_for_cwd(cwd, env)
 
-    if keep_stdin:
+    if stdin is not None:
+        stdin_target = stdin
+    elif keep_stdin:
         stdin_target = sys.stdin
     else:
         stdin_target = open('/dev/null', 'r')

src/ostbuild/pyostbuild/vcs.py

@@ -71,5 +71,5 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
     return dest
 
 def clean(keytype, checkoutdir):
-    assert keytype == 'git'
+    assert keytype in ('git', 'dirty-git')
     run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)