ostbuild: Propagate buildroot version, finish add-artifacts

We need to track what buildroot each artifact was created in.

Makefile-ostbuild.am

@@ -17,6 +17,7 @@
 
 bin_SCRIPTS += \
 	src/ostbuild/ostbuild-autodiscover-meta \
+	src/ostbuild/ostbuild-commit-artifacts \
 	src/ostbuild/ostbuild-compile-one-impl \
 	src/ostbuild/ostbuild-chroot-compile-one-impl \
 	src/ostbuild/ostbuild-nice-and-log-output \

src/ostbuild/ostbuild-autodiscover-meta

@@ -47,6 +47,16 @@ def _discover_version_from_git():
     return None
 _register_discover_func('VERSION', _discover_version_from_git)
 
+def _discover_branch_from_git():
+    if os.path.isdir('.git'):
+        try:
+            ref = subprocess.check_output(['git', 'symbolic-ref', 'HEAD'])
+            return ref.replace('refs/heads/', '').strip()
+        except subprocess.CalledProcessError, e:
+            return None
+    return None
+_register_discover_func('BRANCH', _discover_branch_from_git)
+
 if args.meta:
     f = open(args.meta)
     for line in f.readlines():

src/ostbuild/ostbuild-chroot-compile-one-impl

@@ -18,6 +18,7 @@
 # Boston, MA 02111-1307, USA.
 
 import os,sys,re,subprocess,tempfile,shutil
+from StringIO import StringIO
 import argparse
 
 sys.path
@@ -35,6 +36,7 @@ def get_build_env():
 
 
 parser = argparse.ArgumentParser(description="Build a module in a given root")
+parser.add_argument('--workdir')
 parser.add_argument('--repo')
 parser.add_argument('--resultdir')
 parser.add_argument('--branch')
@@ -48,28 +50,70 @@ def log(m):
     sys.stdout.write('\n')
     sys.stdout.flush()
 
-basename = os.path.basename(os.getcwd())
+if args.meta is None:
+    output = subprocess.check_output(['ostbuild-autodiscover-meta'])
+    ostbuild_meta_f = StringIO(output)
+else:
+    ostbuild_meta_f = open(args.meta)
 
-tmpdir = tempfile.mkdtemp(prefix='ostree-chroot-compile-')
+metadata = {}
-log("Using temporary directory: %s" % (tmpdir, ))
+for line in ostbuild_meta_f:
+    (k,v) = line.split('=', 1)
+    metadata[k.strip()] = v.strip()
 
-child_tmpdir=os.path.join(tmpdir, 'tmp')
+for k in ['NAME']:
+    if k not in metadata:
+        sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
+        sys.exit(1)
+
+workdir_is_tmp = (args.workdir is None)
+if workdir_is_tmp:
+    workdir = tempfile.mkdtemp(prefix='ostree-chroot-compile-')
+else:
+    workdir = args.workdir
+    
+log("Using working directory: %s" % (workdir, ))
+
+child_tmpdir=os.path.join(workdir, 'tmp')
+if os.path.isdir(child_tmpdir):
+    log("Cleaning up previous tmpdir: %r" % (child_tmpdir, ))
+    shutil.rmtree(child_tmpdir)
 os.mkdir(child_tmpdir)
 
 rev = subprocess.check_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.branch])
 rev=rev.strip()
 
-rootdir = os.path.join(tmpdir, 'root-' + rev)
+metadata['BUILDROOT'] = args.branch
-subprocess.check_call(['ostree', '--repo=' + args.repo, 'checkout', '-U', rev, rootdir])
+metadata['BUILDROOT_VERSION'] = rev
-log("Checked out root: %s" % (rootdir, ))
 
+rootdir = os.path.join(workdir, 'root-' + rev)
+rootdir_tmp = rootdir + '.tmp'
 builddir = os.path.join(rootdir, 'ostbuild');
-os.mkdir(builddir)
+if not os.path.isdir(rootdir):
-os.mkdir(os.path.join(builddir, 'source'))
+    if os.path.isdir(rootdir_tmp):
-os.mkdir(os.path.join(builddir, 'source', basename))
+        shutil.rmtree(rootdir_tmp)
-os.mkdir(os.path.join(builddir, 'results'))
+    child_args = ['ostree', '--repo=' + args.repo, 'checkout', '-U', rev, rootdir_tmp]
+    log("Running: %r" % (child_args, ))
+    subprocess.check_call(child_args)
+    builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
+    os.mkdir(builddir_tmp)
+    os.mkdir(os.path.join(builddir_tmp, 'source'))
+    os.mkdir(os.path.join(builddir_tmp, 'results'))
+    os.rename(rootdir_tmp, rootdir)
+    log("Checked out root: %s" % (rootdir, ))
+else:
+    log("Using existing root: %s" % (rootdir, ))
 
-chroot_sourcedir = os.path.join('/ostbuild', 'source', basename)
+sourcedir=os.path.join(builddir, 'source', metadata['NAME'])
+if not os.path.isdir(sourcedir):
+    os.mkdir(sourcedir)
+
+output_metadata = open('_ostbuild-meta', 'w')
+for (k,v) in metadata.iteritems():
+    output_metadata.write('%s=%s\n' % (k, v))
+output_metadata.close()
+
+chroot_sourcedir = os.path.join('/ostbuild', 'source', metadata['NAME'])
 
 # We need to search PATH here manually so we correctly pick up an
 # ostree install in e.g. ~/bin even though we're going to set PATH
@@ -94,10 +138,11 @@ child_args = [ostbuild_user_chroot_path, '--unshare-pid', '--unshare-net', '--un
               '/bin/sh']
 if not args.debug_shell:
     child_args += ['-c',
-             'cd "%s" && ostbuild-compile-one-impl OSTBUILD_RESULTDIR=/ostbuild/results' % (chroot_sourcedir, )
+             'cd "%s" && ostbuild-compile-one-impl OSTBUILD_RESULTDIR=/ostbuild/results OSTBUILD_META=_ostbuild-meta' % (chroot_sourcedir, )
              ]
 log("Running: %r" % (child_args, ))
 subprocess.check_call(child_args, env=get_build_env())
 
-shutil.rmtree(tmpdir)
+if workdir_is_tmp:
+    shutil.rmtree(workdir)
                        

src/ostbuild/ostbuild-commit-artifacts

@@ -25,25 +25,27 @@ import os,sys,subprocess,tempfile,re
 i=1
 repo=sys.argv[i]
 
-artifact_re = re.compile(r'^artifact-([^,]+)-([^,]+),(.+).tar.gz$')
+artifact_re = re.compile(r'^artifact-([^,]+),([^,]+),([^,]+),([^,]+),([^.]+)\.tar\.gz$')
-
-if os.getuid() != 0:
-    print "This program must be run as root."
-    sys.exit(1)
 
 def call_ostree_sync(*args):
-    subprocess.check_call(['ostree', '--repo=' + repo] + args)
+    subprocess.check_call(['ostree', '--repo=' + repo] + list(args))
 
-for arg in sys.argv[1:]:
+for arg in sys.argv[2:]:
-    match = artifact_re.match(arg)
+    basename = os.path.basename(arg)
-    if match is None
+    match = artifact_re.match(basename)
+    if match is None:
         print "Invalid artifact name: %s" % (arg, )
         sys.exit(1)
-    arch = match.group(1)
+    buildroot = match.group(1)
-    name = match.group(2)
+    buildroot_version = match.group(2)
-    version = match.group(3)
+    name = match.group(3)
+    branch = match.group(4)
+    version = match.group(5)
     
-    branch_name = 'artifact-%s-%s' % (arch, name)
+    branch_name = 'artifacts/%s/%s/%s' % (buildroot, name, branch)
 
-    call_ostree_sync('commit', '-b', branch_name, '-s', version, 
+    call_ostree_sync('commit', '-b', branch_name, '-s', 'Build ' + version,
+                     '--add-metadata-string=ostree-buildroot-version=' + buildroot_version,
+                     '--add-metadata-string=ostree-artifact-version=' + version,
+                     '--skip-if-unchanged', '--tar-autocreate-parents', '--tree=tar=' + arg)
                      

src/ostbuild/ostbuild-compile-one-impl

@@ -20,7 +20,7 @@
 # ostbuild-compile-one-make wraps systems that implement the GNOME build API:
 # http://people.gnome.org/~walters/docs/build-api.txt
 
-import os,sys,subprocess,tempfile,re
+import os,sys,subprocess,tempfile,re,shutil
 from StringIO import StringIO
 from multiprocessing import cpu_count
 import select,time
@@ -291,10 +291,21 @@ def make_artifact(name, from_files, tempdir=None, resultdir=None):
 
 def phase_make_artifacts(builddir=None):
     name = metadata['NAME']
+    assert ',' not in name
+    branch = metadata['BRANCH']
+    assert ',' not in name
     version = metadata['VERSION']
     assert ',' not in version
 
-    artifact_prefix='artifact-%s-%s,%s' % (build_target, name, version)
+    root_name = metadata.get('BUILDROOT', None)
+    # TODO - pick up current sysroot version from ostree
+    if root_name is None:
+        root_name = 'unknown-' + build_target
+        root_version = 'UNKNOWN'
+    else:
+        root_version = metadata.get('BUILDROOT_VERSION')
+
+    artifact_prefix='artifact-%s,%s,%s,%s,%s' % (root_name, root_version, name, branch, version)
 
     tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (name,))
     tempfiles.append(tempdir)
@@ -334,6 +345,8 @@ def phase_make_artifacts(builddir=None):
         make_artifact(artifact_prefix + '-devel', devel_files, tempdir=tempdir, resultdir=ostbuild_resultdir)
     make_artifact(artifact_prefix + '-runtime', runtime_files, tempdir=tempdir, resultdir=ostbuild_resultdir)
 
+    phase_complete()
+
 def phase_complete():
     for tmpname in tempfiles:
         if os.path.isdir(tmpname):