ostbuild: Switch to JSON metadata, various other fixes

src/ostbuild/pyostbuild/buildutil.py

@@ -26,7 +26,7 @@ def parse_artifact_name(artifact_basename):
     if match is None:
         raise ValueError("Invalid artifact basename %s" % (artifact_basename))
     return {'buildroot': match.group(1),
-            'buildroot_version': match.group(2),
+            'buildroot-version': match.group(2),
             'name': match.group(3),
             'branch': match.group(4),
             'version': match.group(5),

src/ostbuild/pyostbuild/builtin_autodiscover_meta.py

@@ -19,6 +19,7 @@
 
 import os,sys,re,subprocess,tempfile,shutil
 import argparse
+import json
 
 from . import builtins
 from . import buildutil
@@ -42,15 +43,13 @@ class OstbuildAutodiscoverMeta(builtins.Builtin):
                 AUTODISCOVERED_KEYS[key] = []
             AUTODISCOVERED_KEYS[key].append(func)
 
-        _register_discover_func('NAME', self._discover_name_from_cwd)
+        _register_discover_func('name', self._discover_name_from_cwd)
-        _register_discover_func('VERSION', self._discover_version_from_git)
+        _register_discover_func('version', self._discover_version_from_git)
-        _register_discover_func('BRANCH', self._discover_branch_from_git)
+        _register_discover_func('branch', self._discover_branch_from_git)
 
         if args.meta:
             f = open(args.meta)
-            for line in f.readlines():
+            KEYS = json.load(f)
-                (k,v) = line.split('=', 1)
-                KEYS[k.strip()] = v.strip()
             f.close()
             
         for (key,hooks) in AUTODISCOVERED_KEYS.iteritems():
@@ -65,8 +64,7 @@ class OstbuildAutodiscoverMeta(builtins.Builtin):
                     KEYS[key] = value
                     break
             
-        for (key,value) in KEYS.iteritems():
+        json.dump(KEYS, sys.stdout, indent=2)
-            print "%s=%s" % (key, value)
         
     def _discover_name_from_cwd(self):
         return os.path.basename(os.getcwd())
@@ -79,8 +77,11 @@ class OstbuildAutodiscoverMeta(builtins.Builtin):
     def _discover_branch_from_git(self):
         if os.path.isdir('.git'):
             try:
-                ref = subprocess.check_output(['git', 'symbolic-ref', 'HEAD'])
+                try:
-                return ref.replace('refs/heads/', '').strip()
+                    ref = subprocess.check_output(['git', 'symbolic-ref', 'HEAD'], stderr=open('/dev/null', 'w'))
+                    return ref.replace('refs/heads/', '').strip()
+                except subprocess.CalledProcessError, e:
+                    return subprocess.check_output(['git', 'describe', '--tags', '--exact-match', 'HEAD']).strip()
             except subprocess.CalledProcessError, e:
                 return None
         return None

src/ostbuild/pyostbuild/builtin_build.py

@@ -68,16 +68,10 @@ class OstbuildBuild(builtins.Builtin):
         if idx < 0:
             raise ValueError("Invalid SRC uri=%s" % (srckey, ))
         keytype = srckey[:idx]
-        if keytype not in ('git'):
+        if keytype not in ['git']:
             raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
         uri = srckey[idx+1:]
-        idx = uri.rfind('#')
+        return (keytype, uri)
-        if idx < 0:
-            branch = "master"
-        else:
-            branch = uri[idx+1:]
-            uri = uri[0:idx]
-        return (keytype, uri, branch)
 
     def _get_ostbuild_chroot_args(self, architecture):
         current_machine = os.uname()[4]
@@ -128,7 +122,8 @@ class OstbuildBuild(builtins.Builtin):
     def _build_one_component(self, meta, architecture):
         name = meta['name']
 
-        (keytype, uri, branch) = self._parse_src_key(meta['src'])
+        (keytype, uri) = self._parse_src_key(meta['src'])
+        branch = meta.get('branch', 'master')
 
         buildroot = '%s-%s-devel' % (self.manifest['name'], architecture)
         runtime_branchname = 'artifacts/%s/%s/%s/runtime' % (buildroot, name, branch)
@@ -137,7 +132,7 @@ class OstbuildBuild(builtins.Builtin):
         current_buildroot_version = current_buildroot_version.strip()
 
         artifact_base = {'buildroot': buildroot,
-                         'buildroot_version': current_buildroot_version,
+                         'buildroot-version': current_buildroot_version,
                          'name': name,
                          'branch': branch,
                          }
@@ -148,6 +143,14 @@ class OstbuildBuild(builtins.Builtin):
         current_vcs_version = buildutil.get_git_version_describe(component_src)
         artifact_base['version'] = current_vcs_version
 
+        metadata_dir = os.path.join(self.workdir, 'meta')
+        if not os.path.isdir(metadata_dir):
+            os.makedirs(metadata_dir)
+        metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, ))
+        f = open(metadata_path, 'w')
+        json.dump(artifact_base, f)
+        f.close()
+
         previous_commit_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                        'rev-parse', runtime_branchname],
                                                       stderr=open('/dev/null', 'w'),
@@ -164,7 +167,7 @@ class OstbuildBuild(builtins.Builtin):
 
             previous_artifact_base = dict(artifact_base)
             previous_artifact_base['version'] = previous_artifact_version
-            previous_artifact_base['buildroot_version'] = previous_buildroot_version
+            previous_artifact_base['buildroot-version'] = previous_buildroot_version
 
             previous_artifact_runtime = dict(previous_artifact_base)
             previous_artifact_runtime['type'] = 'runtime'
@@ -198,7 +201,7 @@ class OstbuildBuild(builtins.Builtin):
                 patch_path = os.path.join(self.manifestdir, patch)
                 run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src)
         
-        component_resultdir = os.path.join(self.workdir, name, 'results')
+        component_resultdir = os.path.join(self.workdir, 'results', name)
         if os.path.isdir(component_resultdir):
             shutil.rmtree(component_resultdir)
         os.makedirs(component_resultdir)
@@ -206,10 +209,14 @@ class OstbuildBuild(builtins.Builtin):
         chroot_args = self._get_ostbuild_chroot_args(architecture)
         chroot_args.extend(['--buildroot=' + buildroot,
                             '--workdir=' + self.workdir,
-                            '--resultdir=' + component_resultdir])
+                            '--resultdir=' + component_resultdir,
+                            '--meta=' + metadata_path])
         global_config_opts = self.manifest.get('config-opts')
         if global_config_opts is not None:
             chroot_args.extend(global_config_opts)
+        component_config_opts = meta.get('config-opts')
+        if component_config_opts is not None:
+            chroot_args.extend(component_config_opts)
         if self.buildopts.shell_on_failure:
             ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False)
             if ecode != 0:
@@ -246,6 +253,7 @@ class OstbuildBuild(builtins.Builtin):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--manifest', required=True)
         parser.add_argument('--skip-built', action='store_true')
+        parser.add_argument('--start-at')
         parser.add_argument('--shell-on-failure', action='store_true')
         parser.add_argument('--debug-shell', action='store_true')
 
@@ -265,16 +273,27 @@ class OstbuildBuild(builtins.Builtin):
             self._launch_debug_shell(debug_shell_arch, debug_shell_buildroot)
 
         self.manifestdir = os.path.dirname(args.manifest)
+
+        self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
+
+        start_at_index = -1
+        if args.start_at is not None:
+            for i,component in enumerate(self.resolved_components):
+                if component['name'] == args.start_at:
+                    start_at_index = i
+                    break
+            if start_at_index == -1:
+                fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
+        else:
+            start_at_index = 0
             
-        for component in self.manifest['components']:
+        for component in self.resolved_components[start_at_index:]:
             for architecture in self.manifest['architectures']:
-                component_meta = self._resolve_component_meta(component)
+                (runtime_artifact,devel_artifact) = self._build_one_component(component, architecture)
-    
-                (runtime_artifact,devel_artifact) = self._build_one_component(component_meta, architecture)
                 runtime_branch = buildutil.branch_name_for_artifact(runtime_artifact)
                 devel_branch = buildutil.branch_name_for_artifact(devel_artifact)
     
-                target_component = component_meta.get('component')
+                target_component = component.get('component')
                 if target_component != 'devel':
                     self._compose(architecture + '-runtime', [runtime_branch])
                 self._compose(architecture + '-devel', [runtime_branch, devel_branch])

src/ostbuild/pyostbuild/builtin_chroot_compile_one.py

@@ -18,6 +18,7 @@
 import os,sys,re,subprocess,tempfile,shutil
 from StringIO import StringIO
 import argparse
+import json
 
 from . import builtins
 from .ostbuildlog import log, fatal
@@ -51,17 +52,14 @@ class OstbuildChrootCompileOne(builtins.Builtin):
 
         if args.meta is None:
             output = subprocess.check_output(['ostbuild', 'autodiscover-meta'])
-            ostbuild_meta_f = StringIO(output)
+            self.metadata = json.loads(output)
         else:
-            ostbuild_meta_f = open(args.meta)
+            f = open(args.meta)
+            self.metadata = json.load(f)
+            f.close()
 
-        metadata = {}
+        for k in ['name']:
-        for line in ostbuild_meta_f:
+            if k not in self.metadata:
-            (k,v) = line.split('=', 1)
-            metadata[k.strip()] = v.strip()
-        
-        for k in ['NAME']:
-            if k not in metadata:
                 sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
                 sys.exit(1)
         
@@ -82,10 +80,14 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         rev = subprocess.check_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.buildroot])
         rev=rev.strip()
         
-        metadata['BUILDROOT'] = args.buildroot
+        self.metadata['buildroot'] = args.buildroot
-        metadata['BUILDROOT_VERSION'] = rev
+        self.metadata['buildroot-version'] = rev
+        
+        rootdir_prefix = os.path.join(workdir, 'roots')
+        if not os.path.isdir(rootdir_prefix):
+            os.makedirs(rootdir_prefix)
+        rootdir = os.path.join(rootdir_prefix, rev)
         
-        rootdir = os.path.join(workdir, 'root-' + rev)
         rootdir_tmp = rootdir + '.tmp'
         builddir = os.path.join(rootdir, 'ostbuild');
         if not os.path.isdir(rootdir):
@@ -102,16 +104,15 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         else:
             log("Using existing root: %s" % (rootdir, ))
         
-        sourcedir=os.path.join(builddir, 'source', metadata['NAME'])
+        sourcedir=os.path.join(builddir, 'source', self.metadata['name'])
         if not os.path.isdir(sourcedir):
             os.mkdir(sourcedir)
         
         output_metadata = open('_ostbuild-meta', 'w')
-        for (k,v) in metadata.iteritems():
+        json.dump(self.metadata, output_metadata)
-            output_metadata.write('%s=%s\n' % (k, v))
         output_metadata.close()
         
-        chroot_sourcedir = os.path.join('/ostbuild', 'source', metadata['NAME'])
+        chroot_sourcedir = os.path.join('/ostbuild', 'source', self.metadata['name'])
         
         # We need to search PATH here manually so we correctly pick up an
         # ostree install in e.g. ~/bin even though we're going to set PATH

src/ostbuild/pyostbuild/builtin_compile_one.py

@@ -20,6 +20,7 @@
 
 import os,sys,stat,subprocess,tempfile,re,shutil
 from StringIO import StringIO
+import json
 from multiprocessing import cpu_count
 import select,time
 
@@ -109,19 +110,13 @@ class OstbuildCompileOne(builtins.Builtin):
 
         if self.ostbuild_meta is None:
             output = subprocess.check_output(['ostbuild', 'autodiscover-meta'])
-            ostbuild_meta_f = StringIO(output)
+            self.metadata = json.loads(output)
         else:
-            ostbuild_meta_f = open(self.ostbuild_meta)
+            f = open(self.ostbuild_meta)
+            self.metadata = json.load(f)
+            f.close()
 
-        for line in ostbuild_meta_f:
+        for k in ['name', 'version']:
-            if line == '':
-                continue
-            (k,v) = line.split('=', 1)
-            self.metadata[k.strip()] = v.strip()
-
-        ostbuild_meta_f.close()
-
-        for k in ['NAME', 'VERSION']:
             if k not in self.metadata:
                 fatal('Missing required key "%s" in metadata' % (k, ))
 
@@ -192,20 +187,20 @@ class OstbuildCompileOne(builtins.Builtin):
     
         run_sync(args, cwd=builddir)
 
-        name = self.metadata['NAME']
+        name = self.metadata['name']
         assert ',' not in name
-        branch = self.metadata['BRANCH']
+        branch = self.metadata['branch']
         assert ',' not in name
-        version = self.metadata['VERSION']
+        version = self.metadata['version']
         assert ',' not in version
     
-        root_name = self.metadata.get('BUILDROOT', None)
+        root_name = self.metadata.get('buildroot', None)
         # TODO - pick up current sysroot version from ostree
         if root_name is None:
             root_name = 'unknown-' + self.build_target
             root_version = 'UNKNOWN'
         else:
-            root_version = self.metadata.get('BUILDROOT_VERSION')
+            root_version = self.metadata.get('buildroot-version')
     
         artifact_prefix=os.path.join('artifacts', root_name, name, branch)