ostbuild: Move more intelligence into chroot-compile-one

Let's have chroot-compile-one be the core logic for composing a
buildroot, etc.  This allows us to use it as a developer tool by
checking out a tree, then doing a chroot build.

src/ostbuild/pyostbuild/buildutil.py

@@ -18,6 +18,7 @@
 import os
 import re
 import urlparse
+import tempfile
 
 from .subprocess_helpers import run_sync_get_output
 
@@ -90,3 +91,33 @@ def manifest_buildroot_name(manifest, component, architecture):
     return 'buildroots/%s/%s/%s' % (manifest_target (manifest, architecture),
                                     component['name'],
                                     component['branch'])
+
+def find_component_in_manifest(manifest, component_name):
+    for component in manifest['components']:
+        if component['name'] == component_name:
+            return component
+    return None
+
+def compose(repo, target, artifacts):
+    child_args = ['ostree', '--repo=' + repo, 'compose',
+                  '-b', target, '-s', 'Compose']
+    (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+    f = os.fdopen(fd, 'w')
+    for artifact in artifacts:
+        f.write(artifact)
+        f.write('\n')
+    f.close()
+    child_args.extend(['-F', path])
+    revision = run_sync_get_output(child_args, log_initiation=True).strip()
+    os.unlink(path)
+    return revision
+
+def compose_buildroot(manifest, repo, buildroot_name, component, dependencies, architecture):
+    base = manifest_base(manifest, 'devel', architecture)
+    buildroot_contents = [base + ':/']
+    for dep in dependencies:
+        dep_buildname = manifest_buildname(manifest, dep, architecture)
+        buildroot_contents.append(dep_buildname + ':/runtime')
+        buildroot_contents.append(dep_buildname + ':/devel')
+
+    return compose(repo, buildroot_name, buildroot_contents)

src/ostbuild/pyostbuild/builtin_build.py

@@ -48,8 +48,7 @@ class OstbuildBuild(builtins.Builtin):
             args = ['setarch', architecture]
         else:
             args = []
-        args.extend(['ostbuild', 'chroot-compile-one',
+        args.extend(['ostbuild', 'chroot-compile-one'])
-                     '--repo=' + self.repo])
         return args
 
     def _launch_debug_shell(self, architecture, buildroot, cwd=None):
@@ -92,17 +91,7 @@ class OstbuildBuild(builtins.Builtin):
 
         return result
 
-    def _compose_buildroot(self, buildroot_name, component, dependencies, architecture):
+    def _build_one_component(self, meta, architecture):
-        base = buildutil.manifest_base(self.manifest, 'devel', architecture)
-        buildroot_contents = [base + ':/']
-        for dep in dependencies:
-            dep_buildname = buildutil.manifest_buildname(self.manifest, dep, architecture)
-            buildroot_contents.append(dep_buildname + ':/runtime')
-            buildroot_contents.append(dep_buildname + ':/devel')
-
-        return self._compose(buildroot_name, buildroot_contents)
-
-    def _build_one_component(self, meta, dependencies, architecture):
         name = meta['name']
         branch = meta['branch']
 
@@ -117,10 +106,6 @@ class OstbuildBuild(builtins.Builtin):
         component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, branch,
                                              overwrite=not self.args.debug_shell)
 
-        if self.args.debug_shell:
-            buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
-            self._launch_debug_shell(architecture, buildroot_name, cwd=component_src)
-
         current_vcs_version = meta['revision']
 
         previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
@@ -146,39 +131,19 @@ class OstbuildBuild(builtins.Builtin):
         else:
             log("No previous build for '%s' found" % (buildname, ))
 
-        buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
+        artifact_meta = dict(meta)
 
-        if meta.get('rm-configure', False):
+        metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
-            configure_path = os.path.join(component_src, 'configure')
-            if os.path.exists(configure_path):
-                os.unlink(configure_path)
-
-        artifact_meta = {'buildroot': buildroot_name,
-                         'buildroot-version': buildroot_version,
-                         'name': name,
-                         'branch': branch,
-                         'version': current_vcs_version
-                         }
-
-        metadata_dir = os.path.join(self.workdir, 'meta')
-        if not os.path.isdir(metadata_dir):
-            os.makedirs(metadata_dir)
-        metadata_path = os.path.join(metadata_dir, '%s-meta.json' % (name, ))
         f = open(metadata_path, 'w')
         json.dump(artifact_meta, f)
         f.close()
-
+        
         patches = meta.get('patches')
         if patches is not None:
             for patch in patches:
                 patch_path = os.path.join(self.patchdir, patch)
                 run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src)
         
-        component_resultdir = os.path.join(self.workdir, 'results', name)
-        if os.path.isdir(component_resultdir):
-            shutil.rmtree(component_resultdir)
-        os.makedirs(component_resultdir)
-
         logdir = os.path.join(self.workdir, 'logs', 'compile', name)
         old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name)
         if not os.path.isdir(logdir):
@@ -194,16 +159,7 @@ class OstbuildBuild(builtins.Builtin):
         log("Logging to %s" % (log_path, ))
         f = open(log_path, 'w')
         chroot_args = self._get_ostbuild_chroot_args(architecture)
-        chroot_args.extend(['--buildroot=' + buildroot_name,
+        chroot_args.extend(['--meta=' + metadata_path])
-                            '--workdir=' + self.workdir,
-                            '--resultdir=' + component_resultdir,
-                            '--meta=' + metadata_path])
-        global_config_opts = self.manifest.get('config-opts')
-        if global_config_opts is not None:
-            chroot_args.extend(global_config_opts)
-        component_config_opts = meta.get('config-opts')
-        if component_config_opts is not None:
-            chroot_args.extend(component_config_opts)
         if self.buildopts.shell_on_failure:
             ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
             if ecode != 0:
@@ -211,14 +167,19 @@ class OstbuildBuild(builtins.Builtin):
         else:
             run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
 
+        # Reread metadata to get buildroot version
+        f = open(metadata_path)
+        artifact_meta = json.load(f)
+        f.close()
+
         args = ['ostree', '--repo=' + self.repo,
-                'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['version'],
+                'commit', '-b', buildname, '-s', 'Build ' + artifact_meta['revision'],
-                '--add-metadata-string=ostbuild-buildroot-version=' + buildroot_version,
+                '--add-metadata-string=ostbuild-buildroot-version=' + artifact_meta['buildroot-version'],
-                '--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['version'],
+                '--add-metadata-string=ostbuild-artifact-version=' + artifact_meta['revision'],
                 '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
                 '--skip-if-unchanged']
 
-        setuid_files = meta.get('setuid', [])
+        setuid_files = artifact_meta.get('setuid', [])
         statoverride_path = None
         if len(setuid_files) > 0:
             (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
@@ -227,26 +188,14 @@ class OstbuildBuild(builtins.Builtin):
                 f.write('+2048 ' + path)
             f.close()
             args.append('--statoverride=' + statoverride_path)
+
+        component_resultdir = os.path.join(self.workdir, 'results', name)
             
         run_sync(args, cwd=component_resultdir)
         if statoverride_path is not None:
             os.unlink(statoverride_path)
         return True
 
-    def _compose(self, target, artifacts):
-        child_args = ['ostree', '--repo=' + self.repo, 'compose',
-                      '-b', target, '-s', 'Compose']
-        (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
-        f = os.fdopen(fd, 'w')
-        for artifact in artifacts:
-            f.write(artifact)
-            f.write('\n')
-        f.close()
-        child_args.extend(['-F', path])
-        revision = run_sync_get_output(child_args, log_initiation=True).strip()
-        os.unlink(path)
-        return revision
-
     def _compose_arch(self, architecture, components):
         runtime_base = buildutil.manifest_base(self.manifest, 'runtime', architecture)
         devel_base = buildutil.manifest_base(self.manifest, 'devel', architecture)
@@ -261,10 +210,10 @@ class OstbuildBuild(builtins.Builtin):
             devel_contents.append(branch + ':/doc')
             devel_contents.append(branch + ':/devel')
 
-        self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'),
+        buildutil.compose(self.repo, '%s-%s-%s' % (self.manifest['name'], architecture, 'runtime'),
-                      runtime_contents)
+                          runtime_contents)
-        self._compose('%s-%s-%s' % (self.manifest['name'], architecture, 'devel'),
+        buildutil.compose(self.repo, '%s-%s-%s' % (self.manifest['name'], architecture, 'devel'),
-                      devel_contents)
+                          devel_contents)
     
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
@@ -321,9 +270,8 @@ class OstbuildBuild(builtins.Builtin):
 
         for component in build_components[start_at_index:]:
             index = components.index(component)
-            dependencies = components[:index]
             for architecture in self.manifest['architectures']:
-                self._build_one_component(component, dependencies, architecture)
+                self._build_one_component(component, architecture)
 
         for architecture in self.manifest['architectures']:
             self._compose_arch(architecture, components)

src/ostbuild/pyostbuild/builtin_chroot_compile_one.py

@@ -31,33 +31,48 @@ class OstbuildChrootCompileOne(builtins.Builtin):
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--workdir')
-        parser.add_argument('--repo', required=True)
-        parser.add_argument('--resultdir')
-        parser.add_argument('--buildroot', required=True)
         parser.add_argument('--meta')
         parser.add_argument('--debug-shell', action='store_true')
         
-        (args, rest_args) = parser.parse_known_args(argv)
+        args = parser.parse_args(argv)
 
-        if args.meta is None:
+        self.parse_config()
-            output = run_sync_get_output(['ostbuild', 'autodiscover-meta'])
+
-            self.metadata = json.loads(output)
+        component_name = os.path.basename(os.getcwd())
-        else:
+        build_manifest_path = os.path.join(self.workdir, 'manifest.json')
+        self.manifest = json.load(open(build_manifest_path))
+
+        if args.meta is not None:
             f = open(args.meta)
             self.metadata = json.load(f)
             f.close()
 
-        for k in ['name']:
+            component = buildutil.find_component_in_manifest(self.manifest, self.metadata['name'])
-            if k not in self.metadata:
-                sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
-                sys.exit(1)
-        
-        workdir_is_tmp = (args.workdir is None)
-        if workdir_is_tmp:
-            workdir = tempfile.mkdtemp(prefix='ostbuild-chroot-compile-')
         else:
-            workdir = args.workdir
+            component = buildutil.find_component_in_manifest(self.manifest, component_name)
+            if component is None:
+                fatal("Couldn't find component '%s' in manifest" % (component_name, ))
+
+            self.metadata = component
+            self.metadata['src'] = 'dirty:worktree'
+            self.metadata['revision'] = 'dirty-worktree'
+
+        components = self.manifest['components']
+        index = components.index(component)
+        dependencies = components[:index]
+
+        architecture = os.uname()[4]
+        buildroot_name = buildutil.manifest_buildroot_name(self.manifest, self.metadata, architecture)
+        buildroot_version = buildutil.compose_buildroot(self.manifest, self.repo, buildroot_name,
+                                                        self.metadata, dependencies, architecture)
+        self.metadata['buildroot-name'] = buildroot_name
+        self.metadata['buildroot-version'] = buildroot_version
+
+        if 'name' not in self.metadata:
+            sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
+            sys.exit(1)
+
+        workdir = self.workdir
             
         log("Using working directory: %s" % (workdir, ))
         
@@ -66,24 +81,23 @@ class OstbuildChrootCompileOne(builtins.Builtin):
             log("Cleaning up previous tmpdir: %r" % (child_tmpdir, ))
             shutil.rmtree(child_tmpdir)
         os.mkdir(child_tmpdir)
-        
+
-        rev = run_sync_get_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.buildroot])
+        resultdir = os.path.join(self.workdir, 'results', component['name'])
-        rev = rev.strip()
+        if os.path.isdir(resultdir):
-        
+            shutil.rmtree(resultdir)
-        self.metadata['buildroot'] = args.buildroot
+        os.makedirs(resultdir)
-        self.metadata['buildroot-version'] = rev
         
         rootdir_prefix = os.path.join(workdir, 'roots')
         if not os.path.isdir(rootdir_prefix):
             os.makedirs(rootdir_prefix)
-        rootdir = os.path.join(rootdir_prefix, rev)
+        rootdir = os.path.join(rootdir_prefix, buildroot_version)
         
         rootdir_tmp = rootdir + '.tmp'
         builddir = os.path.join(rootdir, 'ostbuild');
         if not os.path.isdir(rootdir):
             if os.path.isdir(rootdir_tmp):
                 shutil.rmtree(rootdir_tmp)
-            child_args = ['ostree', '--repo=' + args.repo, 'checkout', '-U', rev, rootdir_tmp]
+            child_args = ['ostree', '--repo=' + self.repo, 'checkout', '-U', buildroot_version, rootdir_tmp]
             run_sync(child_args)
             child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
             run_sync(child_args)
@@ -100,7 +114,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         if not os.path.isdir(sourcedir):
             os.mkdir(sourcedir)
         
-        output_metadata = open('_ostbuild-meta', 'w')
+        output_metadata = open('_ostbuild-meta.json', 'w')
         json.dump(self.metadata, output_metadata)
         output_metadata.close()
         
@@ -114,22 +128,18 @@ class OstbuildChrootCompileOne(builtins.Builtin):
                       '--mount-bind', '/dev', '/dev',
                       '--mount-bind', child_tmpdir, '/tmp',
                       '--mount-bind', os.getcwd(), chroot_sourcedir,
+                      '--mount-bind', resultdir, '/ostbuild/results',
                       '--chdir', chroot_sourcedir]
-        if args.resultdir:
-            child_args.extend(['--mount-bind', args.resultdir, '/ostbuild/results'])
         if args.debug_shell:
             child_args.extend([rootdir, '/bin/sh'])
         else:
             child_args.extend([rootdir, '/usr/bin/ostbuild',
                                'compile-one',
                                '--ostbuild-resultdir=/ostbuild/results',
-                               '--ostbuild-meta=_ostbuild-meta'])
+                               '--ostbuild-meta=_ostbuild-meta.json'])
-            child_args.extend(rest_args)
+            child_args.extend(self.metadata.get('config-opts', []))
         env_copy = dict(buildutil.BUILD_ENV)
         env_copy['PWD'] = chroot_sourcedir
         run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
         
-        if workdir_is_tmp:
-            shutil.rmtree(workdir)
-                       
 builtins.register(OstbuildChrootCompileOne)

src/ostbuild/pyostbuild/builtin_compile_one.py

@@ -96,21 +96,22 @@ class OstbuildCompileOne(builtins.Builtin):
         
         if self.ostbuild_resultdir is None:
             fatal("Must specify --ostbuild-resultdir=")
-
-        self.metadata = {}
-
         if self.ostbuild_meta is None:
-            output = run_sync_get_output(['ostbuild', 'autodiscover-meta'])
+            fatal("Must specify --ostbuild-meta=")
-            self.metadata = json.loads(output)
-        else:
-            f = open(self.ostbuild_meta)
-            self.metadata = json.load(f)
-            f.close()
 
-        for k in ['name', 'version']:
+        f = open(self.ostbuild_meta)
+        self.metadata = json.load(f)
+        f.close()
+
+        for k in ['name', 'revision']:
             if k not in self.metadata:
                 fatal('Missing required key "%s" in metadata' % (k, ))
 
+        if self.metadata.get('rm-configure', False):
+            configure_path = 'configure'
+            if os.path.exists(configure_path):
+                os.unlink(configure_path)
+
         autogen_script = None
         if not os.path.exists('configure'):
             log("No 'configure' script found, looking for autogen/bootstrap")
@@ -181,18 +182,6 @@ class OstbuildCompileOne(builtins.Builtin):
 
         name = self.metadata['name']
         assert ',' not in name
-        branch = self.metadata['branch']
-        assert ',' not in name
-        version = self.metadata['version']
-        assert ',' not in version
-    
-        root_name = self.metadata.get('buildroot', None)
-        # TODO - pick up current sysroot version from ostree
-        if root_name is None:
-            root_name = 'unknown-' + self.build_target
-            root_version = 'UNKNOWN'
-        else:
-            root_version = self.metadata.get('buildroot-version')
     
         tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,))
         self.tempfiles.append(tempdir)