ostbuild: Make explicit resolve (and fetch steps)

Build should only work on already-downloaded sources.

Makefile-ostbuild.am

@@ -27,6 +27,7 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/builtin_chroot_compile_one.py	\
 	src/ostbuild/pyostbuild/builtin_commit_artifacts.py	\
 	src/ostbuild/pyostbuild/builtin_compile_one.py	\
+	src/ostbuild/pyostbuild/builtin_resolve.py	\
 	src/ostbuild/pyostbuild/builtins.py		\
 	src/ostbuild/pyostbuild/__init__.py		\
 	src/ostbuild/pyostbuild/kvfile.py		\

src/ostbuild/pyostbuild/buildutil.py

@@ -26,7 +26,9 @@ def branch_name_for_artifact(a):
                                    a['name'],
                                    a['branch'])
 
-def get_git_version_describe(dirpath):
+def get_git_version_describe(dirpath, commit=None):
-    version = run_sync_get_output(['git', 'describe', '--long', '--abbrev=42', '--always'],
+    args = ['git', 'describe', '--long', '--abbrev=42', '--always']
-                                  cwd=dirpath)
+    if commit is not None:
+        args.append(commit)
+    version = run_sync_get_output(args, cwd=dirpath)
     return version.strip()

src/ostbuild/pyostbuild/builtin_build.py

@@ -162,10 +162,7 @@ class OstbuildBuild(builtins.Builtin):
 
         (keytype, uri) = self._parse_src_key(meta['src'])
 
-        component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch)
+        current_vcs_version = meta['revision']
-        component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch)
-
-        current_vcs_version = buildutil.get_git_version_describe(component_src)
 
         previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                       'rev-parse', buildname],
@@ -190,6 +187,9 @@ class OstbuildBuild(builtins.Builtin):
         else:
             log("No previous build for '%s' found" % (buildname, ))
 
+        mirror = os.path.join(self.mirrordir, name)
+        component_src = self._get_vcs_checkout(name, keytype, mirror, branch)
+
         buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)
 
         artifact_meta = {'buildroot': buildroot_name,
@@ -210,7 +210,7 @@ class OstbuildBuild(builtins.Builtin):
         patches = meta.get('patches')
         if patches is not None:
             for patch in patches:
-                patch_path = os.path.join(self.manifestdir, patch)
+                patch_path = os.path.join(self.patchdir, patch)
                 run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src)
         
         component_resultdir = os.path.join(self.workdir, 'results', name)
@@ -283,7 +283,6 @@ class OstbuildBuild(builtins.Builtin):
     
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--manifest', required=True)
         parser.add_argument('--skip-built', action='store_true')
         parser.add_argument('--start-at')
         parser.add_argument('--shell-on-failure', action='store_true')
@@ -299,19 +298,19 @@ class OstbuildBuild(builtins.Builtin):
         self.buildopts.shell_on_failure = args.shell_on_failure
         self.buildopts.skip_built = args.skip_built
 
-        self.manifest = json.load(open(args.manifest))
+        build_manifest_path = os.path.join(self.workdir, 'manifest.json')
+        self.manifest = json.load(open(build_manifest_path))
 
-        self.manifestdir = os.path.dirname(args.manifest)
+        self.patchdir = os.path.join(self.workdir, 'patches')
-
-        self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
 
+        components = self.manifest['components']
         if len(args.components) == 0:
-            build_components = self.resolved_components
+            build_components = components
         else:
             build_components = []
             for name in args.components:
                 found = False
-                for child in self.resolved_components:
+                for child in components:
                     if child['name'] == name:
                         found = True
                         build_components.append(child)
@@ -321,7 +320,7 @@ class OstbuildBuild(builtins.Builtin):
 
         start_at_index = -1
         if args.start_at is not None:
-            if build_components != self.resolved_components:
+            if build_components != components:
                 fatal("Can't specify --start-at with component list")
             for i,component in enumerate(build_components):
                 if component['name'] == args.start_at:
@@ -333,12 +332,12 @@ class OstbuildBuild(builtins.Builtin):
             start_at_index = 0
 
         for component in build_components[start_at_index:]:
-            index = self.resolved_components.index(component)
+            index = components.index(component)
-            dependencies = self.resolved_components[:index]
+            dependencies = components[:index]
             for architecture in self.manifest['architectures']:
                 self._build_one_component(component, dependencies, architecture)
 
         for architecture in self.manifest['architectures']:
-            self._compose_arch(architecture, self.resolved_components)
+            self._compose_arch(architecture, components)
         
 builtins.register(OstbuildBuild)

src/ostbuild/pyostbuild/builtin_resolve.py

@@ -0,0 +1,139 @@
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import ostbuildrc
+from . import buildutil
+from . import kvfile
+from . import odict
+
+class OstbuildResolve(builtins.Builtin):
+    name = "resolve"
+    short_description = "Download the source code for a given manifest"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def _ensure_vcs_mirror(self, name, keytype, uri, branch):
+        assert keytype == 'git'
+        mirror = os.path.join(self.mirrordir, name)
+        tmp_mirror = mirror + '.tmp'
+        if os.path.isdir(tmp_mirror):
+            shutil.rmtree(tmp_mirror)
+        if not os.path.isdir(mirror):
+            run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
+            os.rename(tmp_mirror, mirror)
+        elif self.args.fetch:
+            log("Running git fetch for %s" % (name, ))
+            run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False)
+        return mirror
+
+    def _parse_src_key(self, srckey):
+        idx = srckey.find(':')
+        if idx < 0:
+            raise ValueError("Invalid SRC uri=%s" % (srckey, ))
+        keytype = srckey[:idx]
+        if keytype not in ['git']:
+            raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
+        uri = srckey[idx+1:]
+        return (keytype, uri)
+
+    def _resolve_component_meta(self, component_meta):
+        result = dict(component_meta)
+        orig_src = component_meta['src']
+
+        did_expand = False
+        for (vcsprefix, expansion) in self.manifest['vcsconfig'].iteritems():
+            prefix = vcsprefix + ':'
+            if orig_src.startswith(prefix):
+                result['src'] = expansion + orig_src[len(prefix):]
+                did_expand = True
+                break
+
+        name = component_meta.get('name')
+        if name is None:
+            if did_expand:
+                src = orig_src
+                idx = src.rindex(':')
+                name = src[idx+1:]
+            else:
+                src = result['src']
+                idx = src.rindex('/')
+                name = src[idx+1:]
+            if name.endswith('.git'):
+                name = name[:-4]
+            name = name.replace('/', '-')
+            result['name'] = name
+
+        if 'branch' not in result:
+            result['branch'] = 'master'
+
+        return result
+    
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('--fetch', action='store_true')
+
+        args = parser.parse_args(argv)
+        self.args = args
+        
+        self.parse_config()
+
+        manifest_path = self.ostbuildrc.get_key('manifest')
+        self.manifest = json.load(open(manifest_path))
+
+        self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
+
+        for component in self.resolved_components:
+            (keytype, uri) = self._parse_src_key(component['src'])
+            mirrordir = self._ensure_vcs_mirror(component['name'],
+                                                keytype, uri,
+                                                component['branch'])
+            revision = buildutil.get_git_version_describe(mirrordir,
+                                                          component['branch'])
+            component['revision'] = revision
+
+        self.manifest['components'] = self.resolved_components
+
+        out_manifest = os.path.join(self.workdir, 'manifest.json')
+        patchdir = os.path.join(self.workdir, 'patches')
+        if not os.path.isdir(patchdir):
+            os.mkdir(patchdir)
+        all_patches = {}
+        for component in self.resolved_components:
+            patches = component.get('patches', [])
+            for patch in patches:
+                all_patches[patch] = True
+        for patch in all_patches:
+            src = os.path.join(os.path.dirname(manifest_path),
+                               patch)
+            dest = os.path.join(patchdir, patch)
+            shutil.copy(src, dest)
+        
+        f = open(out_manifest, 'w')
+        json.dump(self.manifest, f, indent=4)
+        f.close()
+        print "Created: %s, %d patches" % (out_manifest, len(all_patches.keys()))
+        
+builtins.register(OstbuildResolve)

src/ostbuild/pyostbuild/builtins.py

@@ -31,6 +31,7 @@ class Builtin(object):
     short_description = None
 
     def parse_config(self):
+        self.ostbuildrc = ostbuildrc
         self.repo = ostbuildrc.get_key('repo')
         self.mirrordir = ostbuildrc.get_key('mirrordir')
         if not os.path.isdir(self.mirrordir):

src/ostbuild/pyostbuild/main.py

@@ -27,6 +27,7 @@ from . import builtin_build
 from . import builtin_chroot_compile_one
 from . import builtin_commit_artifacts
 from . import builtin_compile_one
+from . import builtin_resolve
 
 def usage(ecode):
     print "Builtins:"

src/ostbuild/pyostbuild/subprocess_helpers.py

@@ -67,8 +67,9 @@ def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=Fal
     return None
 
 def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
-             log_success=True):
+             log_success=True, log_initiation=True):
-    log("running: %s" % (subprocess.list2cmdline(args),))
+    if log_initiation:
+        log("running: %s" % (subprocess.list2cmdline(args),))
     # This dance is necessary because we want to keep the PWD
     # environment variable up to date.  Not doing so is a recipie
     # for triggering edge conditions in pwd lookup.