ostbuild: Add 'checkout' builtin

This is useful for the developer to get the source code of
several things at exactly the revision specified in the manifest.

Makefile-ostbuild.am

@@ -24,6 +24,7 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/buildutil.py		\
 	src/ostbuild/pyostbuild/builtin_autodiscover_meta.py	\
 	src/ostbuild/pyostbuild/builtin_build.py	\
+	src/ostbuild/pyostbuild/builtin_checkout.py	\
 	src/ostbuild/pyostbuild/builtin_chroot_compile_one.py	\
 	src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py	\
 	src/ostbuild/pyostbuild/builtin_commit_artifacts.py	\
@@ -40,5 +41,6 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/ostbuildrc.py		\
 	src/ostbuild/pyostbuild/warningfilter.py	\
 	src/ostbuild/pyostbuild/subprocess_helpers.py	\
+	src/ostbuild/pyostbuild/vcs.py			\
 	$(NULL)
 

src/ostbuild/pyostbuild/buildutil.py

@@ -32,6 +32,17 @@ BUILD_ENV = {
     'TZ': 'EST5EDT'
     }
 
+def parse_src_key(srckey):
+    idx = srckey.find(':')
+    if idx < 0:
+        raise ValueError("Invalid SRC uri=%s" % (srckey, ))
+    keytype = srckey[:idx]
+    if keytype not in ['git']:
+        raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
+    uri = srckey[idx+1:]
+    return (keytype, uri)
+
+
 def get_mirrordir(mirrordir, keytype, uri, prefix=''):
     assert keytype == 'git'
     parsed = urlparse.urlsplit(uri)

src/ostbuild/pyostbuild/builtin_build.py

@@ -30,6 +30,7 @@ from . import ostbuildrc
 from . import buildutil
 from . import kvfile
 from . import odict
+from . import vcs
 
 class BuildOptions(object):
     pass
@@ -41,59 +42,6 @@ class OstbuildBuild(builtins.Builtin):
     def __init__(self):
         builtins.Builtin.__init__(self)
 
-    def _fixup_submodule_references(self, cwd):
-        submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd)
-        submodule_status_lines = submodules_status_text.split('\n')
-        have_submodules = False
-        for line in submodule_status_lines:
-            if line == '': continue
-            have_submodules = True
-            line = line[1:]
-            (sub_checksum, sub_name) = line.split(' ', 1)
-            sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
-                                           'submodule.%s.url' % (sub_name, )], cwd=cwd)
-            mirrordir = buildutil.get_mirrordir(self.mirrordir, 'git', sub_url)
-            run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd)
-        return have_submodules
-
-    def _get_vcs_checkout(self, name, keytype, mirrordir, branch, overwrite=True):
-        checkoutdir = os.path.join(self.workdir, 'src')
-        if not os.path.isdir(checkoutdir):
-            os.makedirs(checkoutdir)
-        dest = os.path.join(checkoutdir, name)
-        tmp_dest = dest + '.tmp'
-        if os.path.isdir(dest):
-            if not overwrite:
-                return dest
-            shutil.rmtree(dest)
-        if os.path.isdir(tmp_dest):
-            shutil.rmtree(tmp_dest)
-        git_mirrors_path = os.path.join(self.mirrordir, 'gitconfig')
-        f = open(git_mirrors_path)
-        git_mirrors = f.read()
-        f.close()
-        run_sync(['git', 'clone', '-q',
-                  '--no-checkout', mirrordir, tmp_dest])
-        run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest)
-        run_sync(['git', 'submodule', 'init'], cwd=tmp_dest)
-        have_submodules = self._fixup_submodule_references(tmp_dest)
-        if have_submodules:
-            run_sync(['linux-user-chroot',
-                      '--unshare-net', '--chdir', tmp_dest, '/',
-                      '/usr/bin/git', 'submodule', 'update'])
-        os.rename(tmp_dest, dest)
-        return dest
-
-    def _parse_src_key(self, srckey):
-        idx = srckey.find(':')
-        if idx < 0:
-            raise ValueError("Invalid SRC uri=%s" % (srckey, ))
-        keytype = srckey[:idx]
-        if keytype not in ['git']:
-            raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
-        uri = srckey[idx+1:]
-        return (keytype, uri)
-
     def _get_ostbuild_chroot_args(self, architecture):
         current_machine = os.uname()[4]
         if current_machine != architecture:
@@ -179,11 +127,12 @@ class OstbuildBuild(builtins.Builtin):
         buildname = self._get_buildname(meta, architecture)
         buildroot_name = self._get_buildroot_name(meta, architecture)
 
-        (keytype, uri) = self._parse_src_key(meta['src'])
+        (keytype, uri) = buildutil.parse_src_key(meta['src'])
 
         mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
-        component_src = self._get_vcs_checkout(name, keytype, mirror, branch,
+        checkoutdir = os.path.join(self.workdir, 'src', name)
-                                               overwrite=not self.args.debug_shell)
+        component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, branch,
+                                             overwrite=not self.args.debug_shell)
 
         if self.args.debug_shell:
             buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture)

src/ostbuild/pyostbuild/builtin_checkout.py

@@ -0,0 +1,66 @@
+# Copyright (C) 2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import json
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import ostbuildrc
+from . import buildutil
+from . import odict
+from . import vcs
+
+class OstbuildCheckout(builtins.Builtin):
+    name = "checkout"
+    short_description = "Check out specified modules"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('components', nargs='*')
+
+        args = parser.parse_args(argv)
+        self.args = args
+        
+        self.parse_config()
+
+        build_manifest_path = os.path.join(self.workdir, 'manifest.json')
+        self.manifest = json.load(open(build_manifest_path))
+
+        for component_name in args.components:
+            found = False
+            for component in self.manifest['components']:
+                if component['name'] == component_name:
+                    found = True
+                    break
+            if not found:
+                fatal("Unknown component %r" % (component_name, ))
+            (keytype, uri) = buildutil.parse_src_key(component['src'])
+            checkoutdir = os.path.join(os.getcwd(), component['name'])
+            component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+                                                 component['revision'],
+                                                 overwrite=False)
+            print "Checked out: %r" % (component_src, )
+        
+builtins.register(OstbuildCheckout)

src/ostbuild/pyostbuild/main.py

@@ -24,6 +24,7 @@ import argparse
 from . import builtins
 from . import builtin_autodiscover_meta
 from . import builtin_build
+from . import builtin_checkout
 from . import builtin_chroot_compile_one
 from . import builtin_chroot_run_triggers
 from . import builtin_commit_artifacts

src/ostbuild/pyostbuild/vcs.py

@@ -0,0 +1,67 @@
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import re
+import urlparse
+import shutil
+
+from .subprocess_helpers import run_sync_get_output, run_sync
+
+def get_mirrordir(mirrordir, keytype, uri, prefix=''):
+    assert keytype == 'git'
+    parsed = urlparse.urlsplit(uri)
+    return os.path.join(mirrordir, prefix, keytype, parsed.scheme, parsed.netloc, parsed.path[1:])
+
+def _fixup_submodule_references(mirrordir, cwd):
+    submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd)
+    submodule_status_lines = submodules_status_text.split('\n')
+    have_submodules = False
+    for line in submodule_status_lines:
+        if line == '': continue
+        have_submodules = True
+        line = line[1:]
+        (sub_checksum, sub_name) = line.split(' ', 1)
+        sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
+                                       'submodule.%s.url' % (sub_name, )], cwd=cwd)
+        mirrordir = get_mirrordir(mirrordir, 'git', sub_url)
+        run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd)
+    return have_submodules
+
+def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
+    module_mirror = get_mirrordir(mirrordir, keytype, uri)
+    assert keytype == 'git'
+    checkoutdir_parent=os.path.dirname(dest)
+    if not os.path.isdir(checkoutdir_parent):
+        os.makedirs(checkoutdir_parent)
+    tmp_dest = dest + '.tmp'
+    if os.path.isdir(dest) and overwrite:
+        shutil.rmtree(tmp_dest)
+    if not os.path.isdir(tmp_dest):
+        run_sync(['git', 'clone', '-q',
+                  '--no-checkout', module_mirror, tmp_dest])
+    run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest)
+    run_sync(['git', 'submodule', 'init'], cwd=tmp_dest)
+    have_submodules = _fixup_submodule_references(mirrordir, tmp_dest)
+    if have_submodules:
+        run_sync(['linux-user-chroot',
+                  '--unshare-net', '--chdir', tmp_dest, '/',
+                  '/usr/bin/git', 'submodule', 'update'])
+    os.rename(tmp_dest, dest)
+    return dest
+
+