diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index f5e666f2..054bf9c4 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -24,6 +24,7 @@ pyostbuild_PYTHON = \ src/ostbuild/pyostbuild/buildutil.py \ src/ostbuild/pyostbuild/builtin_autodiscover_meta.py \ src/ostbuild/pyostbuild/builtin_build.py \ + src/ostbuild/pyostbuild/builtin_checkout.py \ src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \ src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py \ src/ostbuild/pyostbuild/builtin_commit_artifacts.py \ @@ -40,5 +41,6 @@ pyostbuild_PYTHON = \ src/ostbuild/pyostbuild/ostbuildrc.py \ src/ostbuild/pyostbuild/warningfilter.py \ src/ostbuild/pyostbuild/subprocess_helpers.py \ + src/ostbuild/pyostbuild/vcs.py \ $(NULL) diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py index e3199a86..70c3cd04 100755 --- a/src/ostbuild/pyostbuild/buildutil.py +++ b/src/ostbuild/pyostbuild/buildutil.py @@ -32,6 +32,17 @@ BUILD_ENV = { 'TZ': 'EST5EDT' } +def parse_src_key(srckey): + idx = srckey.find(':') + if idx < 0: + raise ValueError("Invalid SRC uri=%s" % (srckey, )) + keytype = srckey[:idx] + if keytype not in ['git']: + raise ValueError("Unsupported SRC uri=%s" % (srckey, )) + uri = srckey[idx+1:] + return (keytype, uri) + + def get_mirrordir(mirrordir, keytype, uri, prefix=''): assert keytype == 'git' parsed = urlparse.urlsplit(uri) diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index ffce8b63..02371d6b 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -30,6 +30,7 @@ from . import ostbuildrc from . import buildutil from . import kvfile from . import odict +from . import vcs class BuildOptions(object): pass @@ -41,59 +42,6 @@ class OstbuildBuild(builtins.Builtin): def __init__(self): builtins.Builtin.__init__(self) - def _fixup_submodule_references(self, cwd): - submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd) - submodule_status_lines = submodules_status_text.split('\n') - have_submodules = False - for line in submodule_status_lines: - if line == '': continue - have_submodules = True - line = line[1:] - (sub_checksum, sub_name) = line.split(' ', 1) - sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules', - 'submodule.%s.url' % (sub_name, )], cwd=cwd) - mirrordir = buildutil.get_mirrordir(self.mirrordir, 'git', sub_url) - run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd) - return have_submodules - - def _get_vcs_checkout(self, name, keytype, mirrordir, branch, overwrite=True): - checkoutdir = os.path.join(self.workdir, 'src') - if not os.path.isdir(checkoutdir): - os.makedirs(checkoutdir) - dest = os.path.join(checkoutdir, name) - tmp_dest = dest + '.tmp' - if os.path.isdir(dest): - if not overwrite: - return dest - shutil.rmtree(dest) - if os.path.isdir(tmp_dest): - shutil.rmtree(tmp_dest) - git_mirrors_path = os.path.join(self.mirrordir, 'gitconfig') - f = open(git_mirrors_path) - git_mirrors = f.read() - f.close() - run_sync(['git', 'clone', '-q', - '--no-checkout', mirrordir, tmp_dest]) - run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest) - run_sync(['git', 'submodule', 'init'], cwd=tmp_dest) - have_submodules = self._fixup_submodule_references(tmp_dest) - if have_submodules: - run_sync(['linux-user-chroot', - '--unshare-net', '--chdir', tmp_dest, '/', - '/usr/bin/git', 'submodule', 'update']) - os.rename(tmp_dest, dest) - return dest - - def _parse_src_key(self, srckey): - idx = srckey.find(':') - if idx < 0: - raise ValueError("Invalid SRC uri=%s" % (srckey, )) - keytype = srckey[:idx] - if keytype not in ['git']: - raise ValueError("Unsupported SRC uri=%s" % (srckey, )) - uri = srckey[idx+1:] - return (keytype, uri) - def _get_ostbuild_chroot_args(self, architecture): current_machine = os.uname()[4] if current_machine != architecture: @@ -179,11 +127,12 @@ class OstbuildBuild(builtins.Builtin): buildname = self._get_buildname(meta, architecture) buildroot_name = self._get_buildroot_name(meta, architecture) - (keytype, uri) = self._parse_src_key(meta['src']) + (keytype, uri) = buildutil.parse_src_key(meta['src']) mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri) - component_src = self._get_vcs_checkout(name, keytype, mirror, branch, - overwrite=not self.args.debug_shell) + checkoutdir = os.path.join(self.workdir, 'src', name) + component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, branch, + overwrite=not self.args.debug_shell) if self.args.debug_shell: buildroot_version = self._compose_buildroot(buildroot_name, meta, dependencies, architecture) diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py new file mode 100755 index 00000000..691cfb22 --- /dev/null +++ b/src/ostbuild/pyostbuild/builtin_checkout.py @@ -0,0 +1,66 @@ +# Copyright (C) 2012 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import os,sys,subprocess,tempfile,re,shutil +import argparse +import json +import urlparse +from StringIO import StringIO + +from . import builtins +from .ostbuildlog import log, fatal +from .subprocess_helpers import run_sync, run_sync_get_output +from . import ostbuildrc +from . import buildutil +from . import odict +from . import vcs + +class OstbuildCheckout(builtins.Builtin): + name = "checkout" + short_description = "Check out specified modules" + + def __init__(self): + builtins.Builtin.__init__(self) + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('components', nargs='*') + + args = parser.parse_args(argv) + self.args = args + + self.parse_config() + + build_manifest_path = os.path.join(self.workdir, 'manifest.json') + self.manifest = json.load(open(build_manifest_path)) + + for component_name in args.components: + found = False + for component in self.manifest['components']: + if component['name'] == component_name: + found = True + break + if not found: + fatal("Unknown component %r" % (component_name, )) + (keytype, uri) = buildutil.parse_src_key(component['src']) + checkoutdir = os.path.join(os.getcwd(), component['name']) + component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, + component['revision'], + overwrite=False) + print "Checked out: %r" % (component_src, ) + +builtins.register(OstbuildCheckout) diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py index eccf8795..87d114e9 100755 --- a/src/ostbuild/pyostbuild/main.py +++ b/src/ostbuild/pyostbuild/main.py @@ -24,6 +24,7 @@ import argparse from . import builtins from . import builtin_autodiscover_meta from . import builtin_build +from . import builtin_checkout from . import builtin_chroot_compile_one from . import builtin_chroot_run_triggers from . import builtin_commit_artifacts diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py new file mode 100755 index 00000000..81b3e87e --- /dev/null +++ b/src/ostbuild/pyostbuild/vcs.py @@ -0,0 +1,67 @@ +# Copyright (C) 2011 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import os +import re +import urlparse +import shutil + +from .subprocess_helpers import run_sync_get_output, run_sync + +def get_mirrordir(mirrordir, keytype, uri, prefix=''): + assert keytype == 'git' + parsed = urlparse.urlsplit(uri) + return os.path.join(mirrordir, prefix, keytype, parsed.scheme, parsed.netloc, parsed.path[1:]) + +def _fixup_submodule_references(mirrordir, cwd): + submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd) + submodule_status_lines = submodules_status_text.split('\n') + have_submodules = False + for line in submodule_status_lines: + if line == '': continue + have_submodules = True + line = line[1:] + (sub_checksum, sub_name) = line.split(' ', 1) + sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules', + 'submodule.%s.url' % (sub_name, )], cwd=cwd) + mirrordir = get_mirrordir(mirrordir, 'git', sub_url) + run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd) + return have_submodules + +def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True): + module_mirror = get_mirrordir(mirrordir, keytype, uri) + assert keytype == 'git' + checkoutdir_parent=os.path.dirname(dest) + if not os.path.isdir(checkoutdir_parent): + os.makedirs(checkoutdir_parent) + tmp_dest = dest + '.tmp' + if os.path.isdir(dest) and overwrite: + shutil.rmtree(tmp_dest) + if not os.path.isdir(tmp_dest): + run_sync(['git', 'clone', '-q', + '--no-checkout', module_mirror, tmp_dest]) + run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest) + run_sync(['git', 'submodule', 'init'], cwd=tmp_dest) + have_submodules = _fixup_submodule_references(mirrordir, tmp_dest) + if have_submodules: + run_sync(['linux-user-chroot', + '--unshare-net', '--chdir', tmp_dest, '/', + '/usr/bin/git', 'submodule', 'update']) + os.rename(tmp_dest, dest) + return dest + +