From 55f6998d37fad490d314d2b8e70a2b80e1d725d8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 18 Jan 2012 13:35:53 -0500 Subject: [PATCH] ostbuild: Ensure we don't retrieve submodules from the network This is a bit gross...we have to manually frob the submodule URLs to point to our mirror. --- src/ostbuild/pyostbuild/builtin_build.py | 35 ++++++++-- src/ostbuild/pyostbuild/builtin_resolve.py | 77 +++++++++++++++++++--- 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index 670b4e8c..900a170f 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -18,6 +18,7 @@ import os,sys,subprocess,tempfile,re,shutil import argparse import time +import urlparse import json from StringIO import StringIO @@ -40,6 +41,22 @@ class OstbuildBuild(builtins.Builtin): def __init__(self): builtins.Builtin.__init__(self) + def _mirror_for_url(self, url): + parsed = urlparse.urlsplit(url) + return os.path.join(self.mirrordir, 'git', parsed.scheme, parsed.netloc, parsed.path[1:]) + + def _fixup_submodule_references(self, cwd): + submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd) + submodule_status_lines = submodules_status_text.split('\n') + for line in submodule_status_lines: + if line == '': continue + line = line[1:] + (sub_checksum, sub_name) = line.split(' ', 1) + sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules', + 'submodule.%s.url' % (sub_name, )], cwd=cwd) + mirrordir = self._mirror_for_url(sub_url) + run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd) + def _get_vcs_checkout(self, name, keytype, mirrordir, branch): checkoutdir = os.path.join(self.workdir, 'src') if not os.path.isdir(checkoutdir): @@ -50,9 +67,18 @@ class OstbuildBuild(builtins.Builtin): shutil.rmtree(dest) if os.path.isdir(tmp_dest): shutil.rmtree(tmp_dest) - subprocess.check_call(['git', 'clone', '-q', '--recursive', mirrordir, tmp_dest]) - subprocess.check_call(['git', 'checkout', '-q', branch], cwd=tmp_dest) - subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=tmp_dest) + git_mirrors_path = os.path.join(self.mirrordir, 'gitconfig') + f = open(git_mirrors_path) + git_mirrors = f.read() + f.close() + run_sync(['git', 'clone', '-q', + '--no-checkout', mirrordir, tmp_dest]) + run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest) + run_sync(['git', 'submodule', 'init'], cwd=tmp_dest) + self._fixup_submodule_references(tmp_dest) + run_sync(['linux-user-chroot', + '--unshare-net', '--chdir', tmp_dest, '/', + '/usr/bin/git', 'submodule', 'update']) os.rename(tmp_dest, dest) return dest @@ -178,7 +204,8 @@ class OstbuildBuild(builtins.Builtin): else: log("No previous build for '%s' found" % (buildname, )) - mirror = os.path.join(self.mirrordir, name) + + mirror = self._mirror_for_url(uri) component_src = self._get_vcs_checkout(name, keytype, mirror, branch) if meta.get('rm-configure', False): diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py index 516d144e..c1f20c0e 100755 --- a/src/ostbuild/pyostbuild/builtin_resolve.py +++ b/src/ostbuild/pyostbuild/builtin_resolve.py @@ -18,6 +18,7 @@ import os,sys,subprocess,tempfile,re,shutil import argparse import json +import urlparse from StringIO import StringIO from . import builtins @@ -35,15 +36,51 @@ class OstbuildResolve(builtins.Builtin): def __init__(self): builtins.Builtin.__init__(self) - def _ensure_vcs_mirror(self, name, keytype, uri, branch): + def _ensure_vcs_mirror(self, keytype, uri, branch): assert keytype == 'git' - mirror = os.path.join(self.mirrordir, name) + parsed = urlparse.urlsplit(uri) + mirror = os.path.join(self.mirrordir, keytype, parsed.scheme, parsed.netloc, parsed.path[1:]) tmp_mirror = mirror + '.tmp' if os.path.isdir(tmp_mirror): shutil.rmtree(tmp_mirror) if not os.path.isdir(mirror): run_sync(['git', 'clone', '--mirror', uri, tmp_mirror]) + run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror) os.rename(tmp_mirror, mirror) + last_fetch_path = mirror + '.lastfetch' + if os.path.exists(last_fetch_path): + f = open(last_fetch_path) + last_fetch_contents = f.read() + f.close() + last_fetch_contents = last_fetch_contents.strip() + else: + last_fetch_contents = None + current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror) + current_vcs_version = current_vcs_version.strip() + if current_vcs_version != last_fetch_contents: + log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version)) + tmp_checkout = os.path.join(self.mirrordir, '_tmp-checkouts', keytype, parsed.netloc, parsed.path[1:]) + if os.path.isdir(tmp_checkout): + shutil.rmtree(tmp_checkout) + parent = os.path.dirname(tmp_checkout) + if not os.path.isdir(parent): + os.makedirs(parent) + run_sync(['git', 'clone', '-b', branch, mirror, tmp_checkout]) + run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout) + submodules = [] + submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout) + submodule_status_lines = submodules_status_text.split('\n') + for line in submodule_status_lines: + if line == '': continue + line = line[1:] + (sub_checksum, sub_name) = line.split(' ', 1) + sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules', + 'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout) + self._ensure_vcs_mirror(keytype, sub_url, sub_checksum) + shutil.rmtree(tmp_checkout) + f = open(last_fetch_path, 'w') + f.write(current_vcs_version + '\n') + f.close() return mirror def _parse_src_key(self, srckey): @@ -117,21 +154,45 @@ class OstbuildResolve(builtins.Builtin): if not found: fatal("Unknown component %r" % (component_name, )) (keytype, uri) = self._parse_src_key(component['src']) - mirrordir = self._ensure_vcs_mirror(component['name'], - keytype, uri, - component['branch']) + mirrordir = self._ensure_vcs_mirror(keytype, uri, component['branch']) log("Running git fetch for %s" % (component['name'], )) run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False) + else: + fetch_components = [] for component in self.resolved_components: (keytype, uri) = self._parse_src_key(component['src']) - mirrordir = self._ensure_vcs_mirror(component['name'], - keytype, uri, - component['branch']) + try: + fetch_components.index(component['name']) + continue + except ValueError, e: + pass + mirrordir = self._ensure_vcs_mirror(keytype, uri, component['branch']) revision = buildutil.get_git_version_describe(mirrordir, component['branch']) component['revision'] = revision + mirror_gitconfig_path = os.path.join(self.mirrordir, 'gitconfig') + git_mirrordir = os.path.join(self.mirrordir, 'git') + f = open(mirror_gitconfig_path, 'w') + find_proc = subprocess.Popen(['find', '-type', 'f', '-name', 'HEAD'], + cwd=git_mirrordir, stdout=subprocess.PIPE) + path_to_url_re = re.compile(r'^([^/]+)/([^/]+)/(.+)$') + for line in find_proc.stdout: + assert line.startswith('./') + path = line[2:-6] + f.write('[url "') + f.write('file://' + os.path.join(git_mirrordir, path) + '/') + f.write('"]\n') + f.write(' insteadOf = ') + match = path_to_url_re.match(path) + assert match is not None + url = urlparse.urlunparse([match.group(1), match.group(2), match.group(3), + None, None, None]) + f.write(url) + f.write('/\n') + print "Generated git mirror config: %s" % (mirror_gitconfig_path, ) + self.manifest['components'] = self.resolved_components out_manifest = os.path.join(self.workdir, 'manifest.json')