ostbuild: Ensure we don't retrieve submodules from the network
This is a bit gross...we have to manually frob the submodule URLs to point to our mirror.
This commit is contained in:
Colin Walters
parent
f6d18d9ab0
commit
55f6998d37
|
|
@ -18,6 +18,7 @@
|
|||
import os,sys,subprocess,tempfile,re,shutil
|
||||
import argparse
|
||||
import time
|
||||
import urlparse
|
||||
import json
|
||||
from StringIO import StringIO
|
||||
|
||||
|
|
@ -40,6 +41,22 @@ class OstbuildBuild(builtins.Builtin):
|
|||
def __init__(self):
|
||||
builtins.Builtin.__init__(self)
|
||||
|
||||
def _mirror_for_url(self, url):
|
||||
parsed = urlparse.urlsplit(url)
|
||||
return os.path.join(self.mirrordir, 'git', parsed.scheme, parsed.netloc, parsed.path[1:])
|
||||
|
||||
def _fixup_submodule_references(self, cwd):
|
||||
submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd)
|
||||
submodule_status_lines = submodules_status_text.split('\n')
|
||||
for line in submodule_status_lines:
|
||||
if line == '': continue
|
||||
line = line[1:]
|
||||
(sub_checksum, sub_name) = line.split(' ', 1)
|
||||
sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
|
||||
'submodule.%s.url' % (sub_name, )], cwd=cwd)
|
||||
mirrordir = self._mirror_for_url(sub_url)
|
||||
run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd)
|
||||
|
||||
def _get_vcs_checkout(self, name, keytype, mirrordir, branch):
|
||||
checkoutdir = os.path.join(self.workdir, 'src')
|
||||
if not os.path.isdir(checkoutdir):
|
||||
|
|
@ -50,9 +67,18 @@ class OstbuildBuild(builtins.Builtin):
|
|||
shutil.rmtree(dest)
|
||||
if os.path.isdir(tmp_dest):
|
||||
shutil.rmtree(tmp_dest)
|
||||
subprocess.check_call(['git', 'clone', '-q', '--recursive', mirrordir, tmp_dest])
|
||||
subprocess.check_call(['git', 'checkout', '-q', branch], cwd=tmp_dest)
|
||||
subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=tmp_dest)
|
||||
git_mirrors_path = os.path.join(self.mirrordir, 'gitconfig')
|
||||
f = open(git_mirrors_path)
|
||||
git_mirrors = f.read()
|
||||
f.close()
|
||||
run_sync(['git', 'clone', '-q',
|
||||
'--no-checkout', mirrordir, tmp_dest])
|
||||
run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest)
|
||||
run_sync(['git', 'submodule', 'init'], cwd=tmp_dest)
|
||||
self._fixup_submodule_references(tmp_dest)
|
||||
run_sync(['linux-user-chroot',
|
||||
'--unshare-net', '--chdir', tmp_dest, '/',
|
||||
'/usr/bin/git', 'submodule', 'update'])
|
||||
os.rename(tmp_dest, dest)
|
||||
return dest
|
||||
|
||||
|
|
@ -178,7 +204,8 @@ class OstbuildBuild(builtins.Builtin):
|
|||
else:
|
||||
log("No previous build for '%s' found" % (buildname, ))
|
||||
|
||||
mirror = os.path.join(self.mirrordir, name)
|
||||
|
||||
mirror = self._mirror_for_url(uri)
|
||||
component_src = self._get_vcs_checkout(name, keytype, mirror, branch)
|
||||
|
||||
if meta.get('rm-configure', False):
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
import os,sys,subprocess,tempfile,re,shutil
|
||||
import argparse
|
||||
import json
|
||||
import urlparse
|
||||
from StringIO import StringIO
|
||||
|
||||
from . import builtins
|
||||
|
|
@ -35,15 +36,51 @@ class OstbuildResolve(builtins.Builtin):
|
|||
def __init__(self):
|
||||
builtins.Builtin.__init__(self)
|
||||
|
||||
def _ensure_vcs_mirror(self, name, keytype, uri, branch):
|
||||
def _ensure_vcs_mirror(self, keytype, uri, branch):
|
||||
assert keytype == 'git'
|
||||
mirror = os.path.join(self.mirrordir, name)
|
||||
parsed = urlparse.urlsplit(uri)
|
||||
mirror = os.path.join(self.mirrordir, keytype, parsed.scheme, parsed.netloc, parsed.path[1:])
|
||||
tmp_mirror = mirror + '.tmp'
|
||||
if os.path.isdir(tmp_mirror):
|
||||
shutil.rmtree(tmp_mirror)
|
||||
if not os.path.isdir(mirror):
|
||||
run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
|
||||
run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
|
||||
os.rename(tmp_mirror, mirror)
|
||||
last_fetch_path = mirror + '.lastfetch'
|
||||
if os.path.exists(last_fetch_path):
|
||||
f = open(last_fetch_path)
|
||||
last_fetch_contents = f.read()
|
||||
f.close()
|
||||
last_fetch_contents = last_fetch_contents.strip()
|
||||
else:
|
||||
last_fetch_contents = None
|
||||
current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
|
||||
current_vcs_version = current_vcs_version.strip()
|
||||
if current_vcs_version != last_fetch_contents:
|
||||
log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
|
||||
tmp_checkout = os.path.join(self.mirrordir, '_tmp-checkouts', keytype, parsed.netloc, parsed.path[1:])
|
||||
if os.path.isdir(tmp_checkout):
|
||||
shutil.rmtree(tmp_checkout)
|
||||
parent = os.path.dirname(tmp_checkout)
|
||||
if not os.path.isdir(parent):
|
||||
os.makedirs(parent)
|
||||
run_sync(['git', 'clone', '-b', branch, mirror, tmp_checkout])
|
||||
run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
|
||||
submodules = []
|
||||
submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
|
||||
submodule_status_lines = submodules_status_text.split('\n')
|
||||
for line in submodule_status_lines:
|
||||
if line == '': continue
|
||||
line = line[1:]
|
||||
(sub_checksum, sub_name) = line.split(' ', 1)
|
||||
sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
|
||||
'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
|
||||
self._ensure_vcs_mirror(keytype, sub_url, sub_checksum)
|
||||
shutil.rmtree(tmp_checkout)
|
||||
f = open(last_fetch_path, 'w')
|
||||
f.write(current_vcs_version + '\n')
|
||||
f.close()
|
||||
return mirror
|
||||
|
||||
def _parse_src_key(self, srckey):
|
||||
|
|
@ -117,21 +154,45 @@ class OstbuildResolve(builtins.Builtin):
|
|||
if not found:
|
||||
fatal("Unknown component %r" % (component_name, ))
|
||||
(keytype, uri) = self._parse_src_key(component['src'])
|
||||
mirrordir = self._ensure_vcs_mirror(component['name'],
|
||||
keytype, uri,
|
||||
component['branch'])
|
||||
mirrordir = self._ensure_vcs_mirror(keytype, uri, component['branch'])
|
||||
log("Running git fetch for %s" % (component['name'], ))
|
||||
run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
|
||||
else:
|
||||
fetch_components = []
|
||||
|
||||
for component in self.resolved_components:
|
||||
(keytype, uri) = self._parse_src_key(component['src'])
|
||||
mirrordir = self._ensure_vcs_mirror(component['name'],
|
||||
keytype, uri,
|
||||
component['branch'])
|
||||
try:
|
||||
fetch_components.index(component['name'])
|
||||
continue
|
||||
except ValueError, e:
|
||||
pass
|
||||
mirrordir = self._ensure_vcs_mirror(keytype, uri, component['branch'])
|
||||
revision = buildutil.get_git_version_describe(mirrordir,
|
||||
component['branch'])
|
||||
component['revision'] = revision
|
||||
|
||||
mirror_gitconfig_path = os.path.join(self.mirrordir, 'gitconfig')
|
||||
git_mirrordir = os.path.join(self.mirrordir, 'git')
|
||||
f = open(mirror_gitconfig_path, 'w')
|
||||
find_proc = subprocess.Popen(['find', '-type', 'f', '-name', 'HEAD'],
|
||||
cwd=git_mirrordir, stdout=subprocess.PIPE)
|
||||
path_to_url_re = re.compile(r'^([^/]+)/([^/]+)/(.+)$')
|
||||
for line in find_proc.stdout:
|
||||
assert line.startswith('./')
|
||||
path = line[2:-6]
|
||||
f.write('[url "')
|
||||
f.write('file://' + os.path.join(git_mirrordir, path) + '/')
|
||||
f.write('"]\n')
|
||||
f.write(' insteadOf = ')
|
||||
match = path_to_url_re.match(path)
|
||||
assert match is not None
|
||||
url = urlparse.urlunparse([match.group(1), match.group(2), match.group(3),
|
||||
None, None, None])
|
||||
f.write(url)
|
||||
f.write('/\n')
|
||||
print "Generated git mirror config: %s" % (mirror_gitconfig_path, )
|
||||
|
||||
self.manifest['components'] = self.resolved_components
|
||||
|
||||
out_manifest = os.path.join(self.workdir, 'manifest.json')
|
||||
|
|
|
|||
Loading…
Reference in New Issue