diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index e978754a..58739be1 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -65,7 +65,6 @@ class OstbuildBuild(builtins.Builtin): def _build_one_component(self, component, architecture): basename = component['name'] - branch = component['branch'] buildname = '%s/%s/%s' % (self.snapshot['prefix'], basename, architecture) build_ref = 'components/%s' % (buildname, ) diff --git a/src/ostbuild/pyostbuild/builtin_git_mirror.py b/src/ostbuild/pyostbuild/builtin_git_mirror.py index 9c64d411..0b7f0d58 100755 --- a/src/ostbuild/pyostbuild/builtin_git_mirror.py +++ b/src/ostbuild/pyostbuild/builtin_git_mirror.py @@ -18,7 +18,7 @@ # ostbuild-compile-one-make wraps systems that implement the GNOME build API: # http://people.gnome.org/~walters/docs/build-api.txt -import os,sys,stat,subprocess,tempfile,re,shutil +import os,sys,stat,subprocess,tempfile,re,shutil,time import argparse from StringIO import StringIO import json @@ -40,8 +40,12 @@ class OstbuildGitMirror(builtins.Builtin): parser = argparse.ArgumentParser(description=self.short_description) parser.add_argument('--prefix') parser.add_argument('--src-snapshot') - parser.add_argument('--start-at') - parser.add_argument('--fetch', action='store_true') + parser.add_argument('--start-at', + help="Start at the given component") + parser.add_argument('--fetch-skip-secs', type=int, default=0, + help="Don't perform a fetch if we have done so in the last N seconds") + parser.add_argument('--fetch', action='store_true', + help="Also do a git fetch for components") parser.add_argument('components', nargs='*') args = parser.parse_args(argv) @@ -64,10 +68,33 @@ class OstbuildGitMirror(builtins.Builtin): component = self.get_component(name) src = component['src'] (keytype, uri) = vcs.parse_src_key(src) - mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch']) + branch = component.get('branch') + tag = component.get('tag') + branch_or_tag = branch or tag + mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag) - if args.fetch: - log("Running git fetch for %s" % (name, )) - run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False) + if not args.fetch: + continue + + if tag is not None: + log("Skipping fetch for %s at tag %s" % (name, tag)) + continue + + curtime = time.time() + if args.fetch_skip_secs > 0: + last_fetch_path = vcs.get_lastfetch_path(self.mirrordir, keytype, uri, branch_or_tag) + try: + stbuf = os.stat(last_fetch_path) + except OSError, e: + stbuf = None + if stbuf is not None: + mtime = stbuf.st_mtime + delta = curtime - mtime + if delta < args.fetch_skip_secs: + log("Skipping fetch for %s updated in last %d seconds" % (name, delta)) + continue + + log("Running git fetch for %s" % (name, )) + vcs.fetch(self.mirrordir, keytype, uri, branch_or_tag) builtins.register(OstbuildGitMirror) diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py index 0cff9d1d..a5160f96 100755 --- a/src/ostbuild/pyostbuild/builtin_resolve.py +++ b/src/ostbuild/pyostbuild/builtin_resolve.py @@ -67,7 +67,8 @@ class OstbuildResolve(builtins.Builtin): name = name.replace('/', '-') result['name'] = name - if 'branch' not in result: + branch_or_tag = result.get('branch') or result.get('tag') + if branch_or_tag is None: result['branch'] = 'master' return result @@ -107,9 +108,9 @@ class OstbuildResolve(builtins.Builtin): fatal("Duplicate component name '%s'" % (name, )) unique_component_names.add(name) - mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch']) - revision = buildutil.get_git_version_describe(mirrordir, - component['branch']) + branch_or_tag = component.get('branch') or component.get('tag') + mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag) + revision = buildutil.get_git_version_describe(mirrordir, branch_or_tag) component['revision'] = revision src_db = self.get_src_snapshot_db() diff --git a/src/ostbuild/pyostbuild/builtin_source_diff.py b/src/ostbuild/pyostbuild/builtin_source_diff.py index 1ffc448b..d35f0187 100755 --- a/src/ostbuild/pyostbuild/builtin_source_diff.py +++ b/src/ostbuild/pyostbuild/builtin_source_diff.py @@ -117,7 +117,8 @@ class OstbuildSourceDiff(builtins.Builtin): if keytype == 'local': log("Component %r has local URI" % (name, )) continue - mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, from_component['branch']) + branch_or_tag = from_component.get('branch') or from_component.get('tag') + mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag) to_component = self.find_component_in_snapshot(name, to_snap) if to_component is None: diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py index 1fa618f7..f47c283e 100755 --- a/src/ostbuild/pyostbuild/vcs.py +++ b/src/ostbuild/pyostbuild/vcs.py @@ -89,6 +89,11 @@ def parse_src_key(srckey): uri = srckey[idx+1:] return (keytype, uri) +def get_lastfetch_path(mirrordir, keytype, uri, branch): + mirror = buildutil.get_mirrordir(mirrordir, keytype, uri) + branch_safename = branch.replace('/','_').replace('.', '_') + return mirror + '.lastfetch-%s' % (branch_safename, ) + def ensure_vcs_mirror(mirrordir, keytype, uri, branch): mirror = buildutil.get_mirrordir(mirrordir, keytype, uri) tmp_mirror = mirror + '.tmp' @@ -100,8 +105,7 @@ def ensure_vcs_mirror(mirrordir, keytype, uri, branch): os.rename(tmp_mirror, mirror) if branch is None: return mirror - branch_safename = branch.replace('/','_').replace('.', '_') - last_fetch_path = mirror + '.lastfetch-%s' % (branch_safename, ) + last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch) if os.path.exists(last_fetch_path): f = open(last_fetch_path) last_fetch_contents = f.read() @@ -136,3 +140,14 @@ def ensure_vcs_mirror(mirrordir, keytype, uri, branch): f.write(current_vcs_version + '\n') f.close() return mirror + +def fetch(mirrordir, keytype, uri, branch): + mirror = buildutil.get_mirrordir(mirrordir, keytype, uri) + last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch) + run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False) + current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror) + current_vcs_version = current_vcs_version.strip() + f = open(last_fetch_path, 'w') + f.write(current_vcs_version + '\n') + f.close() +