From 633477806c4c62a3f5485fdf5f9eb9e5bec0cd8c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Dec 2011 07:46:08 -0500 Subject: [PATCH] ostbuild: Propagate buildroot version, finish add-artifacts We need to track what buildroot each artifact was created in. --- Makefile-ostbuild.am | 1 + src/ostbuild/ostbuild-autodiscover-meta | 10 +++ src/ostbuild/ostbuild-chroot-compile-one-impl | 73 +++++++++++++++---- ...dd-artifacts => ostbuild-commit-artifacts} | 30 ++++---- src/ostbuild/ostbuild-compile-one-impl | 17 ++++- 5 files changed, 101 insertions(+), 30 deletions(-) rename src/ostbuild/{ostbuild-add-artifacts => ostbuild-commit-artifacts} (57%) diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index 51a2815a..188246ec 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -17,6 +17,7 @@ bin_SCRIPTS += \ src/ostbuild/ostbuild-autodiscover-meta \ + src/ostbuild/ostbuild-commit-artifacts \ src/ostbuild/ostbuild-compile-one-impl \ src/ostbuild/ostbuild-chroot-compile-one-impl \ src/ostbuild/ostbuild-nice-and-log-output \ diff --git a/src/ostbuild/ostbuild-autodiscover-meta b/src/ostbuild/ostbuild-autodiscover-meta index fb8a8099..15f2311c 100755 --- a/src/ostbuild/ostbuild-autodiscover-meta +++ b/src/ostbuild/ostbuild-autodiscover-meta @@ -47,6 +47,16 @@ def _discover_version_from_git(): return None _register_discover_func('VERSION', _discover_version_from_git) +def _discover_branch_from_git(): + if os.path.isdir('.git'): + try: + ref = subprocess.check_output(['git', 'symbolic-ref', 'HEAD']) + return ref.replace('refs/heads/', '').strip() + except subprocess.CalledProcessError, e: + return None + return None +_register_discover_func('BRANCH', _discover_branch_from_git) + if args.meta: f = open(args.meta) for line in f.readlines(): diff --git a/src/ostbuild/ostbuild-chroot-compile-one-impl b/src/ostbuild/ostbuild-chroot-compile-one-impl index 4a5bc063..4311dd2d 100755 --- a/src/ostbuild/ostbuild-chroot-compile-one-impl +++ b/src/ostbuild/ostbuild-chroot-compile-one-impl @@ -18,6 +18,7 @@ # Boston, MA 02111-1307, USA. import os,sys,re,subprocess,tempfile,shutil +from StringIO import StringIO import argparse sys.path @@ -35,6 +36,7 @@ def get_build_env(): parser = argparse.ArgumentParser(description="Build a module in a given root") +parser.add_argument('--workdir') parser.add_argument('--repo') parser.add_argument('--resultdir') parser.add_argument('--branch') @@ -48,28 +50,70 @@ def log(m): sys.stdout.write('\n') sys.stdout.flush() -basename = os.path.basename(os.getcwd()) +if args.meta is None: + output = subprocess.check_output(['ostbuild-autodiscover-meta']) + ostbuild_meta_f = StringIO(output) +else: + ostbuild_meta_f = open(args.meta) -tmpdir = tempfile.mkdtemp(prefix='ostree-chroot-compile-') -log("Using temporary directory: %s" % (tmpdir, )) +metadata = {} +for line in ostbuild_meta_f: + (k,v) = line.split('=', 1) + metadata[k.strip()] = v.strip() -child_tmpdir=os.path.join(tmpdir, 'tmp') +for k in ['NAME']: + if k not in metadata: + sys.stderr.write('Missing required key "%s" in metadata' % (k, )) + sys.exit(1) + +workdir_is_tmp = (args.workdir is None) +if workdir_is_tmp: + workdir = tempfile.mkdtemp(prefix='ostree-chroot-compile-') +else: + workdir = args.workdir + +log("Using working directory: %s" % (workdir, )) + +child_tmpdir=os.path.join(workdir, 'tmp') +if os.path.isdir(child_tmpdir): + log("Cleaning up previous tmpdir: %r" % (child_tmpdir, )) + shutil.rmtree(child_tmpdir) os.mkdir(child_tmpdir) rev = subprocess.check_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.branch]) rev=rev.strip() -rootdir = os.path.join(tmpdir, 'root-' + rev) -subprocess.check_call(['ostree', '--repo=' + args.repo, 'checkout', '-U', rev, rootdir]) -log("Checked out root: %s" % (rootdir, )) +metadata['BUILDROOT'] = args.branch +metadata['BUILDROOT_VERSION'] = rev +rootdir = os.path.join(workdir, 'root-' + rev) +rootdir_tmp = rootdir + '.tmp' builddir = os.path.join(rootdir, 'ostbuild'); -os.mkdir(builddir) -os.mkdir(os.path.join(builddir, 'source')) -os.mkdir(os.path.join(builddir, 'source', basename)) -os.mkdir(os.path.join(builddir, 'results')) +if not os.path.isdir(rootdir): + if os.path.isdir(rootdir_tmp): + shutil.rmtree(rootdir_tmp) + child_args = ['ostree', '--repo=' + args.repo, 'checkout', '-U', rev, rootdir_tmp] + log("Running: %r" % (child_args, )) + subprocess.check_call(child_args) + builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild') + os.mkdir(builddir_tmp) + os.mkdir(os.path.join(builddir_tmp, 'source')) + os.mkdir(os.path.join(builddir_tmp, 'results')) + os.rename(rootdir_tmp, rootdir) + log("Checked out root: %s" % (rootdir, )) +else: + log("Using existing root: %s" % (rootdir, )) -chroot_sourcedir = os.path.join('/ostbuild', 'source', basename) +sourcedir=os.path.join(builddir, 'source', metadata['NAME']) +if not os.path.isdir(sourcedir): + os.mkdir(sourcedir) + +output_metadata = open('_ostbuild-meta', 'w') +for (k,v) in metadata.iteritems(): + output_metadata.write('%s=%s\n' % (k, v)) +output_metadata.close() + +chroot_sourcedir = os.path.join('/ostbuild', 'source', metadata['NAME']) # We need to search PATH here manually so we correctly pick up an # ostree install in e.g. ~/bin even though we're going to set PATH @@ -94,10 +138,11 @@ child_args = [ostbuild_user_chroot_path, '--unshare-pid', '--unshare-net', '--un '/bin/sh'] if not args.debug_shell: child_args += ['-c', - 'cd "%s" && ostbuild-compile-one-impl OSTBUILD_RESULTDIR=/ostbuild/results' % (chroot_sourcedir, ) + 'cd "%s" && ostbuild-compile-one-impl OSTBUILD_RESULTDIR=/ostbuild/results OSTBUILD_META=_ostbuild-meta' % (chroot_sourcedir, ) ] log("Running: %r" % (child_args, )) subprocess.check_call(child_args, env=get_build_env()) -shutil.rmtree(tmpdir) +if workdir_is_tmp: + shutil.rmtree(workdir) diff --git a/src/ostbuild/ostbuild-add-artifacts b/src/ostbuild/ostbuild-commit-artifacts similarity index 57% rename from src/ostbuild/ostbuild-add-artifacts rename to src/ostbuild/ostbuild-commit-artifacts index f5069ec2..5fcb21d8 100644 --- a/src/ostbuild/ostbuild-add-artifacts +++ b/src/ostbuild/ostbuild-commit-artifacts @@ -25,25 +25,27 @@ import os,sys,subprocess,tempfile,re i=1 repo=sys.argv[i] -artifact_re = re.compile(r'^artifact-([^,]+)-([^,]+),(.+).tar.gz$') - -if os.getuid() != 0: - print "This program must be run as root." - sys.exit(1) +artifact_re = re.compile(r'^artifact-([^,]+),([^,]+),([^,]+),([^,]+),([^.]+)\.tar\.gz$') def call_ostree_sync(*args): - subprocess.check_call(['ostree', '--repo=' + repo] + args) + subprocess.check_call(['ostree', '--repo=' + repo] + list(args)) -for arg in sys.argv[1:]: - match = artifact_re.match(arg) - if match is None +for arg in sys.argv[2:]: + basename = os.path.basename(arg) + match = artifact_re.match(basename) + if match is None: print "Invalid artifact name: %s" % (arg, ) sys.exit(1) - arch = match.group(1) - name = match.group(2) - version = match.group(3) + buildroot = match.group(1) + buildroot_version = match.group(2) + name = match.group(3) + branch = match.group(4) + version = match.group(5) - branch_name = 'artifact-%s-%s' % (arch, name) + branch_name = 'artifacts/%s/%s/%s' % (buildroot, name, branch) - call_ostree_sync('commit', '-b', branch_name, '-s', version, + call_ostree_sync('commit', '-b', branch_name, '-s', 'Build ' + version, + '--add-metadata-string=ostree-buildroot-version=' + buildroot_version, + '--add-metadata-string=ostree-artifact-version=' + version, + '--skip-if-unchanged', '--tar-autocreate-parents', '--tree=tar=' + arg) diff --git a/src/ostbuild/ostbuild-compile-one-impl b/src/ostbuild/ostbuild-compile-one-impl index d4e8a50c..381d45c2 100755 --- a/src/ostbuild/ostbuild-compile-one-impl +++ b/src/ostbuild/ostbuild-compile-one-impl @@ -20,7 +20,7 @@ # ostbuild-compile-one-make wraps systems that implement the GNOME build API: # http://people.gnome.org/~walters/docs/build-api.txt -import os,sys,subprocess,tempfile,re +import os,sys,subprocess,tempfile,re,shutil from StringIO import StringIO from multiprocessing import cpu_count import select,time @@ -291,10 +291,21 @@ def make_artifact(name, from_files, tempdir=None, resultdir=None): def phase_make_artifacts(builddir=None): name = metadata['NAME'] + assert ',' not in name + branch = metadata['BRANCH'] + assert ',' not in name version = metadata['VERSION'] assert ',' not in version - artifact_prefix='artifact-%s-%s,%s' % (build_target, name, version) + root_name = metadata.get('BUILDROOT', None) + # TODO - pick up current sysroot version from ostree + if root_name is None: + root_name = 'unknown-' + build_target + root_version = 'UNKNOWN' + else: + root_version = metadata.get('BUILDROOT_VERSION') + + artifact_prefix='artifact-%s,%s,%s,%s,%s' % (root_name, root_version, name, branch, version) tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (name,)) tempfiles.append(tempdir) @@ -334,6 +345,8 @@ def phase_make_artifacts(builddir=None): make_artifact(artifact_prefix + '-devel', devel_files, tempdir=tempdir, resultdir=ostbuild_resultdir) make_artifact(artifact_prefix + '-runtime', runtime_files, tempdir=tempdir, resultdir=ostbuild_resultdir) + phase_complete() + def phase_complete(): for tmpname in tempfiles: if os.path.isdir(tmpname):