From 5f3b0296384789361a6dcc312b73bcfa839b2b6c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Dec 2011 21:44:32 -0500 Subject: [PATCH] ostbuild: Flesh out chroot build to use ostbuild-user-chroot One thing that made this take significantly longer than it might have otherwise is that we have to keep PWD "up to date" - otherwise we hit bugs in glibc's getcwd() implementation. --- Makefile-ostbuild.am | 2 + Makefile-ostree.am | 1 - src/ostbuild/ostbuild-chroot-compile-one | 0 src/ostbuild/ostbuild-chroot-compile-one-impl | 100 +++++++++++++----- src/ostbuild/ostbuild-compile-one-impl | 63 ++++++----- 5 files changed, 104 insertions(+), 62 deletions(-) mode change 100644 => 100755 src/ostbuild/ostbuild-chroot-compile-one mode change 100644 => 100755 src/ostbuild/ostbuild-chroot-compile-one-impl diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index 6ff0dbba..8eff0632 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -17,6 +17,8 @@ bin_SCRIPTS += src/ostbuild/ostbuild-compile-one \ src/ostbuild/ostbuild-compile-one-impl \ + src/ostbuild/ostbuild-chroot-compile-one \ + src/ostbuild/ostbuild-chroot-compile-one-impl \ src/ostbuild/ostbuild-nice-and-log-output \ $(NULL) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index eb84940d..05498101 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -54,4 +54,3 @@ ostree_pull_SOURCES = src/ostree/ot-main.h \ ostree_pull_CFLAGS = $(ostree_bin_shared_cflags) $(OT_DEP_SOUP_CFLAGS) ostree_pull_LDADD = $(ostree_bin_shared_ldadd) $(OT_DEP_SOUP_LIBS) endif - diff --git a/src/ostbuild/ostbuild-chroot-compile-one b/src/ostbuild/ostbuild-chroot-compile-one old mode 100644 new mode 100755 diff --git a/src/ostbuild/ostbuild-chroot-compile-one-impl b/src/ostbuild/ostbuild-chroot-compile-one-impl old mode 100644 new mode 100755 index 9c253ea5..5e25c343 --- a/src/ostbuild/ostbuild-chroot-compile-one-impl +++ b/src/ostbuild/ostbuild-chroot-compile-one-impl @@ -17,37 +17,79 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -import os,sys,re,subprocess +import os,sys,re,subprocess,tempfile,shutil +import argparse -i=1 -repo=sys.argv[i] -i += 1 -chroot_path=sys.argv[i] -i += 1 -args=sys.argv[i:] - -if os.getuid() != 0: - print "This program must be run as root." - sys.exit(1) - -rootdir=os.path.join(chroot_path, 'root') - -if not os.path.isdir(rootdir): - print "Not a directory: %s" % (rootdir, ) - sys.exit(1) - -builddir = os.path.join(rootdir, 'ostree-build') -if not os.path.isdir(builddir): - os.mkdir(builddir) +def get_build_env(): + return {'HOME' : '/', + 'HOSTNAME' : 'ostbuild', + 'LANG': 'C', + 'PATH' : '/usr/bin:/bin:/usr/sbin:/sbin', + 'SHELL' : '/bin/bash', + 'TERM' : 'vt100', + 'TMPDIR' : '/tmp', + 'TZ': 'EST5EDT' + } -def run_in_chroot(args): - proc_path=os.path.join(chroot_path, 'proc') - subprocess.check_call(['mount', '-t', 'proc', 'proc', proc_path]) +parser = argparse.ArgumentParser(description="Build a module in a given root") +parser.add_argument('--repo') +parser.add_argument('--resultdir') +parser.add_argument('--branch') +parser.add_argument('--debug-shell', type=bool) - try: - subprocess.check_call(['chroot', chroot_path]) - finally: - subprocess.call(['umount', proc_path]) +args = parser.parse_args() -run_in_chroot(args) +def log(m): + sys.stdout.write(m) + sys.stdout.write('\n') + sys.stdout.flush() + +tmpdir = tempfile.mkdtemp(prefix='ostree-chroot-compile-') +log("Using temporary directory: %s" % (tmpdir, )) + +child_tmpdir=os.path.join(tmpdir, 'tmp') +os.mkdir(child_tmpdir) + +rev = subprocess.check_output(['ostree', '--repo=' + args.repo, 'rev-parse', args.branch]) +rev=rev.strip() + +rootdir = os.path.join(tmpdir, 'root-' + rev) +subprocess.check_call(['ostree', '--repo=' + args.repo, 'checkout', '-U', rev, rootdir]) +log("Checked out root: %s" % (rootdir, )) + +builddir = os.path.join(rootdir, 'ostbuild'); +os.mkdir(builddir) +os.mkdir(os.path.join(builddir, 'source')) +os.mkdir(os.path.join(builddir, 'results')) + +# We need to search PATH here manually so we correctly pick up an +# ostree install in e.g. ~/bin even though we're going to set PATH +# below for our children inside the chroot. +ostbuild_user_chroot_path = None +for dirname in os.environ['PATH'].split(':'): + path = os.path.join(dirname, 'ostbuild-user-chroot') + if os.access(path, os.X_OK): + ostbuild_user_chroot_path = path + break +if ostbuild_user_chroot_path is None: + ostbuild_user_chroot_path = 'ostbuild-user-chroot' + +child_args = [ostbuild_user_chroot_path, '--unshare-pid', '--unshare-net', '--unshare-ipc', + '--mount-readonly', '/', + '--mount-proc', '/proc', + '--mount-bind', '/dev', '/dev', + '--mount-bind', child_tmpdir, '/tmp', + '--mount-bind', os.getcwd(), '/ostbuild/source', + '--mount-bind', args.resultdir, '/ostbuild/results', + rootdir, + '/bin/sh'] +if not args.debug_shell: + child_args += ['-c', + 'cd /ostbuild/source && ostbuild-compile-one-impl OSTBUILD_RESULTDIR=/ostbuild/results' + ] +log("Running: %r" % (child_args, )) +subprocess.check_call(child_args, env=get_build_env()) + +shutil.rmtree(tmpdir) + diff --git a/src/ostbuild/ostbuild-compile-one-impl b/src/ostbuild/ostbuild-compile-one-impl index 57e481ba..1a8ac540 100755 --- a/src/ostbuild/ostbuild-compile-one-impl +++ b/src/ostbuild/ostbuild-compile-one-impl @@ -69,7 +69,7 @@ ostbuild_resultdir=top_srcdir for arg in sys.argv[1:]: if arg.startswith('OSTBUILD_RESULTDIR='): - ostbuild_resultdir=arg[20:] + ostbuild_resultdir=arg[len('OSTBUILD_RESULTDIR='):] elif arg.startswith('--'): configargs.append(arg) else: @@ -87,8 +87,22 @@ def fatal(msg): def run_sync(args, cwd=None, env=None): log("running: %r" % (args,)) f = open('/dev/null', 'r') + # This dance is necessary because we want to keep the PWD + # environment variable up to date. Not doing so is a recipie + # for triggering edge conditions in pwd lookup. + if (cwd is not None) and (env is None or ('PWD' in env)): + if env is None: + env_copy = os.environ.copy() + else: + env_copy = env.copy() + if ('PWD' in env_copy) and (not cwd.startswith('/')): + env_copy['PWD'] = os.path.join(env_copy['PWD'], cwd) + else: + env_copy['PWD'] = cwd + else: + env_copy = env proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr, - close_fds=True, cwd=cwd, env=env) + close_fds=True, cwd=cwd, env=env_copy) f.close() returncode = proc.wait() log("pid %d exited with code %d" % (proc.pid, returncode)) @@ -167,7 +181,6 @@ def phase_bootstrap(): if bootstrap: log("Detected bootstrap script: %s, using it" % (bootstrap, )) args = [bootstrap] - args.extend(configargs) # Add NOCONFIGURE; GNOME style scripts use this env = dict(os.environ) env['NOCONFIGURE'] = '1' @@ -187,24 +200,24 @@ def phase_configure(): shutil.copytree('.', '_build', symlinks=True, ignore=shutil.ignore_patterns('_build')) use_builddir = False - builddir = '.' - else: - builddir = '_build' - if not use_builddir: - configdir = './' - else: - configdir = os.getcwd() - builddir = builddir + if use_builddir: + builddir = '_build' log("Using build directory %r" % (builddir, )) if not os.path.isdir(builddir): os.mkdir(builddir) configstatus = 'config.status' if not os.path.exists(configstatus): - args = [os.path.join(configdir, 'configure')] + if use_builddir: + args = ['../configure'] + else: + args = ['./configure'] args.extend(configargs) - run_sync(args, cwd=builddir) + if use_builddir: + run_sync(args, cwd=builddir) + else: + run_sync(args) else: log("Found %s, skipping configure" % (configstatus, )) phase_build(builddir=builddir) @@ -231,7 +244,7 @@ def phase_build(builddir=None): phase_make_artifacts(builddir=builddir) -def make_artifact(name, from_files, fakeroot_temp=None, tempdir=None, resultdir=None): +def make_artifact(name, from_files, tempdir=None, resultdir=None): targz_name = name + '.tar.gz' (fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s' % (name, )) os.close(fd) @@ -242,15 +255,11 @@ def make_artifact(name, from_files, fakeroot_temp=None, tempdir=None, resultdir= f.write(filename) f.write('\n') f.close() - if fakeroot_temp: - args = ['fakeroot', '-i', fakeroot_temp] - else: - args = [] if resultdir: result_path = os.path.join(resultdir, targz_name) else: result_path = targz_name - args.extend(['tar', '-c', '-z', '-C', tempdir, '-f', result_path, '-T', filelist_temp]) + args = ['tar', '-c', '-z', '-C', tempdir, '-f', result_path, '-T', filelist_temp] run_sync(args) log("created: %s" % (os.path.abspath (result_path), )) @@ -267,19 +276,9 @@ def phase_make_artifacts(builddir=None): artifact_prefix='artifact-%s-%s,%s' % (build_target, basename, version) - if os.getuid() != 0: - (fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,)) - os.close(fd) - tempfiles.append(fakeroot_temp) - else: - fakeroot_temp = None tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,)) tempfiles.append(tempdir) - if fakeroot_temp is not None: - args = ['fakeroot', '-s', fakeroot_temp] - else: - args = [] - args.extend(['make', 'install', 'DESTDIR=' + tempdir]) + args = ['make', 'install', 'DESTDIR=' + tempdir] run_sync(args, cwd=builddir) devel_files = set() @@ -302,8 +301,8 @@ def phase_make_artifacts(builddir=None): os.chdir(oldpwd) if devel_files: - make_artifact(artifact_prefix + '-devel', devel_files, fakeroot_temp=fakeroot_temp, tempdir=tempdir, resultdir=ostbuild_resultdir) - make_artifact(artifact_prefix + '-runtime', runtime_files, fakeroot_temp=fakeroot_temp, tempdir=tempdir, resultdir=ostbuild_resultdir) + make_artifact(artifact_prefix + '-devel', devel_files, tempdir=tempdir, resultdir=ostbuild_resultdir) + make_artifact(artifact_prefix + '-runtime', runtime_files, tempdir=tempdir, resultdir=ostbuild_resultdir) def phase_complete(): for tmpname in tempfiles: