From ea858ab5586a1f16192fc769950de0a6af28b0a6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 27 Nov 2011 13:07:33 -0500 Subject: [PATCH] ostbuild: Split nice/logger program out, merge make/makeinstall into one executable --- Makefile-ostbuild.am | 6 +- src/ostbuild/ostbuild-chroot | 20 +++- src/ostbuild/ostbuild-compile-one | 7 ++ ...ild-one-make => ostbuild-compile-one-impl} | 110 +++++++++++++++--- ...build-one => ostbuild-nice-and-log-output} | 57 +++------ .../ostbuild-one-makeinstall-split-artifacts | 105 ----------------- 6 files changed, 130 insertions(+), 175 deletions(-) create mode 100755 src/ostbuild/ostbuild-compile-one rename src/ostbuild/{ostbuild-one-make => ostbuild-compile-one-impl} (61%) rename src/ostbuild/{ostbuild-one => ostbuild-nice-and-log-output} (81%) delete mode 100755 src/ostbuild/ostbuild-one-makeinstall-split-artifacts diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index 5f7c28f2..0a5cd568 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -15,7 +15,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -bin_SCRIPTS += src/ostbuild/ostbuild-one \ - src/ostbuild/ostbuild-one-make \ - src/ostbuild/ostbuild-one-makeinstall-split-artifacts \ +bin_SCRIPTS += src/ostbuild/ostbuild-compile-one \ + src/ostbuild/ostbuild-compile-one-impl \ + src/ostbuild/ostbuild-nice-and-log-output \ $(NULL) diff --git a/src/ostbuild/ostbuild-chroot b/src/ostbuild/ostbuild-chroot index 23537980..9a46a8ac 100755 --- a/src/ostbuild/ostbuild-chroot +++ b/src/ostbuild/ostbuild-chroot @@ -2,12 +2,20 @@ import os,sys,re,subprocess -chroot_path=sys.argv[1] +i=1 +repo=sys.argv[i] +i += 1 +chroot_path=sys.argv[i] +i += 1 +args=sys.argv[i:] -proc_path=os.path.join(chroot_path, 'proc') -subprocess.check_call(['mount', '-t', 'proc', 'proc', proc_path]) +def run_in_chroot(args): + proc_path=os.path.join(chroot_path, 'proc') + subprocess.check_call(['mount', '-t', 'proc', 'proc', proc_path]) -subprocess.call(['chroot', chroot_path]) - -subprocess.check_call(['umount', proc_path]) + try: + subprocess.check_call(['chroot', chroot_path]) + finally: + subprocess.call(['umount', proc_path]) +run_in_chroot(args) diff --git a/src/ostbuild/ostbuild-compile-one b/src/ostbuild/ostbuild-compile-one new file mode 100755 index 00000000..486ee18c --- /dev/null +++ b/src/ostbuild/ostbuild-compile-one @@ -0,0 +1,7 @@ +#!/bin/sh +# +# Copyright 2010, 2011 Colin Walters +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) + +bn=$(basename $(pwd)) +ostbuild-nice-and-log-output "compile-${bn}.log" ostbuild-compile-one-impl "$@" diff --git a/src/ostbuild/ostbuild-one-make b/src/ostbuild/ostbuild-compile-one-impl similarity index 61% rename from src/ostbuild/ostbuild-one-make rename to src/ostbuild/ostbuild-compile-one-impl index b6e89f56..f71c1e19 100755 --- a/src/ostbuild/ostbuild-one-make +++ b/src/ostbuild/ostbuild-compile-one-impl @@ -1,16 +1,23 @@ #!/usr/bin/python -# ostree-buildone-raw: Generic build system wrapper # Copyright 2010, 2011 Colin Walters # Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) -# ostree-buildone-raw wraps systems that implement the GNOME build API: +# ostbuild-compile-one-make wraps systems that implement the GNOME build API: # http://people.gnome.org/~walters/docs/build-api.txt import os,sys,subprocess,tempfile,re from multiprocessing import cpu_count import select,time +tempfiles = [] + +_devel_regexps = map(re.compile, + [r'/usr/include/', + r'/usr/share/pkgconfig/', + r'/.*lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc', + r'/.*lib(?:|(?:32)|(?:64))/.*\.so$']) + root = None prefix = '/usr' @@ -44,7 +51,7 @@ for arg in sys.argv[1:]: makeargs.append(arg) def log(msg): - fullmsg = 'ostree-buildone: ' + msg + '\n' + fullmsg = '%s: %s\n' % (sys.argv[0], msg) sys.stdout.write(fullmsg) sys.stdout.flush() @@ -52,11 +59,11 @@ def fatal(msg): log(msg) sys.exit(1) -def run_sync(args, env=None): - log("Running: %r" % (args, )) +def run_sync(args, cwd=None, env=None): + log("running: %r" % (args,)) f = open('/dev/null', 'r') proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr, - close_fds=True, env=env) + close_fds=True, cwd=cwd, env=env) f.close() returncode = proc.wait() log("pid %d exited with code %d" % (proc.pid, returncode)) @@ -105,13 +112,15 @@ def _search_file(filename, pattern): f.close() return None -def _find_buildapi_makevariable(name): +def _find_buildapi_makevariable(name, builddir='.'): var = '.%s:' % (name, ) line = None - if os.path.exists('Makefile.in'): - line = _search_file('Makefile.in', var) - if not line and os.path.exists('Makefile'): - line = _search_file('Makefile', var) + path = os.path.join(builddir, 'Makefile.in') + if os.path.exists(path): + line = _search_file(path, var) + path = os.path.join(builddir, 'Makefile') + if not line and os.path.exists(path): + line = _search_file(path, var) return line is not None def phase_bootstrap(): @@ -165,21 +174,20 @@ def phase_configure(): log("Using build directory %r" % (builddir, )) if not os.path.isdir(builddir): os.mkdir(builddir) - os.chdir(builddir) configstatus = 'config.status' if not os.path.exists(configstatus): args = [os.path.join(configdir, 'configure')] args.extend(configargs) - run_sync(args) + run_sync(args, cwd=builddir) else: log("Found %s, skipping configure" % (configstatus, )) - phase_build() + phase_build(builddir=builddir) build_status = False -def phase_build(): - if not os.path.exists('Makefile'): +def phase_build(builddir=None): + if not os.path.exists(os.path.join(builddir, 'Makefile')): log("No Makefile found") sys.exit(1) args = makeargs @@ -189,14 +197,82 @@ def phase_build(): user_specified_jobs = True if not user_specified_jobs: - notparallel = _find_buildapi_makevariable('NOTPARALLEL') + notparallel = _find_buildapi_makevariable('NOTPARALLEL', builddir=builddir) if not notparallel: log("Didn't find NOTPARALLEL, using parallel make by default") args.extend(default_buildapi_jobs) + run_sync(args, cwd=builddir) + + phase_make_artifacts(builddir=builddir) + +def make_artifact(name, from_files, fakeroot_temp=None, tempdir=None): + targz_name = name + '.tar.gz' + (fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s' % (name, )) + os.close(fd) + tempfiles.append(filelist_temp) + f = open(filelist_temp, 'w') + for filename in from_files: + assert ('\n' not in filename) + f.write(filename) + f.write('\n') + f.close() + args = ['fakeroot', '-i', fakeroot_temp, 'tar', '-c', '-z', '-C', tempdir, '-f', targz_name, '-T', filelist_temp] run_sync(args) + log("created: %s" % (os.path.abspath (targz_name), )) + +def phase_make_artifacts(builddir=None): + basename=os.path.basename(os.getcwd()) + + try: + version = subprocess.check_output(['git', 'describe']) + except subprocess.CalledProcessError, e: + version = subprocess.check_output(['git', 'rev-parse', 'HEAD']) + version = version.strip() + + artifact_prefix='artifact-%s,%s' % (basename, version) + + (fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,)) + os.close(fd) + tempfiles.append(fakeroot_temp) + tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,)) + tempfiles.append(tempdir) + args = ['fakeroot', '-s', fakeroot_temp, 'make', 'install', 'DESTDIR=' + tempdir] + run_sync(args, cwd=builddir) + + devel_files = set() + runtime_files = set() + + oldpwd=os.getcwd() + os.chdir(tempdir) + for root, dirs, files in os.walk('.'): + for filename in files: + path = os.path.join(root, filename) + matched = False + for r in _devel_regexps: + if not r.match(path[1:]): + continue + devel_files.add(path) + matched = True + break + if not matched: + runtime_files.add(path) + os.chdir(oldpwd) + + if devel_files: + make_artifact(artifact_prefix + '-devel', devel_files, fakeroot_temp=fakeroot_temp, tempdir=tempdir) + make_artifact(artifact_prefix + '-runtime', runtime_files, fakeroot_temp=fakeroot_temp, tempdir=tempdir) def phase_complete(): + for tmpname in tempfiles: + if os.path.isdir(tmpname): + shutil.rmtree(tmpname) + else: + try: + os.unlink(tmpname) + pass + except OSError, e: + pass sys.exit(0) log("invocation arguments: %r" % (sys.argv, )) diff --git a/src/ostbuild/ostbuild-one b/src/ostbuild/ostbuild-nice-and-log-output similarity index 81% rename from src/ostbuild/ostbuild-one rename to src/ostbuild/ostbuild-nice-and-log-output index 252870be..d5281ce9 100755 --- a/src/ostbuild/ostbuild-one +++ b/src/ostbuild/ostbuild-nice-and-log-output @@ -3,13 +3,12 @@ # Copyright 2010, 2011 Colin Walters # Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) -# The build output is automatically logged to $TMPDIR/build-$(PWD).log. -# For example, invoking metabuild in a directory named "foo" will log -# to /tmp/build-foo.log - import os,sys,subprocess,tempfile,re import select,time,stat,fcntl +log_name = sys.argv[1] +subprocess_args = sys.argv[2:] + subprocess_nice_args = [] # In the future we should test for this better; possibly implement a @@ -215,51 +214,18 @@ class OutputFilter(object): self._warning_count, )) self.output.write("ostbuild: full log path: %s\n" % (logfile_path, )) - if successful: - for f in os.listdir('_build'): - path = os.path.join('_build', f) - if f.startswith('artifact-'): - self.output.write("ostbuild: created artifact: %s\n" % (f, )) sys.exit(0 if successful else 1) -def _on_makeinstall_exit(pid, estatus): +def _on_subprocess_exit(pid, estatus): _output_filter.finish(estatus == 0) -def _on_make_exit(pid, estatus): - if estatus == 0: - args = list(subprocess_nice_args) - args.append('ostbuild-one-makeinstall-split-artifacts') - _logfile_f.write("Running: %r\n" % (args, )) - _logfile_f.flush() - proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd) - _loop.watch_pid(proc.pid, _on_makeinstall_exit) - else: - _output_filter.finish(False) - -def _get_version(): - if not os.path.isdir('.git'): - sys.stderr.write("ostbuild-one: error: Couldn't find .git directory") - sys.exit(1) - - proc = subprocess.Popen(['git', 'describe'], stdout=subprocess.PIPE) - output = proc.communicate()[0].strip() - if proc.wait() != 0: - proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE) - if proc.wait() != 0: - sys.stderr.write("ostbuild-one: error: git rev-parse HEAD failed") - sys.exit(1) - output = proc.communicate()[0].strip() - return output - if __name__ == '__main__': user_tmpdir = os.environ.get('XDG_RUNTIME_DIR') if user_tmpdir is None: - user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'metabuild-%s' % (os.getuid(), )) + user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'ostbuild-%s' % (os.getuid(), )) else: user_tmpdir = os.path.join(user_tmpdir, 'ostbuild') - os.environ['OSBUILD_VERSION'] = _get_version() - if os.path.isdir('_build'): for filename in os.listdir('_build'): path = os.path.join('_build', filename) @@ -268,7 +234,11 @@ if __name__ == '__main__': if not os.path.isdir(user_tmpdir): os.makedirs(user_tmpdir) - logfile_path = os.path.join(user_tmpdir, '%s.log' % (os.path.basename(os.getcwd()), )) + stbuf = os.stat(user_tmpdir) + if stbuf.st_uid != os.getuid(): + sys.stderr.write('Directory %s not owned by me!' % (user_tmpdir, )) + sys.exit(1) + logfile_path = os.path.join(user_tmpdir, log_name) try: os.unlink(logfile_path) except OSError, e: @@ -284,14 +254,13 @@ if __name__ == '__main__': _output_filter.start() args = list(subprocess_nice_args) - args.append('ostbuild-one-make') - args.extend(sys.argv[1:]) + args.extend(subprocess_args) devnull=open('/dev/null') - _logfile_f.write("Running: %r\n" % (args, )) + _logfile_f.write("%s: running: %r\n" % (sys.argv[0], args, )) _logfile_f.flush() proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd) global _loop _loop = Mainloop.get(None) - _loop.watch_pid(proc.pid, _on_make_exit) + _loop.watch_pid(proc.pid, _on_subprocess_exit) _loop.run() diff --git a/src/ostbuild/ostbuild-one-makeinstall-split-artifacts b/src/ostbuild/ostbuild-one-makeinstall-split-artifacts deleted file mode 100755 index 1d7b9d33..00000000 --- a/src/ostbuild/ostbuild-one-makeinstall-split-artifacts +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/python - -# ostree-buildone-raw: Generic build system wrapper -# Copyright 2010, 2011 Colin Walters -# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) - -import os,sys,re,subprocess -import tempfile,shutil - -_devel_regexps = map(re.compile, - [r'/usr/include/', - r'/usr/share/pkgconfig/', - r'/.*lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc', - r'/.*lib(?:|(?:32)|(?:64))/.*\.so$']) - -def log(msg): - fullmsg = 'ostree-buildone: ' + msg + '\n' - sys.stdout.write(fullmsg) - sys.stdout.flush() - -tempfiles = [] - -def do_exit(code): - for tmpname in tempfiles: - if os.path.isdir(tmpname): - shutil.rmtree(tmpname) - else: - try: - os.unlink(tmpname) - pass - except OSError, e: - pass - sys.exit(code) - -def fatal(msg): - log(msg) - do_exit(1) - -def run_sync(args, env=None): - log("Running: %r" % (args, )) - f = open('/dev/null', 'r') - proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr, - close_fds=True, env=env) - f.close() - returncode = proc.wait() - if returncode == 0: - func = log - else: - func = fatal - func("pid %d exited with code %d" % (proc.pid, returncode)) - -basename=os.path.basename(os.getcwd()) -artifact_prefix='artifact-%s,%s' % (basename, os.environ['OSBUILD_VERSION']) -origdir=os.getcwd() -os.chdir('_build') - -if not os.path.exists('Makefile'): - log("No Makefile found") - do_exit(1) - -(fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,)) -os.close(fd) -tempfiles.append(fakeroot_temp) -tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,)) -tempfiles.append(tempdir) -args = ['fakeroot', '-s', fakeroot_temp, 'make', 'install', 'DESTDIR=' + tempdir] -run_sync(args) - -devel_files = set() -runtime_files = set() - -oldpwd=os.getcwd() -os.chdir(tempdir) -for root, dirs, files in os.walk('.'): - for filename in files: - path = os.path.join(root, filename) - matched = False - for r in _devel_regexps: - if r.match(path[1:]): - devel_files.add(path) - matched = True - break - if not matched: - runtime_files.add(path) -os.chdir(oldpwd) - -def make_artifact(name, from_files): - artifact_target = '%s-%s.tar.gz' % (artifact_prefix, name) - (fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s-%s' % (basename, name)) - os.close(fd) - tempfiles.append(filelist_temp) - f = open(filelist_temp, 'w') - for filename in from_files: - assert ('\n' not in filename) - f.write(filename) - f.write('\n') - f.close() - args = ['fakeroot', '-i', fakeroot_temp, 'tar', '-c', '-z', '-C', tempdir, '-f', artifact_target, '-T', filelist_temp] - run_sync(args) - -if devel_files: - make_artifact('devel', devel_files) -make_artifact('runtime', runtime_files) - -do_exit(0)