diff --git a/Makefile-osbuild.am b/Makefile-osbuild.am index 1ef77531..3f2e56b5 100644 --- a/Makefile-osbuild.am +++ b/Makefile-osbuild.am @@ -1,5 +1,3 @@ -# Makefile for C source code -# # Copyright (C) 2011 Colin Walters # # This library is free software; you can redistribute it and/or @@ -17,21 +15,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -bin_PROGRAMS += ostree-build - -osbuild_common_cflags = -I$(srcdir)/libotutil -I$(srcdir)/osbuild -DLOCALEDIR=\"$(datadir)/locale\" -DG_LOG_DOMAIN=\"osbuild\" $(GIO_UNIX_CFLAGS) -osbuild_common_ldadd = libotutil.la $(GIO_UNIX_LIBS) - -ostree_build_SOURCES = osbuild/main.c \ - osbuild/ob-builtins.h \ - osbuild/ob-builtin-buildone.c \ +bin_SCRIPTS += osbuild/ostree-buildone \ + osbuild/ostree-buildone-make \ + osbuild/ostree-buildone-makeinstall-split-artifacts \ $(NULL) -ostree_build_CFLAGS = $(osbuild_common_cflags) -ostree_build_LDADD = $(osbuild_common_ldadd) - -bin_PROGRAMS += osbuild-raw-makeinstall - -osbuild_raw_makeinstall_SOURCES = osbuild/osbuild-raw-makeinstall.c \ - $(NULL) -osbuild_raw_makeinstall_CFLAGS = $(osbuild_common_cflags) -osbuild_raw_makeinstall_LDADD = $(osbuild_common_ldadd) diff --git a/osbuild/ostree-buildone b/osbuild/ostree-buildone new file mode 100644 index 00000000..077be412 --- /dev/null +++ b/osbuild/ostree-buildone @@ -0,0 +1,280 @@ +#!/usr/bin/python +# +# ostree-buildone: +# Copyright 2010, 2011 Colin Walters +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) + +# The build output is automatically logged to $TMPDIR/build-$(PWD).log. +# For example, invoking metabuild in a directory named "foo" will log +# to /tmp/build-foo.log +# +# You can pass arguments to metabuild; if they start with '--', they're +# given to configure. Otherwise, they're passed to make. +# +# $ metabuild --enable-libfoo # passed to configure +# $ metabuild -j 1 # passed to make + +import os,sys,subprocess,tempfile,re +import select,time,stat,fcntl + +subprocess_nice_args = [] + +# In the future we should test for this better; possibly implement a +# custom fork handler +try: + chrt_args = ['chrt', '--idle', '0'] + proc = subprocess.Popen(chrt_args + ['true']) + if proc.wait() == 0: + subprocess_nice_args.extend(chrt_args) +except OSError, e: + pass + +try: + ionice_args = ['ionice', '-c', '3', '-t'] + proc = subprocess.Popen(ionice_args + ['true']) + if proc.wait() == 0: + subprocess_nice_args.extend(ionice_args) +except OSError, e: + pass + +warning_re = re.compile(r'(: ((warning)|(error)|(fatal error)): )|(make(\[[0-9]+\])?: \*\*\*)') +output_whitelist_re = re.compile(r'^(make(\[[0-9]+\])?: Entering directory)|(ostree-build )') + +_bold_sequence = None +_normal_sequence = None +if os.isatty(1): + _bold_sequence = subprocess.Popen(['tput', 'bold'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0] + _normal_sequence = subprocess.Popen(['tput', 'sgr0'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0] +def _bold(text): + if _bold_sequence is not None: + return '%s%s%s' % (_bold_sequence, text, _normal_sequence) + else: + return text + +class Mainloop(object): + DEFAULT = None + def __init__(self): + self._running = True + self.poll = select.poll() + self._timeouts = [] + self._pid_watches = {} + self._fd_callbacks = {} + + @classmethod + def get(cls, context): + if context is None: + if cls.DEFAULT is None: + cls.DEFAULT = cls() + return cls.DEFAULT + raise NotImplementedError("Unknown context %r" % (context, )) + + def watch_fd(self, fd, callback): + self.poll.register(fd) + self._fd_callbacks[fd] = callback + + def unwatch_fd(self, fd): + self.poll.unregister(fd) + del self._fd_callbacks[fd] + + def watch_pid(self, pid, callback): + self._pid_watches[pid] = callback + + def timeout_add(self, ms, callback): + self._timeouts.append((ms, callback)) + + def quit(self): + self._running = False + + def run_once(self): + min_timeout = None + if len(self._pid_watches) > 0: + min_timeout = 500 + for (ms, callback) in self._timeouts: + if (min_timeout is None) or (ms < min_timeout): + min_timeout = ms + origtime = time.time() * 1000 + fds = self.poll.poll(min_timeout) + for fd in fds: + self._fd_callbacks[fd]() + for pid in self._pid_watches: + (opid, status) = os.waitpid(pid, os.WNOHANG) + if opid != 0: + self._pid_watches[pid](opid, status) + del self._pid_watches[pid] + newtime = time.time() * 1000 + diff = int(newtime - origtime) + if diff < 0: diff = 0 + for i,(ms, callback) in enumerate(self._timeouts): + remaining_ms = ms - diff + if remaining_ms <= 0: + callback() + else: + self._timeouts[i] = (remaining_ms, callback) + + def run(self): + while self._running: + self.run_once() + +class FileMonitor(object): + def __init__(self): + self._paths = {} + self._path_modtimes = {} + self._timeout = 1000 + self._timeout_installed = False + self._loop = Mainloop.get(None) + + def _stat(self, path): + try: + st = os.stat(path) + return st[stat.ST_MTIME] + except OSError, e: + return None + + def add(self, path, callback): + if path not in self._paths: + self._paths[path] = [] + self._path_modtimes[path] = self._stat(path) + self._paths[path].append(callback) + if not self._timeout_installed: + self._timeout_installed = True + self._loop.timeout_add(self._timeout, self._check_files) + + def _check_files(self): + for (path,callbacks) in self._paths.iteritems(): + mtime = self._stat(path) + orig_mtime = self._path_modtimes[path] + if (mtime is not None) and (orig_mtime is None or (mtime > orig_mtime)): + self._path_modtimes[path] = mtime + for cb in callbacks: + cb() + +_filemon = FileMonitor() + +class OutputFilter(object): + def __init__(self, filename, output): + self.filename = filename + self.output = output + + # inherit globals + self._warning_re = warning_re + self._nonfilter_re = output_whitelist_re + + self._buf = '' + self._warning_count = 0 + self._filtered_line_count = 0 + _filemon.add(filename, self._on_changed) + self._fd = os.open(filename, os.O_RDONLY) + fcntl.fcntl(self._fd, fcntl.F_SETFL, os.O_NONBLOCK) + + def _do_read(self): + while True: + buf = os.read(self._fd, 4096) + if buf == '': + break + self._buf += buf + self._flush() + + def _write_last_log_lines(self): + _last_line_limit = 100 + f = open(logfile_path) + lines = [] + for line in f: + if line.startswith('ostree-build '): + continue + lines.append(line) + if len(lines) > _last_line_limit: + lines.pop(0) + f.close() + for line in lines: + self.output.write('| ') + self.output.write(line) + + def _flush(self): + while True: + p = self._buf.find('\n') + if p < 0: + break + line = self._buf[0:p] + self._buf = self._buf[p+1:] + match = self._warning_re.search(line) + if match: + self._warning_count += 1 + self.output.write(line + '\n') + else: + match = self._nonfilter_re.search(line) + if match: + self.output.write(line + '\n') + else: + self._filtered_line_count += 1 + + def _on_changed(self): + self._do_read() + + def start(self): + self._do_read() + + def finish(self, successful): + self._do_read() + if not successful: + self._write_last_log_lines() + pass + self.output.write("ostree-build %s: %d warnings\n" % ('success' if successful else _bold('failed'), + self._warning_count, )) + self.output.write("ostree-build: full log path: %s\n" % (logfile_path, )) + + if successful: + for f in os.listdir('_build'): + path = os.path.join('_build', f) + if f.startswith('artifact-'): + self.output.write("ostree-build: created artifact: %s\n" % (f, )) + sys.exit(0 if successful else 1) + +def _on_makeinstall_exit(pid, estatus): + _output_filter.finish(estatus == 0) + +def _on_make_exit(pid, estatus): + if estatus == 0: + args = list(subprocess_nice_args) + args.append('ostree-buildone-makeinstall-split-artifacts') + _logfile_f.write("Running: %r\n" % (args, )) + _logfile_f.flush() + proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd) + _loop.watch_pid(proc.pid, _on_makeinstall_exit) + else: + _output_filter.finish(False) + +if __name__ == '__main__': + user_tmpdir = os.environ.get('XDG_RUNTIME_DIR') + if user_tmpdir is None: + user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'metabuild-%s' % (os.getuid(), )) + else: + user_tmpdir = os.path.join(user_tmpdir, 'ostree-build') + if not os.path.isdir(user_tmpdir): + os.makedirs(user_tmpdir) + logfile_path = os.path.join(user_tmpdir, '%s.log' % (os.path.basename(os.getcwd()), )) + try: + os.unlink(logfile_path) + except OSError, e: + pass + logfile_write_fd = os.open(logfile_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + global _logfile_f + _logfile_f = os.fdopen(logfile_write_fd, "w") + sys.stdout.write('ostree-build: logging to %r\n' % (logfile_path, )) + sys.stdout.flush() + + global _output_filter + _output_filter = OutputFilter(logfile_path, sys.stdout) + _output_filter.start() + + args = list(subprocess_nice_args) + args.append('ostree-buildone-make') + args.extend(sys.argv[1:]) + devnull=open('/dev/null') + _logfile_f.write("Running: %r\n" % (args, )) + _logfile_f.flush() + proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd) + + global _loop + _loop = Mainloop.get(None) + _loop.watch_pid(proc.pid, _on_make_exit) + _loop.run() diff --git a/osbuild/ostree-buildone-make b/osbuild/ostree-buildone-make new file mode 100644 index 00000000..b6e89f56 --- /dev/null +++ b/osbuild/ostree-buildone-make @@ -0,0 +1,205 @@ +#!/usr/bin/python + +# ostree-buildone-raw: Generic build system wrapper +# Copyright 2010, 2011 Colin Walters +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) + +# ostree-buildone-raw wraps systems that implement the GNOME build API: +# http://people.gnome.org/~walters/docs/build-api.txt + +import os,sys,subprocess,tempfile,re +from multiprocessing import cpu_count +import select,time + +root = None + +prefix = '/usr' + +# libdir detection +if os.path.isdir('/lib64'): + libdir=os.path.join(prefix, 'lib64') +else: + libdir=os.path.join(prefix, 'lib') + +default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )] +configargs = ['--prefix=' + prefix, + '--libdir=' + libdir, + '--sysconfdir=/etc', + '--localstatedir=/var', + '--bindir=' + os.path.join(prefix, 'bin'), + '--sbindir=' + os.path.join(prefix, 'sbin'), + '--datadir=' + os.path.join(prefix, 'share'), + '--includedir=' + os.path.join(prefix, 'include'), + '--libexecdir=' + os.path.join(prefix, 'libexec'), + '--mandir=' + os.path.join(prefix, 'share', 'man'), + '--infodir=' + os.path.join(prefix, 'share', 'info')] +makeargs = ['make'] + +top_srcdir=os.getcwd() + +for arg in sys.argv[1:]: + if arg.startswith('--'): + configargs.append(arg) + else: + makeargs.append(arg) + +def log(msg): + fullmsg = 'ostree-buildone: ' + msg + '\n' + sys.stdout.write(fullmsg) + sys.stdout.flush() + +def fatal(msg): + log(msg) + sys.exit(1) + +def run_sync(args, env=None): + log("Running: %r" % (args, )) + f = open('/dev/null', 'r') + proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr, + close_fds=True, env=env) + f.close() + returncode = proc.wait() + log("pid %d exited with code %d" % (proc.pid, returncode)) + if returncode != 0: + sys.exit(1) + +class BuildSystemScanner(object): + @classmethod + def _find_file(cls, names): + for name in names: + if os.path.exists(name): + return name + return None + + @classmethod + def get_configure_source_script(cls): + return cls._find_file(('./configure.ac', './configure.in')) + + @classmethod + def get_configure_script(cls): + return cls._find_file(('./configure', )) + + @classmethod + def get_bootstrap_script(cls): + return cls._find_file(('./autogen.sh', )) + + @classmethod + def get_silent_rules(cls): + src = cls.get_configure_source_script() + if not src: + return False + f = open(src) + for line in f: + if line.find('AM_SILENT_RULES') >= 0: + f.close() + return True + f.close() + return False + +def _search_file(filename, pattern): + f = open(filename) + for line in f: + if line.startswith(pattern): + f.close() + return line + f.close() + return None + +def _find_buildapi_makevariable(name): + var = '.%s:' % (name, ) + line = None + if os.path.exists('Makefile.in'): + line = _search_file('Makefile.in', var) + if not line and os.path.exists('Makefile'): + line = _search_file('Makefile', var) + return line is not None + +def phase_bootstrap(): + have_configure = BuildSystemScanner.get_configure_script() + have_configure_source = BuildSystemScanner.get_configure_source_script() + if not (have_configure or have_configure_source): + fatal("No configure or bootstrap script detected; unknown buildsystem") + return + + need_v1 = BuildSystemScanner.get_silent_rules() + if need_v1: + log("Detected AM_SILENT_RULES, adding --disable-silent-rules to configure") + configargs.append('--disable-silent-rules') + + if have_configure: + phase_configure() + else: + bootstrap = BuildSystemScanner.get_bootstrap_script() + if bootstrap: + log("Detected bootstrap script: %s, using it" % (bootstrap, )) + args = [bootstrap] + args.extend(configargs) + # Add NOCONFIGURE; GNOME style scripts use this + env = dict(os.environ) + env['NOCONFIGURE'] = '1' + run_sync(args, env=env) + else: + log("No bootstrap script found; using generic autoreconf") + run_sync(['autoreconf', '-f', '-i']) + phase_configure() + +def phase_configure(): + use_builddir = True + doesnot_support_builddir = _find_buildapi_makevariable('buildapi-no-builddir') + if doesnot_support_builddir: + log("Found .buildapi-no-builddir; copying source tree to _build") + shutil.rmtree('_build') + os.mkdir('_build') + shutil.copytree('.', '_build', symlinks=True, + ignore=shutil.ignore_patterns('_build')) + use_builddir = False + builddir = '.' + else: + builddir = '_build' + + if not use_builddir: + configdir = './' + else: + configdir = os.getcwd() + builddir = builddir + log("Using build directory %r" % (builddir, )) + if not os.path.isdir(builddir): + os.mkdir(builddir) + os.chdir(builddir) + + configstatus = 'config.status' + if not os.path.exists(configstatus): + args = [os.path.join(configdir, 'configure')] + args.extend(configargs) + run_sync(args) + else: + log("Found %s, skipping configure" % (configstatus, )) + phase_build() + +build_status = False + +def phase_build(): + if not os.path.exists('Makefile'): + log("No Makefile found") + sys.exit(1) + args = makeargs + user_specified_jobs = False + for arg in args: + if arg == '-j': + user_specified_jobs = True + + if not user_specified_jobs: + notparallel = _find_buildapi_makevariable('NOTPARALLEL') + if not notparallel: + log("Didn't find NOTPARALLEL, using parallel make by default") + args.extend(default_buildapi_jobs) + + run_sync(args) + +def phase_complete(): + sys.exit(0) + +log("invocation arguments: %r" % (sys.argv, )) + +# Start off the process +phase_bootstrap() diff --git a/osbuild/ostree-buildone-makeinstall-split-artifacts b/osbuild/ostree-buildone-makeinstall-split-artifacts new file mode 100644 index 00000000..accb5275 --- /dev/null +++ b/osbuild/ostree-buildone-makeinstall-split-artifacts @@ -0,0 +1,104 @@ +#!/usr/bin/python + +# ostree-buildone-raw: Generic build system wrapper +# Copyright 2010, 2011 Colin Walters +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) + +import os,sys,re,subprocess +import tempfile,shutil + +_devel_regexps = map(re.compile, + [r'/usr/include/', + r'/usr/share/pkgconfig/', + r'/.*lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc', + r'/.*lib(?:|(?:32)|(?:64))/.*\.so$']) + +def log(msg): + fullmsg = 'ostree-buildone: ' + msg + '\n' + sys.stdout.write(fullmsg) + sys.stdout.flush() + +tempfiles = [] + +def do_exit(code): + for tmpname in tempfiles: + if os.path.isdir(tmpname): + shutil.rmtree(tmpname) + else: + try: + os.unlink(tmpname) + pass + except OSError, e: + pass + sys.exit(code) + +def fatal(msg): + log(msg) + do_exit(1) + +def run_sync(args, env=None): + log("Running: %r" % (args, )) + f = open('/dev/null', 'r') + proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr, + close_fds=True, env=env) + f.close() + returncode = proc.wait() + if returncode == 0: + func = log + else: + func = fatal + func("pid %d exited with code %d" % (proc.pid, returncode)) + +basename=os.path.basename(os.getcwd()) +artifact_prefix='artifact-%s' % (basename, ) +origdir=os.getcwd() +os.chdir('_build') + +if not os.path.exists('Makefile'): + log("No Makefile found") + do_exit(1) + +(fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,)) +os.close(fd) +tempfiles.append(fakeroot_temp) +tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,)) +tempfiles.append(tempdir) +args = ['fakeroot', '-s', fakeroot_temp, 'make', 'install', 'DESTDIR=' + tempdir] +run_sync(args) + +devel_files = set() +runtime_files = set() + +oldpwd=os.getcwd() +os.chdir(tempdir) +for root, dirs, files in os.walk('.'): + for filename in files: + path = os.path.join(root, filename) + matched = False + for r in _devel_regexps: + if r.match(path[1:]): + devel_files.add(path) + matched = True + break + if not matched: + runtime_files.add(path) +os.chdir(oldpwd) + +def make_artifact(name, from_files): + artifact_target = '%s-%s.tar.gz' % (artifact_prefix, name) + (fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s-%s' % (basename, name)) + os.close(fd) + tempfiles.append(filelist_temp) + f = open(filelist_temp, 'w') + for filename in from_files: + f.write(filename) + f.write('\n') + f.close() + args = ['fakeroot', '-i', fakeroot_temp, 'tar', '-c', '-z', '-C', tempdir, '-f', artifact_target, '-T', filelist_temp] + run_sync(args) + +if devel_files: + make_artifact('devel', devel_files) +make_artifact('runtime', runtime_files) + +do_exit(0)