ostree/osbuild/ostree-buildone

#!/usr/bin/python
#
# ostree-buildone:
# Copyright 2010, 2011 Colin Walters <walters@verbum.org>
# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)

# The build output is automatically logged to $TMPDIR/build-$(PWD).log.
# For example, invoking metabuild in a directory named "foo" will log
# to /tmp/build-foo.log
#
# You can pass arguments to metabuild; if they start with '--', they're
# given to configure.  Otherwise, they're passed to make.
#
# $ metabuild --enable-libfoo  # passed to configure
# $ metabuild -j 1             # passed to make

import os,sys,subprocess,tempfile,re
import select,time,stat,fcntl

subprocess_nice_args = []

# In the future we should test for this better; possibly implement a
# custom fork handler
try:
    chrt_args = ['chrt', '--idle', '0']
    proc = subprocess.Popen(chrt_args + ['true'])
    if proc.wait() == 0:
        subprocess_nice_args.extend(chrt_args)
except OSError, e:
    pass

try:
    ionice_args = ['ionice', '-c', '3', '-t']
    proc = subprocess.Popen(ionice_args + ['true'])
    if proc.wait() == 0:
        subprocess_nice_args.extend(ionice_args)
except OSError, e:
    pass

warning_re = re.compile(r'(: ((warning)|(error)|(fatal error)): )|(make(\[[0-9]+\])?: \*\*\*)')
output_whitelist_re = re.compile(r'^(make(\[[0-9]+\])?: Entering directory)|(ostree-build )')

_bold_sequence = None
_normal_sequence = None
if os.isatty(1):
    _bold_sequence = subprocess.Popen(['tput', 'bold'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0]
    _normal_sequence = subprocess.Popen(['tput', 'sgr0'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0]
def _bold(text):
    if _bold_sequence is not None:
        return '%s%s%s' % (_bold_sequence, text, _normal_sequence)
    else:
        return text

class Mainloop(object):
    DEFAULT = None
    def __init__(self):
        self._running = True
        self.poll = select.poll()
        self._timeouts = []
        self._pid_watches = {}
        self._fd_callbacks = {}

    @classmethod
    def get(cls, context):
        if context is None:
            if cls.DEFAULT is None:
                cls.DEFAULT = cls()
            return cls.DEFAULT
        raise NotImplementedError("Unknown context %r" % (context, ))

    def watch_fd(self, fd, callback):
        self.poll.register(fd)
        self._fd_callbacks[fd] = callback

    def unwatch_fd(self, fd):
        self.poll.unregister(fd)
        del self._fd_callbacks[fd]

    def watch_pid(self, pid, callback):
        self._pid_watches[pid] = callback

    def timeout_add(self, ms, callback):
        self._timeouts.append((ms, callback))

    def quit(self):
        self._running = False

    def run_once(self):
        min_timeout = None
        if len(self._pid_watches) > 0:
            min_timeout = 500
        for (ms, callback) in self._timeouts:
            if (min_timeout is None) or (ms < min_timeout):
                min_timeout = ms
        origtime = time.time() * 1000
        fds = self.poll.poll(min_timeout)
        for fd in fds:
            self._fd_callbacks[fd]()
        for pid in self._pid_watches:
            (opid, status) = os.waitpid(pid, os.WNOHANG)
            if opid != 0:
                self._pid_watches[pid](opid, status)
                del self._pid_watches[pid]
        newtime = time.time() * 1000
        diff = int(newtime - origtime)
        if diff < 0: diff = 0
        for i,(ms, callback) in enumerate(self._timeouts):
            remaining_ms = ms - diff
            if remaining_ms <= 0:
                callback()
            else:
                self._timeouts[i] = (remaining_ms, callback)

    def run(self):
        while self._running:
            self.run_once()

class FileMonitor(object):
    def __init__(self):
        self._paths = {}
        self._path_modtimes = {}
        self._timeout = 1000
        self._timeout_installed = False
        self._loop = Mainloop.get(None)

    def _stat(self, path):
        try:
            st = os.stat(path)
            return st[stat.ST_MTIME]
        except OSError, e:
            return None

    def add(self, path, callback):
        if path not in self._paths:
            self._paths[path] = []
            self._path_modtimes[path] = self._stat(path)
        self._paths[path].append(callback)
        if not self._timeout_installed:
            self._timeout_installed = True
            self._loop.timeout_add(self._timeout, self._check_files)

    def _check_files(self):
        for (path,callbacks) in self._paths.iteritems():
            mtime = self._stat(path)
            orig_mtime = self._path_modtimes[path]
            if (mtime is not None) and (orig_mtime is None or (mtime > orig_mtime)):
                self._path_modtimes[path] = mtime
                for cb in callbacks:
                    cb()

_filemon = FileMonitor()

class OutputFilter(object):
    def __init__(self, filename, output):
        self.filename = filename
        self.output = output

        # inherit globals
        self._warning_re = warning_re
        self._nonfilter_re = output_whitelist_re

        self._buf = ''
        self._warning_count = 0
        self._filtered_line_count = 0
        _filemon.add(filename, self._on_changed)
        self._fd = os.open(filename, os.O_RDONLY)
        fcntl.fcntl(self._fd, fcntl.F_SETFL, os.O_NONBLOCK)

    def _do_read(self):
        while True:
            buf = os.read(self._fd, 4096)
            if buf == '':
                break
            self._buf += buf
        self._flush()

    def _write_last_log_lines(self):
        _last_line_limit = 100
        f = open(logfile_path)
        lines = []
        for line in f:
            if line.startswith('ostree-build '):
                continue
            lines.append(line)
            if len(lines) > _last_line_limit:
                lines.pop(0)
        f.close()
        for line in lines:
            self.output.write('| ')
            self.output.write(line)

    def _flush(self):
        while True:
            p = self._buf.find('\n')
            if p < 0:
                break
            line = self._buf[0:p]
            self._buf = self._buf[p+1:]
            match = self._warning_re.search(line)
            if match:
                self._warning_count += 1
                self.output.write(line + '\n')
            else:    
                match = self._nonfilter_re.search(line)
                if match:
                    self.output.write(line + '\n')
                else:
                    self._filtered_line_count += 1

    def _on_changed(self):
        self._do_read()

    def start(self):
        self._do_read()

    def finish(self, successful):
        self._do_read()
        if not successful:
            self._write_last_log_lines()
            pass
        self.output.write("ostree-build %s: %d warnings\n" % ('success' if successful else _bold('failed'),
                                                            self._warning_count, ))
        self.output.write("ostree-build: full log path: %s\n" % (logfile_path, ))
        
        if successful:
            for f in os.listdir('_build'):
                path = os.path.join('_build', f)
                if f.startswith('artifact-'):
                    self.output.write("ostree-build: created artifact: %s\n" % (f, ))
        sys.exit(0 if successful else 1)

def _on_makeinstall_exit(pid, estatus):
    _output_filter.finish(estatus == 0)

def _on_make_exit(pid, estatus):
    if estatus == 0:
        args = list(subprocess_nice_args)
        args.append('ostree-buildone-makeinstall-split-artifacts')
        _logfile_f.write("Running: %r\n" % (args, ))
        _logfile_f.flush()
        proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd)
        _loop.watch_pid(proc.pid, _on_makeinstall_exit)
    else:
        _output_filter.finish(False)

def _get_version():
    if not os.path.isdir('.git'):
        sys.stderr.write("ostree-buildone: error: Couldn't find .git directory")
        sys.exit(1)

    proc = subprocess.Popen(['git', 'describe'], stdout=subprocess.PIPE)
    output = proc.communicate()[0].strip()
    if proc.wait() != 0:
        proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
        if proc.wait() != 0:
            sys.stderr.write("ostree-buildone: error: git rev-parse HEAD failed")
            sys.exit(1)
        output = proc.communicate()[0].strip()
    return output

if __name__ == '__main__':
    user_tmpdir = os.environ.get('XDG_RUNTIME_DIR')
    if user_tmpdir is None:
        user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'metabuild-%s' % (os.getuid(), ))
    else:
        user_tmpdir = os.path.join(user_tmpdir, 'ostree-build') 

    os.environ['OSBUILD_VERSION'] = _get_version()

    if os.path.isdir('_build'):
        for filename in os.listdir('_build'):
            path = os.path.join('_build', filename)
            if filename.startswith('artifact-'):
                os.unlink(path)

    if not os.path.isdir(user_tmpdir):
        os.makedirs(user_tmpdir)
    logfile_path = os.path.join(user_tmpdir, '%s.log' % (os.path.basename(os.getcwd()), ))
    try:
        os.unlink(logfile_path)
    except OSError, e:
        pass
    logfile_write_fd = os.open(logfile_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
    global _logfile_f
    _logfile_f = os.fdopen(logfile_write_fd, "w")
    sys.stdout.write('ostree-build: logging to %r\n' % (logfile_path, ))
    sys.stdout.flush()

    global _output_filter
    _output_filter = OutputFilter(logfile_path, sys.stdout)
    _output_filter.start()

    args = list(subprocess_nice_args)
    args.append('ostree-buildone-make')
    args.extend(sys.argv[1:])
    devnull=open('/dev/null')
    _logfile_f.write("Running: %r\n" % (args, ))
    _logfile_f.flush()
    proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd)

    global _loop
    _loop = Mainloop.get(None)
    _loop.watch_pid(proc.pid, _on_make_exit)
    _loop.run()