ostbuild: Split nice/logger program out, merge make/makeinstall into one executable

Makefile-ostbuild.am

@@ -15,7 +15,7 @@
 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 # Boston, MA 02111-1307, USA.
 
-bin_SCRIPTS += src/ostbuild/ostbuild-one \
+bin_SCRIPTS += src/ostbuild/ostbuild-compile-one \
-	src/ostbuild/ostbuild-one-make \
+	src/ostbuild/ostbuild-compile-one-impl \
-	src/ostbuild/ostbuild-one-makeinstall-split-artifacts \
+	src/ostbuild/ostbuild-nice-and-log-output \
 	$(NULL)

src/ostbuild/ostbuild-chroot

@@ -2,12 +2,20 @@
 
 import os,sys,re,subprocess
 
-chroot_path=sys.argv[1]
+i=1
+repo=sys.argv[i]
+i += 1
+chroot_path=sys.argv[i]
+i += 1
+args=sys.argv[i:]
 
-proc_path=os.path.join(chroot_path, 'proc')
+def run_in_chroot(args):
-subprocess.check_call(['mount', '-t', 'proc', 'proc', proc_path])
+    proc_path=os.path.join(chroot_path, 'proc')
+    subprocess.check_call(['mount', '-t', 'proc', 'proc', proc_path])
 
-subprocess.call(['chroot', chroot_path])
+    try:
-
+        subprocess.check_call(['chroot', chroot_path])
-subprocess.check_call(['umount', proc_path])
+    finally:
+        subprocess.call(['umount', proc_path])
 
+run_in_chroot(args)

src/ostbuild/ostbuild-compile-one

@@ -0,0 +1,7 @@
+#!/bin/sh
+#
+# Copyright 2010, 2011 Colin Walters <walters@verbum.org>
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+
+bn=$(basename $(pwd))
+ostbuild-nice-and-log-output "compile-${bn}.log" ostbuild-compile-one-impl "$@"

src/ostbuild/ostbuild-compile-one-impl

@@ -1,16 +1,23 @@
 #!/usr/bin/python
 
-# ostree-buildone-raw: Generic build system wrapper
 # Copyright 2010, 2011 Colin Walters <walters@verbum.org>
 # Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
 
-# ostree-buildone-raw wraps systems that implement the GNOME build API:
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
 # http://people.gnome.org/~walters/docs/build-api.txt
 
 import os,sys,subprocess,tempfile,re
 from multiprocessing import cpu_count
 import select,time
 
+tempfiles = []
+
+_devel_regexps = map(re.compile,
+                     [r'/usr/include/',
+                      r'/usr/share/pkgconfig/',
+                      r'/.*lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc',
+                      r'/.*lib(?:|(?:32)|(?:64))/.*\.so$'])
+
 root = None
 
 prefix = '/usr'
@@ -44,7 +51,7 @@ for arg in sys.argv[1:]:
         makeargs.append(arg)
 
 def log(msg):
-    fullmsg = 'ostree-buildone: ' + msg + '\n'
+    fullmsg = '%s: %s\n' % (sys.argv[0], msg)
     sys.stdout.write(fullmsg)
     sys.stdout.flush()
 
@@ -52,11 +59,11 @@ def fatal(msg):
     log(msg)
     sys.exit(1)
 
-def run_sync(args, env=None):
+def run_sync(args, cwd=None, env=None):
-    log("Running: %r" % (args, ))
+    log("running: %r" % (args,))
     f = open('/dev/null', 'r')
     proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
-                            close_fds=True, env=env)
+                            close_fds=True, cwd=cwd, env=env)
     f.close()
     returncode = proc.wait()
     log("pid %d exited with code %d" % (proc.pid, returncode))
@@ -105,13 +112,15 @@ def _search_file(filename, pattern):
     f.close()
     return None
 
-def _find_buildapi_makevariable(name):
+def _find_buildapi_makevariable(name, builddir='.'):
     var = '.%s:' % (name, )
     line = None
-    if os.path.exists('Makefile.in'):
+    path = os.path.join(builddir, 'Makefile.in')
-        line = _search_file('Makefile.in', var)
+    if os.path.exists(path):
-    if not line and os.path.exists('Makefile'):
+        line = _search_file(path, var)
-        line = _search_file('Makefile', var)
+    path = os.path.join(builddir, 'Makefile')
+    if not line and os.path.exists(path):
+        line = _search_file(path, var)
     return line is not None
 
 def phase_bootstrap():        
@@ -165,21 +174,20 @@ def phase_configure():
         log("Using build directory %r" % (builddir, ))
         if not os.path.isdir(builddir):
             os.mkdir(builddir)
-        os.chdir(builddir)
 
     configstatus = 'config.status'
     if not os.path.exists(configstatus):
         args = [os.path.join(configdir, 'configure')]
         args.extend(configargs)
-        run_sync(args)
+        run_sync(args, cwd=builddir)
     else:
         log("Found %s, skipping configure" % (configstatus, ))
-    phase_build()
+    phase_build(builddir=builddir)
 
 build_status = False
 
-def phase_build():
+def phase_build(builddir=None):
-    if not os.path.exists('Makefile'):
+    if not os.path.exists(os.path.join(builddir, 'Makefile')):
         log("No Makefile found")
         sys.exit(1)
     args = makeargs
@@ -189,14 +197,82 @@ def phase_build():
             user_specified_jobs = True
 
     if not user_specified_jobs:
-        notparallel = _find_buildapi_makevariable('NOTPARALLEL')
+        notparallel = _find_buildapi_makevariable('NOTPARALLEL', builddir=builddir)
         if not notparallel:
             log("Didn't find NOTPARALLEL, using parallel make by default")
             args.extend(default_buildapi_jobs)
 
+    run_sync(args, cwd=builddir)
+
+    phase_make_artifacts(builddir=builddir)
+
+def make_artifact(name, from_files, fakeroot_temp=None, tempdir=None):
+    targz_name = name + '.tar.gz'
+    (fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s' % (name, ))
+    os.close(fd)
+    tempfiles.append(filelist_temp)
+    f = open(filelist_temp, 'w')
+    for filename in from_files:
+        assert ('\n' not in filename)
+        f.write(filename)
+        f.write('\n')
+    f.close()
+    args = ['fakeroot', '-i', fakeroot_temp, 'tar', '-c', '-z', '-C', tempdir, '-f', targz_name, '-T', filelist_temp]
     run_sync(args)
+    log("created: %s" % (os.path.abspath (targz_name), ))
+
+def phase_make_artifacts(builddir=None):
+    basename=os.path.basename(os.getcwd())
+    
+    try:
+        version = subprocess.check_output(['git', 'describe'])
+    except subprocess.CalledProcessError, e:
+        version = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
+    version = version.strip()
+
+    artifact_prefix='artifact-%s,%s' % (basename, version)
+
+    (fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,))
+    os.close(fd)
+    tempfiles.append(fakeroot_temp)
+    tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,))
+    tempfiles.append(tempdir)
+    args = ['fakeroot', '-s', fakeroot_temp, 'make', 'install', 'DESTDIR=' + tempdir]
+    run_sync(args, cwd=builddir)
+
+    devel_files = set()
+    runtime_files = set()
+
+    oldpwd=os.getcwd()
+    os.chdir(tempdir)
+    for root, dirs, files in os.walk('.'):
+        for filename in files:
+            path = os.path.join(root, filename)
+            matched = False
+            for r in _devel_regexps:
+                if not r.match(path[1:]):
+                    continue
+                devel_files.add(path)
+                matched = True
+                break
+            if not matched:    
+                runtime_files.add(path)
+    os.chdir(oldpwd)
+
+    if devel_files:
+        make_artifact(artifact_prefix + '-devel', devel_files, fakeroot_temp=fakeroot_temp, tempdir=tempdir)
+    make_artifact(artifact_prefix + '-runtime', runtime_files, fakeroot_temp=fakeroot_temp, tempdir=tempdir)
 
 def phase_complete():
+    for tmpname in tempfiles:
+        if os.path.isdir(tmpname):
+            shutil.rmtree(tmpname)
+        else:
+            try:
+                os.unlink(tmpname)
+                pass
+            except OSError, e:
+                pass
     sys.exit(0)
 
 log("invocation arguments: %r" % (sys.argv, ))

src/ostbuild/ostbuild-nice-and-log-output

@@ -3,13 +3,12 @@
 # Copyright 2010, 2011 Colin Walters <walters@verbum.org>
 # Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
 
-# The build output is automatically logged to $TMPDIR/build-$(PWD).log.
-# For example, invoking metabuild in a directory named "foo" will log
-# to /tmp/build-foo.log
-
 import os,sys,subprocess,tempfile,re
 import select,time,stat,fcntl
 
+log_name = sys.argv[1]
+subprocess_args = sys.argv[2:]
+
 subprocess_nice_args = []
 
 # In the future we should test for this better; possibly implement a
@@ -215,51 +214,18 @@ class OutputFilter(object):
                                                             self._warning_count, ))
         self.output.write("ostbuild: full log path: %s\n" % (logfile_path, ))
         
-        if successful:
-            for f in os.listdir('_build'):
-                path = os.path.join('_build', f)
-                if f.startswith('artifact-'):
-                    self.output.write("ostbuild: created artifact: %s\n" % (f, ))
         sys.exit(0 if successful else 1)
 
-def _on_makeinstall_exit(pid, estatus):
+def _on_subprocess_exit(pid, estatus):
     _output_filter.finish(estatus == 0)
 
-def _on_make_exit(pid, estatus):
-    if estatus == 0:
-        args = list(subprocess_nice_args)
-        args.append('ostbuild-one-makeinstall-split-artifacts')
-        _logfile_f.write("Running: %r\n" % (args, ))
-        _logfile_f.flush()
-        proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd)
-        _loop.watch_pid(proc.pid, _on_makeinstall_exit)
-    else:
-        _output_filter.finish(False)
-
-def _get_version():
-    if not os.path.isdir('.git'):
-        sys.stderr.write("ostbuild-one: error: Couldn't find .git directory")
-        sys.exit(1)
-
-    proc = subprocess.Popen(['git', 'describe'], stdout=subprocess.PIPE)
-    output = proc.communicate()[0].strip()
-    if proc.wait() != 0:
-        proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
-        if proc.wait() != 0:
-            sys.stderr.write("ostbuild-one: error: git rev-parse HEAD failed")
-            sys.exit(1)
-        output = proc.communicate()[0].strip()
-    return output
-
 if __name__ == '__main__':
     user_tmpdir = os.environ.get('XDG_RUNTIME_DIR')
     if user_tmpdir is None:
-        user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'metabuild-%s' % (os.getuid(), ))
+        user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'ostbuild-%s' % (os.getuid(), ))
     else:
         user_tmpdir = os.path.join(user_tmpdir, 'ostbuild') 
 
-    os.environ['OSBUILD_VERSION'] = _get_version()
-
     if os.path.isdir('_build'):
         for filename in os.listdir('_build'):
             path = os.path.join('_build', filename)
@@ -268,7 +234,11 @@ if __name__ == '__main__':
 
     if not os.path.isdir(user_tmpdir):
         os.makedirs(user_tmpdir)
-    logfile_path = os.path.join(user_tmpdir, '%s.log' % (os.path.basename(os.getcwd()), ))
+    stbuf = os.stat(user_tmpdir)
+    if stbuf.st_uid != os.getuid():
+        sys.stderr.write('Directory %s not owned by me!' % (user_tmpdir, ))
+        sys.exit(1)
+    logfile_path = os.path.join(user_tmpdir, log_name)
     try:
         os.unlink(logfile_path)
     except OSError, e:
@@ -284,14 +254,13 @@ if __name__ == '__main__':
     _output_filter.start()
 
     args = list(subprocess_nice_args)
-    args.append('ostbuild-one-make')
+    args.extend(subprocess_args)
-    args.extend(sys.argv[1:])
     devnull=open('/dev/null')
-    _logfile_f.write("Running: %r\n" % (args, ))
+    _logfile_f.write("%s: running: %r\n" % (sys.argv[0], args, ))
     _logfile_f.flush()
     proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd)
 
     global _loop
     _loop = Mainloop.get(None)
-    _loop.watch_pid(proc.pid, _on_make_exit)
+    _loop.watch_pid(proc.pid, _on_subprocess_exit)
     _loop.run()

src/ostbuild/ostbuild-one-makeinstall-split-artifacts

@@ -1,105 +0,0 @@
-#!/usr/bin/python
-
-# ostree-buildone-raw: Generic build system wrapper
-# Copyright 2010, 2011 Colin Walters <walters@verbum.org>
-# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
-
-import os,sys,re,subprocess
-import tempfile,shutil
-
-_devel_regexps = map(re.compile,
-                     [r'/usr/include/',
-                      r'/usr/share/pkgconfig/',
-                      r'/.*lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc',
-                      r'/.*lib(?:|(?:32)|(?:64))/.*\.so$'])
-
-def log(msg):
-    fullmsg = 'ostree-buildone: ' + msg + '\n'
-    sys.stdout.write(fullmsg)
-    sys.stdout.flush()
-
-tempfiles = []
-
-def do_exit(code):
-    for tmpname in tempfiles:
-        if os.path.isdir(tmpname):
-            shutil.rmtree(tmpname)
-        else:
-            try:
-                os.unlink(tmpname)
-                pass
-            except OSError, e:
-                pass
-    sys.exit(code)
-
-def fatal(msg):
-    log(msg)
-    do_exit(1)
-
-def run_sync(args, env=None):
-    log("Running: %r" % (args, ))
-    f = open('/dev/null', 'r')
-    proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
-                            close_fds=True, env=env)
-    f.close()
-    returncode = proc.wait()
-    if returncode == 0:
-        func = log
-    else:
-        func = fatal
-    func("pid %d exited with code %d" % (proc.pid, returncode))
-
-basename=os.path.basename(os.getcwd())
-artifact_prefix='artifact-%s,%s' % (basename, os.environ['OSBUILD_VERSION'])
-origdir=os.getcwd()
-os.chdir('_build')
-
-if not os.path.exists('Makefile'):
-    log("No Makefile found")
-    do_exit(1)
-
-(fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,))
-os.close(fd)
-tempfiles.append(fakeroot_temp)
-tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,))
-tempfiles.append(tempdir)
-args = ['fakeroot', '-s', fakeroot_temp, 'make', 'install', 'DESTDIR=' + tempdir]
-run_sync(args)
-
-devel_files = set()
-runtime_files = set()
-
-oldpwd=os.getcwd()
-os.chdir(tempdir)
-for root, dirs, files in os.walk('.'):
-    for filename in files:
-        path = os.path.join(root, filename)
-        matched = False
-        for r in _devel_regexps:
-            if r.match(path[1:]):
-                devel_files.add(path)
-                matched = True
-                break
-        if not matched:    
-            runtime_files.add(path)
-os.chdir(oldpwd)
-
-def make_artifact(name, from_files):
-    artifact_target = '%s-%s.tar.gz' % (artifact_prefix, name)
-    (fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s-%s' % (basename, name))
-    os.close(fd)
-    tempfiles.append(filelist_temp)
-    f = open(filelist_temp, 'w')
-    for filename in from_files:
-        assert ('\n' not in filename)
-        f.write(filename)
-        f.write('\n')
-    f.close()
-    args = ['fakeroot', '-i', fakeroot_temp, 'tar', '-c', '-z', '-C', tempdir, '-f', artifact_target, '-T', filelist_temp]
-    run_sync(args)
-
-if devel_files:
-    make_artifact('devel', devel_files)
-make_artifact('runtime', runtime_files)
-
-do_exit(0)