ostbuild: Split nice/logger program out, merge make/makeinstall into one executable
This commit is contained in:
Colin Walters
parent
6cdea44c4b
commit
ea858ab558
|
|
@ -15,7 +15,7 @@
|
|||
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||
# Boston, MA 02111-1307, USA.
|
||||
|
||||
bin_SCRIPTS += src/ostbuild/ostbuild-one \
|
||||
src/ostbuild/ostbuild-one-make \
|
||||
src/ostbuild/ostbuild-one-makeinstall-split-artifacts \
|
||||
bin_SCRIPTS += src/ostbuild/ostbuild-compile-one \
|
||||
src/ostbuild/ostbuild-compile-one-impl \
|
||||
src/ostbuild/ostbuild-nice-and-log-output \
|
||||
$(NULL)
|
||||
|
|
|
|||
|
|
@ -2,12 +2,20 @@
|
|||
|
||||
import os,sys,re,subprocess
|
||||
|
||||
chroot_path=sys.argv[1]
|
||||
i=1
|
||||
repo=sys.argv[i]
|
||||
i += 1
|
||||
chroot_path=sys.argv[i]
|
||||
i += 1
|
||||
args=sys.argv[i:]
|
||||
|
||||
def run_in_chroot(args):
|
||||
proc_path=os.path.join(chroot_path, 'proc')
|
||||
subprocess.check_call(['mount', '-t', 'proc', 'proc', proc_path])
|
||||
|
||||
subprocess.call(['chroot', chroot_path])
|
||||
|
||||
subprocess.check_call(['umount', proc_path])
|
||||
try:
|
||||
subprocess.check_call(['chroot', chroot_path])
|
||||
finally:
|
||||
subprocess.call(['umount', proc_path])
|
||||
|
||||
run_in_chroot(args)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/sh
|
||||
#
|
||||
# Copyright 2010, 2011 Colin Walters <walters@verbum.org>
|
||||
# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
bn=$(basename $(pwd))
|
||||
ostbuild-nice-and-log-output "compile-${bn}.log" ostbuild-compile-one-impl "$@"
|
||||
|
|
@ -1,16 +1,23 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# ostree-buildone-raw: Generic build system wrapper
|
||||
# Copyright 2010, 2011 Colin Walters <walters@verbum.org>
|
||||
# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
# ostree-buildone-raw wraps systems that implement the GNOME build API:
|
||||
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
|
||||
# http://people.gnome.org/~walters/docs/build-api.txt
|
||||
|
||||
import os,sys,subprocess,tempfile,re
|
||||
from multiprocessing import cpu_count
|
||||
import select,time
|
||||
|
||||
tempfiles = []
|
||||
|
||||
_devel_regexps = map(re.compile,
|
||||
[r'/usr/include/',
|
||||
r'/usr/share/pkgconfig/',
|
||||
r'/.*lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc',
|
||||
r'/.*lib(?:|(?:32)|(?:64))/.*\.so$'])
|
||||
|
||||
root = None
|
||||
|
||||
prefix = '/usr'
|
||||
|
|
@ -44,7 +51,7 @@ for arg in sys.argv[1:]:
|
|||
makeargs.append(arg)
|
||||
|
||||
def log(msg):
|
||||
fullmsg = 'ostree-buildone: ' + msg + '\n'
|
||||
fullmsg = '%s: %s\n' % (sys.argv[0], msg)
|
||||
sys.stdout.write(fullmsg)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
|
@ -52,11 +59,11 @@ def fatal(msg):
|
|||
log(msg)
|
||||
sys.exit(1)
|
||||
|
||||
def run_sync(args, env=None):
|
||||
log("Running: %r" % (args, ))
|
||||
def run_sync(args, cwd=None, env=None):
|
||||
log("running: %r" % (args,))
|
||||
f = open('/dev/null', 'r')
|
||||
proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
|
||||
close_fds=True, env=env)
|
||||
close_fds=True, cwd=cwd, env=env)
|
||||
f.close()
|
||||
returncode = proc.wait()
|
||||
log("pid %d exited with code %d" % (proc.pid, returncode))
|
||||
|
|
@ -105,13 +112,15 @@ def _search_file(filename, pattern):
|
|||
f.close()
|
||||
return None
|
||||
|
||||
def _find_buildapi_makevariable(name):
|
||||
def _find_buildapi_makevariable(name, builddir='.'):
|
||||
var = '.%s:' % (name, )
|
||||
line = None
|
||||
if os.path.exists('Makefile.in'):
|
||||
line = _search_file('Makefile.in', var)
|
||||
if not line and os.path.exists('Makefile'):
|
||||
line = _search_file('Makefile', var)
|
||||
path = os.path.join(builddir, 'Makefile.in')
|
||||
if os.path.exists(path):
|
||||
line = _search_file(path, var)
|
||||
path = os.path.join(builddir, 'Makefile')
|
||||
if not line and os.path.exists(path):
|
||||
line = _search_file(path, var)
|
||||
return line is not None
|
||||
|
||||
def phase_bootstrap():
|
||||
|
|
@ -165,21 +174,20 @@ def phase_configure():
|
|||
log("Using build directory %r" % (builddir, ))
|
||||
if not os.path.isdir(builddir):
|
||||
os.mkdir(builddir)
|
||||
os.chdir(builddir)
|
||||
|
||||
configstatus = 'config.status'
|
||||
if not os.path.exists(configstatus):
|
||||
args = [os.path.join(configdir, 'configure')]
|
||||
args.extend(configargs)
|
||||
run_sync(args)
|
||||
run_sync(args, cwd=builddir)
|
||||
else:
|
||||
log("Found %s, skipping configure" % (configstatus, ))
|
||||
phase_build()
|
||||
phase_build(builddir=builddir)
|
||||
|
||||
build_status = False
|
||||
|
||||
def phase_build():
|
||||
if not os.path.exists('Makefile'):
|
||||
def phase_build(builddir=None):
|
||||
if not os.path.exists(os.path.join(builddir, 'Makefile')):
|
||||
log("No Makefile found")
|
||||
sys.exit(1)
|
||||
args = makeargs
|
||||
|
|
@ -189,14 +197,82 @@ def phase_build():
|
|||
user_specified_jobs = True
|
||||
|
||||
if not user_specified_jobs:
|
||||
notparallel = _find_buildapi_makevariable('NOTPARALLEL')
|
||||
notparallel = _find_buildapi_makevariable('NOTPARALLEL', builddir=builddir)
|
||||
if not notparallel:
|
||||
log("Didn't find NOTPARALLEL, using parallel make by default")
|
||||
args.extend(default_buildapi_jobs)
|
||||
|
||||
run_sync(args, cwd=builddir)
|
||||
|
||||
phase_make_artifacts(builddir=builddir)
|
||||
|
||||
def make_artifact(name, from_files, fakeroot_temp=None, tempdir=None):
|
||||
targz_name = name + '.tar.gz'
|
||||
(fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s' % (name, ))
|
||||
os.close(fd)
|
||||
tempfiles.append(filelist_temp)
|
||||
f = open(filelist_temp, 'w')
|
||||
for filename in from_files:
|
||||
assert ('\n' not in filename)
|
||||
f.write(filename)
|
||||
f.write('\n')
|
||||
f.close()
|
||||
args = ['fakeroot', '-i', fakeroot_temp, 'tar', '-c', '-z', '-C', tempdir, '-f', targz_name, '-T', filelist_temp]
|
||||
run_sync(args)
|
||||
log("created: %s" % (os.path.abspath (targz_name), ))
|
||||
|
||||
def phase_make_artifacts(builddir=None):
|
||||
basename=os.path.basename(os.getcwd())
|
||||
|
||||
try:
|
||||
version = subprocess.check_output(['git', 'describe'])
|
||||
except subprocess.CalledProcessError, e:
|
||||
version = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
|
||||
version = version.strip()
|
||||
|
||||
artifact_prefix='artifact-%s,%s' % (basename, version)
|
||||
|
||||
(fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,))
|
||||
os.close(fd)
|
||||
tempfiles.append(fakeroot_temp)
|
||||
tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,))
|
||||
tempfiles.append(tempdir)
|
||||
args = ['fakeroot', '-s', fakeroot_temp, 'make', 'install', 'DESTDIR=' + tempdir]
|
||||
run_sync(args, cwd=builddir)
|
||||
|
||||
devel_files = set()
|
||||
runtime_files = set()
|
||||
|
||||
oldpwd=os.getcwd()
|
||||
os.chdir(tempdir)
|
||||
for root, dirs, files in os.walk('.'):
|
||||
for filename in files:
|
||||
path = os.path.join(root, filename)
|
||||
matched = False
|
||||
for r in _devel_regexps:
|
||||
if not r.match(path[1:]):
|
||||
continue
|
||||
devel_files.add(path)
|
||||
matched = True
|
||||
break
|
||||
if not matched:
|
||||
runtime_files.add(path)
|
||||
os.chdir(oldpwd)
|
||||
|
||||
if devel_files:
|
||||
make_artifact(artifact_prefix + '-devel', devel_files, fakeroot_temp=fakeroot_temp, tempdir=tempdir)
|
||||
make_artifact(artifact_prefix + '-runtime', runtime_files, fakeroot_temp=fakeroot_temp, tempdir=tempdir)
|
||||
|
||||
def phase_complete():
|
||||
for tmpname in tempfiles:
|
||||
if os.path.isdir(tmpname):
|
||||
shutil.rmtree(tmpname)
|
||||
else:
|
||||
try:
|
||||
os.unlink(tmpname)
|
||||
pass
|
||||
except OSError, e:
|
||||
pass
|
||||
sys.exit(0)
|
||||
|
||||
log("invocation arguments: %r" % (sys.argv, ))
|
||||
|
|
@ -3,13 +3,12 @@
|
|||
# Copyright 2010, 2011 Colin Walters <walters@verbum.org>
|
||||
# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
# The build output is automatically logged to $TMPDIR/build-$(PWD).log.
|
||||
# For example, invoking metabuild in a directory named "foo" will log
|
||||
# to /tmp/build-foo.log
|
||||
|
||||
import os,sys,subprocess,tempfile,re
|
||||
import select,time,stat,fcntl
|
||||
|
||||
log_name = sys.argv[1]
|
||||
subprocess_args = sys.argv[2:]
|
||||
|
||||
subprocess_nice_args = []
|
||||
|
||||
# In the future we should test for this better; possibly implement a
|
||||
|
|
@ -215,51 +214,18 @@ class OutputFilter(object):
|
|||
self._warning_count, ))
|
||||
self.output.write("ostbuild: full log path: %s\n" % (logfile_path, ))
|
||||
|
||||
if successful:
|
||||
for f in os.listdir('_build'):
|
||||
path = os.path.join('_build', f)
|
||||
if f.startswith('artifact-'):
|
||||
self.output.write("ostbuild: created artifact: %s\n" % (f, ))
|
||||
sys.exit(0 if successful else 1)
|
||||
|
||||
def _on_makeinstall_exit(pid, estatus):
|
||||
def _on_subprocess_exit(pid, estatus):
|
||||
_output_filter.finish(estatus == 0)
|
||||
|
||||
def _on_make_exit(pid, estatus):
|
||||
if estatus == 0:
|
||||
args = list(subprocess_nice_args)
|
||||
args.append('ostbuild-one-makeinstall-split-artifacts')
|
||||
_logfile_f.write("Running: %r\n" % (args, ))
|
||||
_logfile_f.flush()
|
||||
proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd)
|
||||
_loop.watch_pid(proc.pid, _on_makeinstall_exit)
|
||||
else:
|
||||
_output_filter.finish(False)
|
||||
|
||||
def _get_version():
|
||||
if not os.path.isdir('.git'):
|
||||
sys.stderr.write("ostbuild-one: error: Couldn't find .git directory")
|
||||
sys.exit(1)
|
||||
|
||||
proc = subprocess.Popen(['git', 'describe'], stdout=subprocess.PIPE)
|
||||
output = proc.communicate()[0].strip()
|
||||
if proc.wait() != 0:
|
||||
proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
|
||||
if proc.wait() != 0:
|
||||
sys.stderr.write("ostbuild-one: error: git rev-parse HEAD failed")
|
||||
sys.exit(1)
|
||||
output = proc.communicate()[0].strip()
|
||||
return output
|
||||
|
||||
if __name__ == '__main__':
|
||||
user_tmpdir = os.environ.get('XDG_RUNTIME_DIR')
|
||||
if user_tmpdir is None:
|
||||
user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'metabuild-%s' % (os.getuid(), ))
|
||||
user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'ostbuild-%s' % (os.getuid(), ))
|
||||
else:
|
||||
user_tmpdir = os.path.join(user_tmpdir, 'ostbuild')
|
||||
|
||||
os.environ['OSBUILD_VERSION'] = _get_version()
|
||||
|
||||
if os.path.isdir('_build'):
|
||||
for filename in os.listdir('_build'):
|
||||
path = os.path.join('_build', filename)
|
||||
|
|
@ -268,7 +234,11 @@ if __name__ == '__main__':
|
|||
|
||||
if not os.path.isdir(user_tmpdir):
|
||||
os.makedirs(user_tmpdir)
|
||||
logfile_path = os.path.join(user_tmpdir, '%s.log' % (os.path.basename(os.getcwd()), ))
|
||||
stbuf = os.stat(user_tmpdir)
|
||||
if stbuf.st_uid != os.getuid():
|
||||
sys.stderr.write('Directory %s not owned by me!' % (user_tmpdir, ))
|
||||
sys.exit(1)
|
||||
logfile_path = os.path.join(user_tmpdir, log_name)
|
||||
try:
|
||||
os.unlink(logfile_path)
|
||||
except OSError, e:
|
||||
|
|
@ -284,14 +254,13 @@ if __name__ == '__main__':
|
|||
_output_filter.start()
|
||||
|
||||
args = list(subprocess_nice_args)
|
||||
args.append('ostbuild-one-make')
|
||||
args.extend(sys.argv[1:])
|
||||
args.extend(subprocess_args)
|
||||
devnull=open('/dev/null')
|
||||
_logfile_f.write("Running: %r\n" % (args, ))
|
||||
_logfile_f.write("%s: running: %r\n" % (sys.argv[0], args, ))
|
||||
_logfile_f.flush()
|
||||
proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd)
|
||||
|
||||
global _loop
|
||||
_loop = Mainloop.get(None)
|
||||
_loop.watch_pid(proc.pid, _on_make_exit)
|
||||
_loop.watch_pid(proc.pid, _on_subprocess_exit)
|
||||
_loop.run()
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# ostree-buildone-raw: Generic build system wrapper
|
||||
# Copyright 2010, 2011 Colin Walters <walters@verbum.org>
|
||||
# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
import os,sys,re,subprocess
|
||||
import tempfile,shutil
|
||||
|
||||
_devel_regexps = map(re.compile,
|
||||
[r'/usr/include/',
|
||||
r'/usr/share/pkgconfig/',
|
||||
r'/.*lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc',
|
||||
r'/.*lib(?:|(?:32)|(?:64))/.*\.so$'])
|
||||
|
||||
def log(msg):
|
||||
fullmsg = 'ostree-buildone: ' + msg + '\n'
|
||||
sys.stdout.write(fullmsg)
|
||||
sys.stdout.flush()
|
||||
|
||||
tempfiles = []
|
||||
|
||||
def do_exit(code):
|
||||
for tmpname in tempfiles:
|
||||
if os.path.isdir(tmpname):
|
||||
shutil.rmtree(tmpname)
|
||||
else:
|
||||
try:
|
||||
os.unlink(tmpname)
|
||||
pass
|
||||
except OSError, e:
|
||||
pass
|
||||
sys.exit(code)
|
||||
|
||||
def fatal(msg):
|
||||
log(msg)
|
||||
do_exit(1)
|
||||
|
||||
def run_sync(args, env=None):
|
||||
log("Running: %r" % (args, ))
|
||||
f = open('/dev/null', 'r')
|
||||
proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
|
||||
close_fds=True, env=env)
|
||||
f.close()
|
||||
returncode = proc.wait()
|
||||
if returncode == 0:
|
||||
func = log
|
||||
else:
|
||||
func = fatal
|
||||
func("pid %d exited with code %d" % (proc.pid, returncode))
|
||||
|
||||
basename=os.path.basename(os.getcwd())
|
||||
artifact_prefix='artifact-%s,%s' % (basename, os.environ['OSBUILD_VERSION'])
|
||||
origdir=os.getcwd()
|
||||
os.chdir('_build')
|
||||
|
||||
if not os.path.exists('Makefile'):
|
||||
log("No Makefile found")
|
||||
do_exit(1)
|
||||
|
||||
(fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,))
|
||||
os.close(fd)
|
||||
tempfiles.append(fakeroot_temp)
|
||||
tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,))
|
||||
tempfiles.append(tempdir)
|
||||
args = ['fakeroot', '-s', fakeroot_temp, 'make', 'install', 'DESTDIR=' + tempdir]
|
||||
run_sync(args)
|
||||
|
||||
devel_files = set()
|
||||
runtime_files = set()
|
||||
|
||||
oldpwd=os.getcwd()
|
||||
os.chdir(tempdir)
|
||||
for root, dirs, files in os.walk('.'):
|
||||
for filename in files:
|
||||
path = os.path.join(root, filename)
|
||||
matched = False
|
||||
for r in _devel_regexps:
|
||||
if r.match(path[1:]):
|
||||
devel_files.add(path)
|
||||
matched = True
|
||||
break
|
||||
if not matched:
|
||||
runtime_files.add(path)
|
||||
os.chdir(oldpwd)
|
||||
|
||||
def make_artifact(name, from_files):
|
||||
artifact_target = '%s-%s.tar.gz' % (artifact_prefix, name)
|
||||
(fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s-%s' % (basename, name))
|
||||
os.close(fd)
|
||||
tempfiles.append(filelist_temp)
|
||||
f = open(filelist_temp, 'w')
|
||||
for filename in from_files:
|
||||
assert ('\n' not in filename)
|
||||
f.write(filename)
|
||||
f.write('\n')
|
||||
f.close()
|
||||
args = ['fakeroot', '-i', fakeroot_temp, 'tar', '-c', '-z', '-C', tempdir, '-f', artifact_target, '-T', filelist_temp]
|
||||
run_sync(args)
|
||||
|
||||
if devel_files:
|
||||
make_artifact('devel', devel_files)
|
||||
make_artifact('runtime', runtime_files)
|
||||
|
||||
do_exit(0)
|
||||
Loading…
Reference in New Issue