ostbuild: Make new 'ostbuild' main entry point, and compile-one a subcommand
The collection of Python scripts here have gotten to the point where we need to share code. Start refactoring things so that we have one main command which imports subcommands as libraries.
This commit is contained in:
Colin Walters
parent
633477806c
commit
122b31ed3b
|
|
@ -15,14 +15,27 @@
|
||||||
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||||
# Boston, MA 02111-1307, USA.
|
# Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
ostbuild: src/ostbuild/ostbuild.in Makefile
|
||||||
|
sed -e s,@libdir\@,$(libdir), -e s,@datarootdir\@,$(datarootdir), -e s,@PYTHON\@,$(PYTHON), $< > $@.tmp && mv $@.tmp $@
|
||||||
|
bin_SCRIPTS += ostbuild
|
||||||
|
|
||||||
bin_SCRIPTS += \
|
bin_SCRIPTS += \
|
||||||
src/ostbuild/ostbuild-autodiscover-meta \
|
src/ostbuild/ostbuild-autodiscover-meta \
|
||||||
src/ostbuild/ostbuild-commit-artifacts \
|
src/ostbuild/ostbuild-commit-artifacts \
|
||||||
src/ostbuild/ostbuild-compile-one-impl \
|
|
||||||
src/ostbuild/ostbuild-chroot-compile-one-impl \
|
src/ostbuild/ostbuild-chroot-compile-one-impl \
|
||||||
src/ostbuild/ostbuild-nice-and-log-output \
|
src/ostbuild/ostbuild-nice-and-log-output \
|
||||||
$(NULL)
|
$(NULL)
|
||||||
|
|
||||||
|
pyostbuilddir=$(libdir)/ostbuild/pyostbuild
|
||||||
|
pyostbuild_PYTHON = \
|
||||||
|
src/ostbuild/pyostbuild/__init__.py \
|
||||||
|
src/ostbuild/pyostbuild/builtins.py \
|
||||||
|
src/ostbuild/pyostbuild/main.py \
|
||||||
|
src/ostbuild/pyostbuild/ostbuildlog.py \
|
||||||
|
src/ostbuild/pyostbuild/subprocess_helpers.py \
|
||||||
|
src/ostbuild/pyostbuild/builtin_compile_one.py \
|
||||||
|
$(NULL)
|
||||||
|
|
||||||
bin_PROGRAMS += src/ostbuild/ostbuild-user-chroot
|
bin_PROGRAMS += src/ostbuild/ostbuild-user-chroot
|
||||||
|
|
||||||
ostbuild_user_chroot_SOURCES = src/ostbuild/ostbuild-user-chroot.c
|
ostbuild_user_chroot_SOURCES = src/ostbuild/ostbuild-user-chroot.c
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,7 @@ if test x$with_libarchive != xno; then
|
||||||
fi
|
fi
|
||||||
AM_CONDITIONAL(USE_LIBARCHIVE, test $with_libarchive != no)
|
AM_CONDITIONAL(USE_LIBARCHIVE, test $with_libarchive != no)
|
||||||
|
|
||||||
AM_PATH_PYTHON
|
AM_PATH_PYTHON([2.7])
|
||||||
|
|
||||||
AC_CONFIG_FILES([
|
AC_CONFIG_FILES([
|
||||||
Makefile
|
Makefile
|
||||||
|
|
|
||||||
|
|
@ -1,365 +0,0 @@
|
||||||
#!/usr/bin/python
|
|
||||||
|
|
||||||
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
|
||||||
#
|
|
||||||
# This library is free software; you can redistribute it and/or
|
|
||||||
# modify it under the terms of the GNU Lesser General Public
|
|
||||||
# License as published by the Free Software Foundation; either
|
|
||||||
# version 2 of the License, or (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This library is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
||||||
# Lesser General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU Lesser General Public
|
|
||||||
# License along with this library; if not, write to the
|
|
||||||
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
||||||
# Boston, MA 02111-1307, USA.
|
|
||||||
|
|
||||||
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
|
|
||||||
# http://people.gnome.org/~walters/docs/build-api.txt
|
|
||||||
|
|
||||||
import os,sys,subprocess,tempfile,re,shutil
|
|
||||||
from StringIO import StringIO
|
|
||||||
from multiprocessing import cpu_count
|
|
||||||
import select,time
|
|
||||||
|
|
||||||
tempfiles = []
|
|
||||||
|
|
||||||
_blacklist_regexps = map(re.compile,
|
|
||||||
[r'.*\.la$',
|
|
||||||
])
|
|
||||||
|
|
||||||
_devel_regexps = map(re.compile,
|
|
||||||
[r'/usr/include/',
|
|
||||||
r'/usr/share/pkgconfig/',
|
|
||||||
r'/(?:usr/)lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc$',
|
|
||||||
r'/(?:usr/)lib(?:|(?:32)|(?:64))/[^/]+\.so$'
|
|
||||||
])
|
|
||||||
|
|
||||||
root = None
|
|
||||||
|
|
||||||
prefix = '/usr'
|
|
||||||
|
|
||||||
uname=os.uname()
|
|
||||||
kernel=uname[0].lower()
|
|
||||||
machine=uname[4]
|
|
||||||
|
|
||||||
build_target='%s-%s' % (machine, kernel)
|
|
||||||
|
|
||||||
# libdir detection
|
|
||||||
if os.path.isdir('/lib64'):
|
|
||||||
libdir=os.path.join(prefix, 'lib64')
|
|
||||||
else:
|
|
||||||
libdir=os.path.join(prefix, 'lib')
|
|
||||||
|
|
||||||
default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
|
|
||||||
configargs = ['--build=' + build_target,
|
|
||||||
'--prefix=' + prefix,
|
|
||||||
'--libdir=' + libdir,
|
|
||||||
'--sysconfdir=/etc',
|
|
||||||
'--localstatedir=/var',
|
|
||||||
'--bindir=' + os.path.join(prefix, 'bin'),
|
|
||||||
'--sbindir=' + os.path.join(prefix, 'sbin'),
|
|
||||||
'--datadir=' + os.path.join(prefix, 'share'),
|
|
||||||
'--includedir=' + os.path.join(prefix, 'include'),
|
|
||||||
'--libexecdir=' + os.path.join(prefix, 'libexec'),
|
|
||||||
'--mandir=' + os.path.join(prefix, 'share', 'man'),
|
|
||||||
'--infodir=' + os.path.join(prefix, 'share', 'info')]
|
|
||||||
makeargs = ['make']
|
|
||||||
|
|
||||||
top_srcdir=os.getcwd()
|
|
||||||
|
|
||||||
ostbuild_resultdir=top_srcdir
|
|
||||||
ostbuild_meta=None
|
|
||||||
|
|
||||||
for arg in sys.argv[1:]:
|
|
||||||
if arg.startswith('OSTBUILD_RESULTDIR='):
|
|
||||||
ostbuild_resultdir=arg[len('OSTBUILD_RESULTDIR='):]
|
|
||||||
elif arg.startswith('OSTBUILD_META='):
|
|
||||||
ostbuild_meta=arg[len('OSTBUILD_META='):]
|
|
||||||
elif arg.startswith('--'):
|
|
||||||
configargs.append(arg)
|
|
||||||
else:
|
|
||||||
makeargs.append(arg)
|
|
||||||
|
|
||||||
metadata = {}
|
|
||||||
|
|
||||||
if ostbuild_meta is None:
|
|
||||||
output = subprocess.check_output(['ostbuild-autodiscover-meta'])
|
|
||||||
ostbuild_meta_f = StringIO(output)
|
|
||||||
else:
|
|
||||||
ostbuild_meta_f = open(ostbuild_meta)
|
|
||||||
|
|
||||||
for line in ostbuild_meta_f:
|
|
||||||
(k,v) = line.split('=', 1)
|
|
||||||
metadata[k.strip()] = v.strip()
|
|
||||||
|
|
||||||
for k in ['NAME', 'VERSION']:
|
|
||||||
if k not in metadata:
|
|
||||||
sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def log(msg):
|
|
||||||
fullmsg = '%s: %s\n' % (sys.argv[0], msg)
|
|
||||||
sys.stdout.write(fullmsg)
|
|
||||||
sys.stdout.flush()
|
|
||||||
|
|
||||||
def fatal(msg):
|
|
||||||
log(msg)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def run_sync(args, cwd=None, env=None):
|
|
||||||
log("running: %r" % (args,))
|
|
||||||
f = open('/dev/null', 'r')
|
|
||||||
# This dance is necessary because we want to keep the PWD
|
|
||||||
# environment variable up to date. Not doing so is a recipie
|
|
||||||
# for triggering edge conditions in pwd lookup.
|
|
||||||
if (cwd is not None) and (env is None or ('PWD' in env)):
|
|
||||||
if env is None:
|
|
||||||
env_copy = os.environ.copy()
|
|
||||||
else:
|
|
||||||
env_copy = env.copy()
|
|
||||||
if ('PWD' in env_copy) and (not cwd.startswith('/')):
|
|
||||||
env_copy['PWD'] = os.path.join(env_copy['PWD'], cwd)
|
|
||||||
else:
|
|
||||||
env_copy['PWD'] = cwd
|
|
||||||
else:
|
|
||||||
env_copy = env
|
|
||||||
proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
|
|
||||||
close_fds=True, cwd=cwd, env=env_copy)
|
|
||||||
f.close()
|
|
||||||
returncode = proc.wait()
|
|
||||||
log("pid %d exited with code %d" % (proc.pid, returncode))
|
|
||||||
if returncode != 0:
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
class BuildSystemScanner(object):
|
|
||||||
@classmethod
|
|
||||||
def _find_file(cls, names):
|
|
||||||
for name in names:
|
|
||||||
if os.path.exists(name):
|
|
||||||
return name
|
|
||||||
return None
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def get_configure_source_script(cls):
|
|
||||||
return cls._find_file(('./configure.ac', './configure.in'))
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def get_configure_script(cls):
|
|
||||||
return cls._find_file(('./configure', ))
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def get_bootstrap_script(cls):
|
|
||||||
return cls._find_file(('./autogen.sh', ))
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def get_silent_rules(cls):
|
|
||||||
src = cls.get_configure_source_script()
|
|
||||||
if not src:
|
|
||||||
return False
|
|
||||||
f = open(src)
|
|
||||||
for line in f:
|
|
||||||
if line.find('AM_SILENT_RULES') >= 0:
|
|
||||||
f.close()
|
|
||||||
return True
|
|
||||||
f.close()
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _search_file(filename, pattern):
|
|
||||||
f = open(filename)
|
|
||||||
for line in f:
|
|
||||||
if line.startswith(pattern):
|
|
||||||
f.close()
|
|
||||||
return line
|
|
||||||
f.close()
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _find_buildapi_makevariable(name, builddir='.'):
|
|
||||||
var = '.%s:' % (name, )
|
|
||||||
line = None
|
|
||||||
path = os.path.join(builddir, 'Makefile.in')
|
|
||||||
if os.path.exists(path):
|
|
||||||
line = _search_file(path, var)
|
|
||||||
path = os.path.join(builddir, 'Makefile')
|
|
||||||
if not line and os.path.exists(path):
|
|
||||||
line = _search_file(path, var)
|
|
||||||
return line is not None
|
|
||||||
|
|
||||||
def phase_bootstrap():
|
|
||||||
have_configure = BuildSystemScanner.get_configure_script()
|
|
||||||
have_configure_source = BuildSystemScanner.get_configure_source_script()
|
|
||||||
if not (have_configure or have_configure_source):
|
|
||||||
fatal("No configure or bootstrap script detected; unknown buildsystem")
|
|
||||||
return
|
|
||||||
|
|
||||||
need_v1 = BuildSystemScanner.get_silent_rules()
|
|
||||||
if need_v1:
|
|
||||||
log("Detected AM_SILENT_RULES, adding --disable-silent-rules to configure")
|
|
||||||
configargs.append('--disable-silent-rules')
|
|
||||||
|
|
||||||
if have_configure:
|
|
||||||
phase_configure()
|
|
||||||
else:
|
|
||||||
bootstrap = BuildSystemScanner.get_bootstrap_script()
|
|
||||||
if bootstrap:
|
|
||||||
log("Detected bootstrap script: %s, using it" % (bootstrap, ))
|
|
||||||
args = [bootstrap]
|
|
||||||
# Add NOCONFIGURE; GNOME style scripts use this
|
|
||||||
env = dict(os.environ)
|
|
||||||
env['NOCONFIGURE'] = '1'
|
|
||||||
run_sync(args, env=env)
|
|
||||||
else:
|
|
||||||
log("No bootstrap script found; using generic autoreconf")
|
|
||||||
run_sync(['autoreconf', '-f', '-i'])
|
|
||||||
phase_configure()
|
|
||||||
|
|
||||||
def phase_configure():
|
|
||||||
use_builddir = True
|
|
||||||
doesnot_support_builddir = _find_buildapi_makevariable('buildapi-no-builddir')
|
|
||||||
if doesnot_support_builddir:
|
|
||||||
log("Found .buildapi-no-builddir; copying source tree to _build")
|
|
||||||
shutil.rmtree('_build')
|
|
||||||
os.mkdir('_build')
|
|
||||||
shutil.copytree('.', '_build', symlinks=True,
|
|
||||||
ignore=shutil.ignore_patterns('_build'))
|
|
||||||
use_builddir = False
|
|
||||||
|
|
||||||
if use_builddir:
|
|
||||||
builddir = '_build'
|
|
||||||
log("Using build directory %r" % (builddir, ))
|
|
||||||
if not os.path.isdir(builddir):
|
|
||||||
os.mkdir(builddir)
|
|
||||||
|
|
||||||
configstatus = 'config.status'
|
|
||||||
if not os.path.exists(configstatus):
|
|
||||||
if use_builddir:
|
|
||||||
args = ['../configure']
|
|
||||||
else:
|
|
||||||
args = ['./configure']
|
|
||||||
args.extend(configargs)
|
|
||||||
if use_builddir:
|
|
||||||
run_sync(args, cwd=builddir)
|
|
||||||
else:
|
|
||||||
run_sync(args)
|
|
||||||
else:
|
|
||||||
log("Found %s, skipping configure" % (configstatus, ))
|
|
||||||
phase_build(builddir=builddir)
|
|
||||||
|
|
||||||
build_status = False
|
|
||||||
|
|
||||||
def phase_build(builddir=None):
|
|
||||||
if not os.path.exists(os.path.join(builddir, 'Makefile')):
|
|
||||||
log("No Makefile found")
|
|
||||||
sys.exit(1)
|
|
||||||
args = makeargs
|
|
||||||
user_specified_jobs = False
|
|
||||||
for arg in args:
|
|
||||||
if arg == '-j':
|
|
||||||
user_specified_jobs = True
|
|
||||||
|
|
||||||
if not user_specified_jobs:
|
|
||||||
notparallel = _find_buildapi_makevariable('NOTPARALLEL', builddir=builddir)
|
|
||||||
if not notparallel:
|
|
||||||
log("Didn't find NOTPARALLEL, using parallel make by default")
|
|
||||||
args.extend(default_buildapi_jobs)
|
|
||||||
|
|
||||||
run_sync(args, cwd=builddir)
|
|
||||||
|
|
||||||
phase_make_artifacts(builddir=builddir)
|
|
||||||
|
|
||||||
def make_artifact(name, from_files, tempdir=None, resultdir=None):
|
|
||||||
targz_name = name + '.tar.gz'
|
|
||||||
(fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s' % (name, ))
|
|
||||||
os.close(fd)
|
|
||||||
tempfiles.append(filelist_temp)
|
|
||||||
f = open(filelist_temp, 'w')
|
|
||||||
for filename in from_files:
|
|
||||||
assert ('\n' not in filename)
|
|
||||||
f.write(filename)
|
|
||||||
f.write('\n')
|
|
||||||
f.close()
|
|
||||||
if resultdir:
|
|
||||||
result_path = os.path.join(resultdir, targz_name)
|
|
||||||
else:
|
|
||||||
result_path = targz_name
|
|
||||||
args = ['tar', '-c', '-z', '-C', tempdir, '-f', result_path, '-T', filelist_temp]
|
|
||||||
run_sync(args)
|
|
||||||
log("created: %s" % (os.path.abspath (result_path), ))
|
|
||||||
|
|
||||||
def phase_make_artifacts(builddir=None):
|
|
||||||
name = metadata['NAME']
|
|
||||||
assert ',' not in name
|
|
||||||
branch = metadata['BRANCH']
|
|
||||||
assert ',' not in name
|
|
||||||
version = metadata['VERSION']
|
|
||||||
assert ',' not in version
|
|
||||||
|
|
||||||
root_name = metadata.get('BUILDROOT', None)
|
|
||||||
# TODO - pick up current sysroot version from ostree
|
|
||||||
if root_name is None:
|
|
||||||
root_name = 'unknown-' + build_target
|
|
||||||
root_version = 'UNKNOWN'
|
|
||||||
else:
|
|
||||||
root_version = metadata.get('BUILDROOT_VERSION')
|
|
||||||
|
|
||||||
artifact_prefix='artifact-%s,%s,%s,%s,%s' % (root_name, root_version, name, branch, version)
|
|
||||||
|
|
||||||
tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (name,))
|
|
||||||
tempfiles.append(tempdir)
|
|
||||||
args = ['make', 'install', 'DESTDIR=' + tempdir]
|
|
||||||
run_sync(args, cwd=builddir)
|
|
||||||
|
|
||||||
devel_files = set()
|
|
||||||
runtime_files = set()
|
|
||||||
|
|
||||||
oldpwd=os.getcwd()
|
|
||||||
os.chdir(tempdir)
|
|
||||||
for root, dirs, files in os.walk('.'):
|
|
||||||
for filename in files:
|
|
||||||
path = os.path.join(root, filename)
|
|
||||||
|
|
||||||
blacklisted = False
|
|
||||||
for r in _blacklist_regexps:
|
|
||||||
if r.match(path):
|
|
||||||
blacklisted = True
|
|
||||||
break
|
|
||||||
|
|
||||||
if blacklisted:
|
|
||||||
continue
|
|
||||||
|
|
||||||
matched = False
|
|
||||||
for r in _devel_regexps:
|
|
||||||
if not r.match(path[1:]):
|
|
||||||
continue
|
|
||||||
devel_files.add(path)
|
|
||||||
matched = True
|
|
||||||
break
|
|
||||||
if not matched:
|
|
||||||
runtime_files.add(path)
|
|
||||||
os.chdir(oldpwd)
|
|
||||||
|
|
||||||
if devel_files:
|
|
||||||
make_artifact(artifact_prefix + '-devel', devel_files, tempdir=tempdir, resultdir=ostbuild_resultdir)
|
|
||||||
make_artifact(artifact_prefix + '-runtime', runtime_files, tempdir=tempdir, resultdir=ostbuild_resultdir)
|
|
||||||
|
|
||||||
phase_complete()
|
|
||||||
|
|
||||||
def phase_complete():
|
|
||||||
for tmpname in tempfiles:
|
|
||||||
if os.path.isdir(tmpname):
|
|
||||||
shutil.rmtree(tmpname)
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
os.unlink(tmpname)
|
|
||||||
pass
|
|
||||||
except OSError, e:
|
|
||||||
pass
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
log("invocation arguments: %r" % (sys.argv, ))
|
|
||||||
|
|
||||||
# Start off the process
|
|
||||||
phase_bootstrap()
|
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
#
|
||||||
|
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
||||||
|
#
|
||||||
|
# This library is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU Lesser General Public
|
||||||
|
# License as published by the Free Software Foundation; either
|
||||||
|
# version 2 of the License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This library is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
# Lesser General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Lesser General Public
|
||||||
|
# License along with this library; if not, write to the
|
||||||
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||||
|
# Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import __builtin__
|
||||||
|
|
||||||
|
__builtin__.__dict__['DATADIR'] = '@datarootdir@'
|
||||||
|
# This is a private directory, we don't want to pollute the global
|
||||||
|
# namespace.
|
||||||
|
path = os.path.join('@libdir@', 'ostbuild')
|
||||||
|
sys.path.insert(0, path)
|
||||||
|
|
||||||
|
from pyostbuild.main import main
|
||||||
|
|
||||||
|
sys.exit(main(sys.argv[1:]))
|
||||||
|
|
@ -0,0 +1,335 @@
|
||||||
|
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
||||||
|
#
|
||||||
|
# This library is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU Lesser General Public
|
||||||
|
# License as published by the Free Software Foundation; either
|
||||||
|
# version 2 of the License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This library is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
# Lesser General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Lesser General Public
|
||||||
|
# License along with this library; if not, write to the
|
||||||
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||||
|
# Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
|
||||||
|
# http://people.gnome.org/~walters/docs/build-api.txt
|
||||||
|
|
||||||
|
import os,sys,subprocess,tempfile,re,shutil
|
||||||
|
from StringIO import StringIO
|
||||||
|
from multiprocessing import cpu_count
|
||||||
|
import select,time
|
||||||
|
|
||||||
|
from . import builtins
|
||||||
|
from .ostbuildlog import log, fatal
|
||||||
|
from .subprocess_helpers import run_sync
|
||||||
|
|
||||||
|
PREFIX = '/usr'
|
||||||
|
|
||||||
|
_BLACKLIST_REGEXPS = map(re.compile,
|
||||||
|
[r'.*\.la$',
|
||||||
|
])
|
||||||
|
|
||||||
|
_DEVEL_REGEXPS = map(re.compile,
|
||||||
|
[r'/usr/include/',
|
||||||
|
r'/usr/share/pkgconfig/',
|
||||||
|
r'/(?:usr/)lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc$',
|
||||||
|
r'/(?:usr/)lib(?:|(?:32)|(?:64))/[^/]+\.so$'
|
||||||
|
])
|
||||||
|
|
||||||
|
class BuildSystemScanner(object):
|
||||||
|
@classmethod
|
||||||
|
def _find_file(cls, names):
|
||||||
|
for name in names:
|
||||||
|
if os.path.exists(name):
|
||||||
|
return name
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_configure_source_script(cls):
|
||||||
|
return cls._find_file(('./configure.ac', './configure.in'))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_configure_script(cls):
|
||||||
|
return cls._find_file(('./configure', ))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_bootstrap_script(cls):
|
||||||
|
return cls._find_file(('./autogen.sh', ))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_silent_rules(cls):
|
||||||
|
src = cls.get_configure_source_script()
|
||||||
|
if not src:
|
||||||
|
return False
|
||||||
|
f = open(src)
|
||||||
|
for line in f:
|
||||||
|
if line.find('AM_SILENT_RULES') >= 0:
|
||||||
|
f.close()
|
||||||
|
return True
|
||||||
|
f.close()
|
||||||
|
return False
|
||||||
|
|
||||||
|
class OstbuildCompileOne(builtins.Builtin):
|
||||||
|
name = "compile-one"
|
||||||
|
short_description = "Build artifacts from the current source directory"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
builtins.Builtin.__init__(self)
|
||||||
|
self.tempfiles = []
|
||||||
|
|
||||||
|
def _search_file(self, filename, pattern):
|
||||||
|
f = open(filename)
|
||||||
|
for line in f:
|
||||||
|
if line.startswith(pattern):
|
||||||
|
f.close()
|
||||||
|
return line
|
||||||
|
f.close()
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _find_buildapi_makevariable(self, name, builddir='.'):
|
||||||
|
var = '.%s:' % (name, )
|
||||||
|
line = None
|
||||||
|
path = os.path.join(builddir, 'Makefile.in')
|
||||||
|
if os.path.exists(path):
|
||||||
|
line = self._search_file(path, var)
|
||||||
|
path = os.path.join(builddir, 'Makefile')
|
||||||
|
if not line and os.path.exists(path):
|
||||||
|
line = self._search_file(path, var)
|
||||||
|
return line is not None
|
||||||
|
|
||||||
|
def execute(self, args):
|
||||||
|
self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
|
||||||
|
|
||||||
|
uname=os.uname()
|
||||||
|
kernel=uname[0].lower()
|
||||||
|
machine=uname[4]
|
||||||
|
self.build_target='%s-%s' % (machine, kernel)
|
||||||
|
|
||||||
|
# libdir detection
|
||||||
|
if os.path.isdir('/lib64'):
|
||||||
|
libdir=os.path.join(PREFIX, 'lib64')
|
||||||
|
else:
|
||||||
|
libdir=os.path.join(PREFIX, 'lib')
|
||||||
|
|
||||||
|
self.configargs = ['--build=' + self.build_target,
|
||||||
|
'--prefix=' + PREFIX,
|
||||||
|
'--libdir=' + libdir,
|
||||||
|
'--sysconfdir=/etc',
|
||||||
|
'--localstatedir=/var',
|
||||||
|
'--bindir=' + os.path.join(PREFIX, 'bin'),
|
||||||
|
'--sbindir=' + os.path.join(PREFIX, 'sbin'),
|
||||||
|
'--datadir=' + os.path.join(PREFIX, 'share'),
|
||||||
|
'--includedir=' + os.path.join(PREFIX, 'include'),
|
||||||
|
'--libexecdir=' + os.path.join(PREFIX, 'libexec'),
|
||||||
|
'--mandir=' + os.path.join(PREFIX, 'share', 'man'),
|
||||||
|
'--infodir=' + os.path.join(PREFIX, 'share', 'info')]
|
||||||
|
self.makeargs = ['make']
|
||||||
|
|
||||||
|
self.ostbuild_resultdir=os.getcwd()
|
||||||
|
self.ostbuild_meta=None
|
||||||
|
|
||||||
|
for arg in args:
|
||||||
|
if arg.startswith('--ostbuild-resultdir='):
|
||||||
|
self.ostbuild_resultdir=arg[len('ostbuild-resultdir='):]
|
||||||
|
elif arg.startswith('ostbuild-meta='):
|
||||||
|
self.ostbuild_meta=arg[len('ostbuild-meta='):]
|
||||||
|
elif arg.startswith('--'):
|
||||||
|
self.configargs.append(arg)
|
||||||
|
else:
|
||||||
|
self.makeargs.append(arg)
|
||||||
|
|
||||||
|
self.metadata = {}
|
||||||
|
|
||||||
|
if self.ostbuild_meta is None:
|
||||||
|
output = subprocess.check_output(['ostbuild-autodiscover-meta'])
|
||||||
|
ostbuild_meta_f = StringIO(output)
|
||||||
|
else:
|
||||||
|
ostbuild_meta_f = open(ostbuild_meta)
|
||||||
|
|
||||||
|
for line in ostbuild_meta_f:
|
||||||
|
(k,v) = line.split('=', 1)
|
||||||
|
self.metadata[k.strip()] = v.strip()
|
||||||
|
|
||||||
|
ostbuild_meta_f.close()
|
||||||
|
|
||||||
|
for k in ['NAME', 'VERSION']:
|
||||||
|
if k not in self.metadata:
|
||||||
|
fatal('Missing required key "%s" in metadata' % (k, ))
|
||||||
|
|
||||||
|
self.phase_bootstrap()
|
||||||
|
|
||||||
|
def phase_bootstrap(self):
|
||||||
|
have_configure = BuildSystemScanner.get_configure_script()
|
||||||
|
have_configure_source = BuildSystemScanner.get_configure_source_script()
|
||||||
|
if not (have_configure or have_configure_source):
|
||||||
|
fatal("No configure or bootstrap script detected; unknown buildsystem")
|
||||||
|
return
|
||||||
|
|
||||||
|
need_v1 = BuildSystemScanner.get_silent_rules()
|
||||||
|
if need_v1:
|
||||||
|
log("Detected AM_SILENT_RULES, adding --disable-silent-rules to configure")
|
||||||
|
self.configargs.append('--disable-silent-rules')
|
||||||
|
|
||||||
|
if have_configure:
|
||||||
|
self.phase_configure()
|
||||||
|
else:
|
||||||
|
bootstrap = BuildSystemScanner.get_bootstrap_script()
|
||||||
|
if bootstrap:
|
||||||
|
log("Detected bootstrap script: %s, using it" % (bootstrap, ))
|
||||||
|
args = [bootstrap]
|
||||||
|
# Add NOCONFIGURE; GNOME style scripts use this
|
||||||
|
env = dict(os.environ)
|
||||||
|
env['NOCONFIGURE'] = '1'
|
||||||
|
run_sync(args, env=env)
|
||||||
|
else:
|
||||||
|
log("No bootstrap script found; using generic autoreconf")
|
||||||
|
run_sync(['autoreconf', '-f', '-i'])
|
||||||
|
self.phase_configure()
|
||||||
|
|
||||||
|
def phase_configure(self):
|
||||||
|
use_builddir = True
|
||||||
|
doesnot_support_builddir = self._find_buildapi_makevariable('buildapi-no-builddir')
|
||||||
|
if doesnot_support_builddir:
|
||||||
|
log("Found .buildapi-no-builddir; copying source tree to _build")
|
||||||
|
shutil.rmtree('_build')
|
||||||
|
os.mkdir('_build')
|
||||||
|
shutil.copytree('.', '_build', symlinks=True,
|
||||||
|
ignore=shutil.ignore_patterns('_build'))
|
||||||
|
use_builddir = False
|
||||||
|
|
||||||
|
if use_builddir:
|
||||||
|
builddir = '_build'
|
||||||
|
log("Using build directory %r" % (builddir, ))
|
||||||
|
if not os.path.isdir(builddir):
|
||||||
|
os.mkdir(builddir)
|
||||||
|
|
||||||
|
configstatus = 'config.status'
|
||||||
|
if not os.path.exists(configstatus):
|
||||||
|
if use_builddir:
|
||||||
|
args = ['../configure']
|
||||||
|
else:
|
||||||
|
args = ['./configure']
|
||||||
|
args.extend(self.configargs)
|
||||||
|
if use_builddir:
|
||||||
|
run_sync(args, cwd=builddir)
|
||||||
|
else:
|
||||||
|
run_sync(args)
|
||||||
|
else:
|
||||||
|
log("Found %s, skipping configure" % (configstatus, ))
|
||||||
|
self.phase_build(builddir=builddir)
|
||||||
|
|
||||||
|
build_status = False
|
||||||
|
|
||||||
|
def phase_build(self, builddir=None):
|
||||||
|
if not os.path.exists(os.path.join(builddir, 'Makefile')):
|
||||||
|
fatal("No Makefile found")
|
||||||
|
args = list(self.makeargs)
|
||||||
|
user_specified_jobs = False
|
||||||
|
for arg in args:
|
||||||
|
if arg == '-j':
|
||||||
|
user_specified_jobs = True
|
||||||
|
|
||||||
|
if not user_specified_jobs:
|
||||||
|
notparallel = self._find_buildapi_makevariable('NOTPARALLEL', builddir=builddir)
|
||||||
|
if not notparallel:
|
||||||
|
log("Didn't find NOTPARALLEL, using parallel make by default")
|
||||||
|
args.extend(self.default_buildapi_jobs)
|
||||||
|
|
||||||
|
run_sync(args, cwd=builddir)
|
||||||
|
|
||||||
|
self.phase_make_artifacts(builddir=builddir)
|
||||||
|
|
||||||
|
def make_artifact(self, name, from_files, tempdir=None, resultdir=None):
|
||||||
|
targz_name = name + '.tar.gz'
|
||||||
|
(fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s' % (name, ))
|
||||||
|
os.close(fd)
|
||||||
|
self.tempfiles.append(filelist_temp)
|
||||||
|
f = open(filelist_temp, 'w')
|
||||||
|
for filename in from_files:
|
||||||
|
assert ('\n' not in filename)
|
||||||
|
f.write(filename)
|
||||||
|
f.write('\n')
|
||||||
|
f.close()
|
||||||
|
if resultdir:
|
||||||
|
result_path = os.path.join(resultdir, targz_name)
|
||||||
|
else:
|
||||||
|
result_path = targz_name
|
||||||
|
args = ['tar', '-c', '-z', '-C', tempdir, '-f', result_path, '-T', filelist_temp]
|
||||||
|
run_sync(args)
|
||||||
|
log("created: %s" % (os.path.abspath (result_path), ))
|
||||||
|
|
||||||
|
def phase_make_artifacts(self, builddir=None):
|
||||||
|
name = self.metadata['NAME']
|
||||||
|
assert ',' not in name
|
||||||
|
branch = self.metadata['BRANCH']
|
||||||
|
assert ',' not in name
|
||||||
|
version = self.metadata['VERSION']
|
||||||
|
assert ',' not in version
|
||||||
|
|
||||||
|
root_name = self.metadata.get('BUILDROOT', None)
|
||||||
|
# TODO - pick up current sysroot version from ostree
|
||||||
|
if root_name is None:
|
||||||
|
root_name = 'unknown-' + self.build_target
|
||||||
|
root_version = 'UNKNOWN'
|
||||||
|
else:
|
||||||
|
root_version = self.metadata.get('BUILDROOT_VERSION')
|
||||||
|
|
||||||
|
artifact_prefix='artifact-%s,%s,%s,%s,%s' % (root_name, root_version, name, branch, version)
|
||||||
|
|
||||||
|
tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (name,))
|
||||||
|
self.tempfiles.append(tempdir)
|
||||||
|
args = ['make', 'install', 'DESTDIR=' + tempdir]
|
||||||
|
run_sync(args, cwd=builddir)
|
||||||
|
|
||||||
|
devel_files = set()
|
||||||
|
runtime_files = set()
|
||||||
|
|
||||||
|
oldpwd=os.getcwd()
|
||||||
|
os.chdir(tempdir)
|
||||||
|
for root, dirs, files in os.walk('.'):
|
||||||
|
for filename in files:
|
||||||
|
path = os.path.join(root, filename)
|
||||||
|
|
||||||
|
blacklisted = False
|
||||||
|
for r in _BLACKLIST_REGEXPS:
|
||||||
|
if r.match(path):
|
||||||
|
blacklisted = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if blacklisted:
|
||||||
|
continue
|
||||||
|
|
||||||
|
matched = False
|
||||||
|
for r in _DEVEL_REGEXPS:
|
||||||
|
if not r.match(path[1:]):
|
||||||
|
continue
|
||||||
|
devel_files.add(path)
|
||||||
|
matched = True
|
||||||
|
break
|
||||||
|
if not matched:
|
||||||
|
runtime_files.add(path)
|
||||||
|
os.chdir(oldpwd)
|
||||||
|
|
||||||
|
if devel_files:
|
||||||
|
self.make_artifact(artifact_prefix + '-devel', devel_files, tempdir=tempdir, resultdir=self.ostbuild_resultdir)
|
||||||
|
self.make_artifact(artifact_prefix + '-runtime', runtime_files, tempdir=tempdir, resultdir=self.ostbuild_resultdir)
|
||||||
|
|
||||||
|
self.phase_complete()
|
||||||
|
|
||||||
|
def phase_complete(self):
|
||||||
|
for tmpname in self.tempfiles:
|
||||||
|
assert os.path.isabs(tmpname)
|
||||||
|
if os.path.isdir(tmpname):
|
||||||
|
shutil.rmtree(tmpname)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
os.unlink(tmpname)
|
||||||
|
pass
|
||||||
|
except OSError, e:
|
||||||
|
pass
|
||||||
|
|
||||||
|
builtins.register(OstbuildCompileOne)
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
#
|
||||||
|
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
||||||
|
#
|
||||||
|
# This library is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU Lesser General Public
|
||||||
|
# License as published by the Free Software Foundation; either
|
||||||
|
# version 2 of the License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This library is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
# Lesser General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Lesser General Public
|
||||||
|
# License along with this library; if not, write to the
|
||||||
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||||
|
# Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
_all_builtins = {}
|
||||||
|
|
||||||
|
class Builtin(object):
|
||||||
|
name = None
|
||||||
|
short_description = None
|
||||||
|
|
||||||
|
def execute(self, args):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def register(builtin):
|
||||||
|
_all_builtins[builtin.name] = builtin
|
||||||
|
|
||||||
|
def get(name):
|
||||||
|
return _all_builtins.get(name)()
|
||||||
|
|
||||||
|
def get_all():
|
||||||
|
return _all_builtins.itervalues()
|
||||||
|
|
@ -0,0 +1,45 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
#
|
||||||
|
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
||||||
|
#
|
||||||
|
# This library is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU Lesser General Public
|
||||||
|
# License as published by the Free Software Foundation; either
|
||||||
|
# version 2 of the License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This library is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
# Lesser General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Lesser General Public
|
||||||
|
# License along with this library; if not, write to the
|
||||||
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||||
|
# Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
from . import builtins
|
||||||
|
from . import builtin_compile_one
|
||||||
|
|
||||||
|
def usage(ecode):
|
||||||
|
print "Builtins:"
|
||||||
|
for builtin in builtins.get_all():
|
||||||
|
print " %s - %s" % (builtin.name, builtin.short_description)
|
||||||
|
return ecode
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
if len(args) < 1:
|
||||||
|
return usage(1)
|
||||||
|
elif args[0] in ('-h', '--help'):
|
||||||
|
return usage(0)
|
||||||
|
else:
|
||||||
|
builtin = builtins.get(args[0])
|
||||||
|
if builtin is None:
|
||||||
|
print "error: Unknown builtin '%s'" % (args[1], )
|
||||||
|
return usage(1)
|
||||||
|
return builtin.execute(args[1:])
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
#
|
||||||
|
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
||||||
|
#
|
||||||
|
# This library is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU Lesser General Public
|
||||||
|
# License as published by the Free Software Foundation; either
|
||||||
|
# version 2 of the License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This library is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
# Lesser General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Lesser General Public
|
||||||
|
# License along with this library; if not, write to the
|
||||||
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||||
|
# Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
fullmsg = '%s: %s\n' % (sys.argv[0], msg)
|
||||||
|
sys.stdout.write(fullmsg)
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
def fatal(msg):
|
||||||
|
log(msg)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
#
|
||||||
|
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
||||||
|
#
|
||||||
|
# This library is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU Lesser General Public
|
||||||
|
# License as published by the Free Software Foundation; either
|
||||||
|
# version 2 of the License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This library is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
# Lesser General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Lesser General Public
|
||||||
|
# License along with this library; if not, write to the
|
||||||
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||||
|
# Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from .ostbuildlog import log, fatal
|
||||||
|
|
||||||
|
def run_sync(args, cwd=None, env=None):
|
||||||
|
log("running: %r" % (args,))
|
||||||
|
f = open('/dev/null', 'r')
|
||||||
|
# This dance is necessary because we want to keep the PWD
|
||||||
|
# environment variable up to date. Not doing so is a recipie
|
||||||
|
# for triggering edge conditions in pwd lookup.
|
||||||
|
if (cwd is not None) and (env is None or ('PWD' in env)):
|
||||||
|
if env is None:
|
||||||
|
env_copy = os.environ.copy()
|
||||||
|
else:
|
||||||
|
env_copy = env.copy()
|
||||||
|
if ('PWD' in env_copy) and (not cwd.startswith('/')):
|
||||||
|
env_copy['PWD'] = os.path.join(env_copy['PWD'], cwd)
|
||||||
|
else:
|
||||||
|
env_copy['PWD'] = cwd
|
||||||
|
else:
|
||||||
|
env_copy = env
|
||||||
|
proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
|
||||||
|
close_fds=True, cwd=cwd, env=env_copy)
|
||||||
|
f.close()
|
||||||
|
returncode = proc.wait()
|
||||||
|
if returncode != 0:
|
||||||
|
logfn = fatal
|
||||||
|
else:
|
||||||
|
logfn = log
|
||||||
|
logfn("pid %d exited with code %d" % (proc.pid, returncode))
|
||||||
Loading…
Reference in New Issue