From fea117cd5e44472c198d3acf1d2d30b66c057a06 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 5 Jan 2012 19:10:09 -0500 Subject: [PATCH] gnomeos: Move everything into JSON, start plowing through X.org Also: * Stop generating tarballs for artifacts, use plain directories * Commit artifacts as uid/gid 0, with no xattrs Other misc stuff --- gnomeos/3.4/glib.txt | 1 - gnomeos/3.4/gobject-introspection.txt | 1 - gnomeos/3.4/gtk-doc-stub.txt | 2 - gnomeos/3.4/libarchive.txt | 2 - gnomeos/3.4/libxslt.txt | 2 - gnomeos/3.4/manifest.json | 154 ++++++++++++++- ...n-autogen.sh-Support-GNOME-Build-API.patch | 25 +++ gnomeos/3.4/xorg-autogen.patch | 25 +++ .../images/gnomeos-contents-devel.bb | 2 +- src/ostbuild/pyostbuild/buildutil.py | 8 +- .../pyostbuild/builtin_autodiscover_meta.py | 4 +- src/ostbuild/pyostbuild/builtin_build.py | 182 +++++++++++------- .../pyostbuild/builtin_chroot_compile_one.py | 22 ++- .../pyostbuild/builtin_compile_one.py | 50 ++--- src/ostbuild/pyostbuild/builtins.py | 6 +- src/ostbuild/pyostbuild/ostbuildlog.py | 2 +- src/ostbuild/pyostbuild/subprocess_helpers.py | 21 +- 17 files changed, 378 insertions(+), 131 deletions(-) delete mode 100644 gnomeos/3.4/glib.txt delete mode 100644 gnomeos/3.4/gobject-introspection.txt delete mode 100644 gnomeos/3.4/gtk-doc-stub.txt delete mode 100644 gnomeos/3.4/libarchive.txt delete mode 100644 gnomeos/3.4/libxslt.txt create mode 100644 gnomeos/3.4/pixman-autogen.sh-Support-GNOME-Build-API.patch create mode 100644 gnomeos/3.4/xorg-autogen.patch diff --git a/gnomeos/3.4/glib.txt b/gnomeos/3.4/glib.txt deleted file mode 100644 index ddb33f68..00000000 --- a/gnomeos/3.4/glib.txt +++ /dev/null @@ -1 +0,0 @@ -SRC=git:git://git.gnome.org/glib diff --git a/gnomeos/3.4/gobject-introspection.txt b/gnomeos/3.4/gobject-introspection.txt deleted file mode 100644 index 142d75ff..00000000 --- a/gnomeos/3.4/gobject-introspection.txt +++ /dev/null @@ -1 +0,0 @@ -SRC=git:git://git.gnome.org/gobject-introspection diff --git a/gnomeos/3.4/gtk-doc-stub.txt b/gnomeos/3.4/gtk-doc-stub.txt deleted file mode 100644 index 75dd7a4a..00000000 --- a/gnomeos/3.4/gtk-doc-stub.txt +++ /dev/null @@ -1,2 +0,0 @@ -SRC=git:git://git.gnome.org/gtk-doc-stub -COMPONENT=devel diff --git a/gnomeos/3.4/libarchive.txt b/gnomeos/3.4/libarchive.txt deleted file mode 100644 index 99a349c6..00000000 --- a/gnomeos/3.4/libarchive.txt +++ /dev/null @@ -1,2 +0,0 @@ -SRC=svn:http://libarchive.googlecode.com/svn/trunk/libarchive-read-only -CONFIGURE_OPTS=--disable-bsdtar --disable-bsdcpio diff --git a/gnomeos/3.4/libxslt.txt b/gnomeos/3.4/libxslt.txt deleted file mode 100644 index a060dfa5..00000000 --- a/gnomeos/3.4/libxslt.txt +++ /dev/null @@ -1,2 +0,0 @@ -SRC=git:git://git.gnome.org/libxslt -EXTRA_OECONF = "--disable-static" diff --git a/gnomeos/3.4/manifest.json b/gnomeos/3.4/manifest.json index 9f23be13..e2a3a29f 100644 --- a/gnomeos/3.4/manifest.json +++ b/gnomeos/3.4/manifest.json @@ -3,9 +3,157 @@ "architectures": ["i686"], "base": "yocto/gnomeos-3.4", + "config-opts": ["--disable-static"], + + "vcsconfig": {"gnome": "git:git://git.gnome.org/", + "fd": "git:git://anongit.freedesktop.org/git/"}, + "components": [ - "gtk-doc-stub", - "gobject-introspection", - "glib" + {"src": "gnome:gtk-doc-stub", + "component": "devel"}, + + {"src": "gnome:libxml2"}, + + {"src": "gnome:libxslt"}, + + {"src": "git:git://github.com/atgreen/libffi.git"}, + + {"src": "gnome:glib"}, + + {"src": "fd:pixman"}, + + {"src": "fd:xorg/util/macros", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/bigreqsproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/compositeproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/damageproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/dmxproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/dri2proto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/evieproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/fixesproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/fontsproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/glproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/inputproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/kbproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/randrproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/recordproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/renderproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/resourceproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/scrnsaverproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/videoproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xcmiscproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xextproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xf86bigfontproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xf86dgaproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xf86driproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xf86miscproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xf86vidmodeproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xineramaproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/fixesproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/proto/xproto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/lib/libxtrans", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xcb/proto", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xcb/pthread-stubs", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xcb/libxcb", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:xorg/lib/libX11", + "component": "devel", + "patches": ["xorg-autogen.patch"]}, + + {"src": "fd:cairo"}, + + {"src": "gnome:gobject-introspection"} ] } diff --git a/gnomeos/3.4/pixman-autogen.sh-Support-GNOME-Build-API.patch b/gnomeos/3.4/pixman-autogen.sh-Support-GNOME-Build-API.patch new file mode 100644 index 00000000..bdc29e28 --- /dev/null +++ b/gnomeos/3.4/pixman-autogen.sh-Support-GNOME-Build-API.patch @@ -0,0 +1,25 @@ +From e4857495898aa2100f9dda77fbf7db68e61a0c48 Mon Sep 17 00:00:00 2001 +From: Colin Walters +Date: Wed, 4 Jan 2012 08:06:05 -0500 +Subject: [PATCH] autogen.sh: Support GNOME Build API + +http://people.gnome.org/~walters/docs/build-api.txt +--- + autogen.sh | 4 +++- + 1 files changed, 3 insertions(+), 1 deletions(-) + +diff --git a/autogen.sh b/autogen.sh +index 354f254..fc34bd5 100755 +--- a/autogen.sh ++++ b/autogen.sh +@@ -9,4 +9,6 @@ cd $srcdir + autoreconf -v --install || exit 1 + cd $ORIGDIR || exit $? + +-$srcdir/configure "$@" ++if test -z "$NOCONFIGURE"; then ++ $srcdir/configure "$@" ++fi +-- +1.7.6.4 + diff --git a/gnomeos/3.4/xorg-autogen.patch b/gnomeos/3.4/xorg-autogen.patch new file mode 100644 index 00000000..5a3fd93a --- /dev/null +++ b/gnomeos/3.4/xorg-autogen.patch @@ -0,0 +1,25 @@ +From c3f75d8999b343131627a7f857e27f05a0f8d4aa Mon Sep 17 00:00:00 2001 +From: Colin Walters +Date: Wed, 4 Jan 2012 17:37:06 -0500 +Subject: [PATCH] autogen.sh: Implement GNOME Build API + +http://people.gnome.org/~walters/docs/build-api.txt +--- + autogen.sh | 4 +++- + 1 files changed, 3 insertions(+), 1 deletions(-) + +diff --git a/autogen.sh b/autogen.sh +index 904cd67..b47abdc 100755 +--- a/autogen.sh ++++ b/autogen.sh +@@ -9,4 +9,6 @@ cd $srcdir + autoreconf -v --install || exit 1 + cd $ORIGDIR || exit $? + +-$srcdir/configure --enable-maintainer-mode "$@" ++if test -z "$NOCONFIGURE"; then ++ $srcdir/configure --enable-maintainer-mode "$@" ++fi +-- +1.7.6.4 + diff --git a/gnomeos/yocto/recipies-core/images/gnomeos-contents-devel.bb b/gnomeos/yocto/recipies-core/images/gnomeos-contents-devel.bb index 0e123b54..96069549 100644 --- a/gnomeos/yocto/recipies-core/images/gnomeos-contents-devel.bb +++ b/gnomeos/yocto/recipies-core/images/gnomeos-contents-devel.bb @@ -7,7 +7,7 @@ LIC_FILES_CHKSUM = "file://${COREBASE}/LICENSE;md5=3f40d7994397109285ec7b81fdeb3 inherit gnomeos-contents -PACKAGE_INSTALL += "task-core-sdk-dev \ +PACKAGE_INSTALL += "task-core-sdk \ python-dev \ bison flex \ git \ diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py index 26bea32a..36e5954b 100755 --- a/src/ostbuild/pyostbuild/buildutil.py +++ b/src/ostbuild/pyostbuild/buildutil.py @@ -17,7 +17,9 @@ import re -ARTIFACT_RE = re.compile(r'^artifact-([^,]+),([^,]+),([^,]+),([^,]+),(.+)-((?:runtime)|(?:devel))\.tar\.gz$') +from .subprocess_helpers import run_sync_get_output + +ARTIFACT_RE = re.compile(r'^artifact-([^,]+),([^,]+),([^,]+),([^,]+),(.+)-((?:runtime)|(?:devel))\.tar$') def parse_artifact_name(artifact_basename): match = ARTIFACT_RE.match(artifact_basename) @@ -36,3 +38,7 @@ def branch_name_for_artifact(a): a['branch'], a['type']) +def get_git_version_describe(dirpath): + version = run_sync_get_output(['git', 'describe', '--long', '--abbrev=42', '--always'], + cwd=dirpath) + return version.strip() diff --git a/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py b/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py index 3d7ec8f8..fe69be09 100755 --- a/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py +++ b/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py @@ -21,6 +21,7 @@ import os,sys,re,subprocess,tempfile,shutil import argparse from . import builtins +from . import buildutil from .ostbuildlog import log, fatal class OstbuildAutodiscoverMeta(builtins.Builtin): @@ -72,8 +73,7 @@ class OstbuildAutodiscoverMeta(builtins.Builtin): def _discover_version_from_git(self): if os.path.isdir('.git'): - version = subprocess.check_output(['git', 'describe', '--long', '--abbrev=42', '--always']) - return version.strip() + return buildutil.get_git_version_describe(os.getcwd()) return None def _discover_branch_from_git(self): diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index fa13f83d..e781b468 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -38,7 +38,7 @@ class OstbuildBuild(builtins.Builtin): def _ensure_vcs_mirror(self, name, keytype, uri, branch): assert keytype == 'git' - mirror = os.path.join(self.srcdir, name) + mirror = os.path.join(self.mirrordir, name) tmp_mirror = mirror + '.tmp' if os.path.isdir(tmp_mirror): shutil.rmtree(tmp_mirror) @@ -48,7 +48,7 @@ class OstbuildBuild(builtins.Builtin): return mirror def _get_vcs_checkout(self, name, keytype, mirrordir, branch): - checkoutdir = os.path.join(self.srcdir, '_checkouts') + checkoutdir = os.path.join(self.workdir, 'src') if not os.path.isdir(checkoutdir): os.makedirs(checkoutdir) dest = os.path.join(checkoutdir, name) @@ -57,16 +57,12 @@ class OstbuildBuild(builtins.Builtin): shutil.rmtree(dest) if os.path.isdir(tmp_dest): shutil.rmtree(tmp_dest) - subprocess.check_call(['git', 'clone', '--depth=1', '-q', mirrordir, tmp_dest]) + subprocess.check_call(['git', 'clone', '-q', mirrordir, tmp_dest]) subprocess.check_call(['git', 'checkout', '-q', branch], cwd=tmp_dest) subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=tmp_dest) os.rename(tmp_dest, dest) return dest - def _get_vcs_version_from_checkout(self, name): - vcsdir = os.path.join(self.srcdir, name) - return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=vcsdir) - def _parse_src_key(self, srckey): idx = srckey.find(':') if idx < 0: @@ -83,16 +79,6 @@ class OstbuildBuild(builtins.Builtin): uri = uri[0:idx] return (keytype, uri, branch) - def _parse_artifact_vcs_version(self, ver): - idx = ver.rfind('-') - if idx > 0: - vcs_ver = ver[idx+1:] - else: - vcs_ver = ver - if not vcs_ver.startswith('g'): - raise ValueError("Invalid artifact version '%s'" % (ver, )) - return vcs_ver[1:] - def _get_ostbuild_chroot_args(self, architecture): current_machine = os.uname()[4] if current_machine != architecture: @@ -111,46 +97,106 @@ class OstbuildBuild(builtins.Builtin): run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True) fatal("Exiting after debug shell") - def _build_one_component(self, name, architecture, meta): - (keytype, uri, branch) = self._parse_src_key(meta['SRC']) - component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch) - component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch) + def _resolve_component_meta(self, component_meta): + result = dict(component_meta) + orig_src = component_meta['src'] + + did_expand = False + for (vcsprefix, expansion) in self.manifest['vcsconfig'].iteritems(): + prefix = vcsprefix + ':' + if orig_src.startswith(prefix): + result['src'] = expansion + orig_src[len(prefix):] + did_expand = True + break + + name = component_meta.get('name') + if name is None: + if did_expand: + src = orig_src + idx = src.rindex(':') + name = src[idx+1:] + else: + src = result['src'] + idx = src.rindex('/') + name = src[idx+1:] + if name.endswith('.git'): + name = name[:-4] + name = name.replace('/', '-') + result['name'] = name + return result + + def _build_one_component(self, meta, architecture): + name = meta['name'] + + (keytype, uri, branch) = self._parse_src_key(meta['src']) + buildroot = '%s-%s-devel' % (self.manifest['name'], architecture) - branchname = 'artifacts/%s/%s/%s' % (buildroot, name, branch) + runtime_branchname = 'artifacts/%s/%s/%s/runtime' % (buildroot, name, branch) current_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', buildroot]) current_buildroot_version = current_buildroot_version.strip() + + artifact_base = {'buildroot': buildroot, + 'buildroot_version': current_buildroot_version, + 'name': name, + 'branch': branch, + } + + component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch) + component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch) + + current_vcs_version = buildutil.get_git_version_describe(component_src) + artifact_base['version'] = current_vcs_version + previous_commit_version = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', branchname], + 'rev-parse', runtime_branchname], stderr=open('/dev/null', 'w'), none_on_error=True) if previous_commit_version is not None: - log("Previous build of '%s' is %s" % (branchname, previous_commit_version)) + log("Previous build of '%s' is %s" % (runtime_branchname, previous_commit_version)) + previous_artifact_version = run_sync_get_output(['ostree', '--repo=' + self.repo, 'show', '--print-metadata-key=ostbuild-artifact-version', previous_commit_version]) previous_artifact_version = previous_artifact_version.strip() previous_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo, 'show', '--print-metadata-key=ostbuild-buildroot-version', previous_commit_version]) previous_buildroot_version = previous_buildroot_version.strip() - - previous_vcs_version = self._parse_artifact_vcs_version(previous_artifact_version) - current_vcs_version = self._get_vcs_version_from_checkout(name) + + previous_artifact_base = dict(artifact_base) + previous_artifact_base['version'] = previous_artifact_version + previous_artifact_base['buildroot_version'] = previous_buildroot_version + + previous_artifact_runtime = dict(previous_artifact_base) + previous_artifact_runtime['type'] = 'runtime' + previous_artifact_devel = dict(previous_artifact_base) + previous_artifact_devel['type'] = 'devel' + previous_artifacts = [previous_artifact_runtime, + previous_artifact_devel] + vcs_version_matches = False - if previous_vcs_version == current_vcs_version: + if previous_artifact_version == current_vcs_version: vcs_version_matches = True - log("VCS version is unchanged from '%s'" % (previous_vcs_version, )) + log("VCS version is unchanged from '%s'" % (previous_artifact_version, )) + if self.buildopts.skip_built: + return previous_artifacts else: - log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version)) + log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_artifact_version)) buildroot_version_matches = False if vcs_version_matches: buildroot_version_matches = (current_buildroot_version == previous_buildroot_version) if buildroot_version_matches: - log("Already have build '%s' of src commit '%s' for '%s' in buildroot '%s'" % (previous_commit_version, previous_vcs_version, branchname, buildroot)) - return + log("Already have build '%s' of src commit '%s' for '%s' in buildroot '%s'" % (previous_commit_version, previous_artifact_version, runtime_branchname, buildroot)) + return previous_artifacts else: log("Buildroot is now '%s'" % (current_buildroot_version, )) else: - log("No previous build for '%s' found" % (branchname, )) + log("No previous build for '%s' found" % (runtime_branchname, )) + + patches = meta.get('patches') + if patches is not None: + for patch in patches: + patch_path = os.path.join(self.manifestdir, patch) + run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src) component_resultdir = os.path.join(self.workdir, name, 'results') if os.path.isdir(component_resultdir): @@ -161,31 +207,33 @@ class OstbuildBuild(builtins.Builtin): chroot_args.extend(['--buildroot=' + buildroot, '--workdir=' + self.workdir, '--resultdir=' + component_resultdir]) + global_config_opts = self.manifest.get('config-opts') + if global_config_opts is not None: + chroot_args.extend(global_config_opts) if self.buildopts.shell_on_failure: ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False) if ecode != 0: self._launch_debug_shell(architecture, buildroot, cwd=component_src) else: run_sync(chroot_args, cwd=component_src, fatal_on_error=True) - artifact_files = [] - for name in os.listdir(component_resultdir): - if name.startswith('artifact-'): - log("Generated artifact file: %s" % (name, )) - artifact_files.append(os.path.join(component_resultdir, name)) - assert len(artifact_files) >= 1 and len(artifact_files) <= 2 - run_sync(['ostbuild', 'commit-artifacts', - '--repo=' + self.repo] + artifact_files) + artifacts = [] - for filename in artifact_files: - parsed = buildutil.parse_artifact_name(os.path.basename(filename)) - artifacts.append(parsed) - def _sort_artifact(a, b): - if a['type'] == b['type']: - return 0 - elif a['type'] == 'runtime': - return -1 - return 1 - artifacts.sort(_sort_artifact) + for artifact_type in ['runtime', 'devel']: + artifact = dict(artifact_base) + artifacts.append(artifact) + artifact['type'] = artifact_type + + artifact_branch = buildutil.branch_name_for_artifact(artifact) + + artifact_resultdir = os.path.join(component_resultdir, artifact_branch) + + run_sync(['ostree', '--repo=' + self.repo, + 'commit', '-b', artifact_branch, '-s', 'Build ' + artifact_base['version'], + '--add-metadata-string=ostbuild-buildroot-version=' + current_buildroot_version, + '--add-metadata-string=ostbuild-artifact-version=' + artifact_base['version'], + '--owner-uid=0', '--owner-gid=0', '--no-xattrs', + '--skip-if-unchanged'], + cwd=artifact_resultdir) return artifacts def _compose(self, suffix, artifacts): @@ -199,7 +247,7 @@ class OstbuildBuild(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) parser.add_argument('--manifest', required=True) - parser.add_argument('--start-at') + parser.add_argument('--skip-built', action='store_true') parser.add_argument('--shell-on-failure', action='store_true') parser.add_argument('--debug-shell', action='store_true') @@ -209,6 +257,7 @@ class OstbuildBuild(builtins.Builtin): self.buildopts = BuildOptions() self.buildopts.shell_on_failure = args.shell_on_failure + self.buildopts.skip_built = args.skip_built self.manifest = json.load(open(args.manifest)) @@ -217,43 +266,28 @@ class OstbuildBuild(builtins.Builtin): debug_shell_buildroot = '%s-%s-devel' % (self.manifest['name'], debug_shell_arch) self._launch_debug_shell(debug_shell_arch, debug_shell_buildroot) - dirname = os.path.dirname(args.manifest) - components = self.manifest['components'] + self.manifestdir = os.path.dirname(args.manifest) runtime_components = [] devel_components = [] runtime_artifacts = [] devel_artifacts = [] - if args.start_at: - start_at_index = -1 - for i,component_name in enumerate(components): - if component_name == args.start_at: - start_at_index = i - break - if start_at_index == -1: - fatal("Unknown component '%s' for --start-at" % (args.start_at, )) - else: - start_at_index = 0 - for component_name in components[start_at_index:]: + for component in self.manifest['components']: for architecture in self.manifest['architectures']: - path = os.path.join(dirname, component_name + '.txt') - f = open(path) - component_meta = kvfile.parse(f) + component_meta = self._resolve_component_meta(component) - artifact_branches = self._build_one_component(component_name, architecture, component_meta) + artifact_branches = self._build_one_component(component_meta, architecture) - target_component = component_meta.get('COMPONENT') + target_component = component_meta.get('component') if target_component == 'devel': - devel_components.append(component_name) + devel_components.append(component_meta['name']) else: - runtime_components.append(component_name) + runtime_components.append(component_meta['name']) for branch in artifact_branches: if branch['type'] == 'runtime': runtime_artifacts.append(branch) devel_artifacts.extend(artifact_branches) - f.close() - devel_branches = map(buildutil.branch_name_for_artifact, devel_artifacts) self._compose(architecture + '-devel', devel_branches) runtime_branches = map(buildutil.branch_name_for_artifact, runtime_artifacts) diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py index 37b15ac0..d78399c2 100755 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py @@ -47,10 +47,10 @@ class OstbuildChrootCompileOne(builtins.Builtin): parser.add_argument('--meta') parser.add_argument('--debug-shell', action='store_true') - args = parser.parse_args(argv) + (args, rest_args) = parser.parse_known_args(argv) if args.meta is None: - output = subprocess.check_output(['ostbuild-autodiscover-meta']) + output = subprocess.check_output(['ostbuild', 'autodiscover-meta']) ostbuild_meta_f = StringIO(output) else: ostbuild_meta_f = open(args.meta) @@ -133,12 +133,18 @@ class OstbuildChrootCompileOne(builtins.Builtin): '--mount-bind', os.getcwd(), chroot_sourcedir] if args.resultdir: child_args.extend(['--mount-bind', args.resultdir, '/ostbuild/results']) - child_args.extend([rootdir, '/bin/sh']) - if not args.debug_shell: - child_args.extend(['-c', - 'cd "%s" && ostbuild compile-one --ostbuild-resultdir=/ostbuild/results --ostbuild-meta=_ostbuild-meta' % (chroot_sourcedir, ) - ]) - run_sync(child_args, env=BUILD_ENV) + if args.debug_shell: + child_args.extend([rootdir, '/bin/sh']) + else: + child_args.extend([rootdir, '/usr/bin/ostbuild', + 'compile-one', + '--chdir=' + chroot_sourcedir, + '--ostbuild-resultdir=/ostbuild/results', + '--ostbuild-meta=_ostbuild-meta']) + child_args.extend(rest_args) + env_copy = dict(BUILD_ENV) + env_copy['PWD'] = chroot_sourcedir + run_sync(child_args, env=env_copy) if workdir_is_tmp: shutil.rmtree(workdir) diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py index 43067065..58d9be92 100755 --- a/src/ostbuild/pyostbuild/builtin_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_compile_one.py @@ -85,18 +85,25 @@ class OstbuildCompileOne(builtins.Builtin): '--infodir=' + os.path.join(PREFIX, 'share', 'info')] self.makeargs = ['make'] - self.ostbuild_resultdir=os.getcwd() + self.ostbuild_resultdir=None self.ostbuild_meta=None + chdir = None + for arg in args: if arg.startswith('--ostbuild-resultdir='): self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):] elif arg.startswith('--ostbuild-meta='): self.ostbuild_meta=arg[len('--ostbuild-meta='):] + elif arg.startswith('--chdir='): + os.chdir(arg[len('--chdir='):]) elif arg.startswith('--'): self.configargs.append(arg) else: self.makeargs.append(arg) + + if self.ostbuild_resultdir is None: + fatal("Must specify --ostbuild-resultdir=") self.metadata = {} @@ -107,6 +114,8 @@ class OstbuildCompileOne(builtins.Builtin): ostbuild_meta_f = open(self.ostbuild_meta) for line in ostbuild_meta_f: + if line == '': + continue (k,v) = line.split('=', 1) self.metadata[k.strip()] = v.strip() @@ -198,7 +207,7 @@ class OstbuildCompileOne(builtins.Builtin): else: root_version = self.metadata.get('BUILDROOT_VERSION') - artifact_prefix='artifact-%s,%s,%s,%s,%s' % (root_name, root_version, name, branch, version) + artifact_prefix=os.path.join('artifacts', root_name, name, branch) tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,)) self.tempfiles.append(tempdir) @@ -206,6 +215,7 @@ class OstbuildCompileOne(builtins.Builtin): run_sync(args, cwd=builddir) devel_files = set() + dbg_files = set() runtime_files = set() oldpwd=os.getcwd() @@ -234,9 +244,8 @@ class OstbuildCompileOne(builtins.Builtin): runtime_files.add(path) os.chdir(oldpwd) - if devel_files: - self.make_artifact(artifact_prefix + '-devel', devel_files, tempdir=tempdir, resultdir=self.ostbuild_resultdir) - self.make_artifact(artifact_prefix + '-runtime', runtime_files, tempdir=tempdir, resultdir=self.ostbuild_resultdir) + self.make_artifact(artifact_prefix, 'devel', devel_files, tempdir=tempdir) + self.make_artifact(artifact_prefix, 'runtime', runtime_files, tempdir=tempdir) for tmpname in self.tempfiles: assert os.path.isabs(tmpname) @@ -245,27 +254,22 @@ class OstbuildCompileOne(builtins.Builtin): else: try: os.unlink(tmpname) - pass except OSError, e: pass - def make_artifact(self, name, from_files, tempdir=None, resultdir=None): - targz_name = name + '.tar.gz' - (fd,filelist_temp)=tempfile.mkstemp(prefix='ostbuild-filelist-%s' % (name, )) - os.close(fd) - self.tempfiles.append(filelist_temp) - f = open(filelist_temp, 'w') + def make_artifact(self, prefix, dirtype, from_files, tempdir): + resultdir = os.path.join(self.ostbuild_resultdir, prefix, dirtype) + if os.path.isdir(resultdir): + shutil.rmtree(resultdir) + os.makedirs(resultdir) + for filename in from_files: - assert ('\n' not in filename) - f.write(filename) - f.write('\n') - f.close() - if resultdir: - result_path = os.path.join(resultdir, targz_name) - else: - result_path = targz_name - args = ['tar', '-c', '-z', '-C', tempdir, '-f', result_path, '-T', filelist_temp] - run_sync(args) - log("created: %s" % (os.path.abspath (result_path), )) + src_path = os.path.join(tempdir, filename) + dest_path = os.path.join(resultdir, filename) + dest_dir = os.path.dirname(dest_path) + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + shutil.move(src_path, dest_path) + log("created: %s" % (os.path.abspath (resultdir), )) builtins.register(OstbuildCompileOne) diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py index b997c07e..5fd4b331 100755 --- a/src/ostbuild/pyostbuild/builtins.py +++ b/src/ostbuild/pyostbuild/builtins.py @@ -32,9 +32,9 @@ class Builtin(object): def parse_config(self): self.repo = ostbuildrc.get_key('repo') - self.srcdir = ostbuildrc.get_key('srcdir') - if not os.path.isdir(self.srcdir): - fatal("Specified srcdir '%s' is not a directory" % (self.srcdir, )) + self.mirrordir = ostbuildrc.get_key('mirrordir') + if not os.path.isdir(self.mirrordir): + fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, )) self.workdir = ostbuildrc.get_key('workdir') if not os.path.isdir(self.workdir): fatal("Specified workdir '%s' is not a directory", (self.workdir, )) diff --git a/src/ostbuild/pyostbuild/ostbuildlog.py b/src/ostbuild/pyostbuild/ostbuildlog.py index 2174d719..a4af2a6a 100755 --- a/src/ostbuild/pyostbuild/ostbuildlog.py +++ b/src/ostbuild/pyostbuild/ostbuildlog.py @@ -21,7 +21,7 @@ import os import sys def log(msg): - fullmsg = '%s: %s\n' % (sys.argv[0], msg) + fullmsg = '%s: %s\n' % (os.path.basename(sys.argv[0]), msg) sys.stdout.write(fullmsg) sys.stdout.flush() diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py index 5b1557f0..efa06c2e 100755 --- a/src/ostbuild/pyostbuild/subprocess_helpers.py +++ b/src/ostbuild/pyostbuild/subprocess_helpers.py @@ -40,8 +40,8 @@ def _get_env_for_cwd(cwd=None, env=None): env_copy = env return env_copy -def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=False): - log("running: %s" % (subprocess.list2cmdline(args),)) +def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=False, + log_success=False): env_copy = _get_env_for_cwd(cwd, env) f = open('/dev/null', 'r') if stderr is None: @@ -54,14 +54,18 @@ def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=Fal output = proc.communicate()[0].strip() if proc.returncode != 0 and not none_on_error: logfn = fatal - else: + elif log_success: logfn = log - logfn("pid %d exited with code %d, %d bytes of output" % (proc.pid, proc.returncode, len(output))) + else: + logfn = None + if logfn is not None: + logfn("cmd '%s' exited with code %d, %d bytes of output" % (subprocess.list2cmdline(args), proc.returncode, len(output))) if proc.returncode == 0: return output return None -def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False): +def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False, + log_success=True): log("running: %s" % (subprocess.list2cmdline(args),)) # This dance is necessary because we want to keep the PWD # environment variable up to date. Not doing so is a recipie @@ -88,7 +92,10 @@ def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False): returncode = proc.wait() if fatal_on_error and returncode != 0: logfn = fatal - else: + elif log_success: logfn = log - logfn("pid %d exited with code %d" % (proc.pid, returncode)) + else: + logfn = None + if logfn is not None: + logfn("pid %d exited with code %d" % (proc.pid, returncode)) return returncode