Commit: a95b833c
This commit is contained in:
jimmy committed 2022-08-23 20:51:43 -04:00
commit a8d8515b75
71 files changed
+6908

No files matched your search

+394
View File
@@ -0,0 +1,394 @@
#!/usr/bin/python3
import argparse
import os
import platform
import select
import shlex
import socket
import subprocess
import sys
import tempfile
import threading
import time
import signal
import shutil
import json
BLOCK_SIZE = 64*1024
def read_manifest(path):
if path == "-":
manifest = sys.stdin.read()
else:
with open(path) as f:
manifest = f.read()
return manifest
def parse_arguments(sys_argv):
parser = argparse.ArgumentParser(description="Build operating system images")
parser.add_argument("manifest_path", metavar="MANIFEST",
help="json file containing the manifest that should be built, or a '-' to read from stdin")
parser.add_argument("--store", metavar="DIRECTORY", type=os.path.abspath,
default=".osbuild",
help="directory where intermediary os trees are stored")
parser.add_argument("--checkpoint", metavar="ID", action="append", type=str, default=None,
help="stage to commit to the object store during build (can be passed multiple times)")
parser.add_argument("--export", metavar="ID", action="append", type=str, default=None,
help="object to export, can be passed multiple times")
parser.add_argument("--output-directory", metavar="DIRECTORY", type=os.path.abspath,
help="directory where result objects are stored")
parser.add_argument("--arch", metavar="ARCH", type=str, default=platform.machine(),
help="Arch to build for")
return parser.parse_args(sys_argv[1:])
def local_osbuild(manifest, opts):
cmd = ['osbuild'] + opts + ['-']
try:
p = subprocess.run(cmd, check=True, input=manifest.encode("utf8"), capture_output=True )
except subprocess.CalledProcessError as e:
print(e.output)
sys.exit(e.returncode)
lines = p.stdout.decode("utf8").splitlines()
checkpoints = {}
for l in lines:
p = l.split()
checkpoint = p[0][:-1]
checkpoints[checkpoint] = p[1]
return checkpoints
def extract_dependencies(manifest):
j = json.loads(manifest)
version = j.get("version", None);
if version != '2':
print(f"Unsupported manifest version {version}, only version 2 supported")
sys.exit(1)
sources = j.get("sources", {});
curl = sources.get("org.osbuild.curl", {});
curl_items = curl.get("items", {});
shas = curl_items.keys()
return list(shas)
def find_images(arch):
base_image=f"osbuildvm-{arch}.img"
base_kernel=f"osbuildvm-{arch}.vmlinuz"
base_initrd=f"osbuildvm-{arch}.initramfs"
image_dir=None
image_dirs = [os.getcwd(), os.path.join(os.getcwd(), "_build"), "/usr/share/osbuildvm"]
for i in image_dirs:
if os.path.exists(os.path.join(i, base_image)):
image_dir = i
break
if not image_dir:
print(f"Unable to find {base_image}, tried: {image_dirs}", file=sys.stderr)
sys.exit(1)
image = os.path.join(image_dir, base_image)
kernel = os.path.join(image_dir, base_kernel)
initrd = os.path.join(image_dir, base_initrd)
return (image, kernel, initrd)
def qemu_img(*args):
res = subprocess.run(["qemu-img"] + [*args],
stdout=subprocess.PIPE,
check=True)
class QEmu(object):
def __init__(self, image1, image2, kernel, initrd, arch):
# This is where we store the sockets and the pidfile
self.tmpdir = tempfile.TemporaryDirectory(prefix="tmp-qemu-")
self.args = []
self.pid = 0
self.host_arch = platform.machine()
self.arch = arch
debug_serial = False
qemu_kvm_path = self.find_qemu()
self.args.append(qemu_kvm_path)
# Virtio serial ports
self.args.extend(["-device", "virtio-serial"])
out_socket_path = self.add_socket("output")
sync_socket_path = self.add_socket("sync")
stdout_socket_path = self.add_socket("stdout")
debug_cmdline="quiet loglevel=1"
if debug_serial:
self.args.extend(["-serial", "file:/dev/stdout"])
debug_cmdline=""
# Machine details
self.args.extend(["-m", "size=2G",
"-nodefaults",
"-vga", "none", "-vnc", "none"])
if self.kvm_supported():
self.args.extend(["-enable-kvm"])
if self.arch=="x86_64":
machine = "q35"
cpu = "qemu64"
if self.arch=="aarch64":
machine = "virt"
cpu = "cortex-a57"
if self.arch == self.host_arch:
cpu = "host"
self.args.extend(["-machine", machine,
"-cpu", cpu])
if self.arch=="aarch64":
self.args.extend(["-bios", "/usr/share/edk2/aarch64/QEMU_EFI.fd",
"-boot", "efi"])
pid_file = os.path.join(self.tmpdir.name, "qemu.pid")
self.args.extend(["-daemonize", "-pidfile" , pid_file,
"-kernel", kernel,
"-initrd", initrd,
"-append", f'root=/dev/vda console=ttyS0 init=/usr/bin/start.sh {debug_cmdline} ro',
"-drive", f"file={image1},index=0,media=disk,format=qcow2,snapshot=on,if=virtio",
"-drive", f"file={image2},index=1,media=disk,format=raw,if=virtio"])
p = subprocess.run(self.args, check=True)
with open(pid_file, "r") as f:
self.pid = int(f.read())
self.sock_out = self.connect_socket(out_socket_path)
self.sock_sync = self.connect_socket(sync_socket_path)
self.sock_stdout = self.connect_socket(stdout_socket_path)
def connect_socket(self, path):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(path)
return sock
def add_socket(self, id):
socket_path = os.path.join(self.tmpdir.name, id + ".sock")
self.args.extend(["-chardev", f"socket,path={socket_path},server=on,wait=off,id={id}",
"-device", f"virtserialport,chardev={id},name={id}"])
return socket_path
def find_qemu(self):
if self.arch == self.host_arch:
binary_name = "qemu-kvm"
else:
binary_name = f"qemu-system-{self.arch}"
for d in ["/usr/bin", "/usr/libexec"]:
p = os.path.join(d, binary_name)
if os.path.isfile(p):
return p
print(f"Can't find {binary_name}", file=sys.stderr)
sys.exit(1)
def kvm_supported(self):
return self.arch == self.host_arch and os.path.exists("/dev/kvm")
def copy_out(self, destination):
while True:
readable, writable, exceptional = select.select([self.sock_out, self.sock_sync, self.sock_stdout], [], [])
read_something = False
if self.sock_stdout in readable:
data = self.sock_stdout.recv(BLOCK_SIZE)
while len(data) > 0:
res = sys.stdout.buffer.write(data)
data = data[res:]
sys.stdout.flush()
read_something = True
if self.sock_out in readable:
data = self.sock_out.recv(BLOCK_SIZE)
while len(data) > 0:
res = destination.write(data)
data = data[res:]
read_something = True
if read_something:
continue # Don't exit until there is no more to read from sock or sock_stdout
# If we had no buffered data in sock and sync_sock is readable that means we copied everything and can exit
if self.sock_sync in readable:
data = self.sock_sync.recv(BLOCK_SIZE)
break
def kill(self):
if self.pid != 0:
os.kill(self.pid, signal.SIGTERM)
self.pid = 0
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.kill()
self.tmpdir.cleanup()
def create_ext4_image(path, size, root_dir):
with open(path, "w") as f:
f.truncate(size)
cmd = ["mkfs.ext4", "-d", root_dir, "-E", "no_copy_xattrs,root_owner=0:0", "-O", "^has_journal", path]
try:
p = subprocess.run(cmd, check=True,capture_output=True)
except subprocess.CalledProcessError as e:
print(f"Unable to create ext4 work fs: {e}", file=sys.stderr)
sys.exit(e.returncode)
def link_or_copy_file(source_path, dest_path):
try:
os.link(source_path, dest_path)
except Exception as e:
shutil.copyfile(source_path, dest_path, follow_symlinks=False)
def make_work_rootfs(args, tmpdirname, manifest, digests, main_sh, checkpoint_ids):
rootdir = os.path.join(tmpdirname, "root")
os.makedirs(rootdir, exist_ok=True)
with open (os.open(os.path.join(rootdir, "main.sh"), os.O_CREAT | os.O_WRONLY, 0o775), "w") as mainsh:
mainsh.write(main_sh)
with open (os.path.join(rootdir, "image.json"), "w") as image_json:
image_json.write(manifest)
orig_sources_dir = os.path.join(args.store, "sources/org.osbuild.files")
root_sources_dir = os.path.join(rootdir, "osbuild_store/sources/org.osbuild.files")
os.makedirs(root_sources_dir, mode=0o777, exist_ok=True)
root_input_dir = os.path.join(rootdir, "input")
os.makedirs(root_input_dir, mode=0o777, exist_ok=True)
for digest in digests:
link_or_copy_file(os.path.join(orig_sources_dir, digest),
os.path.join(root_sources_dir, digest))
for cp, cp_id in checkpoint_ids.items():
source_path = os.path.join(args.store, "refs_tars/" + cp_id + ".tar.gz")
if os.path.isfile(source_path):
link_or_copy_file(source_path,
os.path.join(root_input_dir, cp_id + ".tar.gz"))
return rootdir
# Moves any files recursively in root_src_dir to root_dst_dir, replacing if needed
# Keeps old files and directories in root_dst_dir
def move_merged(root_src_dir, root_dst_dir):
root_src_dir = os.path.abspath(root_src_dir)
root_dst_dir = os.path.abspath(root_dst_dir)
for src_dir, dirs, files in os.walk(root_src_dir):
dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
for file_ in files:
src_file = os.path.join(src_dir, file_)
dst_file = os.path.join(dst_dir, file_)
if os.path.exists(dst_file):
os.remove(dst_file)
shutil.move(src_file, dst_dir)
def run_in_vm(args, manifest, digests, tmpdirname, shell_to_run, checkpoint_ids):
image,kernel,initrd = find_images(args.arch)
rootdir = make_work_rootfs(args, tmpdirname, manifest, digests, shell_to_run, checkpoint_ids)
work_image = os.path.join(tmpdirname, "work.img")
create_ext4_image(work_image, 100*1024*1024*1024, rootdir)
shutil.rmtree(rootdir)
output_path = args.output_directory
os.makedirs(output_path, exist_ok=True)
exit_status = 1
with tempfile.TemporaryDirectory(prefix="download-", dir=output_path) as output_tmpdir:
with QEmu(image, work_image, kernel, initrd, args.arch) as qemu:
with subprocess.Popen(["tar", "x", "-C", output_tmpdir], stdin=subprocess.PIPE) as proc:
qemu.copy_out(proc.stdin)
exit_status_path = os.path.join(output_tmpdir, "exit_status")
if os.path.isfile(exit_status_path):
with open(exit_status_path, "r") as f:
exit_status = int(f.read())
# Move osbuild outout to the real output dir
osbuild_output = os.path.join(output_tmpdir, "osbuild")
if os.path.isdir(osbuild_output):
move_merged(osbuild_output, output_path)
checkpoints_output = os.path.join(output_tmpdir, "checkpoints")
checkpoints_dest = os.path.join(args.store, "refs_tars")
if os.path.isdir(checkpoints_output):
os.makedirs(checkpoints_dest, exist_ok=True)
for tar in os.listdir(checkpoints_output):
if not os.path.isfile(os.path.join(checkpoints_dest, tar)):
shutil.move(os.path.join(checkpoints_output, tar),
checkpoints_dest)
sys.exit(exit_status)
def main():
args = parse_arguments(sys.argv)
manifest = read_manifest(args.manifest_path)
print("Running osbuild on host to download files")
checkpoint_ids = local_osbuild(manifest, ['--store', args.store])
digests = extract_dependencies(manifest)
mainsh_data = f'''\
#!/bin/bash
mkdir -p /work/osbuild_store
(
echo === Extracting checkpoints in vm ===
for tar in $(find /work/input/ -mindepth 1 -print ); do
echo extracting $(basename $tar)
tar xf $tar --acls --selinux --xattrs -C /work/osbuild_store
done
echo === Running osbuild in vm ===
osbuild --store /work/osbuild_store --output-directory /work/output/osbuild {' '.join(map(lambda e: "--export " + e, args.export))} {' '.join(map(lambda cp: "--checkpoint " + cp, args.checkpoint))} /work/image.json
RES=$?
echo $RES > /work/output/exit_status
echo === Osbuild exit status $RES ===
echo === Saving checkpoints ===
mkdir -p /work/output/checkpoints
for cp in $(find /work/osbuild_store/refs/ -mindepth 1 -printf "%f "); do
if test -f /work/input/$cp.tar.gz; then
continue
fi
obj=$(basename $(readlink /work/osbuild_store/refs/$cp))
tar cSf /work/output/checkpoints/$cp.tar.gz --acls --selinux --xattrs -C /work/osbuild_store/ refs/$cp objects/$obj
echo Saved $cp
done
) > /dev/virtio-ports/stdout 2>&1
tar cSf /dev/virtio-ports/output -C /work/output ./
# Signal output ended
sleep 3
echo DONE > /dev/virtio-ports/sync
# Block for it to be fully read
cat /dev/virtio-ports/sync
'''
tmpdir = os.path.join(args.store, "tmp")
os.makedirs(tmpdir, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="osbuild-qemu-", dir=tmpdir) as tmpdirname:
run_in_vm(args, manifest, digests, tmpdirname, mainsh_data, checkpoint_ids)
if __name__ == "__main__":
main()
@@ -0,0 +1,229 @@
version: '2'
mpp-vars:
rootfs_uuid: 86a22bf4-f153-4541-b6c7-0332c0dfaead
rootfs_size: 2147483648
cs9_baseurl: http://mirror.stream.centos.org/9-stream
cs9_repos:
- id: baseos
baseurl: $cs9_baseurl/BaseOS/$arch/os/
- id: appstream
baseurl: $cs9_baseurl/AppStream/$arch/os/
centos_gpg_key: |
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v2.0.22 (GNU/Linux)
mQINBFzMWxkBEADHrskpBgN9OphmhRkc7P/YrsAGSvvl7kfu+e9KAaU6f5MeAVyn
rIoM43syyGkgFyWgjZM8/rur7EMPY2yt+2q/1ZfLVCRn9856JqTIq0XRpDUe4nKQ
8BlA7wDVZoSDxUZkSuTIyExbDf0cpw89Tcf62Mxmi8jh74vRlPy1PgjWL5494b3X
5fxDidH4bqPZyxTBqPrUFuo+EfUVEqiGF94Ppq6ZUvrBGOVo1V1+Ifm9CGEK597c
aevcGc1RFlgxIgN84UpuDjPR9/zSndwJ7XsXYvZ6HXcKGagRKsfYDWGPkA5cOL/e
f+yObOnC43yPUvpggQ4KaNJ6+SMTZOKikM8yciyBwLqwrjo8FlJgkv8Vfag/2UR7
JINbyqHHoLUhQ2m6HXSwK4YjtwidF9EUkaBZWrrskYR3IRZLXlWqeOi/+ezYOW0m
vufrkcvsh+TKlVVnuwmEPjJ8mwUSpsLdfPJo1DHsd8FS03SCKPaXFdD7ePfEjiYk
nHpQaKE01aWVSLUiygn7F7rYemGqV9Vt7tBw5pz0vqSC72a5E3zFzIIuHx6aANry
Gat3aqU3qtBXOrA/dPkX9cWE+UR5wo/A2UdKJZLlGhM2WRJ3ltmGT48V9CeS6N9Y
m4CKdzvg7EWjlTlFrd/8WJ2KoqOE9leDPeXRPncubJfJ6LLIHyG09h9kKQARAQAB
tDpDZW50T1MgKENlbnRPUyBPZmZpY2lhbCBTaWduaW5nIEtleSkgPHNlY3VyaXR5
QGNlbnRvcy5vcmc+iQI3BBMBAgAhBQJczFsZAhsDBgsJCAcDAgYVCAIJCgsDFgIB
Ah4BAheAAAoJEAW1VbOEg8ZdjOsP/2ygSxH9jqffOU9SKyJDlraL2gIutqZ3B8pl
Gy/Qnb9QD1EJVb4ZxOEhcY2W9VJfIpnf3yBuAto7zvKe/G1nxH4Bt6WTJQCkUjcs
N3qPWsx1VslsAEz7bXGiHym6Ay4xF28bQ9XYIokIQXd0T2rD3/lNGxNtORZ2bKjD
vOzYzvh2idUIY1DgGWJ11gtHFIA9CvHcW+SMPEhkcKZJAO51ayFBqTSSpiorVwTq
a0cB+cgmCQOI4/MY+kIvzoexfG7xhkUqe0wxmph9RQQxlTbNQDCdaxSgwbF2T+gw
byaDvkS4xtR6Soj7BKjKAmcnf5fn4C5Or0KLUqMzBtDMbfQQihn62iZJN6ZZ/4dg
q4HTqyVpyuzMXsFpJ9L/FqH2DJ4exGGpBv00ba/Zauy7GsqOc5PnNBsYaHCply0X
407DRx51t9YwYI/ttValuehq9+gRJpOTTKp6AjZn/a5Yt3h6jDgpNfM/EyLFIY9z
V6CXqQQ/8JRvaik/JsGCf+eeLZOw4koIjZGEAg04iuyNTjhx0e/QHEVcYAqNLhXG
rCTTbCn3NSUO9qxEXC+K/1m1kaXoCGA0UWlVGZ1JSifbbMx0yxq/brpEZPUYm+32
o8XfbocBWljFUJ+6aljTvZ3LQLKTSPW7TFO+GXycAOmCGhlXh2tlc6iTc41PACqy
yy+mHmSv
=kkH7
-----END PGP PUBLIC KEY BLOCK-----
pipelines:
- runner: org.osbuild.centos9
name: build
stages:
- type: org.osbuild.rpm
inputs:
packages:
type: org.osbuild.files
origin: org.osbuild.source
mpp-depsolve:
architecture: $arch
module-platform-id: platform:el9
baseurl: $cs9_baseurl/BaseOS/$arch/os/
repos:
mpp-eval: cs9_repos
packages:
- dnf
- e2fsprogs
- policycoreutils
- python3-iniparse
- python39
- qemu-img
- selinux-policy-targeted
- tar
- xz
options:
gpgkeys:
- mpp-eval: centos_gpg_key
exclude:
docs: true
- type: org.osbuild.selinux
options:
file_contexts: etc/selinux/targeted/contexts/files/file_contexts
labels:
/usr/bin/cp: system_u:object_r:install_exec_t:s0
/usr/bin/tar: system_u:object_r:install_exec_t:s0
- name: rootfs
build: name:build
stages:
- type: org.osbuild.rpm
options:
gpgkeys:
- mpp-eval: centos_gpg_key
inputs:
packages:
type: org.osbuild.files
origin: org.osbuild.source
mpp-depsolve:
architecture: $arch
module-platform-id: platform:el9
baseurl: $cs9_baseurl/BaseOS/$arch/os/
repos:
mpp-join:
- mpp-eval: cs9_repos
- - id: osbuild
baseurl: https://download.copr.fedorainfracloud.org/results/@osbuild/osbuild/centos-stream-9-$arch
packages:
- bash
- dracut-config-generic
- kernel
- langpacks-en
- selinux-policy-targeted
- net-tools
- osbuild
- osbuild-tools
- osbuild-ostree
excludes:
- dracut-config-rescue
- type: org.osbuild.locale
options:
language: en_US.UTF-8
- type: org.osbuild.copy
inputs:
inlinefile:
type: org.osbuild.files
origin: org.osbuild.source
mpp-embed:
id: osbuilder.sh
text: |
#!/usr/bin/bash
function clean_up {
systemctl poweroff -f -f
}
trap clean_up EXIT
if grep -q "osbuilder_bash=1" /proc/cmdline; then bash; exit; fi
mount /dev/vdb /work
/work/main.sh
options:
paths:
- from:
mpp-format-string: input://inlinefile/{embedded['osbuilder.sh']}
to: tree:///usr/bin/start.sh
- type: org.osbuild.chmod
options:
items:
/usr/bin/start.sh:
mode: a+x
- type: org.osbuild.mkdir
options:
paths:
- path: /work
- type: org.osbuild.selinux
options:
file_contexts: etc/selinux/targeted/contexts/files/file_contexts
- type: org.osbuild.selinux
options:
file_contexts: etc/selinux/targeted/contexts/files/file_contexts
- name: image
build: name:build
stages:
- type: org.osbuild.truncate
options:
filename: disk.img
size:
mpp-format-string: '{rootfs_size}'
- type: org.osbuild.mkfs.ext4
devices:
device:
type: org.osbuild.loopback
options:
filename: disk.img
start: 0
size:
mpp-eval: rootfs_size
options:
uuid:
mpp-eval: rootfs_uuid
label: root
- type: org.osbuild.copy
inputs:
tree:
type: org.osbuild.tree
origin: org.osbuild.pipeline
references:
- name:rootfs
options:
paths:
- from: input://tree/
to: mount://root/
devices:
root:
type: org.osbuild.loopback
options:
filename: disk.img
start: 0
size:
mpp-eval: rootfs_size
mounts:
- name: root
type: org.osbuild.ext4
source: root
target: /
- name: osbuildvm
build: name:build
stages:
- type: org.osbuild.qemu
inputs:
image:
type: org.osbuild.files
origin: org.osbuild.pipeline
references:
name:image:
file: disk.img
options:
filename: disk.qcow2
format:
type: qcow2
compat: '1.1'
- type: org.osbuild.copy
inputs:
rootfs:
type: org.osbuild.tree
origin: org.osbuild.pipeline
references:
- name:rootfs
options:
paths:
- from:
mpp-format-string: input://rootfs/usr/lib/modules/{rpms['rootfs']['kernel-core'].evra}/vmlinuz
to: tree:///vmlinuz
- from:
mpp-format-string: input://rootfs/boot/initramfs-{rpms['rootfs']['kernel-core'].evra}.img
to: tree:///initramfs
- type: org.osbuild.chmod
options:
items:
/initramfs:
mode: a+r