582 lines
31 KiB
Python
582 lines
31 KiB
Python
"""Verified, resumable Gentoo builds in a dedicated rootless Podman store."""
|
|
import argparse
|
|
import contextlib
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import lzma
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
class BuildError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def package_job_script(command, state='/var/lib/hydra-build', package_db='/var/db/pkg', inputs=''):
|
|
"""The contained job owns its log/result even if the outer caller disconnects."""
|
|
return f'''import fcntl, json, os, pathlib, subprocess, sys, time
|
|
state = pathlib.Path({str(state)!r})
|
|
database = pathlib.Path({str(package_db)!r})
|
|
state.mkdir(parents=True, exist_ok=True)
|
|
lock = (state / 'packages.lock').open('a')
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
sys.exit(75)
|
|
def write(name, data):
|
|
temporary = state / (name + '.tmp')
|
|
temporary.write_text(json.dumps(data, indent=2) + '\\n')
|
|
temporary.replace(state / name)
|
|
status = {{'running': True, 'pid': os.getpid(), 'inputs': {inputs!r}, 'started': int(time.time())}}
|
|
write('packages.status.json', status)
|
|
with (state / 'packages.log').open('a') as log:
|
|
try:
|
|
result = subprocess.run({command!r}, stdout=log, stderr=subprocess.STDOUT)
|
|
code = result.returncode
|
|
except OSError as error:
|
|
log.write(str(error) + '\\n')
|
|
code = 127
|
|
if code == 0:
|
|
inventory = []
|
|
for use in sorted(database.glob('*/*/USE')):
|
|
package = use.parent
|
|
inventory.append({{'package': str(package.relative_to(database)), 'use': use.read_text().split(),
|
|
'build_time': (package / 'BUILD_TIME').read_text().strip() if (package / 'BUILD_TIME').exists() else None}})
|
|
write('packages.json', inventory)
|
|
status.update(running=False, exit_code=code, finished=int(time.time()))
|
|
write('packages.status.json', status)
|
|
sys.exit(code)
|
|
'''
|
|
|
|
|
|
def rootfs_retry_guard_script():
|
|
"""Refuse deletion of a still-mounted export, including nested mountpoints."""
|
|
return r"""mounts=$(findmnt --raw --noheadings --output TARGET)
|
|
while IFS= read -r mountpoint; do
|
|
case "$mountpoint" in
|
|
"$root"|"$root/"*) echo 'Refusing to remove a mounted rootfs export' >&2; exit 1 ;;
|
|
esac
|
|
done <<< "$mounts"
|
|
if [ -e "$root" ] || [ -L "$root" ]; then
|
|
[ ! -L "$root" ] && [ -f "$root/.hydra-export-owned" ] || { echo 'Unowned rootfs output' >&2; exit 1; }
|
|
rm -rf -- "$root"
|
|
fi
|
|
"""
|
|
|
|
|
|
def verify_hash(path, expected):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
if digest.hexdigest() != expected:
|
|
raise BuildError(f'SHA256 mismatch: {path}')
|
|
|
|
|
|
def require_signature(status, fingerprint):
|
|
valid = [line.split() for line in status.splitlines() if line.startswith('[GNUPG:] VALIDSIG ')]
|
|
if not valid or not any(fields[-1].upper() == fingerprint.upper() for fields in valid):
|
|
raise BuildError('Required primary signature fingerprint is missing')
|
|
if any(token in status for token in ('BADSIG', 'ERRSIG', 'REVKEYSIG', 'EXPKEYSIG')):
|
|
raise BuildError('Invalid release signature')
|
|
|
|
|
|
class Builder:
|
|
def __init__(self, project, work, cache, podman=None, gpg=None, source_dir=None):
|
|
if os.geteuid() == 0:
|
|
raise BuildError('Run as the normal user, never host root')
|
|
self.project = Path(project).resolve()
|
|
self.source_dir = Path(source_dir or project).resolve()
|
|
self.work, self.cache = Path(work).resolve(), Path(cache).resolve()
|
|
for path in (self.work, self.cache):
|
|
if path == self.project or not path.is_relative_to(self.project):
|
|
raise BuildError('Work and cache paths must be inside the project, below its root')
|
|
if self.work == self.cache or self.work.is_relative_to(self.cache) or self.cache.is_relative_to(self.work):
|
|
raise BuildError('Work and cache must be separate directories')
|
|
self.podman = podman or shutil.which('podman') or 'podman'
|
|
self.gpg = gpg or shutil.which('gpg') or 'gpg'
|
|
runtime = Path(os.environ.get('XDG_RUNTIME_DIR', f'/run/user/{os.getuid()}'))
|
|
identifier = hashlib.sha256(str(self.work).encode()).hexdigest()[:12]
|
|
self.runroot = runtime / ('hydra-build-podman' if self.work == self.project / 'build' else f'hydra-build-{identifier}')
|
|
self.name = 'hydra-builder-' + identifier
|
|
self.image = 'localhost/hydra-stage3:20261011'
|
|
self._stage3_layers = {}
|
|
|
|
def podman_argv(self, *args):
|
|
return [self.podman, '--root', str(self.work / 'podman'), '--runroot', str(self.runroot), '--storage-driver', 'overlay', *args]
|
|
|
|
def container_run_args(self):
|
|
return ['run', '--detach', '--name', self.name, '--hostname', 'hydra-builder',
|
|
'--network', 'host', '--cap-add', 'SYS_ADMIN', '--security-opt', 'seccomp=unconfined',
|
|
'--volume', f'{self.source_dir}:/hydra-source:ro', '--volume', f'{self.cache}:/hydra-cache:ro',
|
|
self.image, '/bin/sleep', 'infinity']
|
|
|
|
def run(self, argv, capture=False, check=True):
|
|
print('+ ' + ' '.join(str(arg) for arg in argv), file=sys.stderr, flush=True)
|
|
result = subprocess.run(argv, text=True, stdout=subprocess.PIPE if capture else None,
|
|
stderr=subprocess.PIPE if capture else None)
|
|
if check and result.returncode:
|
|
raise BuildError(f'Command failed ({result.returncode}): {argv[0]}\n{result.stderr or ""}')
|
|
return result
|
|
|
|
def pod(self, *args, **kwargs):
|
|
return self.run(self.podman_argv(*args), **kwargs)
|
|
|
|
def exec(self, command, capture=False):
|
|
return self.pod('exec', self.name, '/bin/bash', '-euc', command, capture=capture)
|
|
|
|
def inspect(self, kind, name):
|
|
result = self.pod(kind, 'inspect', name, capture=True, check=False)
|
|
if result.returncode:
|
|
raise BuildError(f'{kind} provenance is unavailable; use a fresh work directory')
|
|
entries = json.loads(result.stdout)
|
|
if not isinstance(entries, list) or len(entries) != 1 or not isinstance(entries[0], dict):
|
|
raise BuildError(f'Unexpected {kind} provenance')
|
|
return entries[0]
|
|
|
|
def check_mounts(self, container):
|
|
for destination, path, label in (('/hydra-source', self.source_dir, 'source'),
|
|
('/hydra-cache', self.cache, 'cache')):
|
|
mounts = [item for item in container.get('Mounts', []) if item.get('Destination') == destination]
|
|
if len(mounts) != 1 or mounts[0].get('Source') != str(path) or mounts[0].get('RW') is not False:
|
|
raise BuildError(f'Existing builder {label} mount differs; use its original checkout/cache or a fresh work directory')
|
|
|
|
def stage3_layer(self, inputs):
|
|
stage = inputs['stage3']
|
|
archive = self.cache / Path(urllib.parse.urlparse(stage['url']).path).name
|
|
verify_hash(archive, stage['sha256'])
|
|
if stage['sha256'] not in self._stage3_layers:
|
|
digest = hashlib.sha256()
|
|
with lzma.open(archive, 'rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
self._stage3_layers[stage['sha256']] = 'sha256:' + digest.hexdigest()
|
|
return self._stage3_layers[stage['sha256']]
|
|
|
|
def check_base(self, image, inputs):
|
|
expected = self.stage3_layer(inputs)
|
|
if image.get('RootFS', {}).get('Layers') != [expected]:
|
|
raise BuildError('Existing builder stage3 differs from pinned archive; use a fresh work directory')
|
|
return expected
|
|
|
|
def read_container_file(self, path):
|
|
# Podman cp also reads a stopped container, so provenance precedes start.
|
|
with tempfile.TemporaryDirectory(prefix='hydra-read-') as directory:
|
|
target = Path(directory) / 'data'
|
|
result = self.pod('cp', f'{self.name}:{path}', str(target), capture=True, check=False)
|
|
return target.read_text() if result.returncode == 0 else None
|
|
|
|
def check_snapshot(self, inputs, allow_pending=False):
|
|
marker = self.read_container_file('/var/db/repos/gentoo/.hydra-snapshot')
|
|
if marker is None and allow_pending:
|
|
marker = self.read_container_file('/var/db/repos/gentoo/.hydra-snapshot-input')
|
|
if marker is None or marker.strip() != inputs['portage']['commit']:
|
|
raise BuildError('Existing builder snapshot differs or is unverified; use a fresh work directory')
|
|
|
|
def active_package_job(self):
|
|
if not (self.work / 'podman').is_dir():
|
|
return False
|
|
result = self.pod('top', self.name, 'hpid', 'args', capture=True, check=False)
|
|
if result.returncode == 0 and any(re.search(r'(?:^|\s)(?:\S*/)?emerge(?:\s|$)', line)
|
|
for line in result.stdout.splitlines()[1:]):
|
|
return True
|
|
probe = self.pod('exec', self.name, 'python3', '-c', '''import fcntl, pathlib, sys
|
|
path = pathlib.Path('/var/lib/hydra-build/packages.lock')
|
|
if path.exists():
|
|
with path.open('r') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
sys.exit(75)
|
|
''', capture=True, check=False)
|
|
return probe.returncode == 75
|
|
|
|
def manifest(self):
|
|
path = self.work / 'manifest.json'
|
|
return json.loads(path.read_text()) if path.exists() else {}
|
|
|
|
def input_id(self):
|
|
digest = hashlib.sha256()
|
|
for directory in ('image', 'patches'):
|
|
for path in sorted((self.source_dir / directory).rglob('*')):
|
|
if path.is_file():
|
|
digest.update(str(path.relative_to(self.source_dir)).encode())
|
|
digest.update(path.read_bytes())
|
|
return digest.hexdigest()
|
|
|
|
def record(self, step, **details):
|
|
data = self.manifest()
|
|
data[step] = {'complete': True, 'inputs': self.input_id(), **details}
|
|
data['builder'] = self.name
|
|
path = self.work / 'manifest.json'
|
|
temp = path.with_suffix('.tmp')
|
|
temp.write_text(json.dumps(data, indent=2) + '\n')
|
|
temp.replace(path)
|
|
|
|
def complete(self, step):
|
|
entry = self.manifest().get(step, {})
|
|
if not entry.get('complete') or entry.get('inputs') != self.input_id():
|
|
return False
|
|
if step in ('rootfs', 'iso'):
|
|
from hydra import image
|
|
if entry.get('source_inputs') != image.source_identity(self.source_dir):
|
|
return False
|
|
if not image.artifacts_valid(self.work / 'artifacts', entry):
|
|
return False
|
|
return self.complete('packages' if step == 'rootfs' else 'rootfs')
|
|
if step == 'packages':
|
|
try:
|
|
self.package_inventory()
|
|
except (BuildError, OSError):
|
|
return False
|
|
return self.complete('prepare')
|
|
if step == 'prepare':
|
|
try:
|
|
container = self.inspect('container', self.name)
|
|
self.check_mounts(container)
|
|
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
|
|
self.check_base(self.inspect('image', container['Image']), inputs)
|
|
self.check_snapshot(inputs)
|
|
return True
|
|
except (BuildError, OSError, ValueError, KeyError, lzma.LZMAError):
|
|
return False
|
|
if step == 'fetch':
|
|
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
|
|
try:
|
|
for item in [inputs['stage3'], inputs['portage'], *inputs.get('local_packages', [])]:
|
|
path = self.cache / item.get('filename', Path(urllib.parse.urlparse(item['url']).path).name)
|
|
verify_hash(path, item['sha256'])
|
|
except (OSError, BuildError):
|
|
return False
|
|
return True
|
|
return False
|
|
|
|
def package_inventory(self):
|
|
from hydra import image
|
|
checksum = self.manifest().get('packages', {}).get('inventory_sha256')
|
|
if not isinstance(checksum, str) or not re.fullmatch('[a-f0-9]{64}', checksum):
|
|
raise BuildError('Package inventory has no recorded checksum')
|
|
try:
|
|
return image.read_package_inventory(self.work / 'artifacts/packages.json', checksum)
|
|
except (image.ImageError, OSError) as error:
|
|
raise BuildError(f'Package inventory validation failed: {error}') from error
|
|
|
|
def download(self, url, filename=None):
|
|
parsed = urllib.parse.urlparse(url)
|
|
if parsed.scheme != 'https':
|
|
raise BuildError('Build input URLs must use HTTPS')
|
|
target = self.cache / (filename or Path(parsed.path).name)
|
|
if not target.exists():
|
|
temp = target.with_suffix(target.suffix + '.partial')
|
|
print(f'Downloading {url}', flush=True)
|
|
with urllib.request.urlopen(url, timeout=120) as response, temp.open('wb') as stream:
|
|
shutil.copyfileobj(response, stream)
|
|
temp.replace(target)
|
|
return target
|
|
|
|
def fetch(self):
|
|
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
|
|
stage = inputs['stage3']
|
|
archive = self.download(stage['url'])
|
|
verify_hash(archive, stage['sha256'])
|
|
snapshot = self.download(inputs['portage']['url'], inputs['portage']['filename'])
|
|
verify_hash(snapshot, inputs['portage']['sha256'])
|
|
for item in inputs.get('local_packages', []):
|
|
verify_hash(self.download(item['url'], item['filename']), item['sha256'])
|
|
signature = self.download(stage['signature_url'])
|
|
keyring = self.download(inputs['keyring_url'])
|
|
verify_hash(keyring, inputs['keyring_sha256'])
|
|
with tempfile.TemporaryDirectory(prefix='hydra-gpg-', dir=self.work) as home:
|
|
result = self.run([self.gpg, '--batch', '--homedir', home, '--no-default-keyring', '--keyring', str(keyring),
|
|
'--status-fd', '1', '--verify', str(signature), str(archive)], capture=True)
|
|
require_signature(result.stdout, stage['fingerprint'])
|
|
(self.work / 'artifacts/stage3-signature.txt').write_text(result.stdout + result.stderr)
|
|
self.record('fetch', sources=inputs)
|
|
return inputs
|
|
|
|
def prepare(self):
|
|
existing = self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
|
|
container = self.inspect('container', self.name) if existing else None
|
|
if container is not None:
|
|
self.check_mounts(container) # Reject another worktree before changing any checkpoint.
|
|
if self.active_package_job():
|
|
raise BuildError('A contained package job is running; wait for it to finish before preparation')
|
|
inputs = self.fetch() # Reverify cached bytes even when resuming.
|
|
if container is not None:
|
|
layer = self.check_base(self.inspect('image', container['Image']), inputs)
|
|
self.check_snapshot(inputs, allow_pending=True)
|
|
elif self.pod('image', 'exists', self.image, capture=True, check=False).returncode:
|
|
self.pod('import', str(self.cache / Path(urllib.parse.urlparse(inputs['stage3']['url']).path).name), self.image)
|
|
layer = self.check_base(self.inspect('image', self.image), inputs)
|
|
else:
|
|
layer = self.check_base(self.inspect('image', self.image), inputs)
|
|
if container is None:
|
|
self.pod(*self.container_run_args())
|
|
else:
|
|
self.pod('start', self.name)
|
|
snapshot = inputs['portage']['filename']
|
|
# Snapshot extraction happens only inside the container; host paths are read-only.
|
|
self.exec(f'''mkdir -p /var/db/repos/gentoo
|
|
if [ ! -f /var/db/repos/gentoo/.hydra-snapshot ]; then
|
|
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot-input
|
|
tar --extract --gzip --file /hydra-cache/{snapshot} --strip-components=1 --directory /var/db/repos/gentoo
|
|
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot
|
|
fi
|
|
cp -a /hydra-source/image/portage/. /etc/portage/
|
|
mkdir -p /var/db/repos/hydra /var/cache/distfiles
|
|
cp -a /hydra-source/image/overlay/. /var/db/repos/hydra/
|
|
cp /hydra-cache/brightnessctl-0.5.1.tar.gz /var/cache/distfiles/brightnessctl-0.5.1.tar.gz
|
|
mkdir -p /etc/portage/patches/app-emulation/qemu /etc/portage/patches/app-emulation/virt-viewer
|
|
cp /hydra-source/patches/qemu-virtio-refresh-rate.patch /etc/portage/patches/app-emulation/qemu/10-hydra-refresh.patch
|
|
cp /hydra-source/patches/virt-viewer-hydra-session.patch /etc/portage/patches/app-emulation/virt-viewer/10-hydra-session.patch
|
|
if [ -d /etc/portage/gnupg ] && [ ! -s /etc/portage/gnupg/mykeyid ]; then
|
|
mv /etc/portage/gnupg /etc/portage/gnupg.hydra-untrusted
|
|
fi
|
|
getuto
|
|
gpg --batch --homedir /etc/portage/gnupg --import /hydra-cache/service-keys.gpg
|
|
chmod a+r /etc/portage/gnupg/pubring.kbx /etc/portage/gnupg/trustdb.gpg
|
|
''')
|
|
self.record('prepare', image=container['Image'] if container else self.image,
|
|
stage3_sha256=inputs['stage3']['sha256'], stage3_layer=layer,
|
|
source_dir=str(self.source_dir), cache_dir=str(self.cache), snapshot=inputs['portage']['commit'])
|
|
|
|
def packages(self, pretend=False):
|
|
self.prepare()
|
|
atoms = [line.strip() for line in (self.source_dir / 'image/packages').read_text().splitlines()
|
|
if line.strip() and not line.startswith('#')]
|
|
import shlex
|
|
command = ['emerge', '--verbose', '--update', '--newuse', '--deep', '--backtrack=100',
|
|
'--usepkg-exclude=app-emulation/qemu app-emulation/virt-viewer']
|
|
if pretend:
|
|
self.exec(shlex.join([*command, '--pretend', *atoms]))
|
|
return
|
|
inputs = self.input_id()
|
|
self.record('packages', complete=False, job_log='/var/lib/hydra-build/packages.log')
|
|
patches = hashlib.sha256()
|
|
for name in ('qemu-virtio-refresh-rate.patch', 'virt-viewer-hydra-session.patch'):
|
|
patches.update((self.source_dir / 'patches' / name).read_bytes())
|
|
patch_id = patches.hexdigest()
|
|
forced = ['emerge', '--verbose', '--oneshot', '--usepkg=n', '--getbinpkg=n',
|
|
'app-emulation/qemu', 'app-emulation/virt-viewer']
|
|
package_commands = shlex.join([*command, *atoms]) + '\n' + f'''if [ "$(cat /var/lib/hydra-build/patched-source.sha256 2>/dev/null || true)" != {shlex.quote(patch_id)} ]; then
|
|
{shlex.join(forced)}
|
|
printf '%s\\n' {shlex.quote(patch_id)} > /var/lib/hydra-build/patched-source.sha256
|
|
fi'''
|
|
script = package_job_script(['/bin/bash', '-euc', package_commands], inputs=inputs)
|
|
self.exec(shlex.join(['python3', '-c', script]))
|
|
if self.input_id() != inputs:
|
|
raise BuildError('Image inputs changed during package installation; completion was not recorded')
|
|
inventory = self.exec('cat /var/lib/hydra-build/packages.json', capture=True).stdout
|
|
from hydra import image
|
|
try:
|
|
image.validate_package_inventory(json.loads(inventory))
|
|
except (ValueError, image.ImageError) as error:
|
|
raise BuildError(f'Package inventory is malformed: {error}') from error
|
|
target = self.work / 'artifacts/packages.json'
|
|
temporary = target.with_suffix('.tmp')
|
|
temporary.write_text(inventory)
|
|
temporary.replace(target)
|
|
self.record('packages', inventory='artifacts/packages.json', patched_source=patch_id,
|
|
inventory_sha256=image.digest(target))
|
|
|
|
def assembly(self, snapshot_digest=None):
|
|
"""A separate rootless container processes an exported copy, never the builder."""
|
|
if snapshot_digest is None:
|
|
state = self.manifest().get('rootfs', {})
|
|
snapshot_digest = state.get('snapshot_sha256')
|
|
if not isinstance(snapshot_digest, str) or not re.fullmatch('[a-f0-9]{64}', snapshot_digest):
|
|
raise BuildError('Verified exported snapshot is required')
|
|
name = self.name + '-image-' + snapshot_digest[:12]
|
|
image = 'localhost/hydra-image:' + snapshot_digest
|
|
exists = self.pod('container', 'exists', name, capture=True, check=False).returncode == 0
|
|
if not exists:
|
|
self.pod('import', str(self.work / 'artifacts/builder-rootfs.tar'), image)
|
|
self.pod('run', '--detach', '--name', name, '--hostname', 'hydra-image-assembler',
|
|
'--network', 'none', '--cap-add', 'SYS_ADMIN', '--security-opt', 'seccomp=unconfined',
|
|
'--volume', f'{self.source_dir}:/hydra-source:ro',
|
|
'--volume', f'{self.work}:/hydra-output:rw', image, '/bin/sleep', 'infinity')
|
|
container = self.inspect('container', name)
|
|
for destination, source, writable in (('/hydra-source', self.source_dir, False),
|
|
('/hydra-output', self.work, True)):
|
|
mounts = [mount for mount in container.get('Mounts', []) if mount.get('Destination') == destination]
|
|
if len(mounts) != 1 or mounts[0].get('Source') != str(source) or mounts[0].get('RW') is not writable:
|
|
raise BuildError('Assembly mount provenance differs; refusing to write')
|
|
provenance = self.inspect('image', container['Image'])
|
|
if provenance.get('RootFS', {}).get('Layers') != ['sha256:' + snapshot_digest]:
|
|
raise BuildError('Assembly image does not match the exported builder snapshot')
|
|
if exists:
|
|
self.pod('start', name)
|
|
return name
|
|
|
|
def rootfs(self):
|
|
from hydra import image
|
|
if self.active_package_job():
|
|
raise BuildError('A contained package job is running; wait before export')
|
|
if not self.complete('packages'):
|
|
raise BuildError('Verified completed packages are required before rootfs export')
|
|
if self.complete('rootfs'):
|
|
return
|
|
source_inputs = image.source_identity(self.source_dir)
|
|
previous = self.manifest().get('rootfs', {})
|
|
inputs = self.input_id()
|
|
inventory_checksum = image.digest(self.work / 'artifacts/packages.json')
|
|
if inventory_checksum != self.manifest().get('packages', {}).get('inventory_sha256'):
|
|
raise BuildError('Package inventory changed before rootfs export')
|
|
original = self.work / 'artifacts/builder-rootfs.tar'
|
|
if original.is_symlink() or not original.parent.resolve().is_relative_to(self.work):
|
|
raise BuildError('Raw export output must remain inside the dedicated work directory')
|
|
if original.exists() and (not original.is_file() or original.stat().st_nlink != 1):
|
|
raise BuildError('Raw export output must be a singly linked regular file')
|
|
snapshot = None
|
|
previous_artifacts = previous.get('artifacts', {})
|
|
if (previous.get('complete') is True and previous.get('inputs') == inputs
|
|
and isinstance(previous_artifacts, dict)
|
|
and previous_artifacts.get('packages.json') == inventory_checksum
|
|
and isinstance(previous.get('snapshot_sha256'), str)
|
|
and re.fullmatch('[a-f0-9]{64}', previous['snapshot_sha256'])
|
|
and original.is_file()):
|
|
try:
|
|
if image.digest(original) == previous['snapshot_sha256']:
|
|
snapshot = previous['snapshot_sha256']
|
|
except OSError:
|
|
pass # Unreadable cache is never trusted; attempt an ordinary export.
|
|
# Preserve the prior completed provenance until the cache decision is made.
|
|
self.record('rootfs', complete=False)
|
|
original.touch(mode=0o600, exist_ok=True)
|
|
original.chmod(0o600) # The unsanitized intermediate can contain builder-private trust.
|
|
if snapshot is None:
|
|
self.pod('export', '--output', str(original), self.name)
|
|
snapshot = image.digest(original)
|
|
assembler = self.assembly(snapshot)
|
|
commit = self.run(['git', '-C', str(self.source_dir), 'rev-parse', 'HEAD'], capture=True).stdout.strip()
|
|
script = 'root=/hydra-output/rootfs\n' + rootfs_retry_guard_script() + r"""mkdir -p "$root"
|
|
touch "$root/.hydra-export-owned"
|
|
tar --extract --preserve-permissions --numeric-owner --file /hydra-output/artifacts/builder-rootfs.tar --directory "$root"
|
|
/hydra-source/image/configure-rootfs.sh "$root"
|
|
cp "$(python3 /hydra-source/hydra/image.py kernel-image "$root")" /hydra-output/artifacts/vmlinuz
|
|
cp "$root/boot/initramfs-live.img" /hydra-output/artifacts/initramfs-live.img
|
|
cp "$root/usr/share/hydra/build-manifest.json" /hydra-output/artifacts/rootfs-manifest.json
|
|
# Keep ownership for safe retries outside the shipped payload.
|
|
mksquashfs "$root" /hydra-output/artifacts/squashfs.img -noappend -comp zstd -processors 4 -e boot/initramfs-live.img .hydra-export-owned
|
|
chmod 0644 /hydra-output/artifacts/vmlinuz /hydra-output/artifacts/initramfs-live.img /hydra-output/artifacts/squashfs.img /hydra-output/artifacts/rootfs-manifest.json
|
|
"""
|
|
self.pod('exec', '--env', f'HYDRA_SOURCE_COMMIT={commit}', assembler, '/bin/bash', '-euc', script)
|
|
if image.source_identity(self.source_dir) != source_inputs:
|
|
raise BuildError('Source inputs changed during rootfs assembly; completion not recorded')
|
|
manifest = json.loads((self.work / 'artifacts/rootfs-manifest.json').read_text())
|
|
if manifest.get('packages') != self.package_inventory():
|
|
raise BuildError('Payload package inventory differs from the validated package checkpoint')
|
|
if manifest.get('source_files') != image.source_files(self.source_dir):
|
|
raise BuildError('Payload source manifest is incomplete or stale')
|
|
artifacts = {name: image.digest(self.work / 'artifacts' / name) for name in
|
|
('vmlinuz', 'initramfs-live.img', 'squashfs.img', 'rootfs-manifest.json', 'packages.json')}
|
|
self.record('rootfs', source_inputs=source_inputs, snapshot_sha256=snapshot,
|
|
assembler=assembler, kernel=manifest['kernel'], artifacts=artifacts)
|
|
|
|
def iso(self):
|
|
from hydra import image
|
|
if not self.complete('rootfs'):
|
|
raise BuildError('Verified rootfs artifacts are required before ISO assembly')
|
|
if self.complete('iso'):
|
|
return
|
|
source_inputs = image.source_identity(self.source_dir)
|
|
assembler = self.assembly()
|
|
self.record('iso', complete=False)
|
|
script = r"""tree=/hydra-output/iso-tree
|
|
[ ! -L "$tree" ] || { echo 'ISO output cannot be a symlink' >&2; exit 1; }
|
|
mkdir -p "$tree/boot/grub" "$tree/LiveOS"
|
|
cp /hydra-output/artifacts/vmlinuz "$tree/boot/vmlinuz"
|
|
cp /hydra-output/artifacts/initramfs-live.img "$tree/boot/initramfs-live.img"
|
|
cp /hydra-output/artifacts/squashfs.img "$tree/LiveOS/squashfs.img"
|
|
cp /hydra-source/image/grub.cfg "$tree/boot/grub/grub.cfg"
|
|
cp /hydra-output/artifacts/rootfs-manifest.json "$tree/build-manifest.json"
|
|
cp /hydra-output/artifacts/packages.json "$tree/packages.json"
|
|
grub-mkrescue --directory=/usr/lib/grub/x86_64-efi --output=/hydra-output/artifacts/hydra-os.iso "$tree" -- -volid HYDRA_OS
|
|
xorriso -indev /hydra-output/artifacts/hydra-os.iso -report_el_torito plain -pvd_info > /hydra-output/artifacts/iso-inspection.txt 2>&1
|
|
chmod 0644 /hydra-output/artifacts/hydra-os.iso /hydra-output/artifacts/iso-inspection.txt
|
|
"""
|
|
self.pod('exec', assembler, '/bin/bash', '-euc', script)
|
|
inspection = (self.work / 'artifacts/iso-inspection.txt').read_text()
|
|
if 'HYDRA_OS' not in inspection or not re.search(r'El Torito boot img.*UEFI', inspection):
|
|
raise BuildError('ISO does not report the expected volume label and UEFI boot image')
|
|
if image.source_identity(self.source_dir) != source_inputs:
|
|
raise BuildError('Source inputs changed during ISO assembly; completion not recorded')
|
|
checksum = image.digest(self.work / 'artifacts/hydra-os.iso')
|
|
(self.work / 'artifacts/hydra-os.iso.sha256').write_text(checksum + ' hydra-os.iso\n')
|
|
artifacts = {name: image.digest(self.work / 'artifacts' / name) for name in
|
|
('hydra-os.iso', 'hydra-os.iso.sha256', 'iso-inspection.txt', 'packages.json')}
|
|
self.record('iso', source_inputs=source_inputs, label=image.LABEL, artifacts=artifacts)
|
|
|
|
def status(self):
|
|
data = self.manifest()
|
|
for step in ('fetch', 'prepare', 'packages', 'rootfs', 'iso'):
|
|
if step in data:
|
|
data[step]['recorded_complete'] = data[step].get('complete', False)
|
|
data[step]['complete'] = self.complete(step)
|
|
data['mutation_lock_held'] = self.build_running()
|
|
data['package_job_running'] = self.active_package_job()
|
|
data['running'] = data['mutation_lock_held'] or data['package_job_running']
|
|
print(json.dumps(data, indent=2))
|
|
|
|
def build_running(self):
|
|
try:
|
|
with (self.work / '.hydra-build.lock').open('r') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
return True
|
|
except FileNotFoundError:
|
|
pass
|
|
return False
|
|
|
|
@contextlib.contextmanager
|
|
def locked(self):
|
|
self.work.mkdir(parents=True, exist_ok=True)
|
|
self.cache.mkdir(parents=True, exist_ok=True)
|
|
(self.work / 'artifacts').mkdir(exist_ok=True)
|
|
with (self.work / '.hydra-build.lock').open('a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as exc:
|
|
raise BuildError('Another Hydra build is running in this work directory') from exc
|
|
yield
|
|
|
|
|
|
def main(argv=None):
|
|
source = Path(__file__).resolve().parents[1]
|
|
# Linked worktrees share project-owned caches/build state in the primary checkout.
|
|
common = subprocess.run(['git', '-C', str(source), 'rev-parse', '--path-format=absolute', '--git-common-dir'],
|
|
text=True, capture_output=True)
|
|
project = Path(common.stdout.strip()).parent if common.returncode == 0 else source
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('step', choices=('fetch', 'prepare', 'packages', 'rootfs', 'iso', 'status'))
|
|
parser.add_argument('--work', type=Path, default=project / 'build')
|
|
parser.add_argument('--cache', type=Path, default=project / 'cache')
|
|
parser.add_argument('--podman')
|
|
parser.add_argument('--gpg')
|
|
parser.add_argument('--pretend', action='store_true', help='resolve packages without installing')
|
|
options = parser.parse_args(argv)
|
|
try:
|
|
builder = Builder(project, options.work, options.cache, options.podman, options.gpg, source)
|
|
if options.step == 'status':
|
|
builder.status()
|
|
return 0
|
|
with builder.locked():
|
|
if options.step == 'packages':
|
|
builder.packages(options.pretend)
|
|
else:
|
|
getattr(builder, options.step)()
|
|
return 0
|
|
except (BuildError, OSError, ValueError, lzma.LZMAError) as exc:
|
|
print(f'hydra-build: {exc}', file=sys.stderr)
|
|
return 1
|