393 lines
19 KiB
Python
393 lines
19 KiB
Python
"""Verified, resumable Gentoo builds in a dedicated rootless Podman store."""
|
|
import argparse
|
|
import contextlib
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import lzma
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
class BuildError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def package_job_script(command, state='/var/lib/hydra-build', package_db='/var/db/pkg', inputs=''):
|
|
"""The contained job owns its log/result even if the outer caller disconnects."""
|
|
return f'''import fcntl, json, os, pathlib, subprocess, sys, time
|
|
state = pathlib.Path({str(state)!r})
|
|
database = pathlib.Path({str(package_db)!r})
|
|
state.mkdir(parents=True, exist_ok=True)
|
|
lock = (state / 'packages.lock').open('a')
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
sys.exit(75)
|
|
def write(name, data):
|
|
temporary = state / (name + '.tmp')
|
|
temporary.write_text(json.dumps(data, indent=2) + '\\n')
|
|
temporary.replace(state / name)
|
|
status = {{'running': True, 'pid': os.getpid(), 'inputs': {inputs!r}, 'started': int(time.time())}}
|
|
write('packages.status.json', status)
|
|
with (state / 'packages.log').open('a') as log:
|
|
try:
|
|
result = subprocess.run({command!r}, stdout=log, stderr=subprocess.STDOUT)
|
|
code = result.returncode
|
|
except OSError as error:
|
|
log.write(str(error) + '\\n')
|
|
code = 127
|
|
if code == 0:
|
|
inventory = []
|
|
for use in sorted(database.glob('*/*/USE')):
|
|
package = use.parent
|
|
inventory.append({{'package': str(package.relative_to(database)), 'use': use.read_text().split(),
|
|
'build_time': (package / 'BUILD_TIME').read_text().strip() if (package / 'BUILD_TIME').exists() else None}})
|
|
write('packages.json', inventory)
|
|
status.update(running=False, exit_code=code, finished=int(time.time()))
|
|
write('packages.status.json', status)
|
|
sys.exit(code)
|
|
'''
|
|
|
|
|
|
def verify_hash(path, expected):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
if digest.hexdigest() != expected:
|
|
raise BuildError(f'SHA256 mismatch: {path}')
|
|
|
|
|
|
def require_signature(status, fingerprint):
|
|
valid = [line.split() for line in status.splitlines() if line.startswith('[GNUPG:] VALIDSIG ')]
|
|
if not valid or not any(fields[-1].upper() == fingerprint.upper() for fields in valid):
|
|
raise BuildError('Required primary signature fingerprint is missing')
|
|
if any(token in status for token in ('BADSIG', 'ERRSIG', 'REVKEYSIG', 'EXPKEYSIG')):
|
|
raise BuildError('Invalid release signature')
|
|
|
|
|
|
class Builder:
|
|
def __init__(self, project, work, cache, podman=None, gpg=None, source_dir=None):
|
|
if os.geteuid() == 0:
|
|
raise BuildError('Run as the normal user, never host root')
|
|
self.project = Path(project).resolve()
|
|
self.source_dir = Path(source_dir or project).resolve()
|
|
self.work, self.cache = Path(work).resolve(), Path(cache).resolve()
|
|
for path in (self.work, self.cache):
|
|
if path == self.project or not path.is_relative_to(self.project):
|
|
raise BuildError('Work and cache paths must be inside the project, below its root')
|
|
if self.work == self.cache or self.work.is_relative_to(self.cache) or self.cache.is_relative_to(self.work):
|
|
raise BuildError('Work and cache must be separate directories')
|
|
self.podman = podman or shutil.which('podman') or 'podman'
|
|
self.gpg = gpg or shutil.which('gpg') or 'gpg'
|
|
runtime = Path(os.environ.get('XDG_RUNTIME_DIR', f'/run/user/{os.getuid()}'))
|
|
identifier = hashlib.sha256(str(self.work).encode()).hexdigest()[:12]
|
|
self.runroot = runtime / ('hydra-build-podman' if self.work == self.project / 'build' else f'hydra-build-{identifier}')
|
|
self.name = 'hydra-builder-' + identifier
|
|
self.image = 'localhost/hydra-stage3:20261011'
|
|
self._stage3_layers = {}
|
|
|
|
def podman_argv(self, *args):
|
|
return [self.podman, '--root', str(self.work / 'podman'), '--runroot', str(self.runroot), '--storage-driver', 'overlay', *args]
|
|
|
|
def container_run_args(self):
|
|
return ['run', '--detach', '--name', self.name, '--hostname', 'hydra-builder',
|
|
'--network', 'host', '--cap-add', 'SYS_ADMIN', '--security-opt', 'seccomp=unconfined',
|
|
'--volume', f'{self.source_dir}:/hydra-source:ro', '--volume', f'{self.cache}:/hydra-cache:ro',
|
|
self.image, '/bin/sleep', 'infinity']
|
|
|
|
def run(self, argv, capture=False, check=True):
|
|
print('+ ' + ' '.join(str(arg) for arg in argv), file=sys.stderr, flush=True)
|
|
result = subprocess.run(argv, text=True, stdout=subprocess.PIPE if capture else None,
|
|
stderr=subprocess.PIPE if capture else None)
|
|
if check and result.returncode:
|
|
raise BuildError(f'Command failed ({result.returncode}): {argv[0]}\n{result.stderr or ""}')
|
|
return result
|
|
|
|
def pod(self, *args, **kwargs):
|
|
return self.run(self.podman_argv(*args), **kwargs)
|
|
|
|
def exec(self, command, capture=False):
|
|
return self.pod('exec', self.name, '/bin/bash', '-euc', command, capture=capture)
|
|
|
|
def inspect(self, kind, name):
|
|
result = self.pod(kind, 'inspect', name, capture=True, check=False)
|
|
if result.returncode:
|
|
raise BuildError(f'{kind} provenance is unavailable; use a fresh work directory')
|
|
entries = json.loads(result.stdout)
|
|
if not isinstance(entries, list) or len(entries) != 1 or not isinstance(entries[0], dict):
|
|
raise BuildError(f'Unexpected {kind} provenance')
|
|
return entries[0]
|
|
|
|
def check_mounts(self, container):
|
|
for destination, path, label in (('/hydra-source', self.source_dir, 'source'),
|
|
('/hydra-cache', self.cache, 'cache')):
|
|
mounts = [item for item in container.get('Mounts', []) if item.get('Destination') == destination]
|
|
if len(mounts) != 1 or mounts[0].get('Source') != str(path) or mounts[0].get('RW') is not False:
|
|
raise BuildError(f'Existing builder {label} mount differs; use its original checkout/cache or a fresh work directory')
|
|
|
|
def stage3_layer(self, inputs):
|
|
stage = inputs['stage3']
|
|
archive = self.cache / Path(urllib.parse.urlparse(stage['url']).path).name
|
|
verify_hash(archive, stage['sha256'])
|
|
if stage['sha256'] not in self._stage3_layers:
|
|
digest = hashlib.sha256()
|
|
with lzma.open(archive, 'rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
self._stage3_layers[stage['sha256']] = 'sha256:' + digest.hexdigest()
|
|
return self._stage3_layers[stage['sha256']]
|
|
|
|
def check_base(self, image, inputs):
|
|
expected = self.stage3_layer(inputs)
|
|
if image.get('RootFS', {}).get('Layers') != [expected]:
|
|
raise BuildError('Existing builder stage3 differs from pinned archive; use a fresh work directory')
|
|
return expected
|
|
|
|
def read_container_file(self, path):
|
|
# Podman cp also reads a stopped container, so provenance precedes start.
|
|
with tempfile.TemporaryDirectory(prefix='hydra-read-') as directory:
|
|
target = Path(directory) / 'data'
|
|
result = self.pod('cp', f'{self.name}:{path}', str(target), capture=True, check=False)
|
|
return target.read_text() if result.returncode == 0 else None
|
|
|
|
def check_snapshot(self, inputs, allow_pending=False):
|
|
marker = self.read_container_file('/var/db/repos/gentoo/.hydra-snapshot')
|
|
if marker is None and allow_pending:
|
|
marker = self.read_container_file('/var/db/repos/gentoo/.hydra-snapshot-input')
|
|
if marker is None or marker.strip() != inputs['portage']['commit']:
|
|
raise BuildError('Existing builder snapshot differs or is unverified; use a fresh work directory')
|
|
|
|
def active_package_job(self):
|
|
if not (self.work / 'podman').is_dir():
|
|
return False
|
|
result = self.pod('top', self.name, 'hpid', 'args', capture=True, check=False)
|
|
if result.returncode == 0 and any(re.search(r'(?:^|\s)(?:\S*/)?emerge(?:\s|$)', line)
|
|
for line in result.stdout.splitlines()[1:]):
|
|
return True
|
|
probe = self.pod('exec', self.name, 'python3', '-c', '''import fcntl, pathlib, sys
|
|
path = pathlib.Path('/var/lib/hydra-build/packages.lock')
|
|
if path.exists():
|
|
with path.open('r') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
sys.exit(75)
|
|
''', capture=True, check=False)
|
|
return probe.returncode == 75
|
|
|
|
def manifest(self):
|
|
path = self.work / 'manifest.json'
|
|
return json.loads(path.read_text()) if path.exists() else {}
|
|
|
|
def input_id(self):
|
|
digest = hashlib.sha256()
|
|
for path in sorted((self.source_dir / 'image').rglob('*')):
|
|
if path.is_file():
|
|
digest.update(str(path.relative_to(self.source_dir)).encode())
|
|
digest.update(path.read_bytes())
|
|
return digest.hexdigest()
|
|
|
|
def record(self, step, **details):
|
|
data = self.manifest()
|
|
data[step] = {'complete': True, 'inputs': self.input_id(), **details}
|
|
data['builder'] = self.name
|
|
path = self.work / 'manifest.json'
|
|
temp = path.with_suffix('.tmp')
|
|
temp.write_text(json.dumps(data, indent=2) + '\n')
|
|
temp.replace(path)
|
|
|
|
def complete(self, step):
|
|
entry = self.manifest().get(step, {})
|
|
if not entry.get('complete') or entry.get('inputs') != self.input_id():
|
|
return False
|
|
if step == 'packages':
|
|
if not (self.work / 'artifacts/packages.json').is_file():
|
|
return False
|
|
return self.complete('prepare')
|
|
if step == 'prepare':
|
|
try:
|
|
container = self.inspect('container', self.name)
|
|
self.check_mounts(container)
|
|
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
|
|
self.check_base(self.inspect('image', container['Image']), inputs)
|
|
self.check_snapshot(inputs)
|
|
return True
|
|
except (BuildError, OSError, ValueError, KeyError, lzma.LZMAError):
|
|
return False
|
|
if step == 'fetch':
|
|
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
|
|
try:
|
|
for name in ('stage3', 'portage'):
|
|
item = inputs[name]
|
|
path = self.cache / item.get('filename', Path(urllib.parse.urlparse(item['url']).path).name)
|
|
verify_hash(path, item['sha256'])
|
|
except (OSError, BuildError):
|
|
return False
|
|
return True
|
|
return False
|
|
|
|
def download(self, url, filename=None):
|
|
parsed = urllib.parse.urlparse(url)
|
|
if parsed.scheme != 'https':
|
|
raise BuildError('Build input URLs must use HTTPS')
|
|
target = self.cache / (filename or Path(parsed.path).name)
|
|
if not target.exists():
|
|
temp = target.with_suffix(target.suffix + '.partial')
|
|
print(f'Downloading {url}', flush=True)
|
|
with urllib.request.urlopen(url, timeout=120) as response, temp.open('wb') as stream:
|
|
shutil.copyfileobj(response, stream)
|
|
temp.replace(target)
|
|
return target
|
|
|
|
def fetch(self):
|
|
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
|
|
stage = inputs['stage3']
|
|
archive = self.download(stage['url'])
|
|
verify_hash(archive, stage['sha256'])
|
|
snapshot = self.download(inputs['portage']['url'], inputs['portage']['filename'])
|
|
verify_hash(snapshot, inputs['portage']['sha256'])
|
|
signature = self.download(stage['signature_url'])
|
|
keyring = self.download(inputs['keyring_url'])
|
|
verify_hash(keyring, inputs['keyring_sha256'])
|
|
with tempfile.TemporaryDirectory(prefix='hydra-gpg-', dir=self.work) as home:
|
|
result = self.run([self.gpg, '--batch', '--homedir', home, '--no-default-keyring', '--keyring', str(keyring),
|
|
'--status-fd', '1', '--verify', str(signature), str(archive)], capture=True)
|
|
require_signature(result.stdout, stage['fingerprint'])
|
|
(self.work / 'artifacts/stage3-signature.txt').write_text(result.stdout + result.stderr)
|
|
self.record('fetch', sources=inputs)
|
|
return inputs
|
|
|
|
def prepare(self):
|
|
existing = self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
|
|
container = self.inspect('container', self.name) if existing else None
|
|
if container is not None:
|
|
self.check_mounts(container) # Reject another worktree before changing any checkpoint.
|
|
if self.active_package_job():
|
|
raise BuildError('A contained package job is running; wait for it to finish before preparation')
|
|
inputs = self.fetch() # Reverify cached bytes even when resuming.
|
|
if container is not None:
|
|
layer = self.check_base(self.inspect('image', container['Image']), inputs)
|
|
self.check_snapshot(inputs, allow_pending=True)
|
|
elif self.pod('image', 'exists', self.image, capture=True, check=False).returncode:
|
|
self.pod('import', str(self.cache / Path(urllib.parse.urlparse(inputs['stage3']['url']).path).name), self.image)
|
|
layer = self.check_base(self.inspect('image', self.image), inputs)
|
|
else:
|
|
layer = self.check_base(self.inspect('image', self.image), inputs)
|
|
if container is None:
|
|
self.pod(*self.container_run_args())
|
|
else:
|
|
self.pod('start', self.name)
|
|
snapshot = inputs['portage']['filename']
|
|
# Snapshot extraction happens only inside the container; host paths are read-only.
|
|
self.exec(f'''mkdir -p /var/db/repos/gentoo
|
|
if [ ! -f /var/db/repos/gentoo/.hydra-snapshot ]; then
|
|
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot-input
|
|
tar --extract --gzip --file /hydra-cache/{snapshot} --strip-components=1 --directory /var/db/repos/gentoo
|
|
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot
|
|
fi
|
|
cp -a /hydra-source/image/portage/. /etc/portage/
|
|
if [ -d /etc/portage/gnupg ] && [ ! -s /etc/portage/gnupg/mykeyid ]; then
|
|
mv /etc/portage/gnupg /etc/portage/gnupg.hydra-untrusted
|
|
fi
|
|
getuto
|
|
gpg --batch --homedir /etc/portage/gnupg --import /hydra-cache/service-keys.gpg
|
|
chmod a+r /etc/portage/gnupg/pubring.kbx /etc/portage/gnupg/trustdb.gpg
|
|
''')
|
|
self.record('prepare', image=container['Image'] if container else self.image,
|
|
stage3_sha256=inputs['stage3']['sha256'], stage3_layer=layer,
|
|
source_dir=str(self.source_dir), cache_dir=str(self.cache), snapshot=inputs['portage']['commit'])
|
|
|
|
def packages(self, pretend=False):
|
|
self.prepare()
|
|
atoms = [line.strip() for line in (self.source_dir / 'image/packages').read_text().splitlines()
|
|
if line.strip() and not line.startswith('#')]
|
|
import shlex
|
|
command = ['emerge', '--verbose', '--update', '--newuse', '--deep', '--backtrack=100']
|
|
if pretend:
|
|
self.exec(shlex.join([*command, '--pretend', *atoms]))
|
|
return
|
|
inputs = self.input_id()
|
|
self.record('packages', complete=False, job_log='/var/lib/hydra-build/packages.log')
|
|
script = package_job_script([*command, *atoms], inputs=inputs)
|
|
self.exec(shlex.join(['python3', '-c', script]))
|
|
if self.input_id() != inputs:
|
|
raise BuildError('Image inputs changed during package installation; completion was not recorded')
|
|
inventory = self.exec('cat /var/lib/hydra-build/packages.json', capture=True).stdout
|
|
(self.work / 'artifacts/packages.json').write_text(inventory)
|
|
self.record('packages', inventory='artifacts/packages.json')
|
|
|
|
def status(self):
|
|
data = self.manifest()
|
|
for step in ('fetch', 'prepare', 'packages', 'rootfs', 'iso'):
|
|
if step in data:
|
|
data[step]['recorded_complete'] = data[step].get('complete', False)
|
|
data[step]['complete'] = self.complete(step)
|
|
data['mutation_lock_held'] = self.build_running()
|
|
data['package_job_running'] = self.active_package_job()
|
|
data['running'] = data['mutation_lock_held'] or data['package_job_running']
|
|
print(json.dumps(data, indent=2))
|
|
|
|
def build_running(self):
|
|
try:
|
|
with (self.work / '.hydra-build.lock').open('r') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
return True
|
|
except FileNotFoundError:
|
|
pass
|
|
return False
|
|
|
|
@contextlib.contextmanager
|
|
def locked(self):
|
|
self.work.mkdir(parents=True, exist_ok=True)
|
|
self.cache.mkdir(parents=True, exist_ok=True)
|
|
(self.work / 'artifacts').mkdir(exist_ok=True)
|
|
with (self.work / '.hydra-build.lock').open('a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as exc:
|
|
raise BuildError('Another Hydra build is running in this work directory') from exc
|
|
yield
|
|
|
|
|
|
def main(argv=None):
|
|
source = Path(__file__).resolve().parents[1]
|
|
# Linked worktrees share project-owned caches/build state in the primary checkout.
|
|
common = subprocess.run(['git', '-C', str(source), 'rev-parse', '--path-format=absolute', '--git-common-dir'],
|
|
text=True, capture_output=True)
|
|
project = Path(common.stdout.strip()).parent if common.returncode == 0 else source
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('step', choices=('fetch', 'prepare', 'packages', 'rootfs', 'iso', 'status'))
|
|
parser.add_argument('--work', type=Path, default=project / 'build')
|
|
parser.add_argument('--cache', type=Path, default=project / 'cache')
|
|
parser.add_argument('--podman')
|
|
parser.add_argument('--gpg')
|
|
parser.add_argument('--pretend', action='store_true', help='resolve packages without installing')
|
|
options = parser.parse_args(argv)
|
|
try:
|
|
builder = Builder(project, options.work, options.cache, options.podman, options.gpg, source)
|
|
if options.step == 'status':
|
|
builder.status()
|
|
return 0
|
|
with builder.locked():
|
|
if options.step in ('rootfs', 'iso'):
|
|
raise BuildError(f'{options.step} assembly is not configured yet')
|
|
if options.step == 'packages':
|
|
builder.packages(options.pretend)
|
|
else:
|
|
getattr(builder, options.step)()
|
|
return 0
|
|
except (BuildError, OSError, ValueError, lzma.LZMAError) as exc:
|
|
print(f'hydra-build: {exc}', file=sys.stderr)
|
|
return 1
|