"""Verified, resumable Gentoo builds in a dedicated rootless Podman store.""" import argparse import contextlib import fcntl import hashlib import json import os from pathlib import Path import shutil import subprocess import sys import tempfile import urllib.parse import urllib.request class BuildError(RuntimeError): pass def verify_hash(path, expected): digest = hashlib.sha256() with Path(path).open('rb') as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b''): digest.update(chunk) if digest.hexdigest() != expected: raise BuildError(f'SHA256 mismatch: {path}') def require_signature(status, fingerprint): valid = [line.split() for line in status.splitlines() if line.startswith('[GNUPG:] VALIDSIG ')] if not valid or not any(fields[-1].upper() == fingerprint.upper() for fields in valid): raise BuildError('Required primary signature fingerprint is missing') if any(token in status for token in ('BADSIG', 'ERRSIG', 'REVKEYSIG', 'EXPKEYSIG')): raise BuildError('Invalid release signature') class Builder: def __init__(self, project, work, cache, podman=None, gpg=None, source_dir=None): if os.geteuid() == 0: raise BuildError('Run as the normal user, never host root') self.project = Path(project).resolve() self.source_dir = Path(source_dir or project).resolve() self.work, self.cache = Path(work).resolve(), Path(cache).resolve() for path in (self.work, self.cache): if path == self.project or not path.is_relative_to(self.project): raise BuildError('Work and cache paths must be inside the project, below its root') if self.work == self.cache or self.work.is_relative_to(self.cache) or self.cache.is_relative_to(self.work): raise BuildError('Work and cache must be separate directories') self.podman = podman or shutil.which('podman') or 'podman' self.gpg = gpg or shutil.which('gpg') or 'gpg' runtime = Path(os.environ.get('XDG_RUNTIME_DIR', f'/run/user/{os.getuid()}')) identifier = hashlib.sha256(str(self.work).encode()).hexdigest()[:12] self.runroot = runtime / ('hydra-build-podman' if self.work == self.project / 'build' else f'hydra-build-{identifier}') self.name = 'hydra-builder-' + identifier self.image = 'localhost/hydra-stage3:20261011' def podman_argv(self, *args): return [self.podman, '--root', str(self.work / 'podman'), '--runroot', str(self.runroot), '--storage-driver', 'overlay', *args] def container_run_args(self): return ['run', '--detach', '--name', self.name, '--hostname', 'hydra-builder', '--network', 'host', '--cap-add', 'SYS_ADMIN', '--security-opt', 'seccomp=unconfined', '--volume', f'{self.source_dir}:/hydra-source:ro', '--volume', f'{self.cache}:/hydra-cache:ro', self.image, '/bin/sleep', 'infinity'] def run(self, argv, capture=False, check=True): print('+ ' + ' '.join(str(arg) for arg in argv), flush=True) result = subprocess.run(argv, text=True, stdout=subprocess.PIPE if capture else None, stderr=subprocess.PIPE if capture else None) if check and result.returncode: raise BuildError(f'Command failed ({result.returncode}): {argv[0]}\n{result.stderr or ""}') return result def pod(self, *args, **kwargs): return self.run(self.podman_argv(*args), **kwargs) def exec(self, command, capture=False): return self.pod('exec', self.name, '/bin/bash', '-euc', command, capture=capture) def manifest(self): path = self.work / 'manifest.json' return json.loads(path.read_text()) if path.exists() else {} def input_id(self): digest = hashlib.sha256() for path in sorted((self.source_dir / 'image').rglob('*')): if path.is_file(): digest.update(str(path.relative_to(self.source_dir)).encode()) digest.update(path.read_bytes()) return digest.hexdigest() def record(self, step, **details): data = self.manifest() data[step] = {'complete': True, 'inputs': self.input_id(), **details} data['builder'] = self.name path = self.work / 'manifest.json' temp = path.with_suffix('.tmp') temp.write_text(json.dumps(data, indent=2) + '\n') temp.replace(path) def complete(self, step): entry = self.manifest().get(step, {}) if not entry.get('complete') or entry.get('inputs') != self.input_id(): return False if step == 'packages': if not (self.work / 'artifacts/packages.json').is_file(): return False return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0 if step == 'prepare': return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0 if step == 'fetch': inputs = json.loads((self.source_dir / 'image/sources.json').read_text()) try: for name in ('stage3', 'portage'): item = inputs[name] path = self.cache / item.get('filename', Path(urllib.parse.urlparse(item['url']).path).name) verify_hash(path, item['sha256']) except (OSError, BuildError): return False return True return False def download(self, url, filename=None): parsed = urllib.parse.urlparse(url) if parsed.scheme != 'https': raise BuildError('Build input URLs must use HTTPS') target = self.cache / (filename or Path(parsed.path).name) if not target.exists(): temp = target.with_suffix(target.suffix + '.partial') print(f'Downloading {url}', flush=True) with urllib.request.urlopen(url, timeout=120) as response, temp.open('wb') as stream: shutil.copyfileobj(response, stream) temp.replace(target) return target def fetch(self): inputs = json.loads((self.source_dir / 'image/sources.json').read_text()) stage = inputs['stage3'] archive = self.download(stage['url']) verify_hash(archive, stage['sha256']) snapshot = self.download(inputs['portage']['url'], inputs['portage']['filename']) verify_hash(snapshot, inputs['portage']['sha256']) signature = self.download(stage['signature_url']) keyring = self.download(inputs['keyring_url']) verify_hash(keyring, inputs['keyring_sha256']) with tempfile.TemporaryDirectory(prefix='hydra-gpg-', dir=self.work) as home: result = self.run([self.gpg, '--batch', '--homedir', home, '--no-default-keyring', '--keyring', str(keyring), '--status-fd', '1', '--verify', str(signature), str(archive)], capture=True) require_signature(result.stdout, stage['fingerprint']) (self.work / 'artifacts/stage3-signature.txt').write_text(result.stdout + result.stderr) self.record('fetch', sources=inputs) return inputs def prepare(self): inputs = self.fetch() # Reverify cached bytes even when resuming. if self.pod('image', 'exists', self.image, capture=True, check=False).returncode: self.pod('import', str(self.cache / Path(urllib.parse.urlparse(inputs['stage3']['url']).path).name), self.image) if self.pod('container', 'exists', self.name, capture=True, check=False).returncode: self.pod(*self.container_run_args()) else: self.pod('start', self.name) snapshot = inputs['portage']['filename'] # Snapshot extraction happens only inside the container; host paths are read-only. self.exec(f'''mkdir -p /var/db/repos/gentoo if [ ! -f /var/db/repos/gentoo/.hydra-snapshot ]; then tar --extract --gzip --file /hydra-cache/{snapshot} --strip-components=1 --directory /var/db/repos/gentoo printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot fi cp -a /hydra-source/image/portage/. /etc/portage/ if [ -d /etc/portage/gnupg ] && [ ! -s /etc/portage/gnupg/mykeyid ]; then mv /etc/portage/gnupg /etc/portage/gnupg.hydra-untrusted fi getuto gpg --batch --homedir /etc/portage/gnupg --import /hydra-cache/service-keys.gpg chmod a+r /etc/portage/gnupg/pubring.kbx /etc/portage/gnupg/trustdb.gpg ''') self.record('prepare', image=self.image, snapshot=inputs['portage']['commit']) def packages(self, pretend=False): self.prepare() atoms = [line.strip() for line in (self.source_dir / 'image/packages').read_text().splitlines() if line.strip() and not line.startswith('#')] import shlex command = 'emerge --verbose --update --newuse --deep --backtrack=100 ' + ('--pretend ' if pretend else '') + ' '.join(map(shlex.quote, atoms)) self.exec(command) if pretend: return inventory = self.exec('python3 - <<\'PY\'\nimport json,pathlib\nout=[]\nfor use in sorted(pathlib.Path("/var/db/pkg").glob("*/*/USE")):\n p=use.parent\n out.append({"package":str(p.relative_to("/var/db/pkg")),"use":use.read_text().split(),"build_time":(p/"BUILD_TIME").read_text().strip() if (p/"BUILD_TIME").exists() else None})\nprint(json.dumps(out,indent=2))\nPY', capture=True).stdout (self.work / 'artifacts/packages.json').write_text(inventory) self.record('packages', inventory='artifacts/packages.json') def status(self): print(json.dumps(self.manifest(), indent=2)) @contextlib.contextmanager def locked(self): self.work.mkdir(parents=True, exist_ok=True) self.cache.mkdir(parents=True, exist_ok=True) (self.work / 'artifacts').mkdir(exist_ok=True) with (self.work / '.hydra-build.lock').open('a') as lock: try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as exc: raise BuildError('Another Hydra build is running in this work directory') from exc yield def main(argv=None): source = Path(__file__).resolve().parents[1] # Linked worktrees share project-owned caches/build state in the primary checkout. common = subprocess.run(['git', '-C', str(source), 'rev-parse', '--path-format=absolute', '--git-common-dir'], text=True, capture_output=True) project = Path(common.stdout.strip()).parent if common.returncode == 0 else source parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('step', choices=('fetch', 'prepare', 'packages', 'rootfs', 'iso', 'status')) parser.add_argument('--work', type=Path, default=project / 'build') parser.add_argument('--cache', type=Path, default=project / 'cache') parser.add_argument('--podman') parser.add_argument('--gpg') parser.add_argument('--pretend', action='store_true', help='resolve packages without installing') options = parser.parse_args(argv) try: builder = Builder(project, options.work, options.cache, options.podman, options.gpg, source) with builder.locked(): if options.step in ('rootfs', 'iso'): raise BuildError(f'{options.step} assembly is not configured yet') if options.step == 'packages': builder.packages(options.pretend) else: getattr(builder, options.step)() return 0 except (BuildError, OSError, ValueError) as exc: print(f'hydra-build: {exc}', file=sys.stderr) return 1