236 lines
12 KiB
Python
236 lines
12 KiB
Python
"""Verified, resumable Gentoo builds in a dedicated rootless Podman store."""
|
|
import argparse
|
|
import contextlib
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
class BuildError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def verify_hash(path, expected):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
if digest.hexdigest() != expected:
|
|
raise BuildError(f'SHA256 mismatch: {path}')
|
|
|
|
|
|
def require_signature(status, fingerprint):
|
|
valid = [line.split() for line in status.splitlines() if line.startswith('[GNUPG:] VALIDSIG ')]
|
|
if not valid or not any(fields[-1].upper() == fingerprint.upper() for fields in valid):
|
|
raise BuildError('Required primary signature fingerprint is missing')
|
|
if any(token in status for token in ('BADSIG', 'ERRSIG', 'REVKEYSIG', 'EXPKEYSIG')):
|
|
raise BuildError('Invalid release signature')
|
|
|
|
|
|
class Builder:
|
|
def __init__(self, project, work, cache, podman=None, gpg=None, source_dir=None):
|
|
if os.geteuid() == 0:
|
|
raise BuildError('Run as the normal user, never host root')
|
|
self.project = Path(project).resolve()
|
|
self.source_dir = Path(source_dir or project).resolve()
|
|
self.work, self.cache = Path(work).resolve(), Path(cache).resolve()
|
|
for path in (self.work, self.cache):
|
|
if path == self.project or not path.is_relative_to(self.project):
|
|
raise BuildError('Work and cache paths must be inside the project, below its root')
|
|
if self.work == self.cache or self.work.is_relative_to(self.cache) or self.cache.is_relative_to(self.work):
|
|
raise BuildError('Work and cache must be separate directories')
|
|
self.podman = podman or shutil.which('podman') or 'podman'
|
|
self.gpg = gpg or shutil.which('gpg') or 'gpg'
|
|
runtime = Path(os.environ.get('XDG_RUNTIME_DIR', f'/run/user/{os.getuid()}'))
|
|
identifier = hashlib.sha256(str(self.work).encode()).hexdigest()[:12]
|
|
self.runroot = runtime / ('hydra-build-podman' if self.work == self.project / 'build' else f'hydra-build-{identifier}')
|
|
self.name = 'hydra-builder-' + identifier
|
|
self.image = 'localhost/hydra-stage3:20261011'
|
|
|
|
def podman_argv(self, *args):
|
|
return [self.podman, '--root', str(self.work / 'podman'), '--runroot', str(self.runroot), '--storage-driver', 'overlay', *args]
|
|
|
|
def container_run_args(self):
|
|
return ['run', '--detach', '--name', self.name, '--hostname', 'hydra-builder',
|
|
'--network', 'host', '--cap-add', 'SYS_ADMIN', '--security-opt', 'seccomp=unconfined',
|
|
'--volume', f'{self.source_dir}:/hydra-source:ro', '--volume', f'{self.cache}:/hydra-cache:ro',
|
|
self.image, '/bin/sleep', 'infinity']
|
|
|
|
def run(self, argv, capture=False, check=True):
|
|
print('+ ' + ' '.join(str(arg) for arg in argv), flush=True)
|
|
result = subprocess.run(argv, text=True, stdout=subprocess.PIPE if capture else None,
|
|
stderr=subprocess.PIPE if capture else None)
|
|
if check and result.returncode:
|
|
raise BuildError(f'Command failed ({result.returncode}): {argv[0]}\n{result.stderr or ""}')
|
|
return result
|
|
|
|
def pod(self, *args, **kwargs):
|
|
return self.run(self.podman_argv(*args), **kwargs)
|
|
|
|
def exec(self, command, capture=False):
|
|
return self.pod('exec', self.name, '/bin/bash', '-euc', command, capture=capture)
|
|
|
|
def manifest(self):
|
|
path = self.work / 'manifest.json'
|
|
return json.loads(path.read_text()) if path.exists() else {}
|
|
|
|
def input_id(self):
|
|
digest = hashlib.sha256()
|
|
for path in sorted((self.source_dir / 'image').rglob('*')):
|
|
if path.is_file():
|
|
digest.update(str(path.relative_to(self.source_dir)).encode())
|
|
digest.update(path.read_bytes())
|
|
return digest.hexdigest()
|
|
|
|
def record(self, step, **details):
|
|
data = self.manifest()
|
|
data[step] = {'complete': True, 'inputs': self.input_id(), **details}
|
|
data['builder'] = self.name
|
|
path = self.work / 'manifest.json'
|
|
temp = path.with_suffix('.tmp')
|
|
temp.write_text(json.dumps(data, indent=2) + '\n')
|
|
temp.replace(path)
|
|
|
|
def complete(self, step):
|
|
entry = self.manifest().get(step, {})
|
|
if not entry.get('complete') or entry.get('inputs') != self.input_id():
|
|
return False
|
|
if step == 'packages':
|
|
if not (self.work / 'artifacts/packages.json').is_file():
|
|
return False
|
|
return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
|
|
if step == 'prepare':
|
|
return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
|
|
if step == 'fetch':
|
|
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
|
|
try:
|
|
for name in ('stage3', 'portage'):
|
|
item = inputs[name]
|
|
path = self.cache / item.get('filename', Path(urllib.parse.urlparse(item['url']).path).name)
|
|
verify_hash(path, item['sha256'])
|
|
except (OSError, BuildError):
|
|
return False
|
|
return True
|
|
return False
|
|
|
|
def download(self, url, filename=None):
|
|
parsed = urllib.parse.urlparse(url)
|
|
if parsed.scheme != 'https':
|
|
raise BuildError('Build input URLs must use HTTPS')
|
|
target = self.cache / (filename or Path(parsed.path).name)
|
|
if not target.exists():
|
|
temp = target.with_suffix(target.suffix + '.partial')
|
|
print(f'Downloading {url}', flush=True)
|
|
with urllib.request.urlopen(url, timeout=120) as response, temp.open('wb') as stream:
|
|
shutil.copyfileobj(response, stream)
|
|
temp.replace(target)
|
|
return target
|
|
|
|
def fetch(self):
|
|
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
|
|
stage = inputs['stage3']
|
|
archive = self.download(stage['url'])
|
|
verify_hash(archive, stage['sha256'])
|
|
snapshot = self.download(inputs['portage']['url'], inputs['portage']['filename'])
|
|
verify_hash(snapshot, inputs['portage']['sha256'])
|
|
signature = self.download(stage['signature_url'])
|
|
keyring = self.download(inputs['keyring_url'])
|
|
verify_hash(keyring, inputs['keyring_sha256'])
|
|
with tempfile.TemporaryDirectory(prefix='hydra-gpg-', dir=self.work) as home:
|
|
result = self.run([self.gpg, '--batch', '--homedir', home, '--no-default-keyring', '--keyring', str(keyring),
|
|
'--status-fd', '1', '--verify', str(signature), str(archive)], capture=True)
|
|
require_signature(result.stdout, stage['fingerprint'])
|
|
(self.work / 'artifacts/stage3-signature.txt').write_text(result.stdout + result.stderr)
|
|
self.record('fetch', sources=inputs)
|
|
return inputs
|
|
|
|
def prepare(self):
|
|
inputs = self.fetch() # Reverify cached bytes even when resuming.
|
|
if self.pod('image', 'exists', self.image, capture=True, check=False).returncode:
|
|
self.pod('import', str(self.cache / Path(urllib.parse.urlparse(inputs['stage3']['url']).path).name), self.image)
|
|
if self.pod('container', 'exists', self.name, capture=True, check=False).returncode:
|
|
self.pod(*self.container_run_args())
|
|
else:
|
|
self.pod('start', self.name)
|
|
snapshot = inputs['portage']['filename']
|
|
# Snapshot extraction happens only inside the container; host paths are read-only.
|
|
self.exec(f'''mkdir -p /var/db/repos/gentoo
|
|
if [ ! -f /var/db/repos/gentoo/.hydra-snapshot ]; then
|
|
tar --extract --gzip --file /hydra-cache/{snapshot} --strip-components=1 --directory /var/db/repos/gentoo
|
|
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot
|
|
fi
|
|
cp -a /hydra-source/image/portage/. /etc/portage/
|
|
if [ -d /etc/portage/gnupg ] && [ ! -s /etc/portage/gnupg/mykeyid ]; then
|
|
mv /etc/portage/gnupg /etc/portage/gnupg.hydra-untrusted
|
|
fi
|
|
getuto
|
|
gpg --batch --homedir /etc/portage/gnupg --import /hydra-cache/service-keys.gpg
|
|
chmod a+r /etc/portage/gnupg/pubring.kbx /etc/portage/gnupg/trustdb.gpg
|
|
''')
|
|
self.record('prepare', image=self.image, snapshot=inputs['portage']['commit'])
|
|
|
|
def packages(self, pretend=False):
|
|
self.prepare()
|
|
atoms = [line.strip() for line in (self.source_dir / 'image/packages').read_text().splitlines()
|
|
if line.strip() and not line.startswith('#')]
|
|
import shlex
|
|
command = 'emerge --verbose --update --newuse --deep --backtrack=100 ' + ('--pretend ' if pretend else '') + ' '.join(map(shlex.quote, atoms))
|
|
self.exec(command)
|
|
if pretend:
|
|
return
|
|
inventory = self.exec('python3 - <<\'PY\'\nimport json,pathlib\nout=[]\nfor use in sorted(pathlib.Path("/var/db/pkg").glob("*/*/USE")):\n p=use.parent\n out.append({"package":str(p.relative_to("/var/db/pkg")),"use":use.read_text().split(),"build_time":(p/"BUILD_TIME").read_text().strip() if (p/"BUILD_TIME").exists() else None})\nprint(json.dumps(out,indent=2))\nPY', capture=True).stdout
|
|
(self.work / 'artifacts/packages.json').write_text(inventory)
|
|
self.record('packages', inventory='artifacts/packages.json')
|
|
|
|
def status(self):
|
|
print(json.dumps(self.manifest(), indent=2))
|
|
|
|
@contextlib.contextmanager
|
|
def locked(self):
|
|
self.work.mkdir(parents=True, exist_ok=True)
|
|
self.cache.mkdir(parents=True, exist_ok=True)
|
|
(self.work / 'artifacts').mkdir(exist_ok=True)
|
|
with (self.work / '.hydra-build.lock').open('a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as exc:
|
|
raise BuildError('Another Hydra build is running in this work directory') from exc
|
|
yield
|
|
|
|
|
|
def main(argv=None):
|
|
source = Path(__file__).resolve().parents[1]
|
|
# Linked worktrees share project-owned caches/build state in the primary checkout.
|
|
common = subprocess.run(['git', '-C', str(source), 'rev-parse', '--path-format=absolute', '--git-common-dir'],
|
|
text=True, capture_output=True)
|
|
project = Path(common.stdout.strip()).parent if common.returncode == 0 else source
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('step', choices=('fetch', 'prepare', 'packages', 'rootfs', 'iso', 'status'))
|
|
parser.add_argument('--work', type=Path, default=project / 'build')
|
|
parser.add_argument('--cache', type=Path, default=project / 'cache')
|
|
parser.add_argument('--podman')
|
|
parser.add_argument('--gpg')
|
|
parser.add_argument('--pretend', action='store_true', help='resolve packages without installing')
|
|
options = parser.parse_args(argv)
|
|
try:
|
|
builder = Builder(project, options.work, options.cache, options.podman, options.gpg, source)
|
|
with builder.locked():
|
|
if options.step in ('rootfs', 'iso'):
|
|
raise BuildError(f'{options.step} assembly is not configured yet')
|
|
if options.step == 'packages':
|
|
builder.packages(options.pretend)
|
|
else:
|
|
getattr(builder, options.step)()
|
|
return 0
|
|
except (BuildError, OSError, ValueError) as exc:
|
|
print(f'hydra-build: {exc}', file=sys.stderr)
|
|
return 1
|