Emil_Shanaty/Hydra-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Validate builder provenance and guard detached package jobs

Browse Source
This commit is contained in:
Codex
2026-10-13 04:22:23 +03:00
parent 440fe42464
commit 7a23508e0c
2 changed files with 374 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hydra/build.py
+168 -11
View File
@@ -4,8 +4,10 @@ import contextlib
import fcntl
import hashlib
import json
import lzma
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
@@ -18,6 +20,43 @@ class BuildError(RuntimeError):
pass
def package_job_script(command, state='/var/lib/hydra-build', package_db='/var/db/pkg', inputs=''):
"""The contained job owns its log/result even if the outer caller disconnects."""
return f'''import fcntl, json, os, pathlib, subprocess, sys, time
state = pathlib.Path({str(state)!r})
database = pathlib.Path({str(package_db)!r})
state.mkdir(parents=True, exist_ok=True)
lock = (state / 'packages.lock').open('a')
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
sys.exit(75)
def write(name, data):
temporary = state / (name + '.tmp')
temporary.write_text(json.dumps(data, indent=2) + '\\n')
temporary.replace(state / name)
status = {{'running': True, 'pid': os.getpid(), 'inputs': {inputs!r}, 'started': int(time.time())}}
write('packages.status.json', status)
with (state / 'packages.log').open('a') as log:
try:
result = subprocess.run({command!r}, stdout=log, stderr=subprocess.STDOUT)
code = result.returncode
except OSError as error:
log.write(str(error) + '\\n')
code = 127
if code == 0:
inventory = []
for use in sorted(database.glob('*/*/USE')):
package = use.parent
inventory.append({{'package': str(package.relative_to(database)), 'use': use.read_text().split(),
'build_time': (package / 'BUILD_TIME').read_text().strip() if (package / 'BUILD_TIME').exists() else None}})
write('packages.json', inventory)
status.update(running=False, exit_code=code, finished=int(time.time()))
write('packages.status.json', status)
sys.exit(code)
'''
def verify_hash(path, expected):
digest = hashlib.sha256()
with Path(path).open('rb') as stream:
@@ -54,6 +93,7 @@ class Builder:
self.runroot = runtime / ('hydra-build-podman' if self.work == self.project / 'build' else f'hydra-build-{identifier}')
self.name = 'hydra-builder-' + identifier
self.image = 'localhost/hydra-stage3:20261011'
self._stage3_layers = {}
def podman_argv(self, *args):
return [self.podman, '--root', str(self.work / 'podman'), '--runroot', str(self.runroot), '--storage-driver', 'overlay', *args]
@@ -65,7 +105,7 @@ class Builder:
self.image, '/bin/sleep', 'infinity']
def run(self, argv, capture=False, check=True):
print('+ ' + ' '.join(str(arg) for arg in argv), flush=True)
print('+ ' + ' '.join(str(arg) for arg in argv), file=sys.stderr, flush=True)
result = subprocess.run(argv, text=True, stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None)
if check and result.returncode:
@@ -78,6 +118,72 @@ class Builder:
def exec(self, command, capture=False):
return self.pod('exec', self.name, '/bin/bash', '-euc', command, capture=capture)
def inspect(self, kind, name):
result = self.pod(kind, 'inspect', name, capture=True, check=False)
if result.returncode:
raise BuildError(f'{kind} provenance is unavailable; use a fresh work directory')
entries = json.loads(result.stdout)
if not isinstance(entries, list) or len(entries) != 1 or not isinstance(entries[0], dict):
raise BuildError(f'Unexpected {kind} provenance')
return entries[0]
def check_mounts(self, container):
for destination, path, label in (('/hydra-source', self.source_dir, 'source'),
('/hydra-cache', self.cache, 'cache')):
mounts = [item for item in container.get('Mounts', []) if item.get('Destination') == destination]
if len(mounts) != 1 or mounts[0].get('Source') != str(path) or mounts[0].get('RW') is not False:
raise BuildError(f'Existing builder {label} mount differs; use its original checkout/cache or a fresh work directory')
def stage3_layer(self, inputs):
stage = inputs['stage3']
archive = self.cache / Path(urllib.parse.urlparse(stage['url']).path).name
verify_hash(archive, stage['sha256'])
if stage['sha256'] not in self._stage3_layers:
digest = hashlib.sha256()
with lzma.open(archive, 'rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
digest.update(chunk)
self._stage3_layers[stage['sha256']] = 'sha256:' + digest.hexdigest()
return self._stage3_layers[stage['sha256']]
def check_base(self, image, inputs):
expected = self.stage3_layer(inputs)
if image.get('RootFS', {}).get('Layers') != [expected]:
raise BuildError('Existing builder stage3 differs from pinned archive; use a fresh work directory')
return expected
def read_container_file(self, path):
# Podman cp also reads a stopped container, so provenance precedes start.
with tempfile.TemporaryDirectory(prefix='hydra-read-') as directory:
target = Path(directory) / 'data'
result = self.pod('cp', f'{self.name}:{path}', str(target), capture=True, check=False)
return target.read_text() if result.returncode == 0 else None
def check_snapshot(self, inputs, allow_pending=False):
marker = self.read_container_file('/var/db/repos/gentoo/.hydra-snapshot')
if marker is None and allow_pending:
marker = self.read_container_file('/var/db/repos/gentoo/.hydra-snapshot-input')
if marker is None or marker.strip() != inputs['portage']['commit']:
raise BuildError('Existing builder snapshot differs or is unverified; use a fresh work directory')
def active_package_job(self):
if not (self.work / 'podman').is_dir():
return False
result = self.pod('top', self.name, 'hpid', 'args', capture=True, check=False)
if result.returncode == 0 and any(re.search(r'(?:^|\s)(?:\S*/)?emerge(?:\s|$)', line)
for line in result.stdout.splitlines()[1:]):
return True
probe = self.pod('exec', self.name, 'python3', '-c', '''import fcntl, pathlib, sys
path = pathlib.Path('/var/lib/hydra-build/packages.lock')
if path.exists():
with path.open('r') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
except BlockingIOError:
sys.exit(75)
''', capture=True, check=False)
return probe.returncode == 75
def manifest(self):
path = self.work / 'manifest.json'
return json.loads(path.read_text()) if path.exists() else {}
@@ -106,9 +212,17 @@ class Builder:
if step == 'packages':
if not (self.work / 'artifacts/packages.json').is_file():
return False
return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
return self.complete('prepare')
if step == 'prepare':
return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
try:
container = self.inspect('container', self.name)
self.check_mounts(container)
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
self.check_base(self.inspect('image', container['Image']), inputs)
self.check_snapshot(inputs)
return True
except (BuildError, OSError, ValueError, KeyError, lzma.LZMAError):
return False
if step == 'fetch':
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
try:
@@ -153,10 +267,22 @@ class Builder:
return inputs
def prepare(self):
existing = self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
container = self.inspect('container', self.name) if existing else None
if container is not None:
self.check_mounts(container) # Reject another worktree before changing any checkpoint.
if self.active_package_job():
raise BuildError('A contained package job is running; wait for it to finish before preparation')
inputs = self.fetch() # Reverify cached bytes even when resuming.
if self.pod('image', 'exists', self.image, capture=True, check=False).returncode:
if container is not None:
layer = self.check_base(self.inspect('image', container['Image']), inputs)
self.check_snapshot(inputs, allow_pending=True)
elif self.pod('image', 'exists', self.image, capture=True, check=False).returncode:
self.pod('import', str(self.cache / Path(urllib.parse.urlparse(inputs['stage3']['url']).path).name), self.image)
if self.pod('container', 'exists', self.name, capture=True, check=False).returncode:
layer = self.check_base(self.inspect('image', self.image), inputs)
else:
layer = self.check_base(self.inspect('image', self.image), inputs)
if container is None:
self.pod(*self.container_run_args())
else:
self.pod('start', self.name)
@@ -164,6 +290,7 @@ class Builder:
# Snapshot extraction happens only inside the container; host paths are read-only.
self.exec(f'''mkdir -p /var/db/repos/gentoo
if [ ! -f /var/db/repos/gentoo/.hydra-snapshot ]; then
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot-input
tar --extract --gzip --file /hydra-cache/{snapshot} --strip-components=1 --directory /var/db/repos/gentoo
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot
fi
@@ -175,23 +302,50 @@ getuto
gpg --batch --homedir /etc/portage/gnupg --import /hydra-cache/service-keys.gpg
chmod a+r /etc/portage/gnupg/pubring.kbx /etc/portage/gnupg/trustdb.gpg
''')
self.record('prepare', image=self.image, snapshot=inputs['portage']['commit'])
self.record('prepare', image=container['Image'] if container else self.image,
stage3_sha256=inputs['stage3']['sha256'], stage3_layer=layer,
source_dir=str(self.source_dir), cache_dir=str(self.cache), snapshot=inputs['portage']['commit'])
def packages(self, pretend=False):
self.prepare()
atoms = [line.strip() for line in (self.source_dir / 'image/packages').read_text().splitlines()
if line.strip() and not line.startswith('#')]
import shlex
command = 'emerge --verbose --update --newuse --deep --backtrack=100 ' + ('--pretend ' if pretend else '') + ' '.join(map(shlex.quote, atoms))
self.exec(command)
command = ['emerge', '--verbose', '--update', '--newuse', '--deep', '--backtrack=100']
if pretend:
self.exec(shlex.join([*command, '--pretend', *atoms]))
return
inventory = self.exec('python3 - <<\'PY\'\nimport json,pathlib\nout=[]\nfor use in sorted(pathlib.Path("/var/db/pkg").glob("*/*/USE")):\n p=use.parent\n out.append({"package":str(p.relative_to("/var/db/pkg")),"use":use.read_text().split(),"build_time":(p/"BUILD_TIME").read_text().strip() if (p/"BUILD_TIME").exists() else None})\nprint(json.dumps(out,indent=2))\nPY', capture=True).stdout
inputs = self.input_id()
self.record('packages', complete=False, job_log='/var/lib/hydra-build/packages.log')
script = package_job_script([*command, *atoms], inputs=inputs)
self.exec(shlex.join(['python3', '-c', script]))
if self.input_id() != inputs:
raise BuildError('Image inputs changed during package installation; completion was not recorded')
inventory = self.exec('cat /var/lib/hydra-build/packages.json', capture=True).stdout
(self.work / 'artifacts/packages.json').write_text(inventory)
self.record('packages', inventory='artifacts/packages.json')
def status(self):
print(json.dumps(self.manifest(), indent=2))
data = self.manifest()
for step in ('fetch', 'prepare', 'packages', 'rootfs', 'iso'):
if step in data:
data[step]['recorded_complete'] = data[step].get('complete', False)
data[step]['complete'] = self.complete(step)
data['mutation_lock_held'] = self.build_running()
data['package_job_running'] = self.active_package_job()
data['running'] = data['mutation_lock_held'] or data['package_job_running']
print(json.dumps(data, indent=2))
def build_running(self):
try:
with (self.work / '.hydra-build.lock').open('r') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
except BlockingIOError:
return True
except FileNotFoundError:
pass
return False
@contextlib.contextmanager
def locked(self):
@@ -222,6 +376,9 @@ def main(argv=None):
options = parser.parse_args(argv)
try:
builder = Builder(project, options.work, options.cache, options.podman, options.gpg, source)
if options.step == 'status':
builder.status()
return 0
with builder.locked():
if options.step in ('rootfs', 'iso'):
raise BuildError(f'{options.step} assembly is not configured yet')
@@ -230,6 +387,6 @@ def main(argv=None):
else:
getattr(builder, options.step)()
return 0
except (BuildError, OSError, ValueError) as exc:
except (BuildError, OSError, ValueError, lzma.LZMAError) as exc:
print(f'hydra-build: {exc}', file=sys.stderr)
return 1