Add verified rootless Gentoo image builder and package inputs

This commit is contained in:
Codex
2026-10-13 04:12:00 +03:00
parent e6bec555c4
commit 440fe42464
11 changed files with 478 additions and 0 deletions
+235
View File
@@ -0,0 +1,235 @@
"""Verified, resumable Gentoo builds in a dedicated rootless Podman store."""
import argparse
import contextlib
import fcntl
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
import urllib.request
class BuildError(RuntimeError):
pass
def verify_hash(path, expected):
digest = hashlib.sha256()
with Path(path).open('rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
digest.update(chunk)
if digest.hexdigest() != expected:
raise BuildError(f'SHA256 mismatch: {path}')
def require_signature(status, fingerprint):
valid = [line.split() for line in status.splitlines() if line.startswith('[GNUPG:] VALIDSIG ')]
if not valid or not any(fields[-1].upper() == fingerprint.upper() for fields in valid):
raise BuildError('Required primary signature fingerprint is missing')
if any(token in status for token in ('BADSIG', 'ERRSIG', 'REVKEYSIG', 'EXPKEYSIG')):
raise BuildError('Invalid release signature')
class Builder:
def __init__(self, project, work, cache, podman=None, gpg=None, source_dir=None):
if os.geteuid() == 0:
raise BuildError('Run as the normal user, never host root')
self.project = Path(project).resolve()
self.source_dir = Path(source_dir or project).resolve()
self.work, self.cache = Path(work).resolve(), Path(cache).resolve()
for path in (self.work, self.cache):
if path == self.project or not path.is_relative_to(self.project):
raise BuildError('Work and cache paths must be inside the project, below its root')
if self.work == self.cache or self.work.is_relative_to(self.cache) or self.cache.is_relative_to(self.work):
raise BuildError('Work and cache must be separate directories')
self.podman = podman or shutil.which('podman') or 'podman'
self.gpg = gpg or shutil.which('gpg') or 'gpg'
runtime = Path(os.environ.get('XDG_RUNTIME_DIR', f'/run/user/{os.getuid()}'))
identifier = hashlib.sha256(str(self.work).encode()).hexdigest()[:12]
self.runroot = runtime / ('hydra-build-podman' if self.work == self.project / 'build' else f'hydra-build-{identifier}')
self.name = 'hydra-builder-' + identifier
self.image = 'localhost/hydra-stage3:20261011'
def podman_argv(self, *args):
return [self.podman, '--root', str(self.work / 'podman'), '--runroot', str(self.runroot), '--storage-driver', 'overlay', *args]
def container_run_args(self):
return ['run', '--detach', '--name', self.name, '--hostname', 'hydra-builder',
'--network', 'host', '--cap-add', 'SYS_ADMIN', '--security-opt', 'seccomp=unconfined',
'--volume', f'{self.source_dir}:/hydra-source:ro', '--volume', f'{self.cache}:/hydra-cache:ro',
self.image, '/bin/sleep', 'infinity']
def run(self, argv, capture=False, check=True):
print('+ ' + ' '.join(str(arg) for arg in argv), flush=True)
result = subprocess.run(argv, text=True, stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None)
if check and result.returncode:
raise BuildError(f'Command failed ({result.returncode}): {argv[0]}\n{result.stderr or ""}')
return result
def pod(self, *args, **kwargs):
return self.run(self.podman_argv(*args), **kwargs)
def exec(self, command, capture=False):
return self.pod('exec', self.name, '/bin/bash', '-euc', command, capture=capture)
def manifest(self):
path = self.work / 'manifest.json'
return json.loads(path.read_text()) if path.exists() else {}
def input_id(self):
digest = hashlib.sha256()
for path in sorted((self.source_dir / 'image').rglob('*')):
if path.is_file():
digest.update(str(path.relative_to(self.source_dir)).encode())
digest.update(path.read_bytes())
return digest.hexdigest()
def record(self, step, **details):
data = self.manifest()
data[step] = {'complete': True, 'inputs': self.input_id(), **details}
data['builder'] = self.name
path = self.work / 'manifest.json'
temp = path.with_suffix('.tmp')
temp.write_text(json.dumps(data, indent=2) + '\n')
temp.replace(path)
def complete(self, step):
entry = self.manifest().get(step, {})
if not entry.get('complete') or entry.get('inputs') != self.input_id():
return False
if step == 'packages':
if not (self.work / 'artifacts/packages.json').is_file():
return False
return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
if step == 'prepare':
return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
if step == 'fetch':
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
try:
for name in ('stage3', 'portage'):
item = inputs[name]
path = self.cache / item.get('filename', Path(urllib.parse.urlparse(item['url']).path).name)
verify_hash(path, item['sha256'])
except (OSError, BuildError):
return False
return True
return False
def download(self, url, filename=None):
parsed = urllib.parse.urlparse(url)
if parsed.scheme != 'https':
raise BuildError('Build input URLs must use HTTPS')
target = self.cache / (filename or Path(parsed.path).name)
if not target.exists():
temp = target.with_suffix(target.suffix + '.partial')
print(f'Downloading {url}', flush=True)
with urllib.request.urlopen(url, timeout=120) as response, temp.open('wb') as stream:
shutil.copyfileobj(response, stream)
temp.replace(target)
return target
def fetch(self):
inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
stage = inputs['stage3']
archive = self.download(stage['url'])
verify_hash(archive, stage['sha256'])
snapshot = self.download(inputs['portage']['url'], inputs['portage']['filename'])
verify_hash(snapshot, inputs['portage']['sha256'])
signature = self.download(stage['signature_url'])
keyring = self.download(inputs['keyring_url'])
verify_hash(keyring, inputs['keyring_sha256'])
with tempfile.TemporaryDirectory(prefix='hydra-gpg-', dir=self.work) as home:
result = self.run([self.gpg, '--batch', '--homedir', home, '--no-default-keyring', '--keyring', str(keyring),
'--status-fd', '1', '--verify', str(signature), str(archive)], capture=True)
require_signature(result.stdout, stage['fingerprint'])
(self.work / 'artifacts/stage3-signature.txt').write_text(result.stdout + result.stderr)
self.record('fetch', sources=inputs)
return inputs
def prepare(self):
inputs = self.fetch() # Reverify cached bytes even when resuming.
if self.pod('image', 'exists', self.image, capture=True, check=False).returncode:
self.pod('import', str(self.cache / Path(urllib.parse.urlparse(inputs['stage3']['url']).path).name), self.image)
if self.pod('container', 'exists', self.name, capture=True, check=False).returncode:
self.pod(*self.container_run_args())
else:
self.pod('start', self.name)
snapshot = inputs['portage']['filename']
# Snapshot extraction happens only inside the container; host paths are read-only.
self.exec(f'''mkdir -p /var/db/repos/gentoo
if [ ! -f /var/db/repos/gentoo/.hydra-snapshot ]; then
tar --extract --gzip --file /hydra-cache/{snapshot} --strip-components=1 --directory /var/db/repos/gentoo
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot
fi
cp -a /hydra-source/image/portage/. /etc/portage/
if [ -d /etc/portage/gnupg ] && [ ! -s /etc/portage/gnupg/mykeyid ]; then
mv /etc/portage/gnupg /etc/portage/gnupg.hydra-untrusted
fi
getuto
gpg --batch --homedir /etc/portage/gnupg --import /hydra-cache/service-keys.gpg
chmod a+r /etc/portage/gnupg/pubring.kbx /etc/portage/gnupg/trustdb.gpg
''')
self.record('prepare', image=self.image, snapshot=inputs['portage']['commit'])
def packages(self, pretend=False):
self.prepare()
atoms = [line.strip() for line in (self.source_dir / 'image/packages').read_text().splitlines()
if line.strip() and not line.startswith('#')]
import shlex
command = 'emerge --verbose --update --newuse --deep --backtrack=100 ' + ('--pretend ' if pretend else '') + ' '.join(map(shlex.quote, atoms))
self.exec(command)
if pretend:
return
inventory = self.exec('python3 - <<\'PY\'\nimport json,pathlib\nout=[]\nfor use in sorted(pathlib.Path("/var/db/pkg").glob("*/*/USE")):\n p=use.parent\n out.append({"package":str(p.relative_to("/var/db/pkg")),"use":use.read_text().split(),"build_time":(p/"BUILD_TIME").read_text().strip() if (p/"BUILD_TIME").exists() else None})\nprint(json.dumps(out,indent=2))\nPY', capture=True).stdout
(self.work / 'artifacts/packages.json').write_text(inventory)
self.record('packages', inventory='artifacts/packages.json')
def status(self):
print(json.dumps(self.manifest(), indent=2))
@contextlib.contextmanager
def locked(self):
self.work.mkdir(parents=True, exist_ok=True)
self.cache.mkdir(parents=True, exist_ok=True)
(self.work / 'artifacts').mkdir(exist_ok=True)
with (self.work / '.hydra-build.lock').open('a') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
raise BuildError('Another Hydra build is running in this work directory') from exc
yield
def main(argv=None):
source = Path(__file__).resolve().parents[1]
# Linked worktrees share project-owned caches/build state in the primary checkout.
common = subprocess.run(['git', '-C', str(source), 'rev-parse', '--path-format=absolute', '--git-common-dir'],
text=True, capture_output=True)
project = Path(common.stdout.strip()).parent if common.returncode == 0 else source
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('step', choices=('fetch', 'prepare', 'packages', 'rootfs', 'iso', 'status'))
parser.add_argument('--work', type=Path, default=project / 'build')
parser.add_argument('--cache', type=Path, default=project / 'cache')
parser.add_argument('--podman')
parser.add_argument('--gpg')
parser.add_argument('--pretend', action='store_true', help='resolve packages without installing')
options = parser.parse_args(argv)
try:
builder = Builder(project, options.work, options.cache, options.podman, options.gpg, source)
with builder.locked():
if options.step in ('rootfs', 'iso'):
raise BuildError(f'{options.step} assembly is not configured yet')
if options.step == 'packages':
builder.packages(options.pretend)
else:
getattr(builder, options.step)()
return 0
except (BuildError, OSError, ValueError) as exc:
print(f'hydra-build: {exc}', file=sys.stderr)
return 1
+38
View File
@@ -0,0 +1,38 @@
HydraOS builds start from the pinned amd64 systemd stage3 and pinned Gentoo
repository in `sources.json`. Run the builder as the normal user with Python 3,
Podman and GnuPG available; temporary tool environments are sufficient. It
refuses host root and work/cache paths outside the primary project checkout.
```
scripts/hydra-build fetch
scripts/hydra-build prepare
scripts/hydra-build packages --pretend
scripts/hydra-build packages
scripts/hydra-build status
```
`--work PATH`, `--cache PATH`, `--podman PATH` and `--gpg PATH` override defaults.
Linked worktrees use the primary checkout's `build/` and `cache/`. The build
lock prevents simultaneous writes in the same work directory. Interrupted
package installation resumes through Portage; successful steps write
`build/manifest.json`, signature evidence and the installed package/version/USE
inventory in `build/artifacts/`. Downloaded inputs are reverified on every
prepare; missing or changed files cannot be trusted from a checkpoint alone.
The dedicated rootless Podman store is `build/podman`, separate from default
Podman and Distrobox storage. The build container mounts inputs and repository
read-only and retains its writable layer across failures. Host networking makes
the existing local VPN/proxy available. `SYS_ADMIN` operates within the rootless
user namespace; package sandbox features incompatible with it are disabled.
No host disks or `/dev/kvm` are passed into the builder. Hydra never prunes
containers/images or changes host services.
Official Gentoo binpkgs require signatures. USE mismatches and selective testing
keywords can cause source builds; concurrency is four compiler processes and
one emerge job. Keep licenses and Portage's `/var/db/pkg`, world and repository
for installed-system updates. Rolling binhost contents can change; resulting
versions/USE are recorded without claiming bit-identical output. The pinned
service-keyring hash may need deliberate review when Gentoo rotates keys.
`rootfs` and `iso` are reserved assembly commands and currently return an
explicit error until the live-image assembly implementation is integrated.
+30
View File
@@ -0,0 +1,30 @@
# Host, boot medium, and recovery tools. Keep Portage's installed database/world.
app-emulation/qemu
~sys-firmware/edk2-bin-202408
>=gui-wm/labwc-0.20.0
app-emulation/virt-viewer
gui-apps/foot
media-fonts/dejavu
gui-libs/greetd
net-misc/networkmanager
net-dns/dnsmasq
net-firewall/nftables
media-video/pipewire
media-video/wireplumber
app-admin/sudo
sys-kernel/gentoo-kernel-bin
sys-kernel/linux-firmware
sys-firmware/sof-firmware
sys-firmware/intel-microcode
sys-kernel/dracut
sys-boot/grub
sys-fs/squashfs-tools
sys-fs/dosfstools
sys-fs/e2fsprogs
sys-block/parted
sys-apps/util-linux
sys-fs/mtools
dev-libs/libisoburn
dev-lang/python
sec-keys/openpgp-keys-gentoo-release
sec-keys/openpgp-keys-gentoo-auth
+4
View File
@@ -0,0 +1,4 @@
[gentoo]
sync-uri = https://distfiles.gentoo.org/releases/amd64/binpackages/23.0/x86-64/
priority = 10
verify-signature = yes
+16
View File
@@ -0,0 +1,16 @@
COMMON_FLAGS="-O2 -pipe -march=x86-64 -mtune=generic"
CFLAGS="${COMMON_FLAGS}"
CXXFLAGS="${COMMON_FLAGS}"
FCFLAGS="${COMMON_FLAGS}"
FFLAGS="${COMMON_FLAGS}"
MAKEOPTS="-j4"
EMERGE_DEFAULT_OPTS="--jobs=1 --getbinpkg --binpkg-respect-use=y --autounmask=n"
FEATURES="getbinpkg binpkg-request-signature -usersandbox -sandbox -network-sandbox -ipc-sandbox -pid-sandbox"
ACCEPT_LICENSE="@FREE @BINARY-REDISTRIBUTABLE linux-fw-redistributable no-source-code Intel-SDP"
VIDEO_CARDS="intel"
USE="X dbus wayland opengl alsa"
QEMU_SOFTMMU_TARGETS="x86_64"
QEMU_USER_TARGETS=""
GRUB_PLATFORMS="efi-64"
BINPKG_GPG_VERIFY_GPG_HOME="/etc/portage/gnupg"
PORTAGE_NICENESS="10"
@@ -0,0 +1,2 @@
>=gui-wm/labwc-0.20.0 ~amd64
gui-libs/wlroots ~amd64
+8
View File
@@ -0,0 +1,8 @@
app-emulation/qemu spice usb usbredir virgl opengl wayland pipewire -gtk -sdl
app-emulation/virt-viewer spice -libvirt
net-misc/spice-gtk usbredir
media-video/pipewire sound-server bluetooth dbus pipewire-alsa
net-misc/networkmanager wifi
sys-boot/grub mount
media-libs/freetype harfbuzz
sys-kernel/installkernel dracut
+5
View File
@@ -0,0 +1,5 @@
[gentoo]
location = /var/db/repos/gentoo
sync-type = git
sync-uri = https://github.com/gentoo/gentoo.git
auto-sync = no
+17
View File
@@ -0,0 +1,17 @@
{
"stage3": {
"url": "https://distfiles.gentoo.org/releases/amd64/autobuilds/20261011T170049Z/stage3-amd64-systemd-20261011T170049Z.tar.xz",
"sha256": "50634e8cb87384b94a6dbf78cd2707766c45a5a98f3823e7287d114ee998970b",
"signature_url": "https://distfiles.gentoo.org/releases/amd64/autobuilds/20261011T170049Z/stage3-amd64-systemd-20261011T170049Z.tar.xz.asc",
"fingerprint": "13EBBDBEDE7A12775DFDB1BABB572E0E2D182910"
},
"portage": {
"url": "https://github.com/gentoo/gentoo/archive/4696089e782c20b5f0c04952f6698d6a8cc1acb4.tar.gz",
"filename": "gentoo-4696089e782c20b5f0c04952f6698d6a8cc1acb4.tar.gz",
"sha256": "d24b2056f7ae26506193374b0d9f25c82d4bf547e0ee17279940169418bd3bca",
"commit": "4696089e782c20b5f0c04952f6698d6a8cc1acb4"
},
"keyring_url": "https://qa-reports.gentoo.org/output/service-keys.gpg",
"keyring_sha256": "9fe40114c956c4c4d593f6e961ac9a994b0772863f4e685608fbdd8d77e39748",
"binhost": "https://distfiles.gentoo.org/releases/amd64/binpackages/23.0/x86-64/"
}
+9
View File
@@ -0,0 +1,9 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from hydra.build import main
if __name__ == '__main__':
raise SystemExit(main())
+114
View File
@@ -0,0 +1,114 @@
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
class BuilderTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.project = Path(self.tmp.name)
spec = importlib.util.spec_from_file_location('hydra_build', Path(__file__).parents[1] / 'hydra/build.py')
self.assertIsNotNone(spec, 'builder module is required')
self.assertTrue(Path(spec.origin).is_file(), 'builder implementation is missing')
self.module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(self.module)
def tearDown(self):
self.tmp.cleanup()
def builder(self):
return self.module.Builder(self.project, self.project / 'build', self.project / 'cache', podman='podman')
def test_bad_cached_hash_is_rejected(self):
artifact = self.project / 'archive'
artifact.write_bytes(b'tampered')
with self.assertRaisesRegex(self.module.BuildError, 'SHA256'):
self.module.verify_hash(artifact, hashlib.sha256(b'authentic').hexdigest())
def test_wrong_primary_signature_is_rejected(self):
status = '[GNUPG:] VALIDSIG ABC 2026-10-13 123 0 4 0 1 10 00 DEADBEEF\n'
with self.assertRaisesRegex(self.module.BuildError, 'fingerprint'):
self.module.require_signature(status, '13EBBDBEDE7A12775DFDB1BABB572E0E2D182910')
def test_signature_accepts_only_successful_matching_primary(self):
fingerprint = '13EBBDBEDE7A12775DFDB1BABB572E0E2D182910'
self.module.require_signature('[GNUPG:] VALIDSIG ABC 2026-10-13 123 0 4 0 1 10 00 ' + fingerprint, fingerprint)
with self.assertRaises(self.module.BuildError):
self.module.require_signature('[GNUPG:] BADSIG ABC test', fingerprint)
def test_refuses_root_and_outside_project(self):
with patch.object(self.module.os, 'geteuid', return_value=0):
with self.assertRaisesRegex(self.module.BuildError, 'root'):
self.builder()
with patch.object(self.module.os, 'geteuid', return_value=1000):
with self.assertRaisesRegex(self.module.BuildError, 'project'):
self.module.Builder(self.project, self.project.parent / 'unsafe', self.project / 'cache')
def test_podman_store_is_scoped_to_work(self):
argv = self.builder().podman_argv('container', 'exists', 'builder')
self.assertEqual(argv[0], 'podman')
self.assertEqual(argv[argv.index('--root') + 1], str(self.project / 'build/podman'))
self.assertIn('--runroot', argv)
self.assertEqual(argv[-3:], ['container', 'exists', 'builder'])
def test_resume_requires_actual_inventory(self):
builder = self.builder()
builder.work.mkdir()
(builder.work / 'manifest.json').write_text(json.dumps({'packages': {'complete': True}}))
self.assertFalse(builder.complete('packages'))
def test_symlink_escape_is_rejected(self):
(self.project / 'escape').symlink_to(self.project.parent, target_is_directory=True)
with self.assertRaisesRegex(self.module.BuildError, 'project'):
self.module.Builder(self.project, self.project / 'escape/unsafe', self.project / 'cache')
def test_changed_configuration_invalidates_fetch_checkpoint(self):
image = self.project / 'image'
image.mkdir()
config = image / 'sources.json'
config.write_text(json.dumps({'stage3': {'url': 'https://example.test/stage.tar', 'sha256': hashlib.sha256(b'good').hexdigest()}, 'portage': {'url': 'https://example.test/repo.tar', 'sha256': hashlib.sha256(b'repo').hexdigest()}}))
builder = self.builder()
with builder.locked():
(builder.cache / 'stage.tar').write_bytes(b'good')
(builder.cache / 'repo.tar').write_bytes(b'repo')
builder.record('fetch')
self.assertTrue(builder.complete('fetch'))
config.write_text('{"changed":true}')
self.assertFalse(builder.complete('fetch'))
def test_missing_fetch_archive_invalidates_checkpoint(self):
image = self.project / 'image'
image.mkdir()
(image / 'sources.json').write_text(json.dumps({'stage3': {'url': 'https://example.test/stage.tar', 'sha256': hashlib.sha256(b'good').hexdigest()}, 'portage': {'url': 'https://example.test/repo.tar', 'sha256': hashlib.sha256(b'repo').hexdigest()}}))
builder = self.builder()
with builder.locked():
builder.record('fetch')
self.assertFalse(builder.complete('fetch'))
def test_second_builder_cannot_share_active_work(self):
first, second = self.builder(), self.builder()
with first.locked():
with self.assertRaisesRegex(self.module.BuildError, 'running'):
with second.locked():
self.fail('overlapping build was allowed')
def test_container_uses_host_proxy_network_and_readonly_inputs(self):
builder = self.builder()
args = builder.container_run_args()
self.assertEqual(args[args.index('--network') + 1], 'host')
self.assertIn(str(self.project) + ':/hydra-source:ro', args)
self.assertIn(str(self.project / 'cache') + ':/hydra-cache:ro', args)
self.assertNotIn('--privileged', args)
def test_distinct_workspaces_have_distinct_podman_runtime_storage(self):
other = self.module.Builder(self.project, self.project / 'other-build', self.project / 'cache')
self.assertNotEqual(self.builder().runroot, other.runroot)
if __name__ == '__main__':
unittest.main()