Emil_Shanaty/Hydra-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: assemble an installable Gentoo live image in an exported rootfs

Browse Source
This commit is contained in:
Codex
2026-10-13 04:32:18 +03:00
parent 38914cf67f
commit 0f71430b8e
32 changed files with 874 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hydra/build.py
+135 -9
View File
@@ -9,6 +9,7 @@ import os
from pathlib import Path
import re
import shutil
import shlex
import subprocess
import sys
import tempfile
@@ -190,10 +191,11 @@ if path.exists():
def input_id(self):
digest = hashlib.sha256()
for path in sorted((self.source_dir / 'image').rglob('*')):
if path.is_file():
digest.update(str(path.relative_to(self.source_dir)).encode())
digest.update(path.read_bytes())
for directory in ('image', 'patches'):
for path in sorted((self.source_dir / directory).rglob('*')):
if path.is_file():
digest.update(str(path.relative_to(self.source_dir)).encode())
digest.update(path.read_bytes())
return digest.hexdigest()
def record(self, step, **details):
@@ -209,6 +211,13 @@ if path.exists():
entry = self.manifest().get(step, {})
if not entry.get('complete') or entry.get('inputs') != self.input_id():
return False
if step in ('rootfs', 'iso'):
from . import image
if entry.get('source_inputs') != image.source_identity(self.source_dir):
return False
if not image.artifacts_valid(self.work / 'artifacts', entry):
return False
return self.complete('packages' if step == 'rootfs' else 'rootfs')
if step == 'packages':
if not (self.work / 'artifacts/packages.json').is_file():
return False
@@ -295,6 +304,9 @@ if [ ! -f /var/db/repos/gentoo/.hydra-snapshot ]; then
printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot
fi
cp -a /hydra-source/image/portage/. /etc/portage/
mkdir -p /etc/portage/patches/app-emulation/qemu /etc/portage/patches/app-emulation/virt-viewer
cp /hydra-source/patches/qemu-virtio-refresh-rate.patch /etc/portage/patches/app-emulation/qemu/10-hydra-refresh.patch
cp /hydra-source/patches/virt-viewer-hydra-session.patch /etc/portage/patches/app-emulation/virt-viewer/10-hydra-session.patch
if [ -d /etc/portage/gnupg ] && [ ! -s /etc/portage/gnupg/mykeyid ]; then
mv /etc/portage/gnupg /etc/portage/gnupg.hydra-untrusted
fi
@@ -311,19 +323,135 @@ chmod a+r /etc/portage/gnupg/pubring.kbx /etc/portage/gnupg/trustdb.gpg
atoms = [line.strip() for line in (self.source_dir / 'image/packages').read_text().splitlines()
if line.strip() and not line.startswith('#')]
import shlex
command = ['emerge', '--verbose', '--update', '--newuse', '--deep', '--backtrack=100']
command = ['emerge', '--verbose', '--update', '--newuse', '--deep', '--backtrack=100',
'--usepkg-exclude=app-emulation/qemu app-emulation/virt-viewer']
if pretend:
self.exec(shlex.join([*command, '--pretend', *atoms]))
return
inputs = self.input_id()
self.record('packages', complete=False, job_log='/var/lib/hydra-build/packages.log')
script = package_job_script([*command, *atoms], inputs=inputs)
patches = hashlib.sha256()
for name in ('qemu-virtio-refresh-rate.patch', 'virt-viewer-hydra-session.patch'):
patches.update((self.source_dir / 'patches' / name).read_bytes())
patch_id = patches.hexdigest()
forced = ['emerge', '--verbose', '--oneshot', '--usepkg=n', '--getbinpkg=n',
'app-emulation/qemu', 'app-emulation/virt-viewer']
package_commands = shlex.join([*command, *atoms]) + '\n' + f'''if [ "$(cat /var/lib/hydra-build/patched-source.sha256 2>/dev/null || true)" != {shlex.quote(patch_id)} ]; then
{shlex.join(forced)}
printf '%s\\n' {shlex.quote(patch_id)} > /var/lib/hydra-build/patched-source.sha256
fi'''
script = package_job_script(['/bin/bash', '-euc', package_commands], inputs=inputs)
self.exec(shlex.join(['python3', '-c', script]))
if self.input_id() != inputs:
raise BuildError('Image inputs changed during package installation; completion was not recorded')
inventory = self.exec('cat /var/lib/hydra-build/packages.json', capture=True).stdout
(self.work / 'artifacts/packages.json').write_text(inventory)
self.record('packages', inventory='artifacts/packages.json')
self.record('packages', inventory='artifacts/packages.json', patched_source=patch_id)
def assembly(self, snapshot_digest=None):
"""A separate rootless container processes an exported copy, never the builder."""
if snapshot_digest is None:
state = self.manifest().get('rootfs', {})
snapshot_digest = state.get('snapshot_sha256')
if not isinstance(snapshot_digest, str) or not re.fullmatch('[a-f0-9]{64}', snapshot_digest):
raise BuildError('Verified exported snapshot is required')
name = self.name + '-image-' + snapshot_digest[:12]
image = 'localhost/hydra-image:' + snapshot_digest
exists = self.pod('container', 'exists', name, capture=True, check=False).returncode == 0
if not exists:
self.pod('import', str(self.work / 'artifacts/builder-rootfs.tar'), image)
self.pod('run', '--detach', '--name', name, '--hostname', 'hydra-image-assembler',
'--network', 'none', '--cap-add', 'SYS_ADMIN', '--security-opt', 'seccomp=unconfined',
'--volume', f'{self.source_dir}:/hydra-source:ro',
'--volume', f'{self.work}:/hydra-output:rw', image, '/bin/sleep', 'infinity')
container = self.inspect('container', name)
for destination, source, writable in (('/hydra-source', self.source_dir, False),
('/hydra-output', self.work, True)):
mounts = [mount for mount in container.get('Mounts', []) if mount.get('Destination') == destination]
if len(mounts) != 1 or mounts[0].get('Source') != str(source) or mounts[0].get('RW') is not writable:
raise BuildError('Assembly mount provenance differs; refusing to write')
provenance = self.inspect('image', container['Image'])
if provenance.get('RootFS', {}).get('Layers') != ['sha256:' + snapshot_digest]:
raise BuildError('Assembly image does not match the exported builder snapshot')
if exists:
self.pod('start', name)
return name
def rootfs(self):
from . import image
if self.active_package_job():
raise BuildError('A contained package job is running; wait before export')
if not self.complete('packages'):
raise BuildError('Verified completed packages are required before rootfs export')
if self.complete('rootfs'):
return
source_inputs = image.source_identity(self.source_dir)
self.record('rootfs', complete=False)
original = self.work / 'artifacts/builder-rootfs.tar'
self.pod('export', '--output', str(original), self.name)
snapshot = image.digest(original)
assembler = self.assembly(snapshot)
commit = self.run(['git', '-C', str(self.source_dir), 'rev-parse', 'HEAD'], capture=True).stdout.strip()
script = r"""root=/hydra-output/rootfs
if [ -e "$root" ] || [ -L "$root" ]; then
[ ! -L "$root" ] && [ -f "$root/.hydra-export-owned" ] || { echo 'Unowned rootfs output' >&2; exit 1; }
rm -rf -- "$root"
fi
mkdir -p "$root"
touch "$root/.hydra-export-owned"
tar --extract --preserve-permissions --numeric-owner --file /hydra-output/artifacts/builder-rootfs.tar --directory "$root"
/hydra-source/image/configure-rootfs.sh "$root"
cp "$root/boot/vmlinuz-$(python3 /hydra-source/hydra/image.py kernel "$root")" /hydra-output/artifacts/vmlinuz
cp "$root/boot/initramfs-live.img" /hydra-output/artifacts/initramfs-live.img
cp "$root/usr/share/hydra/build-manifest.json" /hydra-output/artifacts/rootfs-manifest.json
# Keep ownership for safe retries outside the shipped payload.
mksquashfs "$root" /hydra-output/artifacts/squashfs.img -noappend -comp zstd -processors 4 -e boot/initramfs-live.img .hydra-export-owned
chmod 0644 /hydra-output/artifacts/vmlinuz /hydra-output/artifacts/initramfs-live.img /hydra-output/artifacts/squashfs.img /hydra-output/artifacts/rootfs-manifest.json
"""
self.pod('exec', '--env', f'HYDRA_SOURCE_COMMIT={commit}', assembler, '/bin/bash', '-euc', script)
if image.source_identity(self.source_dir) != source_inputs:
raise BuildError('Source inputs changed during rootfs assembly; completion not recorded')
manifest = json.loads((self.work / 'artifacts/rootfs-manifest.json').read_text())
if not manifest.get('packages') or manifest.get('source_files') != image.source_files(self.source_dir):
raise BuildError('Payload inventory/source manifest is incomplete or stale')
artifacts = {name: image.digest(self.work / 'artifacts' / name) for name in
('vmlinuz', 'initramfs-live.img', 'squashfs.img', 'rootfs-manifest.json')}
self.record('rootfs', source_inputs=source_inputs, snapshot_sha256=snapshot,
assembler=assembler, kernel=manifest['kernel'], artifacts=artifacts)
def iso(self):
from . import image
if not self.complete('rootfs'):
raise BuildError('Verified rootfs artifacts are required before ISO assembly')
if self.complete('iso'):
return
source_inputs = image.source_identity(self.source_dir)
assembler = self.assembly()
self.record('iso', complete=False)
script = r"""tree=/hydra-output/iso-tree
[ ! -L "$tree" ] || { echo 'ISO output cannot be a symlink' >&2; exit 1; }
mkdir -p "$tree/boot/grub" "$tree/LiveOS"
cp /hydra-output/artifacts/vmlinuz "$tree/boot/vmlinuz"
cp /hydra-output/artifacts/initramfs-live.img "$tree/boot/initramfs-live.img"
cp /hydra-output/artifacts/squashfs.img "$tree/LiveOS/squashfs.img"
cp /hydra-source/image/grub.cfg "$tree/boot/grub/grub.cfg"
cp /hydra-output/artifacts/rootfs-manifest.json "$tree/build-manifest.json"
cp /hydra-output/artifacts/packages.json "$tree/packages.json"
grub-mkrescue --directory=/usr/lib/grub/x86_64-efi --output=/hydra-output/artifacts/hydra-os.iso "$tree" -- -volid HYDRA_OS
xorriso -indev /hydra-output/artifacts/hydra-os.iso -report_el_torito plain -pvd_info > /hydra-output/artifacts/iso-inspection.txt 2>&1
chmod 0644 /hydra-output/artifacts/hydra-os.iso /hydra-output/artifacts/iso-inspection.txt
"""
self.pod('exec', assembler, '/bin/bash', '-euc', script)
inspection = (self.work / 'artifacts/iso-inspection.txt').read_text()
if 'HYDRA_OS' not in inspection or not re.search(r'El Torito boot img.*UEFI', inspection):
raise BuildError('ISO does not report the expected volume label and UEFI boot image')
if image.source_identity(self.source_dir) != source_inputs:
raise BuildError('Source inputs changed during ISO assembly; completion not recorded')
checksum = image.digest(self.work / 'artifacts/hydra-os.iso')
(self.work / 'artifacts/hydra-os.iso.sha256').write_text(checksum + ' hydra-os.iso\n')
artifacts = {name: image.digest(self.work / 'artifacts' / name) for name in
('hydra-os.iso', 'hydra-os.iso.sha256', 'iso-inspection.txt')}
self.record('iso', source_inputs=source_inputs, label=image.LABEL, artifacts=artifacts)
def status(self):
data = self.manifest()
@@ -380,8 +508,6 @@ def main(argv=None):
builder.status()
return 0
with builder.locked():
if options.step in ('rootfs', 'iso'):
raise BuildError(f'{options.step} assembly is not configured yet')
if options.step == 'packages':
builder.packages(options.pretend)
else: