Emil_Shanaty/Hydra-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: preserve installed helpers, adaptations and inventory provenance

Browse Source
This commit is contained in:
Codex
2026-10-13 04:40:18 +03:00
parent 0f71430b8e
commit de1a1698a9
5 changed files with 229 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hydra/build.py
+33 -10
View File
@@ -212,14 +212,16 @@ if path.exists():
if not entry.get('complete') or entry.get('inputs') != self.input_id():
return False
if step in ('rootfs', 'iso'):
from . import image
from hydra import image
if entry.get('source_inputs') != image.source_identity(self.source_dir):
return False
if not image.artifacts_valid(self.work / 'artifacts', entry):
return False
return self.complete('packages' if step == 'rootfs' else 'rootfs')
if step == 'packages':
if not (self.work / 'artifacts/packages.json').is_file():
try:
self.package_inventory()
except (BuildError, OSError):
return False
return self.complete('prepare')
if step == 'prepare':
@@ -244,6 +246,16 @@ if path.exists():
return True
return False
def package_inventory(self):
from hydra import image
checksum = self.manifest().get('packages', {}).get('inventory_sha256')
if not isinstance(checksum, str) or not re.fullmatch('[a-f0-9]{64}', checksum):
raise BuildError('Package inventory has no recorded checksum')
try:
return image.read_package_inventory(self.work / 'artifacts/packages.json', checksum)
except (image.ImageError, OSError) as error:
raise BuildError(f'Package inventory validation failed: {error}') from error
def download(self, url, filename=None):
parsed = urllib.parse.urlparse(url)
if parsed.scheme != 'https':
@@ -345,8 +357,17 @@ fi'''
if self.input_id() != inputs:
raise BuildError('Image inputs changed during package installation; completion was not recorded')
inventory = self.exec('cat /var/lib/hydra-build/packages.json', capture=True).stdout
(self.work / 'artifacts/packages.json').write_text(inventory)
self.record('packages', inventory='artifacts/packages.json', patched_source=patch_id)
from hydra import image
try:
image.validate_package_inventory(json.loads(inventory))
except (ValueError, image.ImageError) as error:
raise BuildError(f'Package inventory is malformed: {error}') from error
target = self.work / 'artifacts/packages.json'
temporary = target.with_suffix('.tmp')
temporary.write_text(inventory)
temporary.replace(target)
self.record('packages', inventory='artifacts/packages.json', patched_source=patch_id,
inventory_sha256=image.digest(target))
def assembly(self, snapshot_digest=None):
"""A separate rootless container processes an exported copy, never the builder."""
@@ -378,7 +399,7 @@ fi'''
return name
def rootfs(self):
from . import image
from hydra import image
if self.active_package_job():
raise BuildError('A contained package job is running; wait before export')
if not self.complete('packages'):
@@ -412,15 +433,17 @@ chmod 0644 /hydra-output/artifacts/vmlinuz /hydra-output/artifacts/initramfs-liv
if image.source_identity(self.source_dir) != source_inputs:
raise BuildError('Source inputs changed during rootfs assembly; completion not recorded')
manifest = json.loads((self.work / 'artifacts/rootfs-manifest.json').read_text())
if not manifest.get('packages') or manifest.get('source_files') != image.source_files(self.source_dir):
raise BuildError('Payload inventory/source manifest is incomplete or stale')
if manifest.get('packages') != self.package_inventory():
raise BuildError('Payload package inventory differs from the validated package checkpoint')
if manifest.get('source_files') != image.source_files(self.source_dir):
raise BuildError('Payload source manifest is incomplete or stale')
artifacts = {name: image.digest(self.work / 'artifacts' / name) for name in
('vmlinuz', 'initramfs-live.img', 'squashfs.img', 'rootfs-manifest.json')}
('vmlinuz', 'initramfs-live.img', 'squashfs.img', 'rootfs-manifest.json', 'packages.json')}
self.record('rootfs', source_inputs=source_inputs, snapshot_sha256=snapshot,
assembler=assembler, kernel=manifest['kernel'], artifacts=artifacts)
def iso(self):
from . import image
from hydra import image
if not self.complete('rootfs'):
raise BuildError('Verified rootfs artifacts are required before ISO assembly')
if self.complete('iso'):
@@ -450,7 +473,7 @@ chmod 0644 /hydra-output/artifacts/hydra-os.iso /hydra-output/artifacts/iso-insp
checksum = image.digest(self.work / 'artifacts/hydra-os.iso')
(self.work / 'artifacts/hydra-os.iso.sha256').write_text(checksum + ' hydra-os.iso\n')
artifacts = {name: image.digest(self.work / 'artifacts' / name) for name in
('hydra-os.iso', 'hydra-os.iso.sha256', 'iso-inspection.txt')}
('hydra-os.iso', 'hydra-os.iso.sha256', 'iso-inspection.txt', 'packages.json')}
self.record('iso', source_inputs=source_inputs, label=image.LABEL, artifacts=artifacts)
def status(self):
hydra/image.py
+50 -1
View File
@@ -154,6 +154,50 @@ def artifacts_valid(directory, state):
return True
def validate_package_inventory(data):
if not isinstance(data, list) or not data:
raise ImageError('Package inventory must be a nonempty array')
seen = set()
for record in data:
if not isinstance(record, dict) or set(record) != {'package', 'use', 'build_time'}:
raise ImageError('Malformed package inventory entry')
package = record['package']
if (not isinstance(package, str) or
not re.fullmatch(r'[A-Za-z0-9+_.-]+/[A-Za-z0-9+_.-]+-[0-9][A-Za-z0-9+_.-]*', package) or
package in seen):
raise ImageError('Invalid or duplicate package inventory identity')
seen.add(package)
flags = record['use']
if (not isinstance(flags, list) or any(not isinstance(flag, str) or
not re.fullmatch(r'[A-Za-z0-9+_.@-]+', flag) for flag in flags) or len(set(flags)) != len(flags)):
raise ImageError('Invalid package inventory USE flags')
built = record['build_time']
if built is not None and (not isinstance(built, str) or not re.fullmatch(r'[0-9]+', built)):
raise ImageError('Invalid package inventory build time')
return data
def read_package_inventory(path, expected_digest=None):
path = Path(path)
if path.is_symlink() or not path.is_file():
raise ImageError('Package inventory must be an ordinary file')
if expected_digest is not None and digest(path) != expected_digest:
raise ImageError('Package inventory checksum differs from checkpoint')
try:
return validate_package_inventory(json.loads(path.read_text()))
except (ValueError, UnicodeError) as error:
raise ImageError('Package inventory is not valid JSON') from error
def enable_user_audio(root):
root_path(root, 'etc/systemd/user')
result = subprocess.run(['systemctl', '--root=' + str(root), '--global', 'enable',
'pipewire.socket', 'pipewire-pulse.socket', 'wireplumber.service'],
text=True, capture_output=True)
if result.returncode:
raise ImageError('Cannot enable user audio sockets: ' + result.stderr.strip())
def configure_rootfs(root, source):
root, source = Path(root), Path(source)
# Refuse incomplete integrations before changing the exported copy.
@@ -169,6 +213,9 @@ def configure_rootfs(root, source):
package.mkdir(parents=True, exist_ok=True)
for module in sorted((source / 'hydra').glob('*.py')):
shutil.copyfile(module, root_path(root, 'usr/lib/hydra/hydra/' + module.name))
# Session helpers derive this checkout-compatible launcher from __file__.
for name in ('hydra', 'hydra-install'):
write(root, 'usr/lib/hydra/bin/' + name, (source / 'bin' / name).read_text(), 0o755)
for name, module in (('hydra', 'cli'), ('hydra-install', 'installer')):
write(root, 'usr/bin/' + name,
'#!/usr/bin/python3\nimport sys\nsys.path.insert(0, "/usr/lib/hydra")\n'
@@ -271,7 +318,7 @@ def digest(path):
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('step', choices=('configure', 'kernel', 'network'))
parser.add_argument('step', choices=('configure', 'kernel', 'network', 'audio'))
parser.add_argument('root', nargs='?', type=Path)
parser.add_argument('--source', type=Path)
options = parser.parse_args(argv)
@@ -282,6 +329,8 @@ def main(argv=None):
raise ImageError('An isolated rootfs directory is required')
if options.step == 'kernel':
print(kernel_version(options.root))
elif options.step == 'audio':
enable_user_audio(options.root)
else:
if options.source is None:
raise ImageError('Source checkout is required')
image/configure-rootfs.sh
+1 -1
View File
@@ -17,7 +17,7 @@ chroot "$root" usermod --lock root
chroot "$root" locale-gen
ln -snf /usr/share/zoneinfo/Europe/Moscow "$root/etc/localtime"
# Enable the ordinary per-user sound services without starting a user manager.
systemctl --root="$root" --global enable pipewire.socket wireplumber.service
python3 /hydra-source/hydra/image.py audio "$root"
# Set rootfs permissions now; capabilities are regenerated in the boot namespace.
chroot "$root" chown root:kvm /usr/libexec/qemu-bridge-helper
chroot "$root" chmod 0750 /usr/libexec/qemu-bridge-helper
image/portage/make.conf
+1 -1
View File
@@ -4,7 +4,7 @@ CXXFLAGS="${COMMON_FLAGS}"
FCFLAGS="${COMMON_FLAGS}"
FFLAGS="${COMMON_FLAGS}"
MAKEOPTS="-j4"
EMERGE_DEFAULT_OPTS="--jobs=1 --getbinpkg --binpkg-respect-use=y --autounmask=n"
EMERGE_DEFAULT_OPTS="--jobs=1 --getbinpkg --binpkg-respect-use=y --autounmask=n --usepkg-exclude='app-emulation/qemu app-emulation/virt-viewer'"
FEATURES="getbinpkg binpkg-request-signature -usersandbox -sandbox -network-sandbox -ipc-sandbox -pid-sandbox"
ACCEPT_LICENSE="@FREE @BINARY-REDISTRIBUTABLE linux-fw-redistributable no-source-code Intel-SDP"
VIDEO_CARDS="intel virgl"
tests/test_image.py
+144
View File
@@ -1,5 +1,8 @@
import importlib.util
import json
import os
import shlex
import sys
from pathlib import Path
import subprocess
import tempfile
@@ -90,6 +93,65 @@ class ImageTests(unittest.TestCase):
self.assertIn('binpkg-request-signature', features)
self.assertEqual((self.root / 'usr/share/hydra/build-manifest.json').is_file(), True)
def test_installed_helper_uses_relocated_payload_and_opens_empty_recovery(self):
self.kernel()
image.configure_rootfs(self.root, Path(__file__).resolve().parents[1])
terminal = Path(self.tmp.name) / 'terminal'
terminal.write_text('#!/bin/sh\nshift\nexec "$@"\n')
terminal.chmod(0o755)
manifest = Path(self.tmp.name) / 'session.json'
manifest.write_text(json.dumps({'config': str(Path(self.tmp.name) / 'config'),
'state': str(Path(self.tmp.name) / 'state'),
'runtime': str(Path(self.tmp.name) / 'runtime'),
'terminal': str(terminal), 'names': []}))
script = r"""import json, pathlib, subprocess, sys
sys.path.insert(0, sys.argv[1])
from hydra.session import _module_argv
helper = _module_argv('clients', pathlib.Path(sys.argv[2]))
help_result = subprocess.run(helper[:2] + ['--help'], text=True, capture_output=True)
print(help_result.stdout)
print(help_result.stderr, file=sys.stderr)
if help_result.returncode:
sys.exit(help_result.returncode)
result = subprocess.run(helper, input=':quit\n', text=True, capture_output=True, timeout=10)
print(result.stdout)
print(result.stderr, file=sys.stderr)
sys.exit(result.returncode)
"""
result = subprocess.run([sys.executable, '-I', '-c', script, str(self.root / 'usr/lib/hydra'), str(manifest)],
cwd=self.tmp.name, text=True, capture_output=True, timeout=20)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('HydraOS local recovery', result.stdout)
self.assertIn('add', result.stdout)
def test_installed_updates_exclude_unadapted_qemu_and_viewer_binaries(self):
self.kernel()
source = Path(__file__).resolve().parents[1]
self.file('etc/portage/make.conf', (source / 'image/portage/make.conf').read_text())
image.configure_rootfs(self.root, source)
settings = subprocess.run(['bash', '-euc', 'source "$1"; printf "%s" "$EMERGE_DEFAULT_OPTS"',
'--', str(self.root / 'etc/portage/make.conf')], text=True, capture_output=True, check=True)
defaults = shlex.split(settings.stdout)
exclusions = [item.split('=', 1)[1].split() for item in defaults if item.startswith('--usepkg-exclude=')]
self.assertEqual(exclusions, [['app-emulation/qemu', 'app-emulation/virt-viewer']])
self.assertIn('--getbinpkg', defaults)
def test_export_enables_pulse_socket_without_starting_user_services(self):
source = Path(__file__).resolve().parents[1]
self.assertTrue(callable(getattr(image, 'enable_user_audio', None)), 'user audio enable is missing')
units = self.root / 'usr/lib/systemd/user'
units.mkdir(parents=True)
for name, section in [('pipewire.socket', 'Socket'), ('pipewire-pulse.socket', 'Socket'),
('wireplumber.service', 'Service')]:
target = 'sockets.target' if section == 'Socket' else 'default.target'
(units / name).write_text(f'[{section}]\n[Install]\nWantedBy={target}\n')
image.enable_user_audio(self.root)
for target, name in [('sockets.target', 'pipewire.socket'), ('sockets.target', 'pipewire-pulse.socket'),
('default.target', 'wireplumber.service')]:
link = self.root / 'etc/systemd/user' / (target + '.wants') / name
self.assertTrue(link.is_symlink(), name)
self.assertEqual(os.readlink(link), '/usr/lib/systemd/user/' + name)
def test_rootfs_configuration_separates_live_overrides_and_installed_auth(self):
source = Path(__file__).resolve().parents[1]
image.copy_overlay(self.root, source / 'image/rootfs')
@@ -154,6 +216,88 @@ class ImageBuildTests(unittest.TestCase):
self.project = Path(self.tmp.name)
self.builder = Builder(self.project, self.project / 'build', self.project / 'cache', podman='podman')
def inventory(self, records=None):
records = records if records is not None else [{'package': 'app-emulation/qemu-9.2.4', 'use': ['spice'], 'build_time': '123'}]
self.builder.work.mkdir(exist_ok=True)
artifacts = self.builder.work / 'artifacts'
artifacts.mkdir(exist_ok=True)
path = artifacts / 'packages.json'
path.write_text(json.dumps(records))
original = self.builder.complete
self.builder.complete = lambda step: True if step == 'prepare' else original(step)
self.builder.record('packages', inventory='artifacts/packages.json', inventory_sha256=image.digest(path))
return path
def test_changed_inventory_invalidates_package_checkpoint(self):
path = self.inventory()
self.assertTrue(self.builder.complete('packages'))
path.write_text('[{"package":"app-emulation/qemu-8.0", "use": [], "build_time": null}]')
self.assertFalse(self.builder.complete('packages'))
def test_malformed_inventory_with_matching_digest_is_refused(self):
for data in ['{', '{}', '[]', '[{"package":"wrong"}]',
'[{"package":"app-emulation/qemu-9.2.4", "use":"spice", "build_time":"123"}]',
'[{"package":"app-emulation/qemu-9.2.4", "use":[], "build_time":42}]']:
with self.subTest(data=data):
path = self.inventory()
path.write_text(data)
self.builder.record('packages', inventory='artifacts/packages.json', inventory_sha256=image.digest(path))
self.assertFalse(self.builder.complete('packages'))
def test_changed_inventory_invalidates_completed_rootfs_and_iso(self):
path = self.inventory()
artifacts = path.parent
self.builder.active_package_job = lambda: False
self.builder.assembly = lambda snapshot=None: 'isolated-assembler'
self.builder.run = lambda *args, **kwargs: subprocess.CompletedProcess(args, 0, 'commit\n', '')
def pod(*args, **kwargs):
if args[0] == 'export':
(artifacts / 'builder-rootfs.tar').write_bytes(b'exported fixture')
elif args[0] == 'exec' and 'configure-rootfs.sh' in args[-1]:
for name in ('vmlinuz', 'initramfs-live.img', 'squashfs.img'):
(artifacts / name).write_bytes(b'artifact')
(artifacts / 'rootfs-manifest.json').write_text(json.dumps({'kernel': '6.12.58', 'source_files': {},
'packages': [{'package': 'app-emulation/qemu-9.2.4', 'use': ['spice'], 'build_time': '123'}]}))
elif args[0] == 'exec' and 'grub-mkrescue' in args[-1]:
(artifacts / 'hydra-os.iso').write_bytes(b'ISO fixture')
(artifacts / 'iso-inspection.txt').write_text('HYDRA_OS\nEl Torito boot img : 1 UEFI\n')
else:
self.fail('Unexpected Podman operation: ' + repr(args))
self.builder.pod = pod
self.builder.rootfs()
self.builder.iso()
self.assertTrue(self.builder.complete('rootfs'))
self.assertTrue(self.builder.complete('iso'))
path.write_text('[{"package":"app-emulation/qemu-9.2.5", "use":[], "build_time":null}]')
self.builder.record('packages', inventory='artifacts/packages.json', inventory_sha256=image.digest(path))
self.assertTrue(self.builder.complete('packages'))
self.assertFalse(self.builder.complete('rootfs'))
self.assertFalse(self.builder.complete('iso'))
def test_export_inventory_must_match_validated_package_checkpoint(self):
from hydra.build import BuildError
path = self.inventory()
self.builder.complete = lambda step: step == 'packages'
self.builder.active_package_job = lambda: False
self.builder.assembly = lambda snapshot: 'isolated-assembler'
self.builder.run = lambda *args, **kwargs: subprocess.CompletedProcess(args, 0, 'commit\n', '')
def pod(*args, **kwargs):
artifacts = self.builder.work / 'artifacts'
if args[0] == 'export':
(artifacts / 'builder-rootfs.tar').write_bytes(b'exported fixture')
elif args[0] == 'exec':
for name in ('vmlinuz', 'initramfs-live.img', 'squashfs.img'):
(artifacts / name).write_bytes(b'artifact')
# Actual exported package differs from the checkpoint inventory.
(artifacts / 'rootfs-manifest.json').write_text(json.dumps({'kernel': '6.12.58', 'source_files': {},
'packages': [{'package': 'app-emulation/qemu-9.2.5', 'use': ['spice'], 'build_time': '456'}]}))
else:
self.fail('Unexpected Podman operation: ' + repr(args))
self.builder.pod = pod
with self.assertRaisesRegex(BuildError, 'inventory'):
self.builder.rootfs()
self.assertFalse(self.builder.manifest()['rootfs']['complete'])
def test_changed_canonical_patch_invalidates_package_input_identity(self):
patches = self.project / 'patches'
patches.mkdir()