From de1a1698a9da800511c778916f76e9481c82757f Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 13 Oct 2026 04:40:18 +0300 Subject: [PATCH] fix: preserve installed helpers, adaptations and inventory provenance --- hydra/build.py | 43 +++++++++--- hydra/image.py | 51 +++++++++++++- image/configure-rootfs.sh | 2 +- image/portage/make.conf | 2 +- tests/test_image.py | 144 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 13 deletions(-) diff --git a/hydra/build.py b/hydra/build.py index 2bc094f..40ec5ff 100644 --- a/hydra/build.py +++ b/hydra/build.py @@ -212,14 +212,16 @@ if path.exists(): if not entry.get('complete') or entry.get('inputs') != self.input_id(): return False if step in ('rootfs', 'iso'): - from . import image + from hydra import image if entry.get('source_inputs') != image.source_identity(self.source_dir): return False if not image.artifacts_valid(self.work / 'artifacts', entry): return False return self.complete('packages' if step == 'rootfs' else 'rootfs') if step == 'packages': - if not (self.work / 'artifacts/packages.json').is_file(): + try: + self.package_inventory() + except (BuildError, OSError): return False return self.complete('prepare') if step == 'prepare': @@ -244,6 +246,16 @@ if path.exists(): return True return False + def package_inventory(self): + from hydra import image + checksum = self.manifest().get('packages', {}).get('inventory_sha256') + if not isinstance(checksum, str) or not re.fullmatch('[a-f0-9]{64}', checksum): + raise BuildError('Package inventory has no recorded checksum') + try: + return image.read_package_inventory(self.work / 'artifacts/packages.json', checksum) + except (image.ImageError, OSError) as error: + raise BuildError(f'Package inventory validation failed: {error}') from error + def download(self, url, filename=None): parsed = urllib.parse.urlparse(url) if parsed.scheme != 'https': @@ -345,8 +357,17 @@ fi''' if self.input_id() != inputs: raise BuildError('Image inputs changed during package installation; completion was not recorded') inventory = self.exec('cat /var/lib/hydra-build/packages.json', capture=True).stdout - (self.work / 'artifacts/packages.json').write_text(inventory) - self.record('packages', inventory='artifacts/packages.json', patched_source=patch_id) + from hydra import image + try: + image.validate_package_inventory(json.loads(inventory)) + except (ValueError, image.ImageError) as error: + raise BuildError(f'Package inventory is malformed: {error}') from error + target = self.work / 'artifacts/packages.json' + temporary = target.with_suffix('.tmp') + temporary.write_text(inventory) + temporary.replace(target) + self.record('packages', inventory='artifacts/packages.json', patched_source=patch_id, + inventory_sha256=image.digest(target)) def assembly(self, snapshot_digest=None): """A separate rootless container processes an exported copy, never the builder.""" @@ -378,7 +399,7 @@ fi''' return name def rootfs(self): - from . import image + from hydra import image if self.active_package_job(): raise BuildError('A contained package job is running; wait before export') if not self.complete('packages'): @@ -412,15 +433,17 @@ chmod 0644 /hydra-output/artifacts/vmlinuz /hydra-output/artifacts/initramfs-liv if image.source_identity(self.source_dir) != source_inputs: raise BuildError('Source inputs changed during rootfs assembly; completion not recorded') manifest = json.loads((self.work / 'artifacts/rootfs-manifest.json').read_text()) - if not manifest.get('packages') or manifest.get('source_files') != image.source_files(self.source_dir): - raise BuildError('Payload inventory/source manifest is incomplete or stale') + if manifest.get('packages') != self.package_inventory(): + raise BuildError('Payload package inventory differs from the validated package checkpoint') + if manifest.get('source_files') != image.source_files(self.source_dir): + raise BuildError('Payload source manifest is incomplete or stale') artifacts = {name: image.digest(self.work / 'artifacts' / name) for name in - ('vmlinuz', 'initramfs-live.img', 'squashfs.img', 'rootfs-manifest.json')} + ('vmlinuz', 'initramfs-live.img', 'squashfs.img', 'rootfs-manifest.json', 'packages.json')} self.record('rootfs', source_inputs=source_inputs, snapshot_sha256=snapshot, assembler=assembler, kernel=manifest['kernel'], artifacts=artifacts) def iso(self): - from . import image + from hydra import image if not self.complete('rootfs'): raise BuildError('Verified rootfs artifacts are required before ISO assembly') if self.complete('iso'): @@ -450,7 +473,7 @@ chmod 0644 /hydra-output/artifacts/hydra-os.iso /hydra-output/artifacts/iso-insp checksum = image.digest(self.work / 'artifacts/hydra-os.iso') (self.work / 'artifacts/hydra-os.iso.sha256').write_text(checksum + ' hydra-os.iso\n') artifacts = {name: image.digest(self.work / 'artifacts' / name) for name in - ('hydra-os.iso', 'hydra-os.iso.sha256', 'iso-inspection.txt')} + ('hydra-os.iso', 'hydra-os.iso.sha256', 'iso-inspection.txt', 'packages.json')} self.record('iso', source_inputs=source_inputs, label=image.LABEL, artifacts=artifacts) def status(self): diff --git a/hydra/image.py b/hydra/image.py index f304e99..cde36f9 100644 --- a/hydra/image.py +++ b/hydra/image.py @@ -154,6 +154,50 @@ def artifacts_valid(directory, state): return True +def validate_package_inventory(data): + if not isinstance(data, list) or not data: + raise ImageError('Package inventory must be a nonempty array') + seen = set() + for record in data: + if not isinstance(record, dict) or set(record) != {'package', 'use', 'build_time'}: + raise ImageError('Malformed package inventory entry') + package = record['package'] + if (not isinstance(package, str) or + not re.fullmatch(r'[A-Za-z0-9+_.-]+/[A-Za-z0-9+_.-]+-[0-9][A-Za-z0-9+_.-]*', package) or + package in seen): + raise ImageError('Invalid or duplicate package inventory identity') + seen.add(package) + flags = record['use'] + if (not isinstance(flags, list) or any(not isinstance(flag, str) or + not re.fullmatch(r'[A-Za-z0-9+_.@-]+', flag) for flag in flags) or len(set(flags)) != len(flags)): + raise ImageError('Invalid package inventory USE flags') + built = record['build_time'] + if built is not None and (not isinstance(built, str) or not re.fullmatch(r'[0-9]+', built)): + raise ImageError('Invalid package inventory build time') + return data + + +def read_package_inventory(path, expected_digest=None): + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise ImageError('Package inventory must be an ordinary file') + if expected_digest is not None and digest(path) != expected_digest: + raise ImageError('Package inventory checksum differs from checkpoint') + try: + return validate_package_inventory(json.loads(path.read_text())) + except (ValueError, UnicodeError) as error: + raise ImageError('Package inventory is not valid JSON') from error + + +def enable_user_audio(root): + root_path(root, 'etc/systemd/user') + result = subprocess.run(['systemctl', '--root=' + str(root), '--global', 'enable', + 'pipewire.socket', 'pipewire-pulse.socket', 'wireplumber.service'], + text=True, capture_output=True) + if result.returncode: + raise ImageError('Cannot enable user audio sockets: ' + result.stderr.strip()) + + def configure_rootfs(root, source): root, source = Path(root), Path(source) # Refuse incomplete integrations before changing the exported copy. @@ -169,6 +213,9 @@ def configure_rootfs(root, source): package.mkdir(parents=True, exist_ok=True) for module in sorted((source / 'hydra').glob('*.py')): shutil.copyfile(module, root_path(root, 'usr/lib/hydra/hydra/' + module.name)) + # Session helpers derive this checkout-compatible launcher from __file__. + for name in ('hydra', 'hydra-install'): + write(root, 'usr/lib/hydra/bin/' + name, (source / 'bin' / name).read_text(), 0o755) for name, module in (('hydra', 'cli'), ('hydra-install', 'installer')): write(root, 'usr/bin/' + name, '#!/usr/bin/python3\nimport sys\nsys.path.insert(0, "/usr/lib/hydra")\n' @@ -271,7 +318,7 @@ def digest(path): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('step', choices=('configure', 'kernel', 'network')) + parser.add_argument('step', choices=('configure', 'kernel', 'network', 'audio')) parser.add_argument('root', nargs='?', type=Path) parser.add_argument('--source', type=Path) options = parser.parse_args(argv) @@ -282,6 +329,8 @@ def main(argv=None): raise ImageError('An isolated rootfs directory is required') if options.step == 'kernel': print(kernel_version(options.root)) + elif options.step == 'audio': + enable_user_audio(options.root) else: if options.source is None: raise ImageError('Source checkout is required') diff --git a/image/configure-rootfs.sh b/image/configure-rootfs.sh index c59c102..c27094b 100755 --- a/image/configure-rootfs.sh +++ b/image/configure-rootfs.sh @@ -17,7 +17,7 @@ chroot "$root" usermod --lock root chroot "$root" locale-gen ln -snf /usr/share/zoneinfo/Europe/Moscow "$root/etc/localtime" # Enable the ordinary per-user sound services without starting a user manager. -systemctl --root="$root" --global enable pipewire.socket wireplumber.service +python3 /hydra-source/hydra/image.py audio "$root" # Set rootfs permissions now; capabilities are regenerated in the boot namespace. chroot "$root" chown root:kvm /usr/libexec/qemu-bridge-helper chroot "$root" chmod 0750 /usr/libexec/qemu-bridge-helper diff --git a/image/portage/make.conf b/image/portage/make.conf index 2254324..666a561 100644 --- a/image/portage/make.conf +++ b/image/portage/make.conf @@ -4,7 +4,7 @@ CXXFLAGS="${COMMON_FLAGS}" FCFLAGS="${COMMON_FLAGS}" FFLAGS="${COMMON_FLAGS}" MAKEOPTS="-j4" -EMERGE_DEFAULT_OPTS="--jobs=1 --getbinpkg --binpkg-respect-use=y --autounmask=n" +EMERGE_DEFAULT_OPTS="--jobs=1 --getbinpkg --binpkg-respect-use=y --autounmask=n --usepkg-exclude='app-emulation/qemu app-emulation/virt-viewer'" FEATURES="getbinpkg binpkg-request-signature -usersandbox -sandbox -network-sandbox -ipc-sandbox -pid-sandbox" ACCEPT_LICENSE="@FREE @BINARY-REDISTRIBUTABLE linux-fw-redistributable no-source-code Intel-SDP" VIDEO_CARDS="intel virgl" diff --git a/tests/test_image.py b/tests/test_image.py index 7eff0c2..b50e7df 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -1,5 +1,8 @@ import importlib.util import json +import os +import shlex +import sys from pathlib import Path import subprocess import tempfile @@ -90,6 +93,65 @@ class ImageTests(unittest.TestCase): self.assertIn('binpkg-request-signature', features) self.assertEqual((self.root / 'usr/share/hydra/build-manifest.json').is_file(), True) + def test_installed_helper_uses_relocated_payload_and_opens_empty_recovery(self): + self.kernel() + image.configure_rootfs(self.root, Path(__file__).resolve().parents[1]) + terminal = Path(self.tmp.name) / 'terminal' + terminal.write_text('#!/bin/sh\nshift\nexec "$@"\n') + terminal.chmod(0o755) + manifest = Path(self.tmp.name) / 'session.json' + manifest.write_text(json.dumps({'config': str(Path(self.tmp.name) / 'config'), + 'state': str(Path(self.tmp.name) / 'state'), + 'runtime': str(Path(self.tmp.name) / 'runtime'), + 'terminal': str(terminal), 'names': []})) + script = r"""import json, pathlib, subprocess, sys +sys.path.insert(0, sys.argv[1]) +from hydra.session import _module_argv +helper = _module_argv('clients', pathlib.Path(sys.argv[2])) +help_result = subprocess.run(helper[:2] + ['--help'], text=True, capture_output=True) +print(help_result.stdout) +print(help_result.stderr, file=sys.stderr) +if help_result.returncode: + sys.exit(help_result.returncode) +result = subprocess.run(helper, input=':quit\n', text=True, capture_output=True, timeout=10) +print(result.stdout) +print(result.stderr, file=sys.stderr) +sys.exit(result.returncode) +""" + result = subprocess.run([sys.executable, '-I', '-c', script, str(self.root / 'usr/lib/hydra'), str(manifest)], + cwd=self.tmp.name, text=True, capture_output=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('HydraOS local recovery', result.stdout) + self.assertIn('add', result.stdout) + + def test_installed_updates_exclude_unadapted_qemu_and_viewer_binaries(self): + self.kernel() + source = Path(__file__).resolve().parents[1] + self.file('etc/portage/make.conf', (source / 'image/portage/make.conf').read_text()) + image.configure_rootfs(self.root, source) + settings = subprocess.run(['bash', '-euc', 'source "$1"; printf "%s" "$EMERGE_DEFAULT_OPTS"', + '--', str(self.root / 'etc/portage/make.conf')], text=True, capture_output=True, check=True) + defaults = shlex.split(settings.stdout) + exclusions = [item.split('=', 1)[1].split() for item in defaults if item.startswith('--usepkg-exclude=')] + self.assertEqual(exclusions, [['app-emulation/qemu', 'app-emulation/virt-viewer']]) + self.assertIn('--getbinpkg', defaults) + + def test_export_enables_pulse_socket_without_starting_user_services(self): + source = Path(__file__).resolve().parents[1] + self.assertTrue(callable(getattr(image, 'enable_user_audio', None)), 'user audio enable is missing') + units = self.root / 'usr/lib/systemd/user' + units.mkdir(parents=True) + for name, section in [('pipewire.socket', 'Socket'), ('pipewire-pulse.socket', 'Socket'), + ('wireplumber.service', 'Service')]: + target = 'sockets.target' if section == 'Socket' else 'default.target' + (units / name).write_text(f'[{section}]\n[Install]\nWantedBy={target}\n') + image.enable_user_audio(self.root) + for target, name in [('sockets.target', 'pipewire.socket'), ('sockets.target', 'pipewire-pulse.socket'), + ('default.target', 'wireplumber.service')]: + link = self.root / 'etc/systemd/user' / (target + '.wants') / name + self.assertTrue(link.is_symlink(), name) + self.assertEqual(os.readlink(link), '/usr/lib/systemd/user/' + name) + def test_rootfs_configuration_separates_live_overrides_and_installed_auth(self): source = Path(__file__).resolve().parents[1] image.copy_overlay(self.root, source / 'image/rootfs') @@ -154,6 +216,88 @@ class ImageBuildTests(unittest.TestCase): self.project = Path(self.tmp.name) self.builder = Builder(self.project, self.project / 'build', self.project / 'cache', podman='podman') + def inventory(self, records=None): + records = records if records is not None else [{'package': 'app-emulation/qemu-9.2.4', 'use': ['spice'], 'build_time': '123'}] + self.builder.work.mkdir(exist_ok=True) + artifacts = self.builder.work / 'artifacts' + artifacts.mkdir(exist_ok=True) + path = artifacts / 'packages.json' + path.write_text(json.dumps(records)) + original = self.builder.complete + self.builder.complete = lambda step: True if step == 'prepare' else original(step) + self.builder.record('packages', inventory='artifacts/packages.json', inventory_sha256=image.digest(path)) + return path + + def test_changed_inventory_invalidates_package_checkpoint(self): + path = self.inventory() + self.assertTrue(self.builder.complete('packages')) + path.write_text('[{"package":"app-emulation/qemu-8.0", "use": [], "build_time": null}]') + self.assertFalse(self.builder.complete('packages')) + + def test_malformed_inventory_with_matching_digest_is_refused(self): + for data in ['{', '{}', '[]', '[{"package":"wrong"}]', + '[{"package":"app-emulation/qemu-9.2.4", "use":"spice", "build_time":"123"}]', + '[{"package":"app-emulation/qemu-9.2.4", "use":[], "build_time":42}]']: + with self.subTest(data=data): + path = self.inventory() + path.write_text(data) + self.builder.record('packages', inventory='artifacts/packages.json', inventory_sha256=image.digest(path)) + self.assertFalse(self.builder.complete('packages')) + + def test_changed_inventory_invalidates_completed_rootfs_and_iso(self): + path = self.inventory() + artifacts = path.parent + self.builder.active_package_job = lambda: False + self.builder.assembly = lambda snapshot=None: 'isolated-assembler' + self.builder.run = lambda *args, **kwargs: subprocess.CompletedProcess(args, 0, 'commit\n', '') + def pod(*args, **kwargs): + if args[0] == 'export': + (artifacts / 'builder-rootfs.tar').write_bytes(b'exported fixture') + elif args[0] == 'exec' and 'configure-rootfs.sh' in args[-1]: + for name in ('vmlinuz', 'initramfs-live.img', 'squashfs.img'): + (artifacts / name).write_bytes(b'artifact') + (artifacts / 'rootfs-manifest.json').write_text(json.dumps({'kernel': '6.12.58', 'source_files': {}, + 'packages': [{'package': 'app-emulation/qemu-9.2.4', 'use': ['spice'], 'build_time': '123'}]})) + elif args[0] == 'exec' and 'grub-mkrescue' in args[-1]: + (artifacts / 'hydra-os.iso').write_bytes(b'ISO fixture') + (artifacts / 'iso-inspection.txt').write_text('HYDRA_OS\nEl Torito boot img : 1 UEFI\n') + else: + self.fail('Unexpected Podman operation: ' + repr(args)) + self.builder.pod = pod + self.builder.rootfs() + self.builder.iso() + self.assertTrue(self.builder.complete('rootfs')) + self.assertTrue(self.builder.complete('iso')) + path.write_text('[{"package":"app-emulation/qemu-9.2.5", "use":[], "build_time":null}]') + self.builder.record('packages', inventory='artifacts/packages.json', inventory_sha256=image.digest(path)) + self.assertTrue(self.builder.complete('packages')) + self.assertFalse(self.builder.complete('rootfs')) + self.assertFalse(self.builder.complete('iso')) + + def test_export_inventory_must_match_validated_package_checkpoint(self): + from hydra.build import BuildError + path = self.inventory() + self.builder.complete = lambda step: step == 'packages' + self.builder.active_package_job = lambda: False + self.builder.assembly = lambda snapshot: 'isolated-assembler' + self.builder.run = lambda *args, **kwargs: subprocess.CompletedProcess(args, 0, 'commit\n', '') + def pod(*args, **kwargs): + artifacts = self.builder.work / 'artifacts' + if args[0] == 'export': + (artifacts / 'builder-rootfs.tar').write_bytes(b'exported fixture') + elif args[0] == 'exec': + for name in ('vmlinuz', 'initramfs-live.img', 'squashfs.img'): + (artifacts / name).write_bytes(b'artifact') + # Actual exported package differs from the checkpoint inventory. + (artifacts / 'rootfs-manifest.json').write_text(json.dumps({'kernel': '6.12.58', 'source_files': {}, + 'packages': [{'package': 'app-emulation/qemu-9.2.5', 'use': ['spice'], 'build_time': '456'}]})) + else: + self.fail('Unexpected Podman operation: ' + repr(args)) + self.builder.pod = pod + with self.assertRaisesRegex(BuildError, 'inventory'): + self.builder.rootfs() + self.assertFalse(self.builder.manifest()['rootfs']['complete']) + def test_changed_canonical_patch_invalidates_package_input_identity(self): patches = self.project / 'patches' patches.mkdir()