Validate builder provenance and guard detached package jobs

hydra/build.py

@@ -4,8 +4,10 @@ import contextlib
 import fcntl
 import hashlib
 import json
+import lzma
 import os
 from pathlib import Path
+import re
 import shutil
 import subprocess
 import sys
@@ -18,6 +20,43 @@ class BuildError(RuntimeError):
     pass
 
 
+def package_job_script(command, state='/var/lib/hydra-build', package_db='/var/db/pkg', inputs=''):
+    """The contained job owns its log/result even if the outer caller disconnects."""
+    return f'''import fcntl, json, os, pathlib, subprocess, sys, time
+state = pathlib.Path({str(state)!r})
+database = pathlib.Path({str(package_db)!r})
+state.mkdir(parents=True, exist_ok=True)
+lock = (state / 'packages.lock').open('a')
+try:
+    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+except BlockingIOError:
+    sys.exit(75)
+def write(name, data):
+    temporary = state / (name + '.tmp')
+    temporary.write_text(json.dumps(data, indent=2) + '\\n')
+    temporary.replace(state / name)
+status = {{'running': True, 'pid': os.getpid(), 'inputs': {inputs!r}, 'started': int(time.time())}}
+write('packages.status.json', status)
+with (state / 'packages.log').open('a') as log:
+    try:
+        result = subprocess.run({command!r}, stdout=log, stderr=subprocess.STDOUT)
+        code = result.returncode
+    except OSError as error:
+        log.write(str(error) + '\\n')
+        code = 127
+if code == 0:
+    inventory = []
+    for use in sorted(database.glob('*/*/USE')):
+        package = use.parent
+        inventory.append({{'package': str(package.relative_to(database)), 'use': use.read_text().split(),
+                          'build_time': (package / 'BUILD_TIME').read_text().strip() if (package / 'BUILD_TIME').exists() else None}})
+    write('packages.json', inventory)
+status.update(running=False, exit_code=code, finished=int(time.time()))
+write('packages.status.json', status)
+sys.exit(code)
+'''
+
+
 def verify_hash(path, expected):
     digest = hashlib.sha256()
     with Path(path).open('rb') as stream:
@@ -54,6 +93,7 @@ class Builder:
         self.runroot = runtime / ('hydra-build-podman' if self.work == self.project / 'build' else f'hydra-build-{identifier}')
         self.name = 'hydra-builder-' + identifier
         self.image = 'localhost/hydra-stage3:20261011'
+        self._stage3_layers = {}
 
     def podman_argv(self, *args):
         return [self.podman, '--root', str(self.work / 'podman'), '--runroot', str(self.runroot), '--storage-driver', 'overlay', *args]
@@ -65,7 +105,7 @@ class Builder:
                 self.image, '/bin/sleep', 'infinity']
 
     def run(self, argv, capture=False, check=True):
-        print('+ ' + ' '.join(str(arg) for arg in argv), flush=True)
+        print('+ ' + ' '.join(str(arg) for arg in argv), file=sys.stderr, flush=True)
         result = subprocess.run(argv, text=True, stdout=subprocess.PIPE if capture else None,
                                 stderr=subprocess.PIPE if capture else None)
         if check and result.returncode:
@@ -78,6 +118,72 @@ class Builder:
     def exec(self, command, capture=False):
         return self.pod('exec', self.name, '/bin/bash', '-euc', command, capture=capture)
 
+    def inspect(self, kind, name):
+        result = self.pod(kind, 'inspect', name, capture=True, check=False)
+        if result.returncode:
+            raise BuildError(f'{kind} provenance is unavailable; use a fresh work directory')
+        entries = json.loads(result.stdout)
+        if not isinstance(entries, list) or len(entries) != 1 or not isinstance(entries[0], dict):
+            raise BuildError(f'Unexpected {kind} provenance')
+        return entries[0]
+
+    def check_mounts(self, container):
+        for destination, path, label in (('/hydra-source', self.source_dir, 'source'),
+                                         ('/hydra-cache', self.cache, 'cache')):
+            mounts = [item for item in container.get('Mounts', []) if item.get('Destination') == destination]
+            if len(mounts) != 1 or mounts[0].get('Source') != str(path) or mounts[0].get('RW') is not False:
+                raise BuildError(f'Existing builder {label} mount differs; use its original checkout/cache or a fresh work directory')
+
+    def stage3_layer(self, inputs):
+        stage = inputs['stage3']
+        archive = self.cache / Path(urllib.parse.urlparse(stage['url']).path).name
+        verify_hash(archive, stage['sha256'])
+        if stage['sha256'] not in self._stage3_layers:
+            digest = hashlib.sha256()
+            with lzma.open(archive, 'rb') as stream:
+                for chunk in iter(lambda: stream.read(1024 * 1024), b''):
+                    digest.update(chunk)
+            self._stage3_layers[stage['sha256']] = 'sha256:' + digest.hexdigest()
+        return self._stage3_layers[stage['sha256']]
+
+    def check_base(self, image, inputs):
+        expected = self.stage3_layer(inputs)
+        if image.get('RootFS', {}).get('Layers') != [expected]:
+            raise BuildError('Existing builder stage3 differs from pinned archive; use a fresh work directory')
+        return expected
+
+    def read_container_file(self, path):
+        # Podman cp also reads a stopped container, so provenance precedes start.
+        with tempfile.TemporaryDirectory(prefix='hydra-read-') as directory:
+            target = Path(directory) / 'data'
+            result = self.pod('cp', f'{self.name}:{path}', str(target), capture=True, check=False)
+            return target.read_text() if result.returncode == 0 else None
+
+    def check_snapshot(self, inputs, allow_pending=False):
+        marker = self.read_container_file('/var/db/repos/gentoo/.hydra-snapshot')
+        if marker is None and allow_pending:
+            marker = self.read_container_file('/var/db/repos/gentoo/.hydra-snapshot-input')
+        if marker is None or marker.strip() != inputs['portage']['commit']:
+            raise BuildError('Existing builder snapshot differs or is unverified; use a fresh work directory')
+
+    def active_package_job(self):
+        if not (self.work / 'podman').is_dir():
+            return False
+        result = self.pod('top', self.name, 'hpid', 'args', capture=True, check=False)
+        if result.returncode == 0 and any(re.search(r'(?:^|\s)(?:\S*/)?emerge(?:\s|$)', line)
+                                         for line in result.stdout.splitlines()[1:]):
+            return True
+        probe = self.pod('exec', self.name, 'python3', '-c', '''import fcntl, pathlib, sys
+path = pathlib.Path('/var/lib/hydra-build/packages.lock')
+if path.exists():
+    with path.open('r') as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
+        except BlockingIOError:
+            sys.exit(75)
+''', capture=True, check=False)
+        return probe.returncode == 75
+
     def manifest(self):
         path = self.work / 'manifest.json'
         return json.loads(path.read_text()) if path.exists() else {}
@@ -106,9 +212,17 @@ class Builder:
         if step == 'packages':
             if not (self.work / 'artifacts/packages.json').is_file():
                 return False
-            return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
+            return self.complete('prepare')
         if step == 'prepare':
-            return self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
+            try:
+                container = self.inspect('container', self.name)
+                self.check_mounts(container)
+                inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
+                self.check_base(self.inspect('image', container['Image']), inputs)
+                self.check_snapshot(inputs)
+                return True
+            except (BuildError, OSError, ValueError, KeyError, lzma.LZMAError):
+                return False
         if step == 'fetch':
             inputs = json.loads((self.source_dir / 'image/sources.json').read_text())
             try:
@@ -153,10 +267,22 @@ class Builder:
         return inputs
 
     def prepare(self):
+        existing = self.pod('container', 'exists', self.name, capture=True, check=False).returncode == 0
+        container = self.inspect('container', self.name) if existing else None
+        if container is not None:
+            self.check_mounts(container)  # Reject another worktree before changing any checkpoint.
+            if self.active_package_job():
+                raise BuildError('A contained package job is running; wait for it to finish before preparation')
         inputs = self.fetch()  # Reverify cached bytes even when resuming.
-        if self.pod('image', 'exists', self.image, capture=True, check=False).returncode:
+        if container is not None:
+            layer = self.check_base(self.inspect('image', container['Image']), inputs)
+            self.check_snapshot(inputs, allow_pending=True)
+        elif self.pod('image', 'exists', self.image, capture=True, check=False).returncode:
             self.pod('import', str(self.cache / Path(urllib.parse.urlparse(inputs['stage3']['url']).path).name), self.image)
-        if self.pod('container', 'exists', self.name, capture=True, check=False).returncode:
+            layer = self.check_base(self.inspect('image', self.image), inputs)
+        else:
+            layer = self.check_base(self.inspect('image', self.image), inputs)
+        if container is None:
             self.pod(*self.container_run_args())
         else:
             self.pod('start', self.name)
@@ -164,6 +290,7 @@ class Builder:
         # Snapshot extraction happens only inside the container; host paths are read-only.
         self.exec(f'''mkdir -p /var/db/repos/gentoo
 if [ ! -f /var/db/repos/gentoo/.hydra-snapshot ]; then
+  printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot-input
   tar --extract --gzip --file /hydra-cache/{snapshot} --strip-components=1 --directory /var/db/repos/gentoo
   printf '%s\\n' '{inputs['portage']['commit']}' > /var/db/repos/gentoo/.hydra-snapshot
 fi
@@ -175,23 +302,50 @@ getuto
 gpg --batch --homedir /etc/portage/gnupg --import /hydra-cache/service-keys.gpg
 chmod a+r /etc/portage/gnupg/pubring.kbx /etc/portage/gnupg/trustdb.gpg
 ''')
-        self.record('prepare', image=self.image, snapshot=inputs['portage']['commit'])
+        self.record('prepare', image=container['Image'] if container else self.image,
+                    stage3_sha256=inputs['stage3']['sha256'], stage3_layer=layer,
+                    source_dir=str(self.source_dir), cache_dir=str(self.cache), snapshot=inputs['portage']['commit'])
 
     def packages(self, pretend=False):
         self.prepare()
         atoms = [line.strip() for line in (self.source_dir / 'image/packages').read_text().splitlines()
                  if line.strip() and not line.startswith('#')]
         import shlex
-        command = 'emerge --verbose --update --newuse --deep --backtrack=100 ' + ('--pretend ' if pretend else '') + ' '.join(map(shlex.quote, atoms))
+        command = ['emerge', '--verbose', '--update', '--newuse', '--deep', '--backtrack=100']
-        self.exec(command)
         if pretend:
+            self.exec(shlex.join([*command, '--pretend', *atoms]))
             return
-        inventory = self.exec('python3 - <<\'PY\'\nimport json,pathlib\nout=[]\nfor use in sorted(pathlib.Path("/var/db/pkg").glob("*/*/USE")):\n p=use.parent\n out.append({"package":str(p.relative_to("/var/db/pkg")),"use":use.read_text().split(),"build_time":(p/"BUILD_TIME").read_text().strip() if (p/"BUILD_TIME").exists() else None})\nprint(json.dumps(out,indent=2))\nPY', capture=True).stdout
+        inputs = self.input_id()
+        self.record('packages', complete=False, job_log='/var/lib/hydra-build/packages.log')
+        script = package_job_script([*command, *atoms], inputs=inputs)
+        self.exec(shlex.join(['python3', '-c', script]))
+        if self.input_id() != inputs:
+            raise BuildError('Image inputs changed during package installation; completion was not recorded')
+        inventory = self.exec('cat /var/lib/hydra-build/packages.json', capture=True).stdout
         (self.work / 'artifacts/packages.json').write_text(inventory)
         self.record('packages', inventory='artifacts/packages.json')
 
     def status(self):
-        print(json.dumps(self.manifest(), indent=2))
+        data = self.manifest()
+        for step in ('fetch', 'prepare', 'packages', 'rootfs', 'iso'):
+            if step in data:
+                data[step]['recorded_complete'] = data[step].get('complete', False)
+                data[step]['complete'] = self.complete(step)
+        data['mutation_lock_held'] = self.build_running()
+        data['package_job_running'] = self.active_package_job()
+        data['running'] = data['mutation_lock_held'] or data['package_job_running']
+        print(json.dumps(data, indent=2))
+
+    def build_running(self):
+        try:
+            with (self.work / '.hydra-build.lock').open('r') as lock:
+                try:
+                    fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
+                except BlockingIOError:
+                    return True
+        except FileNotFoundError:
+            pass
+        return False
 
     @contextlib.contextmanager
     def locked(self):
@@ -222,6 +376,9 @@ def main(argv=None):
     options = parser.parse_args(argv)
     try:
         builder = Builder(project, options.work, options.cache, options.podman, options.gpg, source)
+        if options.step == 'status':
+            builder.status()
+            return 0
         with builder.locked():
             if options.step in ('rootfs', 'iso'):
                 raise BuildError(f'{options.step} assembly is not configured yet')
@@ -230,6 +387,6 @@ def main(argv=None):
             else:
                 getattr(builder, options.step)()
         return 0
-    except (BuildError, OSError, ValueError) as exc:
+    except (BuildError, OSError, ValueError, lzma.LZMAError) as exc:
         print(f'hydra-build: {exc}', file=sys.stderr)
         return 1

tests/test_build.py

@@ -1,6 +1,9 @@
 import hashlib
+import contextlib
+import io
 import importlib.util
 import json
+import lzma
 import os
 from pathlib import Path
 import tempfile
@@ -109,6 +112,209 @@ class BuilderTests(unittest.TestCase):
         other = self.module.Builder(self.project, self.project / 'other-build', self.project / 'cache')
         self.assertNotEqual(self.builder().runroot, other.runroot)
 
+    def existing_builder(self, source=None, cache=None, layer=None, snapshot='old-snapshot'):
+        """A real executable models only the external Podman boundary."""
+        source = source or self.project
+        cache = cache or self.project / 'cache'
+        image = self.project / 'image'
+        image.mkdir(exist_ok=True)
+        inputs = {'stage3': {'url': 'https://example.test/stage.tar.xz',
+                            'sha256': hashlib.sha256(lzma.compress(b'original layer')).hexdigest()},
+                  'portage': {'filename': 'repo.tar.gz', 'commit': 'old-snapshot'}}
+        (image / 'sources.json').write_text(json.dumps(inputs))
+        builder = self.builder()
+        (builder.work / 'podman').mkdir(parents=True)
+        fixture = self.project / 'podman-fixture.json'
+        fixture.write_text(json.dumps({'Image': 'original-image-id',
+            'Mounts': [{'Source': str(source), 'Destination': '/hydra-source', 'RW': False},
+                       {'Source': str(cache), 'Destination': '/hydra-cache', 'RW': False}],
+            'RootFS': {'Layers': [layer or 'sha256:' + hashlib.sha256(b'original layer').hexdigest()]},
+            'snapshot': snapshot}))
+        executable = self.project / 'fixture-podman'
+        executable.write_text('''#!/usr/bin/env python3
+import json, pathlib, sys
+root = pathlib.Path(__file__).parent
+data = json.loads((root / 'podman-fixture.json').read_text())
+args = sys.argv[7:]
+if 'exists' in args:
+    sys.exit(0)
+if 'inspect' in args:
+    print(json.dumps([data]))
+elif args[0] == 'top':
+    print('HPID ARGS')
+    if data.get('active_job'):
+        print('123 /usr/bin/python3 /usr/lib/python-exec/python3/emerge --update package')
+elif args[0] == 'cp':
+    snapshot = data.get('snapshot_pending') if args[1].endswith('.hydra-snapshot-input') else data['snapshot']
+    if snapshot is None:
+        sys.exit(1)
+    pathlib.Path(args[-1]).write_text(snapshot + '\\n')
+elif args[0] == 'exec' and data.get('stopped'):
+    sys.exit(125)
+elif args[0] == 'exec' and 'packages.lock' in args[-1]:
+    sys.exit(75 if data.get('job_lock') else 0)
+elif args[0] == 'exec' and args[-1].startswith('cat '):
+    snapshot = data['snapshot'] or (data.get('snapshot_pending') if '.hydra-snapshot-input' in args[-1] else None)
+    if snapshot is None:
+        sys.exit(1)
+    print(snapshot)
+else:
+    if args[0] == 'start':
+        data['stopped'] = False
+        (root / 'podman-fixture.json').write_text(json.dumps(data))
+    with (root / 'mutations.jsonl').open('a') as stream:
+        stream.write(json.dumps(args) + '\\n')
+''')
+        executable.chmod(0o755)
+        builder.podman = str(executable)
+        with builder.locked():
+            (builder.cache / 'stage.tar.xz').write_bytes(lzma.compress(b'original layer'))
+        return builder, inputs
+
+    def assert_prepare_refused(self, builder, inputs, reason):
+        with patch.object(builder, 'fetch', return_value=inputs):
+            with self.assertRaisesRegex(self.module.BuildError, reason):
+                builder.prepare()
+        self.assertFalse((self.project / 'mutations.jsonl').exists(), 'refusal must precede container/config mutations')
+        self.assertNotIn('prepare', builder.manifest(), 'refusal must not record false provenance')
+
+    def test_other_worktree_mount_is_refused_before_preparation(self):
+        builder, inputs = self.existing_builder(source=self.project / 'other-worktree')
+        self.assert_prepare_refused(builder, inputs, 'source')
+
+    def test_other_cache_mount_is_refused_before_preparation(self):
+        builder, inputs = self.existing_builder(cache=self.project / 'other-cache')
+        self.assert_prepare_refused(builder, inputs, 'cache')
+
+    def test_changed_stage3_pin_refuses_old_container(self):
+        builder, inputs = self.existing_builder()
+        archive = lzma.compress(b'new stage layer')
+        (builder.cache / 'stage.tar.xz').write_bytes(archive)
+        inputs['stage3']['sha256'] = hashlib.sha256(archive).hexdigest()
+        self.assert_prepare_refused(builder, inputs, 'stage3')
+
+    def test_changed_snapshot_pin_refuses_existing_tree(self):
+        builder, inputs = self.existing_builder()
+        inputs['portage']['commit'] = 'new-snapshot'
+        self.assert_prepare_refused(builder, inputs, 'snapshot')
+
+    def test_matching_existing_builder_retains_compatible_provenance(self):
+        builder, inputs = self.existing_builder()
+        with patch.object(builder, 'fetch', return_value=inputs):
+            builder.prepare()
+        self.assertEqual(builder.manifest()['prepare'].get('stage3_sha256'), inputs['stage3']['sha256'])
+        self.assertEqual(builder.manifest()['prepare']['snapshot'], 'old-snapshot')
+
+    def test_status_invalidates_missing_package_inventory(self):
+        builder = self.builder()
+        with builder.locked():
+            builder.record('packages')
+        output = io.StringIO()
+        with contextlib.redirect_stdout(output):
+            builder.status()
+        self.assertFalse(json.loads(output.getvalue())['packages']['complete'])
+
+    def test_status_cli_runs_during_active_build_without_mutations(self):
+        builder = self.builder()
+        self.module.subprocess.run(['git', 'init', '--quiet', str(self.project)], check=True)
+        output = io.StringIO()
+        with builder.locked():
+            builder.record('packages')
+            manifest_before = (builder.work / 'manifest.json').read_bytes()
+            with patch.object(self.module, '__file__', str(self.project / 'hydra/build.py')):
+                with contextlib.redirect_stdout(output):
+                    result = self.module.main(['status'])
+            self.assertEqual(result, 0, 'status must not acquire the exclusive build lock')
+            data = json.loads(output.getvalue())
+            self.assertTrue(data['running'])
+            self.assertFalse(data['packages']['complete'])
+            self.assertEqual((builder.work / 'manifest.json').read_bytes(), manifest_before)
+
+    def test_detached_emerge_refuses_preparation_before_fetch_or_writes(self):
+        builder, inputs = self.existing_builder()
+        fixture = self.project / 'podman-fixture.json'
+        data = json.loads(fixture.read_text())
+        data['active_job'] = True
+        fixture.write_text(json.dumps(data))
+        with patch.object(builder, 'fetch', return_value=inputs) as fetch:
+            with self.assertRaisesRegex(self.module.BuildError, 'package job'):
+                builder.prepare()
+            fetch.assert_not_called()
+        self.assertFalse((self.project / 'mutations.jsonl').exists())
+
+    def test_contained_job_lock_refuses_preparation_before_fetch(self):
+        builder, inputs = self.existing_builder()
+        fixture = self.project / 'podman-fixture.json'
+        data = json.loads(fixture.read_text())
+        data['job_lock'] = True
+        fixture.write_text(json.dumps(data))
+        with patch.object(builder, 'fetch', return_value=inputs) as fetch:
+            with self.assertRaisesRegex(self.module.BuildError, 'package job'):
+                builder.prepare()
+            fetch.assert_not_called()
+
+    def test_status_detects_detached_emerge_without_outer_lock(self):
+        builder, inputs = self.existing_builder()
+        fixture = self.project / 'podman-fixture.json'
+        data = json.loads(fixture.read_text())
+        data['active_job'] = True
+        fixture.write_text(json.dumps(data))
+        output = io.StringIO()
+        with contextlib.redirect_stdout(output):
+            builder.status()
+        status = json.loads(output.getvalue())
+        self.assertTrue(status['running'])
+        self.assertTrue(status['package_job_running'])
+        self.assertFalse(status['mutation_lock_held'])
+
+    def test_package_job_records_failure_and_log_without_host_collection(self):
+        self.assertTrue(hasattr(self.module, 'package_job_script'), 'durable job runner is required')
+        script = self.module.package_job_script(['/bin/sh', '-c', 'echo diagnostic; exit 7'],
+                                               self.project / 'job', self.project / 'pkgdb', 'inputs-one')
+        result = self.module.subprocess.run(['python3', '-c', script])
+        self.assertEqual(result.returncode, 7)
+        status = json.loads((self.project / 'job/packages.status.json').read_text())
+        self.assertFalse(status['running'])
+        self.assertEqual(status['exit_code'], 7)
+        self.assertIn('diagnostic', (self.project / 'job/packages.log').read_text())
+        self.assertFalse((self.project / 'job/packages.json').exists())
+
+    def test_package_job_persists_inventory_after_success_without_host_collection(self):
+        self.assertTrue(hasattr(self.module, 'package_job_script'), 'durable job runner is required')
+        package = self.project / 'pkgdb/category/example-1.0'
+        package.mkdir(parents=True)
+        (package / 'USE').write_text('wayland opengl')
+        (package / 'BUILD_TIME').write_text('123')
+        script = self.module.package_job_script(['/bin/sh', '-c', 'exit 0'], self.project / 'job',
+                                               self.project / 'pkgdb', 'inputs-one')
+        result = self.module.subprocess.run(['python3', '-c', script])
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(json.loads((self.project / 'job/packages.json').read_text()),
+                         [{'package': 'category/example-1.0', 'use': ['wayland', 'opengl'], 'build_time': '123'}])
+        self.assertEqual(json.loads((self.project / 'job/packages.status.json').read_text())['exit_code'], 0)
+
+    def test_matching_stopped_builder_checks_provenance_before_start(self):
+        builder, inputs = self.existing_builder()
+        fixture = self.project / 'podman-fixture.json'
+        data = json.loads(fixture.read_text())
+        data['stopped'] = True
+        fixture.write_text(json.dumps(data))
+        with patch.object(builder, 'fetch', return_value=inputs):
+            try:
+                builder.prepare()
+            except self.module.BuildError as error:
+                self.fail(f'Compatible stopped builder must resume: {error}')
+        self.assertTrue(builder.manifest()['prepare']['complete'])
+
+    def test_unfinished_snapshot_is_not_reported_as_prepared(self):
+        builder, inputs = self.existing_builder(snapshot=None)
+        fixture = self.project / 'podman-fixture.json'
+        data = json.loads(fixture.read_text())
+        data['snapshot_pending'] = 'old-snapshot'
+        fixture.write_text(json.dumps(data))
+        builder.record('prepare')
+        self.assertFalse(builder.complete('prepare'))
+
 
 if __name__ == '__main__':
     unittest.main()