From 05906054cf5c4f6a9346ce0b4c507f256566e43e Mon Sep 17 00:00:00 2001 From: Emil <65846814+emil28092005@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:56:04 +0300 Subject: [PATCH] Validate release lighting sweep provenance and matched controls --- tests/test_p3_lighting_benchmark.py | 171 ++++++++++++++++++++++++++-- tools/benchmark_p3_lighting.py | 147 ++++++++++++++++++++---- 2 files changed, 287 insertions(+), 31 deletions(-) diff --git a/tests/test_p3_lighting_benchmark.py b/tests/test_p3_lighting_benchmark.py index bdfa443..e3ec100 100644 --- a/tests/test_p3_lighting_benchmark.py +++ b/tests/test_p3_lighting_benchmark.py @@ -2,6 +2,7 @@ import csv import json +import hashlib import subprocess import sys import tempfile @@ -15,19 +16,41 @@ sys.path.insert(0, str(ROOT / "tools")) def sample(shadows, visibility, lights, raster, gpu, repeat=1, frame=0): + requested = 6 * lights if shadows == "on" else 0 + rendered = min(12, requested) return { "shadows": shadows, "visibility": visibility, "light_count": str(lights), "run_index": str(repeat), "frame": str(frame), - "gpu_main_raster_ms": str(raster), "gpu_ms": str(gpu), + "gpu_main_raster_ms": str(raster), "gpu_post_raster_ms": "0", + "gpu_post_visible": "0", "gpu_ms": str(gpu), "gpu_shadow_ms": "0", "cpu_ms": "1", "readback_cpu_ms": ".5", "validation_errors": "0", "device": "Fake GPU", "driver": "Fake Driver", "commit": "abc123", "effective_visibility": visibility, "lighting_path": "forward", "submitted_local_lights": str(lights), - "omitted_local_lights": "0", "shadow_tiles": "0", "draw_calls": "1", + "omitted_local_lights": "0", "shadow_tiles": str(rendered), "draw_calls": "1", "gpu_bytes": "4096", "validation_enabled": "0", "width": "1920", "height": "1080", + "build_configuration": "Release", "requested_local_shadow_faces": str(requested), + "rendered_local_shadow_faces": str(rendered), + "dropped_shadow_faces": str(requested - rendered), + "shadow_atlas_full_drops": str(requested - rendered), } +def source_repository(root: Path) -> tuple[Path, str]: + source = root / "source" + source.mkdir() + (source / "README").write_text("fixed source\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", source], check=True) + subprocess.run(["git", "-C", source, "-c", "user.name=Benchmark Test", + "-c", "user.email=benchmark@example.invalid", "add", "README"], check=True) + subprocess.run(["git", "-C", source, "-c", "user.name=Benchmark Test", + "-c", "user.email=benchmark@example.invalid", "commit", "-qm", "Fixture"], + check=True) + revision = subprocess.check_output(["git", "-C", source, "rev-parse", "HEAD"], + text=True).strip() + return source, revision + + FAKE_BENCHMARK = r'''import argparse import csv import json @@ -46,7 +69,10 @@ fieldnames = ["light_count", "shadows", "visibility", "frame", "device", "driver "commit", "gpu_main_raster_ms", "gpu_ms", "gpu_shadow_ms", "cpu_ms", "readback_cpu_ms", "validation_errors", "run_index", "effective_visibility", "lighting_path", "submitted_local_lights", "omitted_local_lights", - "shadow_tiles", "draw_calls", "gpu_bytes", "validation_enabled", "width", "height"] + "shadow_tiles", "draw_calls", "gpu_bytes", "validation_enabled", "width", "height", + "build_configuration", "requested_local_shadow_faces", "rendered_local_shadow_faces", + "dropped_shadow_faces", "shadow_atlas_full_drops", "gpu_post_raster_ms", + "gpu_post_visible"] with path.open("w", newline="", encoding="utf-8") as stream: writer = csv.DictWriter(stream, fieldnames=fieldnames) writer.writeheader() @@ -54,13 +80,20 @@ with path.open("w", newline="", encoding="utf-8") as stream: writer.writerow(dict(light_count=a.lights, shadows=a.shadows, visibility=a.visibility, frame=frame, device="Fake GPU", driver="Fake Driver", commit=a.commit, - gpu_main_raster_ms=.4 + .02 * a.lights, gpu_ms=4 + .02 * a.lights, + gpu_main_raster_ms=.4 + .02 * a.lights, gpu_post_raster_ms=0, + gpu_post_visible=0, gpu_ms=4 + .02 * a.lights, gpu_shadow_ms=0, cpu_ms=1, readback_cpu_ms=.5, validation_errors=0, run_index=a.run_index, effective_visibility=a.visibility, lighting_path="forward", submitted_local_lights=a.lights, omitted_local_lights=0, - shadow_tiles=0, draw_calls=1, gpu_bytes=4096, - validation_enabled=0, width=a.width, height=a.height)) + shadow_tiles=min(12, 6 * a.lights) if a.shadows == "on" else 0, + draw_calls=1, gpu_bytes=4096, + validation_enabled=0, width=a.width, height=a.height, + build_configuration="Release", + requested_local_shadow_faces=6 * a.lights if a.shadows == "on" else 0, + rendered_local_shadow_faces=min(12, 6 * a.lights) if a.shadows == "on" else 0, + dropped_shadow_faces=max(0, 6 * a.lights - 12) if a.shadows == "on" else 0, + shadow_atlas_full_drops=max(0, 6 * a.lights - 12) if a.shadows == "on" else 0)) ''' @@ -83,6 +116,17 @@ class LightingBenchmarkTests(unittest.TestCase): for light in (0, 4, 16, 32, 64, 128) for mode in ("direct", "gpu-frustum", "gpu-occlusion") for repeat in (1, 2, 3)}) + positions = {(run["shadows"], run["visibility"], run["repeat"], run["light_count"]): index + for index, run in enumerate(runs)} + for run in runs: + if run["light_count"] not in (32, 64, 128): + continue + baseline = positions[(run["shadows"], run["visibility"], run["repeat"], 0)] + separation = positions[(run["shadows"], run["visibility"], + run["repeat"], run["light_count"])] - baseline + self.assertGreater(separation, 0) + self.assertLessEqual(separation, 5, + "Each expensive run needs a nearby control on the same repeat") def test_gate_uses_per_run_medians_and_same_mode_shadow_baseline(self): from benchmark_p3_lighting import summarize_rows @@ -107,15 +151,54 @@ class LightingBenchmarkTests(unittest.TestCase): self.assertEqual(hits[("off", "direct", 32)]["zero_light_gpu_ms"], 10) self.assertEqual(hits[("on", "gpu-frustum", 64)]["zero_light_gpu_ms"], 4) + def test_gate_pairs_controls_and_counts_occlusion_post_raster(self): + from benchmark_p3_lighting import summarize_rows + + rows = [] + for repeat, baseline_main, candidate_main in ((1, .4, 1.5), + (2, 1.4, 1.4), + (3, 2.4, 3.5)): + rows.append(sample("off", "direct", 0, baseline_main, 5, repeat)) + rows.append(sample("off", "direct", 32, candidate_main, 6, repeat)) + control = sample("off", "gpu-occlusion", 0, .4, 4, repeat) + candidate = sample("off", "gpu-occlusion", 64, .5, 4.9, repeat) + candidate["gpu_post_raster_ms"] = ".8" + rows.extend((control, candidate)) + summary = summarize_rows(rows) + hits = {(item["visibility"], item["light_count"]): item + for item in summary["forward_plus_gate"]["candidates"]} + self.assertAlmostEqual(hits[("direct", 32)]["overhead_ms"], 1.1) + self.assertAlmostEqual(hits[("gpu-occlusion", 64)]["overhead_ms"], .9) + self.assertEqual(hits[("gpu-occlusion", 64)]["raster_metric"], "main_plus_post") + + def test_shadow_summary_keeps_requested_rendered_and_dropped_faces(self): + from benchmark_p3_lighting import summarize_rows + + rows = [sample("on", "direct", 0, .4, 4, repeat) + for repeat in (1, 2, 3)] + rows += [sample("on", "direct", 64, 1.1, 5, repeat) + for repeat in (1, 2, 3)] + summary = summarize_rows(rows) + high = next(item for item in summary["configurations"] if item["light_count"] == 64) + self.assertEqual(high["shadow_counts"], { + "requested_local_shadow_faces": 384, + "rendered_local_shadow_faces": 12, + "shadow_tiles": 12, + "dropped_shadow_faces": 372, + "shadow_atlas_full_drops": 372, + }) + def test_sweep_runs_fake_executable_and_preserves_all_raw_frames(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) fake = root / "fake_benchmark.py" fake.write_text(FAKE_BENCHMARK, encoding="utf-8") + source, revision = source_repository(root) output = root / "café 世界" process = subprocess.run( [sys.executable, SCRIPT, "--sweep", "--executable", fake, - "--output", output, "--shadows", "off", "--commit", "abc123", + "--output", output, "--shadows", "off", "--commit", revision, + "--source-root", source, "--driver", "Fake Driver"], text=True, capture_output=True) self.assertEqual(process.returncode, 0, process.stderr) @@ -126,10 +209,14 @@ class LightingBenchmarkTests(unittest.TestCase): self.assertEqual(len(merged), 54 * 30) self.assertEqual({row["source_csv"] for row in merged}, {path.name for path in raw}) + self.assertEqual({int(row["acquisition_index"]) for row in merged}, set(range(54))) report = json.loads((output / "summary.json").read_text(encoding="utf-8")) self.assertEqual(report["runs_completed"], 54) self.assertEqual(report["rows"], 54 * 30) self.assertTrue(report["forward_plus_gate"]["triggered"]) + self.assertEqual(report["benchmark_sha256"], hashlib.sha256(fake.read_bytes()).hexdigest()) + self.assertEqual(report["source_revision"], revision) + self.assertFalse(report["source_dirty"]) one = json.loads(next((output / "raw").glob("*.args.json")).read_text()) self.assertEqual((one["width"], one["height"], one["warmup"], one["frames"]), (1920, 1080, 10, 30)) @@ -142,10 +229,12 @@ class LightingBenchmarkTests(unittest.TestCase): fake.write_text(FAKE_BENCHMARK.replace( '"gpu_main_raster_ms", "gpu_ms"', '"gpu_ms"').replace( 'gpu_main_raster_ms=.4 + .02 * a.lights, ', ''), encoding="utf-8") + source, revision = source_repository(root) output = root / "invalid" process = subprocess.run( [sys.executable, SCRIPT, "--sweep", "--executable", fake, - "--output", output, "--shadows", "off", "--commit", "abc123", + "--output", output, "--shadows", "off", "--commit", revision, + "--source-root", source, "--driver", "Fake Driver"], text=True, capture_output=True) self.assertNotEqual(process.returncode, 0) @@ -159,10 +248,12 @@ class LightingBenchmarkTests(unittest.TestCase): fake.write_text(FAKE_BENCHMARK.replace( 'effective_visibility=a.visibility', 'effective_visibility="direct"'), encoding="utf-8") + source, revision = source_repository(root) output = root / "fallback-output" process = subprocess.run( [sys.executable, SCRIPT, "--sweep", "--executable", fake, - "--output", output, "--shadows", "off", "--commit", "abc123", + "--output", output, "--shadows", "off", "--commit", revision, + "--source-root", source, "--driver", "Fake Driver"], text=True, capture_output=True) self.assertNotEqual(process.returncode, 0) @@ -176,10 +267,12 @@ class LightingBenchmarkTests(unittest.TestCase): fake.write_text(FAKE_BENCHMARK.replace( 'submitted_local_lights=a.lights', 'submitted_local_lights=0'), encoding="utf-8") + source, revision = source_repository(root) output = root / "wrong-count-output" process = subprocess.run( [sys.executable, SCRIPT, "--sweep", "--executable", fake, - "--output", output, "--shadows", "off", "--commit", "abc123", + "--output", output, "--shadows", "off", "--commit", revision, + "--source-root", source, "--driver", "Fake Driver"], text=True, capture_output=True) self.assertNotEqual(process.returncode, 0) @@ -194,6 +287,64 @@ class LightingBenchmarkTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "--driver"): sweep(fake, Path(temporary) / "out", "off", "abc123") + def test_sweep_rejects_debug_binary_before_reporting_a_release_gate(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, revision = source_repository(root) + fake = root / "debug.py" + fake.write_text(FAKE_BENCHMARK.replace( + 'build_configuration="Release"', 'build_configuration="Debug"'), + encoding="utf-8") + output = root / "debug-output" + process = subprocess.run( + [sys.executable, SCRIPT, "--sweep", "--executable", fake, + "--output", output, "--source-root", source, "--commit", revision, + "--driver", "Fake Driver", "--shadows", "off"], + text=True, capture_output=True) + self.assertNotEqual(process.returncode, 0) + self.assertIn("Release", process.stderr) + self.assertFalse((output / "summary.json").exists()) + + def test_sweep_rejects_dirty_or_misidentified_source_revision(self): + from benchmark_p3_lighting import sweep + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, revision = source_repository(root) + fake = root / "fake.py" + fake.write_text(FAKE_BENCHMARK, encoding="utf-8") + with self.assertRaisesRegex(ValueError, "revision"): + sweep(fake, root / "wrong-commit", "off", "incorrect", driver="Fake Driver", + source_root=source) + (source / "README").write_text("edited after commit\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "dirty"): + sweep(fake, root / "dirty", "off", revision, driver="Fake Driver", + source_root=source) + + def test_sweep_rejects_mixed_device_driver_path_or_validation(self): + mutations = { + "device": ('device="Fake GPU"', 'device="Other GPU" if a.lights else "Fake GPU"'), + "driver": ('driver="Fake Driver"', 'driver="Wrong Driver"'), + "lighting_path": ('lighting_path="forward"', + 'lighting_path="forward_plus" if a.lights else "forward"'), + "validation_enabled": ('validation_enabled=0', 'validation_enabled=1'), + } + for field, (before, after) in mutations.items(): + with self.subTest(field=field), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, revision = source_repository(root) + fake = root / "changed.py" + fake.write_text(FAKE_BENCHMARK.replace(before, after), encoding="utf-8") + output = root / "changed-output" + process = subprocess.run( + [sys.executable, SCRIPT, "--sweep", "--executable", fake, + "--output", output, "--source-root", source, "--commit", revision, + "--driver", "Fake Driver", "--shadows", "off"], + text=True, capture_output=True) + self.assertNotEqual(process.returncode, 0) + self.assertIn(field, process.stderr) + self.assertFalse((output / "summary.json").exists()) + def real_executable_smoke(executable: Path) -> None: from benchmark_p3_lighting import REQUIRED_COLUMNS diff --git a/tools/benchmark_p3_lighting.py b/tools/benchmark_p3_lighting.py index 66bdad8..fea1bf0 100644 --- a/tools/benchmark_p3_lighting.py +++ b/tools/benchmark_p3_lighting.py @@ -9,6 +9,7 @@ from __future__ import annotations import argparse import csv +import hashlib import json import math from pathlib import Path @@ -20,6 +21,9 @@ import sys LIGHT_COUNTS = (0, 4, 16, 32, 64, 128) VISIBILITY_MODES = ("direct", "gpu-frustum", "gpu-occlusion") REPEATS = (1, 2, 3) +# Pair a zero-light control with every independent repeat, keeping high-light +# measurements nearby rather than comparing the first run with the last. +MEASUREMENT_ORDER = (0, 32, 4, 64, 16, 128) WARMUP_FRAMES = 10 MEASURED_FRAMES = 30 WIDTH, HEIGHT = 1920, 1080 @@ -29,9 +33,15 @@ REQUIRED_COLUMNS = ( "readback_cpu_ms", "validation_errors", "run_index", "effective_visibility", "lighting_path", "submitted_local_lights", "omitted_local_lights", "shadow_tiles", "draw_calls", "gpu_bytes", "validation_enabled", "width", "height", + "build_configuration", "requested_local_shadow_faces", "rendered_local_shadow_faces", + "dropped_shadow_faces", "shadow_atlas_full_drops", "gpu_post_raster_ms", + "gpu_post_visible", ) -TIMING_COLUMNS = ("gpu_main_raster_ms", "gpu_ms", "gpu_shadow_ms", "cpu_ms", - "readback_cpu_ms") +TIMING_COLUMNS = ("gpu_main_raster_ms", "gpu_post_raster_ms", "gpu_ms", + "gpu_shadow_ms", "cpu_ms", "readback_cpu_ms") +SUMMARY_TIMINGS = (*TIMING_COLUMNS, "forward_raster_ms") +SHADOW_COLUMNS = ("requested_local_shadow_faces", "rendered_local_shadow_faces", + "shadow_tiles", "dropped_shadow_faces", "shadow_atlas_full_drops") def build_runs(shadows: str = "both") -> list[dict]: @@ -40,8 +50,8 @@ def build_runs(shadows: str = "both") -> list[dict]: settings = ("off", "on") if shadows == "both" else (shadows,) return [{"shadows": shadow, "visibility": mode, "light_count": lights, "repeat": repeat} - for shadow in settings for mode in VISIBILITY_MODES - for lights in LIGHT_COUNTS for repeat in REPEATS] + for shadow in settings for repeat in REPEATS + for mode in VISIBILITY_MODES for lights in MEASUREMENT_ORDER] def median(values: list[float]) -> float: @@ -67,6 +77,24 @@ def _measurement(row: dict, name: str) -> float: return value +def _timing(row: dict, name: str) -> float: + if name == "forward_raster_ms": + main = _measurement(row, "gpu_main_raster_ms") + return main + (_measurement(row, "gpu_post_raster_ms") + if row["visibility"] == "gpu-occlusion" else 0) + return _measurement(row, name) + + +def _count(row: dict, name: str) -> int: + try: + value = int(row[name]) + except (KeyError, TypeError, ValueError) as error: + raise ValueError(f"Invalid {name} in benchmark CSV") from error + if value < 0: + raise ValueError(f"Invalid {name} in benchmark CSV: {value}") + return value + + def summarize_rows(rows: list[dict]) -> dict: """Use the median of each independent run's median, then compare like baselines.""" grouped: dict[tuple[str, str, int], dict[int, list[dict]]] = {} @@ -80,6 +108,8 @@ def summarize_rows(rows: list[dict]) -> dict: raise ValueError(f"Invalid benchmark configuration: {key}, repeat {repeat}") for name in TIMING_COLUMNS: _measurement(row, name) + for name in SHADOW_COLUMNS: + _count(row, name) grouped.setdefault(key, {}).setdefault(repeat, []).append(row) configurations = [] @@ -89,16 +119,21 @@ def summarize_rows(rows: list[dict]) -> dict: for repeat, samples in sorted(repeats.items()): run_summaries.append({ "run_index": repeat, "frames": len(samples), - "median_ms": {name: median([_measurement(row, name) for row in samples]) - for name in TIMING_COLUMNS}, + "median_ms": {name: median([_timing(row, name) for row in samples]) + for name in SUMMARY_TIMINGS}, }) + shadow_counts = {name: {_count(row, name) for samples in repeats.values() + for row in samples} for name in SHADOW_COLUMNS} + if any(len(values) != 1 for values in shadow_counts.values()): + raise ValueError("Shadow counts changed within a fixed benchmark configuration") entry = { "shadows": shadows, "visibility": visibility, "light_count": lights, "runs": run_summaries, + "shadow_counts": {name: next(iter(values)) for name, values in shadow_counts.items()}, "median_ms": {name: median([run["median_ms"][name] for run in run_summaries]) - for name in TIMING_COLUMNS}, - "p95_ms": {name: p95([_measurement(row, name) for samples in repeats.values() - for row in samples]) for name in TIMING_COLUMNS}, + for name in SUMMARY_TIMINGS}, + "p95_ms": {name: p95([_timing(row, name) for samples in repeats.values() + for row in samples]) for name in SUMMARY_TIMINGS}, } configurations.append(entry) lookup[(shadows, visibility, lights)] = entry @@ -111,13 +146,20 @@ def summarize_rows(rows: list[dict]) -> dict: baseline = lookup.get((entry["shadows"], entry["visibility"], 0)) if baseline is None: raise ValueError("Forward+ gate requires a zero-light baseline for each mode/shadow setting") - zero_gpu = baseline["median_ms"]["gpu_ms"] + base_runs = {run["run_index"]: run for run in baseline["runs"]} + paired = [(run, base_runs[run["run_index"]]) for run in entry["runs"] + if run["run_index"] in base_runs] + if len(paired) != len(entry["runs"]): + raise ValueError("Forward+ gate requires a matched zero-light repeat for every run") + zero_gpu = median([base["median_ms"]["gpu_ms"] for _, base in paired]) if zero_gpu <= 0: raise ValueError("Forward+ gate requires positive zero-light GPU frame timing") - overhead = (entry["median_ms"]["gpu_main_raster_ms"] - - baseline["median_ms"]["gpu_main_raster_ms"]) + overhead = median([run["median_ms"]["forward_raster_ms"] - + base["median_ms"]["forward_raster_ms"] for run, base in paired]) result = {"shadows": entry["shadows"], "visibility": entry["visibility"], "light_count": entry["light_count"], "overhead_ms": overhead, + "raster_metric": ("main_plus_post" if entry["visibility"] == "gpu-occlusion" + else "main"), "zero_light_gpu_ms": zero_gpu, "overhead_percent_of_zero_gpu": 100 * overhead / zero_gpu, "absolute_threshold_reached": overhead >= 1.0, @@ -128,10 +170,11 @@ def summarize_rows(rows: list[dict]) -> dict: return {"configurations": configurations, "forward_plus_gate": {"triggered": bool(candidates), "candidates": candidates, "evaluated": evaluated, - "basis": "median of three independent run medians; same-mode/shadow zero-light GPU baseline"}} + "basis": "median of paired per-run raster overheads; same-mode/shadow/repeat zero-light GPU baseline"}} -def _read_run_csv(path: Path, run: dict, commit: str) -> tuple[list[str], list[dict]]: +def _read_run_csv(path: Path, run: dict, commit: str, driver: str, + validation: str) -> tuple[list[str], list[dict]]: with path.open(newline="", encoding="utf-8") as stream: reader = csv.DictReader(stream) columns = reader.fieldnames or [] @@ -149,6 +192,12 @@ def _read_run_csv(path: Path, run: dict, commit: str) -> tuple[list[str], list[d for name, value in expected.items(): if row[name] != value: raise ValueError(f"{path}: {name} mismatch: expected {value}, got {row[name]}") + if row["build_configuration"] != "Release": + raise ValueError(f"{path}: Forward+ sweep requires a Release benchmark binary") + if row["driver"] != driver: + raise ValueError(f"{path}: driver differs from the declared driver identity") + if row["validation_enabled"] != ("1" if validation == "on" else "0"): + raise ValueError(f"{path}: validation_enabled differs from the requested setting") if row["effective_visibility"] != run["visibility"]: raise ValueError(f"{path}: effective_visibility fell back from {run['visibility']}") if row["submitted_local_lights"] != str(run["light_count"]) or row["omitted_local_lights"] != "0": @@ -163,6 +212,16 @@ def _read_run_csv(path: Path, run: dict, commit: str) -> tuple[list[str], list[d frames.add(frame) if not row["device"] or not row["driver"] or not row["lighting_path"]: raise ValueError(f"{path}: device, driver and effective lighting path are required") + requested = _count(row, "requested_local_shadow_faces") + rendered = _count(row, "rendered_local_shadow_faces") + tiles = _count(row, "shadow_tiles") + dropped = _count(row, "dropped_shadow_faces") + atlas_drops = _count(row, "shadow_atlas_full_drops") + expected_faces = 6 * run["light_count"] if run["shadows"] == "on" else 0 + if (requested != expected_faces or rendered > tiles or tiles > min(requested, 16) or + dropped > requested or atlas_drops > dropped): + raise ValueError(f"{path}: requested/effective shadow face counts disagree with the workload") + _count(row, "gpu_post_visible") for name in TIMING_COLUMNS: _measurement(row, name) return columns, rows @@ -174,21 +233,44 @@ def _git_revision() -> str: text=True).strip() +def _binary_sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def _clean_source_revision(root: Path, expected: str) -> str: + revision = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], + text=True).strip() + if revision != expected: + raise ValueError(f"Source revision {revision} disagrees with benchmark commit {expected}") + status = subprocess.check_output(["git", "-C", str(root), "status", "--porcelain", + "--untracked-files=all"], text=True) + if status: + raise ValueError("Benchmark source checkout is dirty; commit sources before measurement") + return revision + + def sweep(executable: Path, output: Path, shadows: str, commit: str, - validation: str = "off", driver: str | None = None) -> dict: + validation: str = "off", driver: str | None = None, + source_root: Path | None = None) -> dict: if not executable.is_file(): raise ValueError(f"Benchmark executable does not exist: {executable}") if driver is None or not driver.strip() or driver.strip().lower() == "unknown": raise ValueError("A measured sweep requires an explicit --driver identity") + source = (source_root or Path(__file__).resolve().parents[1]).resolve() + revision = _clean_source_revision(source, commit) + binary_sha256 = _binary_sha256(executable) if output.exists() and any(output.iterdir()): raise ValueError(f"Output directory must be new or empty: {output}") raw = output / "raw" raw.mkdir(parents=True) all_rows = [] columns = None + identity = None runs = build_runs(shadows) command_prefix = [sys.executable, str(executable)] if executable.suffix.lower() == ".py" else [str(executable)] - for run in runs: + run_order = [] + for acquisition_index, run in enumerate(runs): filename = (f"shadows-{run['shadows']}_{run['visibility']}_" f"lights-{run['light_count']:03d}_run-{run['repeat']}.csv") target = raw / filename @@ -205,22 +287,42 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str, errors="replace", timeout=180) if result.returncode != 0: raise RuntimeError(f"Benchmark failed for {filename}: {result.stderr[-2000:]}") - run_columns, samples = _read_run_csv(target, run, commit) + run_columns, samples = _read_run_csv(target, run, commit, driver, validation) if columns is None: columns = run_columns elif columns != run_columns: raise ValueError(f"{target}: CSV schema differs from other runs") - all_rows.extend({**row, "source_csv": filename} for row in samples) + for row in samples: + actual = (row["device"], row["driver"], row["lighting_path"], + row["validation_enabled"], row["build_configuration"]) + if identity is None: + identity = actual + elif identity != actual: + changed = next(name for name, before, after in zip( + ("device", "driver", "lighting_path", "validation_enabled", + "build_configuration"), identity, actual) if before != after) + raise ValueError(f"{target}: {changed} changed during the sweep") + all_rows.append({**row, "source_csv": filename, + "acquisition_index": acquisition_index}) + run_order.append(filename) merged = output / "merged.csv" with merged.open("w", newline="", encoding="utf-8") as stream: - writer = csv.DictWriter(stream, fieldnames=[*(columns or []), "source_csv"]) + writer = csv.DictWriter(stream, fieldnames=[*(columns or []), "source_csv", + "acquisition_index"]) writer.writeheader() writer.writerows(all_rows) + if _binary_sha256(executable) != binary_sha256 or _clean_source_revision(source, commit) != revision: + raise ValueError("Benchmark binary or source changed during the sweep") summary = {"format": "faset.p3-lighting-benchmark", "version": 1, "commit": commit, "warmup_frames_per_run": WARMUP_FRAMES, "measured_frames_per_run": MEASURED_FRAMES, "width": WIDTH, "height": HEIGHT, "validation": validation, "driver": driver, + "source_revision": revision, "source_root": str(source), "source_dirty": False, + "benchmark_sha256": binary_sha256, + "device": identity[0], "lighting_path": identity[2], + "validation_enabled": identity[3] == "1", "build_configuration": identity[4], + "run_order": run_order, "runs_completed": len(runs), "rows": len(all_rows), **summarize_rows(all_rows)} (output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", @@ -237,6 +339,8 @@ def main() -> int: parser.add_argument("--executable", type=Path, help="Built C++ benchmark executable") parser.add_argument("--output", type=Path, help="New or empty evidence directory") parser.add_argument("--commit", help="Source revision; defaults to this checkout's HEAD") + parser.add_argument("--source-root", type=Path, + help="Clean source checkout used to build the benchmark; defaults to this repo") parser.add_argument("--driver", help="Required driver identity for a measured sweep") parser.add_argument("--validation", choices=("on", "off"), default="off") args = parser.parse_args() @@ -248,8 +352,9 @@ def main() -> int: parser.error("--sweep requires --executable and --output") try: summary = sweep(args.executable.resolve(), args.output.resolve(), args.shadows, - args.commit or _git_revision(), args.validation, args.driver) - except (OSError, ValueError, RuntimeError) as error: + args.commit or _git_revision(), args.validation, args.driver, + args.source_root) + except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: print(f"P3 lighting benchmark failed: {error}", file=sys.stderr) return 1 print(json.dumps({"summary": str(args.output.resolve() / "summary.json"),