Check real lighting benchmark output and driver identity

cmake/Renderer.cmake

@@ -85,7 +85,12 @@ if(BUILD_TESTING)
   add_executable(faset_p3_lighting_benchmark
     "${PROJECT_SOURCE_DIR}/examples/renderer/p3_lighting_benchmark.cpp")
   target_link_libraries(faset_p3_lighting_benchmark PRIVATE faset_render faset_core)
-  add_test(NAME render_lighting_benchmark_schema COMMAND faset_p3_lighting_benchmark --list-runs)
+  add_test(NAME render_lighting_benchmark_schema COMMAND
-  set_tests_properties(render_lighting_benchmark_schema PROPERTIES LABELS "gpu;p3" TIMEOUT 90)
+    "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tests/test_p3_lighting_benchmark.py")
+  set_tests_properties(render_lighting_benchmark_schema PROPERTIES LABELS "p3" TIMEOUT 90)
+  add_test(NAME render_lighting_benchmark_smoke COMMAND
+    "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tests/test_p3_lighting_benchmark.py"
+    --real-executable "$<TARGET_FILE:faset_p3_lighting_benchmark>")
+  set_tests_properties(render_lighting_benchmark_smoke PROPERTIES LABELS "gpu;p3" TIMEOUT 90)
 endif()
 install(FILES ${FASET_SHADER_OUTPUTS} DESTINATION shaders)

examples/renderer/p3_lighting_benchmark.cpp

@@ -179,6 +179,12 @@ void benchmark(const Options& options) {
         const auto stats = renderer.stats();
         if (stats.validation_errors != 0)
             throw std::runtime_error("Vulkan validation error during benchmark");
+        if (stats.submitted_local_lights != options.lights || stats.omitted_local_lights != 0)
+            throw std::runtime_error("Renderer did not submit every requested local light");
+        if (stats.effective_visibility_mode != options.visibility)
+            throw std::runtime_error("Requested visibility path fell back during benchmark");
+        if (stats.gpu_main_raster_ms <= 0 || stats.gpu_ms <= 0)
+            throw std::runtime_error("GPU raster or frame timestamp was unavailable");
         csv << options.lights << ',' << (options.shadows ? "on" : "off") << ','
             << mode_name(options.visibility) << ',' << mode_name(stats.effective_visibility_mode)
             << ",forward," << options.run_index << ',' << frame << ',';

tests/test_p3_lighting_benchmark.py

@@ -115,7 +115,8 @@ class LightingBenchmarkTests(unittest.TestCase):
             output = root / "café 世界"
             process = subprocess.run(
                 [sys.executable, SCRIPT, "--sweep", "--executable", fake,
-                 "--output", output, "--shadows", "off", "--commit", "abc123"],
+                 "--output", output, "--shadows", "off", "--commit", "abc123",
+                 "--driver", "Fake Driver"],
                 text=True, capture_output=True)
             self.assertEqual(process.returncode, 0, process.stderr)
             raw = list((output / "raw").glob("*.csv"))
@@ -144,7 +145,8 @@ class LightingBenchmarkTests(unittest.TestCase):
             output = root / "invalid"
             process = subprocess.run(
                 [sys.executable, SCRIPT, "--sweep", "--executable", fake,
-                 "--output", output, "--shadows", "off", "--commit", "abc123"],
+                 "--output", output, "--shadows", "off", "--commit", "abc123",
+                 "--driver", "Fake Driver"],
                 text=True, capture_output=True)
             self.assertNotEqual(process.returncode, 0)
             self.assertIn("gpu_main_raster_ms", process.stderr)
@@ -160,7 +162,8 @@ class LightingBenchmarkTests(unittest.TestCase):
             output = root / "fallback-output"
             process = subprocess.run(
                 [sys.executable, SCRIPT, "--sweep", "--executable", fake,
-                 "--output", output, "--shadows", "off", "--commit", "abc123"],
+                 "--output", output, "--shadows", "off", "--commit", "abc123",
+                 "--driver", "Fake Driver"],
                 text=True, capture_output=True)
             self.assertNotEqual(process.returncode, 0)
             self.assertIn("effective_visibility", process.stderr)
@@ -176,12 +179,51 @@ class LightingBenchmarkTests(unittest.TestCase):
             output = root / "wrong-count-output"
             process = subprocess.run(
                 [sys.executable, SCRIPT, "--sweep", "--executable", fake,
-                 "--output", output, "--shadows", "off", "--commit", "abc123"],
+                 "--output", output, "--shadows", "off", "--commit", "abc123",
+                 "--driver", "Fake Driver"],
                 text=True, capture_output=True)
             self.assertNotEqual(process.returncode, 0)
             self.assertIn("submitted_local_lights", process.stderr)
             self.assertFalse((output / "summary.json").exists())
 
+    def test_sweep_requires_known_driver_identity(self):
+        from benchmark_p3_lighting import sweep
+        with tempfile.TemporaryDirectory() as temporary:
+            fake = Path(temporary) / "fake.py"
+            fake.write_text(FAKE_BENCHMARK, encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "--driver"):
+                sweep(fake, Path(temporary) / "out", "off", "abc123")
+
+
+def real_executable_smoke(executable: Path) -> None:
+    from benchmark_p3_lighting import REQUIRED_COLUMNS
+
+    choices = subprocess.run([executable, "--list-runs"], capture_output=True,
+                             text=True, check=True)
+    declared = json.loads(choices.stdout)
+    if declared["lights"] != [0, 4, 16, 32, 64, 128] or len(declared["visibility"]) != 3:
+        raise AssertionError("C++ executable and Python sweep matrix disagree")
+    with tempfile.TemporaryDirectory() as directory:
+        output = Path(directory) / "smoke.csv"
+        subprocess.run([executable, "--lights", "4", "--shadows", "off",
+                        "--visibility", "direct", "--width", "64", "--height", "64",
+                        "--warmup", "0", "--frames", "1", "--validation", "on",
+                        "--commit", "smoke", "--driver", "smoke-driver",
+                        "--csv", output], capture_output=True, text=True, check=True)
+        with output.open(newline="", encoding="utf-8") as stream:
+            reader = csv.DictReader(stream)
+            columns, rows = reader.fieldnames or [], list(reader)
+        if set(REQUIRED_COLUMNS) - set(columns) or len(rows) != 1:
+            raise AssertionError("Real benchmark CSV lacks a complete single-frame row")
+        row = rows[0]
+        if (row["effective_visibility"] != "direct" or row["lighting_path"] != "forward" or
+                row["submitted_local_lights"] != "4" or row["validation_errors"] != "0" or
+                float(row["gpu_main_raster_ms"]) <= 0):
+            raise AssertionError("Real benchmark did not report the measured lighting path")
+
 
 if __name__ == "__main__":
-    unittest.main()
+    if len(sys.argv) == 3 and sys.argv[1] == "--real-executable":
+        real_executable_smoke(Path(sys.argv[2]).resolve())
+    else:
+        unittest.main()

tools/benchmark_p3_lighting.py

@@ -178,6 +178,8 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str,
           validation: str = "off", driver: str | None = None) -> dict:
     if not executable.is_file():
         raise ValueError(f"Benchmark executable does not exist: {executable}")
+    if driver is None or not driver.strip() or driver.strip().lower() == "unknown":
+        raise ValueError("A measured sweep requires an explicit --driver identity")
     if output.exists() and any(output.iterdir()):
         raise ValueError(f"Output directory must be new or empty: {output}")
     raw = output / "raw"
@@ -200,7 +202,7 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str,
         if driver is not None:
             command += ["--driver", driver]
         result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8",
-                                errors="replace")
+                                errors="replace", timeout=180)
         if result.returncode != 0:
             raise RuntimeError(f"Benchmark failed for {filename}: {result.stderr[-2000:]}")
         run_columns, samples = _read_run_csv(target, run, commit)
@@ -218,7 +220,8 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str,
     summary = {"format": "faset.p3-lighting-benchmark", "version": 1,
                "commit": commit, "warmup_frames_per_run": WARMUP_FRAMES,
                "measured_frames_per_run": MEASURED_FRAMES, "width": WIDTH, "height": HEIGHT,
-               "validation": validation, "runs_completed": len(runs), "rows": len(all_rows),
+               "validation": validation, "driver": driver,
+               "runs_completed": len(runs), "rows": len(all_rows),
                **summarize_rows(all_rows)}
     (output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n",
                                           encoding="utf-8")
@@ -234,7 +237,7 @@ def main() -> int:
     parser.add_argument("--executable", type=Path, help="Built C++ benchmark executable")
     parser.add_argument("--output", type=Path, help="New or empty evidence directory")
     parser.add_argument("--commit", help="Source revision; defaults to this checkout's HEAD")
-    parser.add_argument("--driver", help="Explicit driver label to pass through to the benchmark")
+    parser.add_argument("--driver", help="Required driver identity for a measured sweep")
     parser.add_argument("--validation", choices=("on", "off"), default="off")
     args = parser.parse_args()
     if args.list_runs: