diff --git a/cmake/Renderer.cmake b/cmake/Renderer.cmake index bfec8e7..5b02552 100644 --- a/cmake/Renderer.cmake +++ b/cmake/Renderer.cmake @@ -85,7 +85,12 @@ if(BUILD_TESTING) add_executable(faset_p3_lighting_benchmark "${PROJECT_SOURCE_DIR}/examples/renderer/p3_lighting_benchmark.cpp") target_link_libraries(faset_p3_lighting_benchmark PRIVATE faset_render faset_core) - add_test(NAME render_lighting_benchmark_schema COMMAND faset_p3_lighting_benchmark --list-runs) - set_tests_properties(render_lighting_benchmark_schema PROPERTIES LABELS "gpu;p3" TIMEOUT 90) + add_test(NAME render_lighting_benchmark_schema COMMAND + "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tests/test_p3_lighting_benchmark.py") + set_tests_properties(render_lighting_benchmark_schema PROPERTIES LABELS "p3" TIMEOUT 90) + add_test(NAME render_lighting_benchmark_smoke COMMAND + "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tests/test_p3_lighting_benchmark.py" + --real-executable "$") + set_tests_properties(render_lighting_benchmark_smoke PROPERTIES LABELS "gpu;p3" TIMEOUT 90) endif() install(FILES ${FASET_SHADER_OUTPUTS} DESTINATION shaders) diff --git a/examples/renderer/p3_lighting_benchmark.cpp b/examples/renderer/p3_lighting_benchmark.cpp index e183c63..756da04 100644 --- a/examples/renderer/p3_lighting_benchmark.cpp +++ b/examples/renderer/p3_lighting_benchmark.cpp @@ -179,6 +179,12 @@ void benchmark(const Options& options) { const auto stats = renderer.stats(); if (stats.validation_errors != 0) throw std::runtime_error("Vulkan validation error during benchmark"); + if (stats.submitted_local_lights != options.lights || stats.omitted_local_lights != 0) + throw std::runtime_error("Renderer did not submit every requested local light"); + if (stats.effective_visibility_mode != options.visibility) + throw std::runtime_error("Requested visibility path fell back during benchmark"); + if (stats.gpu_main_raster_ms <= 0 || stats.gpu_ms <= 0) + throw std::runtime_error("GPU raster or frame timestamp was unavailable"); csv << options.lights << ',' << (options.shadows ? "on" : "off") << ',' << mode_name(options.visibility) << ',' << mode_name(stats.effective_visibility_mode) << ",forward," << options.run_index << ',' << frame << ','; diff --git a/tests/test_p3_lighting_benchmark.py b/tests/test_p3_lighting_benchmark.py index 4bcd072..4f7dff4 100644 --- a/tests/test_p3_lighting_benchmark.py +++ b/tests/test_p3_lighting_benchmark.py @@ -115,7 +115,8 @@ class LightingBenchmarkTests(unittest.TestCase): output = root / "café 世界" process = subprocess.run( [sys.executable, SCRIPT, "--sweep", "--executable", fake, - "--output", output, "--shadows", "off", "--commit", "abc123"], + "--output", output, "--shadows", "off", "--commit", "abc123", + "--driver", "Fake Driver"], text=True, capture_output=True) self.assertEqual(process.returncode, 0, process.stderr) raw = list((output / "raw").glob("*.csv")) @@ -144,7 +145,8 @@ class LightingBenchmarkTests(unittest.TestCase): output = root / "invalid" process = subprocess.run( [sys.executable, SCRIPT, "--sweep", "--executable", fake, - "--output", output, "--shadows", "off", "--commit", "abc123"], + "--output", output, "--shadows", "off", "--commit", "abc123", + "--driver", "Fake Driver"], text=True, capture_output=True) self.assertNotEqual(process.returncode, 0) self.assertIn("gpu_main_raster_ms", process.stderr) @@ -160,7 +162,8 @@ class LightingBenchmarkTests(unittest.TestCase): output = root / "fallback-output" process = subprocess.run( [sys.executable, SCRIPT, "--sweep", "--executable", fake, - "--output", output, "--shadows", "off", "--commit", "abc123"], + "--output", output, "--shadows", "off", "--commit", "abc123", + "--driver", "Fake Driver"], text=True, capture_output=True) self.assertNotEqual(process.returncode, 0) self.assertIn("effective_visibility", process.stderr) @@ -176,12 +179,51 @@ class LightingBenchmarkTests(unittest.TestCase): output = root / "wrong-count-output" process = subprocess.run( [sys.executable, SCRIPT, "--sweep", "--executable", fake, - "--output", output, "--shadows", "off", "--commit", "abc123"], + "--output", output, "--shadows", "off", "--commit", "abc123", + "--driver", "Fake Driver"], text=True, capture_output=True) self.assertNotEqual(process.returncode, 0) self.assertIn("submitted_local_lights", process.stderr) self.assertFalse((output / "summary.json").exists()) + def test_sweep_requires_known_driver_identity(self): + from benchmark_p3_lighting import sweep + with tempfile.TemporaryDirectory() as temporary: + fake = Path(temporary) / "fake.py" + fake.write_text(FAKE_BENCHMARK, encoding="utf-8") + with self.assertRaisesRegex(ValueError, "--driver"): + sweep(fake, Path(temporary) / "out", "off", "abc123") + + +def real_executable_smoke(executable: Path) -> None: + from benchmark_p3_lighting import REQUIRED_COLUMNS + + choices = subprocess.run([executable, "--list-runs"], capture_output=True, + text=True, check=True) + declared = json.loads(choices.stdout) + if declared["lights"] != [0, 4, 16, 32, 64, 128] or len(declared["visibility"]) != 3: + raise AssertionError("C++ executable and Python sweep matrix disagree") + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "smoke.csv" + subprocess.run([executable, "--lights", "4", "--shadows", "off", + "--visibility", "direct", "--width", "64", "--height", "64", + "--warmup", "0", "--frames", "1", "--validation", "on", + "--commit", "smoke", "--driver", "smoke-driver", + "--csv", output], capture_output=True, text=True, check=True) + with output.open(newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + columns, rows = reader.fieldnames or [], list(reader) + if set(REQUIRED_COLUMNS) - set(columns) or len(rows) != 1: + raise AssertionError("Real benchmark CSV lacks a complete single-frame row") + row = rows[0] + if (row["effective_visibility"] != "direct" or row["lighting_path"] != "forward" or + row["submitted_local_lights"] != "4" or row["validation_errors"] != "0" or + float(row["gpu_main_raster_ms"]) <= 0): + raise AssertionError("Real benchmark did not report the measured lighting path") + if __name__ == "__main__": - unittest.main() + if len(sys.argv) == 3 and sys.argv[1] == "--real-executable": + real_executable_smoke(Path(sys.argv[2]).resolve()) + else: + unittest.main() diff --git a/tools/benchmark_p3_lighting.py b/tools/benchmark_p3_lighting.py index 863792a..66bdad8 100644 --- a/tools/benchmark_p3_lighting.py +++ b/tools/benchmark_p3_lighting.py @@ -178,6 +178,8 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str, validation: str = "off", driver: str | None = None) -> dict: if not executable.is_file(): raise ValueError(f"Benchmark executable does not exist: {executable}") + if driver is None or not driver.strip() or driver.strip().lower() == "unknown": + raise ValueError("A measured sweep requires an explicit --driver identity") if output.exists() and any(output.iterdir()): raise ValueError(f"Output directory must be new or empty: {output}") raw = output / "raw" @@ -200,7 +202,7 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str, if driver is not None: command += ["--driver", driver] result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", - errors="replace") + errors="replace", timeout=180) if result.returncode != 0: raise RuntimeError(f"Benchmark failed for {filename}: {result.stderr[-2000:]}") run_columns, samples = _read_run_csv(target, run, commit) @@ -218,7 +220,8 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str, summary = {"format": "faset.p3-lighting-benchmark", "version": 1, "commit": commit, "warmup_frames_per_run": WARMUP_FRAMES, "measured_frames_per_run": MEASURED_FRAMES, "width": WIDTH, "height": HEIGHT, - "validation": validation, "runs_completed": len(runs), "rows": len(all_rows), + "validation": validation, "driver": driver, + "runs_completed": len(runs), "rows": len(all_rows), **summarize_rows(all_rows)} (output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") @@ -234,7 +237,7 @@ def main() -> int: parser.add_argument("--executable", type=Path, help="Built C++ benchmark executable") parser.add_argument("--output", type=Path, help="New or empty evidence directory") parser.add_argument("--commit", help="Source revision; defaults to this checkout's HEAD") - parser.add_argument("--driver", help="Explicit driver label to pass through to the benchmark") + parser.add_argument("--driver", help="Required driver identity for a measured sweep") parser.add_argument("--validation", choices=("on", "off"), default="off") args = parser.parse_args() if args.list_runs: