Emil_Shanaty/Faset_Engine
1
0
Fork 0
Code Issues Pull Requests Actions 58 Packages Projects Releases Wiki Activity

Check real lighting benchmark output and driver identity

Browse Source
This commit is contained in:
Emil
2026-09-24 02:43:18 +03:00
parent d871db1ded
commit e074713fed
4 changed files with 66 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmake/Renderer.cmake
+7 -2
View File
@@ -85,7 +85,12 @@ if(BUILD_TESTING)
add_executable(faset_p3_lighting_benchmark
"${PROJECT_SOURCE_DIR}/examples/renderer/p3_lighting_benchmark.cpp")
target_link_libraries(faset_p3_lighting_benchmark PRIVATE faset_render faset_core)
add_test(NAME render_lighting_benchmark_schema COMMAND faset_p3_lighting_benchmark --list-runs)
set_tests_properties(render_lighting_benchmark_schema PROPERTIES LABELS "gpu;p3" TIMEOUT 90)
add_test(NAME render_lighting_benchmark_schema COMMAND
"${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tests/test_p3_lighting_benchmark.py")
set_tests_properties(render_lighting_benchmark_schema PROPERTIES LABELS "p3" TIMEOUT 90)
add_test(NAME render_lighting_benchmark_smoke COMMAND
"${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tests/test_p3_lighting_benchmark.py"
--real-executable "$<TARGET_FILE:faset_p3_lighting_benchmark>")
set_tests_properties(render_lighting_benchmark_smoke PROPERTIES LABELS "gpu;p3" TIMEOUT 90)
endif()
install(FILES ${FASET_SHADER_OUTPUTS} DESTINATION shaders)
examples/renderer/p3_lighting_benchmark.cpp
+6
View File
@@ -179,6 +179,12 @@ void benchmark(const Options& options) {
const auto stats = renderer.stats();
if (stats.validation_errors != 0)
throw std::runtime_error("Vulkan validation error during benchmark");
if (stats.submitted_local_lights != options.lights || stats.omitted_local_lights != 0)
throw std::runtime_error("Renderer did not submit every requested local light");
if (stats.effective_visibility_mode != options.visibility)
throw std::runtime_error("Requested visibility path fell back during benchmark");
if (stats.gpu_main_raster_ms <= 0 || stats.gpu_ms <= 0)
throw std::runtime_error("GPU raster or frame timestamp was unavailable");
csv << options.lights << ',' << (options.shadows ? "on" : "off") << ','
<< mode_name(options.visibility) << ',' << mode_name(stats.effective_visibility_mode)
<< ",forward," << options.run_index << ',' << frame << ',';
tests/test_p3_lighting_benchmark.py
+47 -5
View File
@@ -115,7 +115,8 @@ class LightingBenchmarkTests(unittest.TestCase):
output = root / "café 世界"
process = subprocess.run(
[sys.executable, SCRIPT, "--sweep", "--executable", fake,
"--output", output, "--shadows", "off", "--commit", "abc123"],
"--output", output, "--shadows", "off", "--commit", "abc123",
"--driver", "Fake Driver"],
text=True, capture_output=True)
self.assertEqual(process.returncode, 0, process.stderr)
raw = list((output / "raw").glob("*.csv"))
@@ -144,7 +145,8 @@ class LightingBenchmarkTests(unittest.TestCase):
output = root / "invalid"
process = subprocess.run(
[sys.executable, SCRIPT, "--sweep", "--executable", fake,
"--output", output, "--shadows", "off", "--commit", "abc123"],
"--output", output, "--shadows", "off", "--commit", "abc123",
"--driver", "Fake Driver"],
text=True, capture_output=True)
self.assertNotEqual(process.returncode, 0)
self.assertIn("gpu_main_raster_ms", process.stderr)
@@ -160,7 +162,8 @@ class LightingBenchmarkTests(unittest.TestCase):
output = root / "fallback-output"
process = subprocess.run(
[sys.executable, SCRIPT, "--sweep", "--executable", fake,
"--output", output, "--shadows", "off", "--commit", "abc123"],
"--output", output, "--shadows", "off", "--commit", "abc123",
"--driver", "Fake Driver"],
text=True, capture_output=True)
self.assertNotEqual(process.returncode, 0)
self.assertIn("effective_visibility", process.stderr)
@@ -176,12 +179,51 @@ class LightingBenchmarkTests(unittest.TestCase):
output = root / "wrong-count-output"
process = subprocess.run(
[sys.executable, SCRIPT, "--sweep", "--executable", fake,
"--output", output, "--shadows", "off", "--commit", "abc123"],
"--output", output, "--shadows", "off", "--commit", "abc123",
"--driver", "Fake Driver"],
text=True, capture_output=True)
self.assertNotEqual(process.returncode, 0)
self.assertIn("submitted_local_lights", process.stderr)
self.assertFalse((output / "summary.json").exists())
def test_sweep_requires_known_driver_identity(self):
from benchmark_p3_lighting import sweep
with tempfile.TemporaryDirectory() as temporary:
fake = Path(temporary) / "fake.py"
fake.write_text(FAKE_BENCHMARK, encoding="utf-8")
with self.assertRaisesRegex(ValueError, "--driver"):
sweep(fake, Path(temporary) / "out", "off", "abc123")
def real_executable_smoke(executable: Path) -> None:
from benchmark_p3_lighting import REQUIRED_COLUMNS
choices = subprocess.run([executable, "--list-runs"], capture_output=True,
text=True, check=True)
declared = json.loads(choices.stdout)
if declared["lights"] != [0, 4, 16, 32, 64, 128] or len(declared["visibility"]) != 3:
raise AssertionError("C++ executable and Python sweep matrix disagree")
with tempfile.TemporaryDirectory() as directory:
output = Path(directory) / "smoke.csv"
subprocess.run([executable, "--lights", "4", "--shadows", "off",
"--visibility", "direct", "--width", "64", "--height", "64",
"--warmup", "0", "--frames", "1", "--validation", "on",
"--commit", "smoke", "--driver", "smoke-driver",
"--csv", output], capture_output=True, text=True, check=True)
with output.open(newline="", encoding="utf-8") as stream:
reader = csv.DictReader(stream)
columns, rows = reader.fieldnames or [], list(reader)
if set(REQUIRED_COLUMNS) - set(columns) or len(rows) != 1:
raise AssertionError("Real benchmark CSV lacks a complete single-frame row")
row = rows[0]
if (row["effective_visibility"] != "direct" or row["lighting_path"] != "forward" or
row["submitted_local_lights"] != "4" or row["validation_errors"] != "0" or
float(row["gpu_main_raster_ms"]) <= 0):
raise AssertionError("Real benchmark did not report the measured lighting path")
if __name__ == "__main__":
unittest.main()
if len(sys.argv) == 3 and sys.argv[1] == "--real-executable":
real_executable_smoke(Path(sys.argv[2]).resolve())
else:
unittest.main()
tools/benchmark_p3_lighting.py
+6 -3
View File
@@ -178,6 +178,8 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str,
validation: str = "off", driver: str | None = None) -> dict:
if not executable.is_file():
raise ValueError(f"Benchmark executable does not exist: {executable}")
if driver is None or not driver.strip() or driver.strip().lower() == "unknown":
raise ValueError("A measured sweep requires an explicit --driver identity")
if output.exists() and any(output.iterdir()):
raise ValueError(f"Output directory must be new or empty: {output}")
raw = output / "raw"
@@ -200,7 +202,7 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str,
if driver is not None:
command += ["--driver", driver]
result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8",
errors="replace")
errors="replace", timeout=180)
if result.returncode != 0:
raise RuntimeError(f"Benchmark failed for {filename}: {result.stderr[-2000:]}")
run_columns, samples = _read_run_csv(target, run, commit)
@@ -218,7 +220,8 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str,
summary = {"format": "faset.p3-lighting-benchmark", "version": 1,
"commit": commit, "warmup_frames_per_run": WARMUP_FRAMES,
"measured_frames_per_run": MEASURED_FRAMES, "width": WIDTH, "height": HEIGHT,
"validation": validation, "runs_completed": len(runs), "rows": len(all_rows),
"validation": validation, "driver": driver,
"runs_completed": len(runs), "rows": len(all_rows),
**summarize_rows(all_rows)}
(output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n",
encoding="utf-8")
@@ -234,7 +237,7 @@ def main() -> int:
parser.add_argument("--executable", type=Path, help="Built C++ benchmark executable")
parser.add_argument("--output", type=Path, help="New or empty evidence directory")
parser.add_argument("--commit", help="Source revision; defaults to this checkout's HEAD")
parser.add_argument("--driver", help="Explicit driver label to pass through to the benchmark")
parser.add_argument("--driver", help="Required driver identity for a measured sweep")
parser.add_argument("--validation", choices=("on", "off"), default="off")
args = parser.parse_args()
if args.list_runs: