Record build profile and temporal raster cost in lighting benchmark

cmake/Renderer.cmake

@@ -82,5 +82,17 @@ if(BUILD_TESTING)
   target_link_libraries(faset_render_window_tests PRIVATE faset_render SDL3::SDL3)
   add_test(NAME render_window_lifecycle COMMAND faset_render_window_tests "${CMAKE_BINARY_DIR}/window-test")
   set_tests_properties(render_window_lifecycle PROPERTIES LABELS "gpu;window" TIMEOUT 40 SKIP_RETURN_CODE 77)
+  add_executable(faset_p3_lighting_benchmark
+    "${PROJECT_SOURCE_DIR}/examples/renderer/p3_lighting_benchmark.cpp")
+  target_link_libraries(faset_p3_lighting_benchmark PRIVATE faset_render faset_core)
+  target_compile_definitions(faset_p3_lighting_benchmark PRIVATE
+    FASET_BENCHMARK_CONFIGURATION="$<CONFIG>")
+  add_test(NAME render_lighting_benchmark_schema COMMAND
+    "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tests/test_p3_lighting_benchmark.py")
+  set_tests_properties(render_lighting_benchmark_schema PROPERTIES LABELS "p3" TIMEOUT 90)
+  add_test(NAME render_lighting_benchmark_smoke COMMAND
+    "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tests/test_p3_lighting_benchmark.py"
+    --real-executable "$<TARGET_FILE:faset_p3_lighting_benchmark>")
+  set_tests_properties(render_lighting_benchmark_smoke PROPERTIES LABELS "gpu;p3" TIMEOUT 90)
 endif()
 install(FILES ${FASET_SHADER_OUTPUTS} DESTINATION shaders)

examples/renderer/p3_lighting_benchmark.cpp

@@ -0,0 +1,233 @@
+#include <faset/core/io.hpp>
+#include <faset/render/renderer.hpp>
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+
+using namespace faset::render;
+namespace fs = std::filesystem;
+
+namespace {
+struct Options {
+    unsigned lights{}, width{1920}, height{1080}, warmup{10}, frames{30}, run_index{};
+    bool shadows{}, validation{};
+    VisibilityMode visibility{VisibilityMode::Direct};
+    std::string commit{"unknown"}, driver{"unknown"};
+    fs::path csv, capture;
+};
+
+unsigned number(std::string_view text, std::string_view name) {
+    std::size_t end{};
+    const auto value = std::stoul(std::string(text), &end);
+    if (end != text.size() || value > 100000)
+        throw std::invalid_argument("Invalid value for " + std::string(name));
+    return static_cast<unsigned>(value);
+}
+
+Options parse(int argc, char** argv) {
+    Options options;
+    for (int i = 1; i < argc; ++i) {
+        const std::string name = argv[i];
+        if (name == "--list-runs") {
+            std::cout << "{\"lights\":[0,4,16,32,64,128],"
+                         "\"visibility\":[\"direct\",\"gpu-frustum\",\"gpu-occlusion\"],"
+                         "\"shadows\":[\"off\",\"on\"]}\n";
+            std::exit(0);
+        }
+        if (name == "--help") {
+            std::cout << "Usage: faset_p3_lighting_benchmark --lights 0|4|16|32|64|128 "
+                         "--shadows on|off --visibility direct|gpu-frustum|gpu-occlusion "
+                         "--csv PATH [--width N --height N --warmup N --frames N "
+                         "--run-index N --commit SHA --driver NAME --validation on|off "
+                         "--capture PATH]\n";
+            std::exit(0);
+        }
+        if (i + 1 >= argc)
+            throw std::invalid_argument("Missing value for " + name);
+        const std::string value = argv[++i];
+        if (name == "--lights") options.lights = number(value, name);
+        else if (name == "--width") options.width = number(value, name);
+        else if (name == "--height") options.height = number(value, name);
+        else if (name == "--warmup") options.warmup = number(value, name);
+        else if (name == "--frames") options.frames = number(value, name);
+        else if (name == "--run-index") options.run_index = number(value, name);
+        else if (name == "--commit") options.commit = value;
+        else if (name == "--driver") options.driver = value;
+        else if (name == "--csv") options.csv = faset::path_from_utf8(value);
+        else if (name == "--capture") options.capture = faset::path_from_utf8(value);
+        else if (name == "--shadows") {
+            if (value != "on" && value != "off")
+                throw std::invalid_argument("--shadows must be on or off");
+            options.shadows = value == "on";
+        } else if (name == "--validation") {
+            if (value != "on" && value != "off")
+                throw std::invalid_argument("--validation must be on or off");
+            options.validation = value == "on";
+        } else if (name == "--visibility") {
+            if (value == "direct") options.visibility = VisibilityMode::Direct;
+            else if (value == "gpu-frustum") options.visibility = VisibilityMode::GpuFrustum;
+            else if (value == "gpu-occlusion") options.visibility = VisibilityMode::GpuOcclusion;
+            else throw std::invalid_argument("Unknown visibility mode: " + value);
+        } else throw std::invalid_argument("Unknown option: " + name);
+    }
+    constexpr std::array allowed_lights{0u, 4u, 16u, 32u, 64u, 128u};
+    if (options.csv.empty() || options.width == 0 || options.height == 0 ||
+        options.frames == 0 || options.warmup > 1000 ||
+        std::find(allowed_lights.begin(), allowed_lights.end(), options.lights) ==
+            allowed_lights.end())
+        throw std::invalid_argument("Invalid benchmark configuration");
+    return options;
+}
+
+const char* mode_name(VisibilityMode mode) {
+    switch (mode) {
+    case VisibilityMode::Direct: return "direct";
+    case VisibilityMode::GpuFrustum: return "gpu-frustum";
+    case VisibilityMode::GpuOcclusion: return "gpu-occlusion";
+    }
+    return "unknown";
+}
+
+void csv_text(std::ostream& out, std::string_view value) {
+    out << '"';
+    for (const char c : value) {
+        if (c == '"') out << '"';
+        out << c;
+    }
+    out << '"';
+}
+
+Snapshot benchmark_scene(const Options& options) {
+    Snapshot scene;
+    scene.view_id = "p3-lighting-benchmark-fixed-scene";
+    scene.eye = {0, 0, 9};
+    scene.projection = perspective(.9f, float(options.width) / float(options.height), .1f, 100);
+    const auto view = look_at(scene.eye, {0, 0, 0});
+    scene.view_projection = multiply(scene.projection, view);
+    scene.camera_frustum = CameraFrustum{view, scene.projection, .1f, 100, true};
+    scene.authored_lights_present = true;
+    scene.clear_color = {.035f, .04f, .05f, 1};
+    DrawItem receiver;
+    receiver.mesh = cube_mesh();
+    receiver.model = transform({0, 0, -.15f}, {}, {10, 7.5f, .2f});
+    receiver.color = {.65f, .67f, .7f, 1};
+    receiver.roughness = .65f;
+    receiver.cast_shadow = options.shadows;
+    receiver.instance_key = "large-receiver";
+    scene.draws.push_back(receiver);
+    for (int i = 0; i < 9; ++i) {
+        DrawItem object;
+        object.mesh = cube_mesh();
+        object.model = transform({(float(i % 3) - 1.f) * 2.5f,
+                                  (float(i / 3) - 1.f) * 1.8f, .45f},
+                                 {}, {.42f, .42f, .6f});
+        object.color = {.6f + .1f * float(i % 3), .5f, .4f + .1f * float(i / 3), 1};
+        object.cast_shadow = options.shadows;
+        object.instance_key = "caster-" + std::to_string(i);
+        scene.draws.push_back(std::move(object));
+    }
+    for (unsigned i = 0; i < options.lights; ++i) {
+        LocalLight light;
+        light.kind = LocalLight::Kind::Point;
+        light.stable_id = "benchmark-light-" + std::to_string(i);
+        light.position = {(float(i % 8) - 3.5f) * 1.35f,
+                          (float((i / 8) % 8) - 3.5f) * .95f,
+                          2.f + .35f * float(i % 3)};
+        light.color = {.6f + .4f * float(i % 3 == 0),
+                       .6f + .4f * float(i % 3 == 1),
+                       .6f + .4f * float(i % 3 == 2), 1};
+        light.intensity = 5.f;
+        light.range = 8.f;
+        light.casts_shadow = options.shadows;
+        scene.local_lights.push_back(std::move(light));
+    }
+    return scene;
+}
+
+void benchmark(const Options& options) {
+    RendererConfig config;
+    config.width = options.width;
+    config.height = options.height;
+    config.headless = true;
+    config.validation = options.validation;
+    config.visibility_mode = options.visibility;
+    auto renderer = Renderer(config);
+    const auto scene = benchmark_scene(options);
+    for (unsigned i = 0; i < options.warmup; ++i)
+        renderer.render(scene);
+    if (!options.csv.parent_path().empty())
+        fs::create_directories(faset::native_io_path(options.csv.parent_path()));
+    std::ofstream csv(faset::native_io_path(options.csv));
+    if (!csv)
+        throw std::runtime_error("Cannot open benchmark CSV: " + faset::path_to_utf8(options.csv));
+    csv << "light_count,shadows,visibility,effective_visibility,lighting_path,"
+           "build_configuration,run_index,frame,"
+           "device,driver,commit,width,height,validation_enabled,validation_errors,"
+           "submitted_local_lights,omitted_local_lights,shadow_tiles,draw_calls,gpu_bytes,"
+           "gpu_main_raster_ms,gpu_post_raster_ms,gpu_post_visible,visibility_counters_valid,"
+           "gpu_shadow_ms,gpu_ms,cpu_ms,readback_cpu_ms\n";
+    csv << std::fixed << std::setprecision(6);
+    for (unsigned frame = 0; frame < options.frames; ++frame) {
+        renderer.render(scene);
+        const auto stats = renderer.stats();
+        if (stats.validation_errors != 0)
+            throw std::runtime_error("Vulkan validation error during benchmark");
+        if (stats.submitted_local_lights != options.lights || stats.omitted_local_lights != 0)
+            throw std::runtime_error("Renderer did not submit every requested local light");
+        if (stats.effective_visibility_mode != options.visibility)
+            throw std::runtime_error("Requested visibility path fell back during benchmark");
+        if (stats.gpu_main_raster_ms <= 0 || stats.gpu_ms <= 0)
+            throw std::runtime_error("GPU raster or frame timestamp was unavailable");
+        csv << options.lights << ',' << (options.shadows ? "on" : "off") << ','
+            << mode_name(options.visibility) << ',' << mode_name(stats.effective_visibility_mode)
+            << ",forward," << FASET_BENCHMARK_CONFIGURATION << ','
+            << options.run_index << ',' << frame << ',';
+        csv_text(csv, stats.device);
+        csv << ',';
+        csv_text(csv, options.driver);
+        csv << ',';
+        csv_text(csv, options.commit);
+        csv << ',' << options.width << ',' << options.height << ','
+            << (stats.validation_enabled ? 1 : 0) << ',' << stats.validation_errors << ','
+            << stats.submitted_local_lights << ',' << stats.omitted_local_lights
+            << ",0," << stats.draw_calls << ',' << stats.gpu_allocated_bytes << ','
+            << stats.gpu_main_raster_ms << ',' << stats.gpu_post_raster_ms << ','
+            << stats.gpu_post_visible << ',' << (stats.visibility_counters_valid ? 1 : 0)
+            << ",0," << stats.gpu_ms << ','
+            << stats.cpu_ms << ',' << stats.readback_cpu_ms << '\n';
+    }
+    if (!csv)
+        throw std::runtime_error("Cannot finish benchmark CSV: " + faset::path_to_utf8(options.csv));
+    if (!options.capture.empty())
+        renderer.capture(faset::native_io_path(options.capture));
+}
+} // namespace
+
+int benchmark_main(int argc, char** argv) {
+    try {
+        benchmark(parse(argc, argv));
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << "P3 lighting benchmark: " << error.what() << '\n';
+        return 1;
+    }
+}
+
+#ifdef _WIN32
+int wmain(int argc, wchar_t** argv) {
+    return faset::run_utf8_main(argc, argv, benchmark_main);
+}
+#else
+int main(int argc, char** argv) {
+    return benchmark_main(argc, argv);
+}
+#endif

tests/test_p3_lighting_benchmark.py

@@ -0,0 +1,229 @@
+"""Contract and arithmetic tests for the offline P3 lighting sweep wrapper."""
+
+import csv
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "tools/benchmark_p3_lighting.py"
+sys.path.insert(0, str(ROOT / "tools"))
+
+
+def sample(shadows, visibility, lights, raster, gpu, repeat=1, frame=0):
+    return {
+        "shadows": shadows, "visibility": visibility, "light_count": str(lights),
+        "run_index": str(repeat), "frame": str(frame),
+        "gpu_main_raster_ms": str(raster), "gpu_ms": str(gpu),
+        "gpu_shadow_ms": "0", "cpu_ms": "1", "readback_cpu_ms": ".5",
+        "validation_errors": "0", "device": "Fake GPU", "driver": "Fake Driver",
+        "commit": "abc123", "effective_visibility": visibility,
+        "lighting_path": "forward", "submitted_local_lights": str(lights),
+        "omitted_local_lights": "0", "shadow_tiles": "0", "draw_calls": "1",
+        "gpu_bytes": "4096", "validation_enabled": "0", "width": "1920", "height": "1080",
+    }
+
+
+FAKE_BENCHMARK = r'''import argparse
+import csv
+import json
+from pathlib import Path
+
+p = argparse.ArgumentParser()
+for flag in ("lights", "run-index", "width", "height", "warmup", "frames"):
+    p.add_argument("--" + flag, type=int, required=True)
+for flag in ("shadows", "visibility", "csv", "commit", "validation"):
+    p.add_argument("--" + flag, required=True)
+p.add_argument("--driver")
+a = p.parse_args()
+path = Path(a.csv)
+path.with_suffix(".args.json").write_text(json.dumps(vars(a)), encoding="utf-8")
+fieldnames = ["light_count", "shadows", "visibility", "frame", "device", "driver",
+              "commit", "gpu_main_raster_ms", "gpu_ms", "gpu_shadow_ms", "cpu_ms",
+              "readback_cpu_ms", "validation_errors", "run_index", "effective_visibility",
+              "lighting_path", "submitted_local_lights", "omitted_local_lights",
+              "shadow_tiles", "draw_calls", "gpu_bytes", "validation_enabled", "width", "height"]
+with path.open("w", newline="", encoding="utf-8") as stream:
+    writer = csv.DictWriter(stream, fieldnames=fieldnames)
+    writer.writeheader()
+    for frame in range(a.frames):
+        writer.writerow(dict(light_count=a.lights, shadows=a.shadows,
+                             visibility=a.visibility, frame=frame, device="Fake GPU",
+                             driver="Fake Driver", commit=a.commit,
+                             gpu_main_raster_ms=.4 + .02 * a.lights, gpu_ms=4 + .02 * a.lights,
+                             gpu_shadow_ms=0, cpu_ms=1, readback_cpu_ms=.5,
+                             validation_errors=0, run_index=a.run_index,
+                             effective_visibility=a.visibility, lighting_path="forward",
+                             submitted_local_lights=a.lights, omitted_local_lights=0,
+                             shadow_tiles=0, draw_calls=1, gpu_bytes=4096,
+                             validation_enabled=0, width=a.width, height=a.height))
+'''
+
+
+class LightingBenchmarkTests(unittest.TestCase):
+    def test_list_runs_has_three_independent_repeats_for_each_shadow_setting(self):
+        process = subprocess.run([sys.executable, SCRIPT, "--list-runs"],
+                                 text=True, capture_output=True, check=True)
+        runs = json.loads(process.stdout)["runs"]
+        self.assertEqual(len(runs), 108)
+        for shadows in ("off", "on"):
+            group = [run for run in runs if run["shadows"] == shadows]
+            self.assertEqual(len(group), 54)
+            self.assertEqual({(run["light_count"], run["visibility"])
+                              for run in group},
+                             {(light, mode) for light in (0, 4, 16, 32, 64, 128)
+                              for mode in ("direct", "gpu-frustum", "gpu-occlusion")})
+            self.assertEqual({(run["light_count"], run["visibility"], run["repeat"])
+                              for run in group},
+                             {(light, mode, repeat)
+                              for light in (0, 4, 16, 32, 64, 128)
+                              for mode in ("direct", "gpu-frustum", "gpu-occlusion")
+                              for repeat in (1, 2, 3)})
+
+    def test_gate_uses_per_run_medians_and_same_mode_shadow_baseline(self):
+        from benchmark_p3_lighting import summarize_rows
+
+        rows = []
+        for repeat in (1, 2, 3):
+            for frame in (0, 1, 2):
+                rows.append(sample("off", "direct", 0, .5, 10, repeat, frame))
+                rows.append(sample("off", "direct", 32,
+                                   100 if repeat == 3 else 1.5, 11, repeat, frame))
+                rows.append(sample("on", "gpu-frustum", 0, .5, 4, repeat, frame))
+                rows.append(sample("on", "gpu-frustum", 64, 1.1, 4.6, repeat, frame))
+                rows.append(sample("off", "gpu-occlusion", 0, .5, 4, repeat, frame))
+                rows.append(sample("off", "gpu-occlusion", 128, 1.09, 4.59, repeat, frame))
+        summary = summarize_rows(rows)
+        hits = {(item["shadows"], item["visibility"], item["light_count"]): item
+                for item in summary["forward_plus_gate"]["candidates"]}
+        self.assertEqual(set(hits), {("off", "direct", 32),
+                                     ("on", "gpu-frustum", 64)})
+        self.assertAlmostEqual(hits[("off", "direct", 32)]["overhead_ms"], 1.0)
+        self.assertAlmostEqual(hits[("on", "gpu-frustum", 64)]["overhead_ms"], .6)
+        self.assertEqual(hits[("off", "direct", 32)]["zero_light_gpu_ms"], 10)
+        self.assertEqual(hits[("on", "gpu-frustum", 64)]["zero_light_gpu_ms"], 4)
+
+    def test_sweep_runs_fake_executable_and_preserves_all_raw_frames(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            fake = root / "fake_benchmark.py"
+            fake.write_text(FAKE_BENCHMARK, encoding="utf-8")
+            output = root / "café 世界"
+            process = subprocess.run(
+                [sys.executable, SCRIPT, "--sweep", "--executable", fake,
+                 "--output", output, "--shadows", "off", "--commit", "abc123",
+                 "--driver", "Fake Driver"],
+                text=True, capture_output=True)
+            self.assertEqual(process.returncode, 0, process.stderr)
+            raw = list((output / "raw").glob("*.csv"))
+            self.assertEqual(len(raw), 54)
+            with (output / "merged.csv").open(newline="", encoding="utf-8") as stream:
+                merged = list(csv.DictReader(stream))
+            self.assertEqual(len(merged), 54 * 30)
+            self.assertEqual({row["source_csv"] for row in merged},
+                             {path.name for path in raw})
+            report = json.loads((output / "summary.json").read_text(encoding="utf-8"))
+            self.assertEqual(report["runs_completed"], 54)
+            self.assertEqual(report["rows"], 54 * 30)
+            self.assertTrue(report["forward_plus_gate"]["triggered"])
+            one = json.loads(next((output / "raw").glob("*.args.json")).read_text())
+            self.assertEqual((one["width"], one["height"], one["warmup"], one["frames"]),
+                             (1920, 1080, 10, 30))
+            self.assertEqual(one["validation"], "off")
+
+    def test_sweep_rejects_missing_gpu_raster_column(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            fake = root / "bad_benchmark.py"
+            fake.write_text(FAKE_BENCHMARK.replace(
+                '"gpu_main_raster_ms", "gpu_ms"', '"gpu_ms"').replace(
+                'gpu_main_raster_ms=.4 + .02 * a.lights, ', ''), encoding="utf-8")
+            output = root / "invalid"
+            process = subprocess.run(
+                [sys.executable, SCRIPT, "--sweep", "--executable", fake,
+                 "--output", output, "--shadows", "off", "--commit", "abc123",
+                 "--driver", "Fake Driver"],
+                text=True, capture_output=True)
+            self.assertNotEqual(process.returncode, 0)
+            self.assertIn("gpu_main_raster_ms", process.stderr)
+            self.assertFalse((output / "summary.json").exists())
+
+    def test_sweep_rejects_visibility_fallback_as_a_mode_measurement(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            fake = root / "fallback.py"
+            fake.write_text(FAKE_BENCHMARK.replace(
+                'effective_visibility=a.visibility', 'effective_visibility="direct"'),
+                encoding="utf-8")
+            output = root / "fallback-output"
+            process = subprocess.run(
+                [sys.executable, SCRIPT, "--sweep", "--executable", fake,
+                 "--output", output, "--shadows", "off", "--commit", "abc123",
+                 "--driver", "Fake Driver"],
+                text=True, capture_output=True)
+            self.assertNotEqual(process.returncode, 0)
+            self.assertIn("effective_visibility", process.stderr)
+            self.assertFalse((output / "summary.json").exists())
+
+    def test_sweep_rejects_a_scene_that_does_not_submit_requested_lights(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            fake = root / "wrong-count.py"
+            fake.write_text(FAKE_BENCHMARK.replace(
+                'submitted_local_lights=a.lights', 'submitted_local_lights=0'),
+                encoding="utf-8")
+            output = root / "wrong-count-output"
+            process = subprocess.run(
+                [sys.executable, SCRIPT, "--sweep", "--executable", fake,
+                 "--output", output, "--shadows", "off", "--commit", "abc123",
+                 "--driver", "Fake Driver"],
+                text=True, capture_output=True)
+            self.assertNotEqual(process.returncode, 0)
+            self.assertIn("submitted_local_lights", process.stderr)
+            self.assertFalse((output / "summary.json").exists())
+
+    def test_sweep_requires_known_driver_identity(self):
+        from benchmark_p3_lighting import sweep
+        with tempfile.TemporaryDirectory() as temporary:
+            fake = Path(temporary) / "fake.py"
+            fake.write_text(FAKE_BENCHMARK, encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "--driver"):
+                sweep(fake, Path(temporary) / "out", "off", "abc123")
+
+
+def real_executable_smoke(executable: Path) -> None:
+    from benchmark_p3_lighting import REQUIRED_COLUMNS
+
+    choices = subprocess.run([executable, "--list-runs"], capture_output=True,
+                             text=True, check=True)
+    declared = json.loads(choices.stdout)
+    if declared["lights"] != [0, 4, 16, 32, 64, 128] or len(declared["visibility"]) != 3:
+        raise AssertionError("C++ executable and Python sweep matrix disagree")
+    with tempfile.TemporaryDirectory() as directory:
+        output = Path(directory) / "café 世界" / "smoke.csv"
+        subprocess.run([executable, "--lights", "4", "--shadows", "off",
+                        "--visibility", "direct", "--width", "64", "--height", "64",
+                        "--warmup", "0", "--frames", "1", "--validation", "on",
+                        "--commit", "smoke", "--driver", "smoke-driver",
+                        "--csv", output], capture_output=True, text=True, check=True)
+        with output.open(newline="", encoding="utf-8") as stream:
+            reader = csv.DictReader(stream)
+            columns, rows = reader.fieldnames or [], list(reader)
+        if set(REQUIRED_COLUMNS) - set(columns) or len(rows) != 1:
+            raise AssertionError("Real benchmark CSV lacks a complete single-frame row")
+        row = rows[0]
+        if (row["effective_visibility"] != "direct" or row["lighting_path"] != "forward" or
+                row["submitted_local_lights"] != "4" or row["validation_errors"] != "0" or
+                float(row["gpu_main_raster_ms"]) <= 0):
+            raise AssertionError("Real benchmark did not report the measured lighting path")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 3 and sys.argv[1] == "--real-executable":
+        real_executable_smoke(Path(sys.argv[2]).resolve())
+    else:
+        unittest.main()

tools/benchmark_p3_lighting.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+"""Run the fixed P3 lighting sweep and retain raw per-frame GPU measurements.
+
+The Forward+ threshold in the summary is a measurement result, not an automatic
+renderer switch. Apply it to the Linux physical reference GPU; keep other devices
+as separate functional/performance observations.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import math
+from pathlib import Path
+import statistics
+import subprocess
+import sys
+
+
+LIGHT_COUNTS = (0, 4, 16, 32, 64, 128)
+VISIBILITY_MODES = ("direct", "gpu-frustum", "gpu-occlusion")
+REPEATS = (1, 2, 3)
+WARMUP_FRAMES = 10
+MEASURED_FRAMES = 30
+WIDTH, HEIGHT = 1920, 1080
+REQUIRED_COLUMNS = (
+    "light_count", "shadows", "visibility", "frame", "device", "driver",
+    "commit", "gpu_main_raster_ms", "gpu_ms", "gpu_shadow_ms", "cpu_ms",
+    "readback_cpu_ms", "validation_errors", "run_index", "effective_visibility",
+    "lighting_path", "submitted_local_lights", "omitted_local_lights",
+    "shadow_tiles", "draw_calls", "gpu_bytes", "validation_enabled", "width", "height",
+)
+TIMING_COLUMNS = ("gpu_main_raster_ms", "gpu_ms", "gpu_shadow_ms", "cpu_ms",
+                  "readback_cpu_ms")
+
+
+def build_runs(shadows: str = "both") -> list[dict]:
+    if shadows not in ("off", "on", "both"):
+        raise ValueError(f"Unsupported shadow setting: {shadows}")
+    settings = ("off", "on") if shadows == "both" else (shadows,)
+    return [{"shadows": shadow, "visibility": mode, "light_count": lights,
+             "repeat": repeat}
+            for shadow in settings for mode in VISIBILITY_MODES
+            for lights in LIGHT_COUNTS for repeat in REPEATS]
+
+
+def median(values: list[float]) -> float:
+    if not values:
+        raise ValueError("Cannot summarize empty measurements")
+    return float(statistics.median(values))
+
+
+def p95(values: list[float]) -> float:
+    if not values:
+        raise ValueError("Cannot summarize empty measurements")
+    ordered = sorted(values)
+    return ordered[math.ceil(.95 * len(ordered)) - 1]
+
+
+def _measurement(row: dict, name: str) -> float:
+    try:
+        value = float(row[name])
+    except (KeyError, TypeError, ValueError) as error:
+        raise ValueError(f"Invalid {name} in benchmark CSV") from error
+    if not math.isfinite(value) or value < 0:
+        raise ValueError(f"Invalid {name} in benchmark CSV: {value}")
+    return value
+
+
+def summarize_rows(rows: list[dict]) -> dict:
+    """Use the median of each independent run's median, then compare like baselines."""
+    grouped: dict[tuple[str, str, int], dict[int, list[dict]]] = {}
+    for row in rows:
+        try:
+            key = (row["shadows"], row["visibility"], int(row["light_count"]))
+            repeat = int(row["run_index"])
+        except (KeyError, TypeError, ValueError) as error:
+            raise ValueError("Benchmark row lacks shadow/mode/light/repeat identity") from error
+        if key[0] not in ("off", "on") or key[1] not in VISIBILITY_MODES or repeat < 1:
+            raise ValueError(f"Invalid benchmark configuration: {key}, repeat {repeat}")
+        for name in TIMING_COLUMNS:
+            _measurement(row, name)
+        grouped.setdefault(key, {}).setdefault(repeat, []).append(row)
+
+    configurations = []
+    lookup = {}
+    for (shadows, visibility, lights), repeats in sorted(grouped.items()):
+        run_summaries = []
+        for repeat, samples in sorted(repeats.items()):
+            run_summaries.append({
+                "run_index": repeat, "frames": len(samples),
+                "median_ms": {name: median([_measurement(row, name) for row in samples])
+                              for name in TIMING_COLUMNS},
+            })
+        entry = {
+            "shadows": shadows, "visibility": visibility, "light_count": lights,
+            "runs": run_summaries,
+            "median_ms": {name: median([run["median_ms"][name] for run in run_summaries])
+                          for name in TIMING_COLUMNS},
+            "p95_ms": {name: p95([_measurement(row, name) for samples in repeats.values()
+                                   for row in samples]) for name in TIMING_COLUMNS},
+        }
+        configurations.append(entry)
+        lookup[(shadows, visibility, lights)] = entry
+
+    candidates = []
+    evaluated = []
+    for entry in configurations:
+        if entry["light_count"] not in (32, 64, 128):
+            continue
+        baseline = lookup.get((entry["shadows"], entry["visibility"], 0))
+        if baseline is None:
+            raise ValueError("Forward+ gate requires a zero-light baseline for each mode/shadow setting")
+        zero_gpu = baseline["median_ms"]["gpu_ms"]
+        if zero_gpu <= 0:
+            raise ValueError("Forward+ gate requires positive zero-light GPU frame timing")
+        overhead = (entry["median_ms"]["gpu_main_raster_ms"] -
+                    baseline["median_ms"]["gpu_main_raster_ms"])
+        result = {"shadows": entry["shadows"], "visibility": entry["visibility"],
+                  "light_count": entry["light_count"], "overhead_ms": overhead,
+                  "zero_light_gpu_ms": zero_gpu,
+                  "overhead_percent_of_zero_gpu": 100 * overhead / zero_gpu,
+                  "absolute_threshold_reached": overhead >= 1.0,
+                  "relative_threshold_reached": overhead >= .15 * zero_gpu}
+        evaluated.append(result)
+        if result["absolute_threshold_reached"] or result["relative_threshold_reached"]:
+            candidates.append(result)
+    return {"configurations": configurations,
+            "forward_plus_gate": {"triggered": bool(candidates), "candidates": candidates,
+                                  "evaluated": evaluated,
+                                  "basis": "median of three independent run medians; same-mode/shadow zero-light GPU baseline"}}
+
+
+def _read_run_csv(path: Path, run: dict, commit: str) -> tuple[list[str], list[dict]]:
+    with path.open(newline="", encoding="utf-8") as stream:
+        reader = csv.DictReader(stream)
+        columns = reader.fieldnames or []
+        missing = sorted(set(REQUIRED_COLUMNS) - set(columns))
+        if missing:
+            raise ValueError(f"{path}: missing CSV column(s): {', '.join(missing)}")
+        rows = list(reader)
+    if len(rows) != MEASURED_FRAMES:
+        raise ValueError(f"{path}: expected {MEASURED_FRAMES} measured frames, got {len(rows)}")
+    frames = set()
+    for row in rows:
+        expected = {"light_count": str(run["light_count"]), "shadows": run["shadows"],
+                    "visibility": run["visibility"], "run_index": str(run["repeat"]),
+                    "commit": commit, "width": str(WIDTH), "height": str(HEIGHT)}
+        for name, value in expected.items():
+            if row[name] != value:
+                raise ValueError(f"{path}: {name} mismatch: expected {value}, got {row[name]}")
+        if row["effective_visibility"] != run["visibility"]:
+            raise ValueError(f"{path}: effective_visibility fell back from {run['visibility']}")
+        if row["submitted_local_lights"] != str(run["light_count"]) or row["omitted_local_lights"] != "0":
+            raise ValueError(f"{path}: submitted_local_lights or omitted_local_lights disagrees with the workload")
+        try:
+            frame = int(row["frame"])
+            errors = int(row["validation_errors"])
+        except ValueError as error:
+            raise ValueError(f"{path}: invalid frame or validation error count") from error
+        if frame in frames or errors != 0:
+            raise ValueError(f"{path}: duplicate frame or Vulkan validation error")
+        frames.add(frame)
+        if not row["device"] or not row["driver"] or not row["lighting_path"]:
+            raise ValueError(f"{path}: device, driver and effective lighting path are required")
+        for name in TIMING_COLUMNS:
+            _measurement(row, name)
+    return columns, rows
+
+
+def _git_revision() -> str:
+    root = Path(__file__).resolve().parents[1]
+    return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"],
+                                   text=True).strip()
+
+
+def sweep(executable: Path, output: Path, shadows: str, commit: str,
+          validation: str = "off", driver: str | None = None) -> dict:
+    if not executable.is_file():
+        raise ValueError(f"Benchmark executable does not exist: {executable}")
+    if driver is None or not driver.strip() or driver.strip().lower() == "unknown":
+        raise ValueError("A measured sweep requires an explicit --driver identity")
+    if output.exists() and any(output.iterdir()):
+        raise ValueError(f"Output directory must be new or empty: {output}")
+    raw = output / "raw"
+    raw.mkdir(parents=True)
+    all_rows = []
+    columns = None
+    runs = build_runs(shadows)
+    command_prefix = [sys.executable, str(executable)] if executable.suffix.lower() == ".py" else [str(executable)]
+    for run in runs:
+        filename = (f"shadows-{run['shadows']}_{run['visibility']}_"
+                    f"lights-{run['light_count']:03d}_run-{run['repeat']}.csv")
+        target = raw / filename
+        command = command_prefix + [
+            "--lights", str(run["light_count"]), "--shadows", run["shadows"],
+            "--visibility", run["visibility"], "--csv", str(target),
+            "--run-index", str(run["repeat"]), "--commit", commit,
+            "--validation", validation, "--width", str(WIDTH), "--height", str(HEIGHT),
+            "--warmup", str(WARMUP_FRAMES), "--frames", str(MEASURED_FRAMES),
+        ]
+        if driver is not None:
+            command += ["--driver", driver]
+        result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8",
+                                errors="replace", timeout=180)
+        if result.returncode != 0:
+            raise RuntimeError(f"Benchmark failed for {filename}: {result.stderr[-2000:]}")
+        run_columns, samples = _read_run_csv(target, run, commit)
+        if columns is None:
+            columns = run_columns
+        elif columns != run_columns:
+            raise ValueError(f"{target}: CSV schema differs from other runs")
+        all_rows.extend({**row, "source_csv": filename} for row in samples)
+
+    merged = output / "merged.csv"
+    with merged.open("w", newline="", encoding="utf-8") as stream:
+        writer = csv.DictWriter(stream, fieldnames=[*(columns or []), "source_csv"])
+        writer.writeheader()
+        writer.writerows(all_rows)
+    summary = {"format": "faset.p3-lighting-benchmark", "version": 1,
+               "commit": commit, "warmup_frames_per_run": WARMUP_FRAMES,
+               "measured_frames_per_run": MEASURED_FRAMES, "width": WIDTH, "height": HEIGHT,
+               "validation": validation, "driver": driver,
+               "runs_completed": len(runs), "rows": len(all_rows),
+               **summarize_rows(all_rows)}
+    (output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n",
+                                          encoding="utf-8")
+    return summary
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--list-runs", action="store_true", help="Print the deterministic sweep matrix as JSON")
+    mode.add_argument("--sweep", action="store_true", help="Run every configuration and retain raw CSV")
+    parser.add_argument("--shadows", choices=("off", "on", "both"), default="both")
+    parser.add_argument("--executable", type=Path, help="Built C++ benchmark executable")
+    parser.add_argument("--output", type=Path, help="New or empty evidence directory")
+    parser.add_argument("--commit", help="Source revision; defaults to this checkout's HEAD")
+    parser.add_argument("--driver", help="Required driver identity for a measured sweep")
+    parser.add_argument("--validation", choices=("on", "off"), default="off")
+    args = parser.parse_args()
+    if args.list_runs:
+        print(json.dumps({"format": "faset.p3-lighting-run-matrix", "version": 1,
+                          "runs": build_runs(args.shadows)}, indent=2))
+        return 0
+    if args.executable is None or args.output is None:
+        parser.error("--sweep requires --executable and --output")
+    try:
+        summary = sweep(args.executable.resolve(), args.output.resolve(), args.shadows,
+                        args.commit or _git_revision(), args.validation, args.driver)
+    except (OSError, ValueError, RuntimeError) as error:
+        print(f"P3 lighting benchmark failed: {error}", file=sys.stderr)
+        return 1
+    print(json.dumps({"summary": str(args.output.resolve() / "summary.json"),
+                      "runs_completed": summary["runs_completed"],
+                      "forward_plus_threshold_reached": summary["forward_plus_gate"]["triggered"]}))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())