Emil_Shanaty/Faset_Engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Validate release lighting sweep provenance and matched controls

Browse Source
This commit is contained in:
Emil
2026-09-24 03:05:48 +03:00
parent bf2b88a6a2
commit 2d71f8eac2
2 changed files with 287 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/benchmark_p3_lighting.py
+126 -21
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import argparse
import csv
import hashlib
import json
import math
from pathlib import Path
@@ -20,6 +21,9 @@ import sys
LIGHT_COUNTS = (0, 4, 16, 32, 64, 128)
VISIBILITY_MODES = ("direct", "gpu-frustum", "gpu-occlusion")
REPEATS = (1, 2, 3)
# Pair a zero-light control with every independent repeat, keeping high-light
# measurements nearby rather than comparing the first run with the last.
MEASUREMENT_ORDER = (0, 32, 4, 64, 16, 128)
WARMUP_FRAMES = 10
MEASURED_FRAMES = 30
WIDTH, HEIGHT = 1920, 1080
@@ -29,9 +33,15 @@ REQUIRED_COLUMNS = (
"readback_cpu_ms", "validation_errors", "run_index", "effective_visibility",
"lighting_path", "submitted_local_lights", "omitted_local_lights",
"shadow_tiles", "draw_calls", "gpu_bytes", "validation_enabled", "width", "height",
"build_configuration", "requested_local_shadow_faces", "rendered_local_shadow_faces",
"dropped_shadow_faces", "shadow_atlas_full_drops", "gpu_post_raster_ms",
"gpu_post_visible",
)
TIMING_COLUMNS = ("gpu_main_raster_ms", "gpu_ms", "gpu_shadow_ms", "cpu_ms",
"readback_cpu_ms")
TIMING_COLUMNS = ("gpu_main_raster_ms", "gpu_post_raster_ms", "gpu_ms",
"gpu_shadow_ms", "cpu_ms", "readback_cpu_ms")
SUMMARY_TIMINGS = (*TIMING_COLUMNS, "forward_raster_ms")
SHADOW_COLUMNS = ("requested_local_shadow_faces", "rendered_local_shadow_faces",
"shadow_tiles", "dropped_shadow_faces", "shadow_atlas_full_drops")
def build_runs(shadows: str = "both") -> list[dict]:
@@ -40,8 +50,8 @@ def build_runs(shadows: str = "both") -> list[dict]:
settings = ("off", "on") if shadows == "both" else (shadows,)
return [{"shadows": shadow, "visibility": mode, "light_count": lights,
"repeat": repeat}
for shadow in settings for mode in VISIBILITY_MODES
for lights in LIGHT_COUNTS for repeat in REPEATS]
for shadow in settings for repeat in REPEATS
for mode in VISIBILITY_MODES for lights in MEASUREMENT_ORDER]
def median(values: list[float]) -> float:
@@ -67,6 +77,24 @@ def _measurement(row: dict, name: str) -> float:
return value
def _timing(row: dict, name: str) -> float:
if name == "forward_raster_ms":
main = _measurement(row, "gpu_main_raster_ms")
return main + (_measurement(row, "gpu_post_raster_ms")
if row["visibility"] == "gpu-occlusion" else 0)
return _measurement(row, name)
def _count(row: dict, name: str) -> int:
try:
value = int(row[name])
except (KeyError, TypeError, ValueError) as error:
raise ValueError(f"Invalid {name} in benchmark CSV") from error
if value < 0:
raise ValueError(f"Invalid {name} in benchmark CSV: {value}")
return value
def summarize_rows(rows: list[dict]) -> dict:
"""Use the median of each independent run's median, then compare like baselines."""
grouped: dict[tuple[str, str, int], dict[int, list[dict]]] = {}
@@ -80,6 +108,8 @@ def summarize_rows(rows: list[dict]) -> dict:
raise ValueError(f"Invalid benchmark configuration: {key}, repeat {repeat}")
for name in TIMING_COLUMNS:
_measurement(row, name)
for name in SHADOW_COLUMNS:
_count(row, name)
grouped.setdefault(key, {}).setdefault(repeat, []).append(row)
configurations = []
@@ -89,16 +119,21 @@ def summarize_rows(rows: list[dict]) -> dict:
for repeat, samples in sorted(repeats.items()):
run_summaries.append({
"run_index": repeat, "frames": len(samples),
"median_ms": {name: median([_measurement(row, name) for row in samples])
for name in TIMING_COLUMNS},
"median_ms": {name: median([_timing(row, name) for row in samples])
for name in SUMMARY_TIMINGS},
})
shadow_counts = {name: {_count(row, name) for samples in repeats.values()
for row in samples} for name in SHADOW_COLUMNS}
if any(len(values) != 1 for values in shadow_counts.values()):
raise ValueError("Shadow counts changed within a fixed benchmark configuration")
entry = {
"shadows": shadows, "visibility": visibility, "light_count": lights,
"runs": run_summaries,
"shadow_counts": {name: next(iter(values)) for name, values in shadow_counts.items()},
"median_ms": {name: median([run["median_ms"][name] for run in run_summaries])
for name in TIMING_COLUMNS},
"p95_ms": {name: p95([_measurement(row, name) for samples in repeats.values()
for row in samples]) for name in TIMING_COLUMNS},
for name in SUMMARY_TIMINGS},
"p95_ms": {name: p95([_timing(row, name) for samples in repeats.values()
for row in samples]) for name in SUMMARY_TIMINGS},
}
configurations.append(entry)
lookup[(shadows, visibility, lights)] = entry
@@ -111,13 +146,20 @@ def summarize_rows(rows: list[dict]) -> dict:
baseline = lookup.get((entry["shadows"], entry["visibility"], 0))
if baseline is None:
raise ValueError("Forward+ gate requires a zero-light baseline for each mode/shadow setting")
zero_gpu = baseline["median_ms"]["gpu_ms"]
base_runs = {run["run_index"]: run for run in baseline["runs"]}
paired = [(run, base_runs[run["run_index"]]) for run in entry["runs"]
if run["run_index"] in base_runs]
if len(paired) != len(entry["runs"]):
raise ValueError("Forward+ gate requires a matched zero-light repeat for every run")
zero_gpu = median([base["median_ms"]["gpu_ms"] for _, base in paired])
if zero_gpu <= 0:
raise ValueError("Forward+ gate requires positive zero-light GPU frame timing")
overhead = (entry["median_ms"]["gpu_main_raster_ms"] -
baseline["median_ms"]["gpu_main_raster_ms"])
overhead = median([run["median_ms"]["forward_raster_ms"] -
base["median_ms"]["forward_raster_ms"] for run, base in paired])
result = {"shadows": entry["shadows"], "visibility": entry["visibility"],
"light_count": entry["light_count"], "overhead_ms": overhead,
"raster_metric": ("main_plus_post" if entry["visibility"] == "gpu-occlusion"
else "main"),
"zero_light_gpu_ms": zero_gpu,
"overhead_percent_of_zero_gpu": 100 * overhead / zero_gpu,
"absolute_threshold_reached": overhead >= 1.0,
@@ -128,10 +170,11 @@ def summarize_rows(rows: list[dict]) -> dict:
return {"configurations": configurations,
"forward_plus_gate": {"triggered": bool(candidates), "candidates": candidates,
"evaluated": evaluated,
"basis": "median of three independent run medians; same-mode/shadow zero-light GPU baseline"}}
"basis": "median of paired per-run raster overheads; same-mode/shadow/repeat zero-light GPU baseline"}}
def _read_run_csv(path: Path, run: dict, commit: str) -> tuple[list[str], list[dict]]:
def _read_run_csv(path: Path, run: dict, commit: str, driver: str,
validation: str) -> tuple[list[str], list[dict]]:
with path.open(newline="", encoding="utf-8") as stream:
reader = csv.DictReader(stream)
columns = reader.fieldnames or []
@@ -149,6 +192,12 @@ def _read_run_csv(path: Path, run: dict, commit: str) -> tuple[list[str], list[d
for name, value in expected.items():
if row[name] != value:
raise ValueError(f"{path}: {name} mismatch: expected {value}, got {row[name]}")
if row["build_configuration"] != "Release":
raise ValueError(f"{path}: Forward+ sweep requires a Release benchmark binary")
if row["driver"] != driver:
raise ValueError(f"{path}: driver differs from the declared driver identity")
if row["validation_enabled"] != ("1" if validation == "on" else "0"):
raise ValueError(f"{path}: validation_enabled differs from the requested setting")
if row["effective_visibility"] != run["visibility"]:
raise ValueError(f"{path}: effective_visibility fell back from {run['visibility']}")
if row["submitted_local_lights"] != str(run["light_count"]) or row["omitted_local_lights"] != "0":
@@ -163,6 +212,16 @@ def _read_run_csv(path: Path, run: dict, commit: str) -> tuple[list[str], list[d
frames.add(frame)
if not row["device"] or not row["driver"] or not row["lighting_path"]:
raise ValueError(f"{path}: device, driver and effective lighting path are required")
requested = _count(row, "requested_local_shadow_faces")
rendered = _count(row, "rendered_local_shadow_faces")
tiles = _count(row, "shadow_tiles")
dropped = _count(row, "dropped_shadow_faces")
atlas_drops = _count(row, "shadow_atlas_full_drops")
expected_faces = 6 * run["light_count"] if run["shadows"] == "on" else 0
if (requested != expected_faces or rendered > tiles or tiles > min(requested, 16) or
dropped > requested or atlas_drops > dropped):
raise ValueError(f"{path}: requested/effective shadow face counts disagree with the workload")
_count(row, "gpu_post_visible")
for name in TIMING_COLUMNS:
_measurement(row, name)
return columns, rows
@@ -174,21 +233,44 @@ def _git_revision() -> str:
text=True).strip()
def _binary_sha256(path: Path) -> str:
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def _clean_source_revision(root: Path, expected: str) -> str:
revision = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"],
text=True).strip()
if revision != expected:
raise ValueError(f"Source revision {revision} disagrees with benchmark commit {expected}")
status = subprocess.check_output(["git", "-C", str(root), "status", "--porcelain",
"--untracked-files=all"], text=True)
if status:
raise ValueError("Benchmark source checkout is dirty; commit sources before measurement")
return revision
def sweep(executable: Path, output: Path, shadows: str, commit: str,
validation: str = "off", driver: str | None = None) -> dict:
validation: str = "off", driver: str | None = None,
source_root: Path | None = None) -> dict:
if not executable.is_file():
raise ValueError(f"Benchmark executable does not exist: {executable}")
if driver is None or not driver.strip() or driver.strip().lower() == "unknown":
raise ValueError("A measured sweep requires an explicit --driver identity")
source = (source_root or Path(__file__).resolve().parents[1]).resolve()
revision = _clean_source_revision(source, commit)
binary_sha256 = _binary_sha256(executable)
if output.exists() and any(output.iterdir()):
raise ValueError(f"Output directory must be new or empty: {output}")
raw = output / "raw"
raw.mkdir(parents=True)
all_rows = []
columns = None
identity = None
runs = build_runs(shadows)
command_prefix = [sys.executable, str(executable)] if executable.suffix.lower() == ".py" else [str(executable)]
for run in runs:
run_order = []
for acquisition_index, run in enumerate(runs):
filename = (f"shadows-{run['shadows']}_{run['visibility']}_"
f"lights-{run['light_count']:03d}_run-{run['repeat']}.csv")
target = raw / filename
@@ -205,22 +287,42 @@ def sweep(executable: Path, output: Path, shadows: str, commit: str,
errors="replace", timeout=180)
if result.returncode != 0:
raise RuntimeError(f"Benchmark failed for {filename}: {result.stderr[-2000:]}")
run_columns, samples = _read_run_csv(target, run, commit)
run_columns, samples = _read_run_csv(target, run, commit, driver, validation)
if columns is None:
columns = run_columns
elif columns != run_columns:
raise ValueError(f"{target}: CSV schema differs from other runs")
all_rows.extend({**row, "source_csv": filename} for row in samples)
for row in samples:
actual = (row["device"], row["driver"], row["lighting_path"],
row["validation_enabled"], row["build_configuration"])
if identity is None:
identity = actual
elif identity != actual:
changed = next(name for name, before, after in zip(
("device", "driver", "lighting_path", "validation_enabled",
"build_configuration"), identity, actual) if before != after)
raise ValueError(f"{target}: {changed} changed during the sweep")
all_rows.append({**row, "source_csv": filename,
"acquisition_index": acquisition_index})
run_order.append(filename)
merged = output / "merged.csv"
with merged.open("w", newline="", encoding="utf-8") as stream:
writer = csv.DictWriter(stream, fieldnames=[*(columns or []), "source_csv"])
writer = csv.DictWriter(stream, fieldnames=[*(columns or []), "source_csv",
"acquisition_index"])
writer.writeheader()
writer.writerows(all_rows)
if _binary_sha256(executable) != binary_sha256 or _clean_source_revision(source, commit) != revision:
raise ValueError("Benchmark binary or source changed during the sweep")
summary = {"format": "faset.p3-lighting-benchmark", "version": 1,
"commit": commit, "warmup_frames_per_run": WARMUP_FRAMES,
"measured_frames_per_run": MEASURED_FRAMES, "width": WIDTH, "height": HEIGHT,
"validation": validation, "driver": driver,
"source_revision": revision, "source_root": str(source), "source_dirty": False,
"benchmark_sha256": binary_sha256,
"device": identity[0], "lighting_path": identity[2],
"validation_enabled": identity[3] == "1", "build_configuration": identity[4],
"run_order": run_order,
"runs_completed": len(runs), "rows": len(all_rows),
**summarize_rows(all_rows)}
(output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n",
@@ -237,6 +339,8 @@ def main() -> int:
parser.add_argument("--executable", type=Path, help="Built C++ benchmark executable")
parser.add_argument("--output", type=Path, help="New or empty evidence directory")
parser.add_argument("--commit", help="Source revision; defaults to this checkout's HEAD")
parser.add_argument("--source-root", type=Path,
help="Clean source checkout used to build the benchmark; defaults to this repo")
parser.add_argument("--driver", help="Required driver identity for a measured sweep")
parser.add_argument("--validation", choices=("on", "off"), default="off")
args = parser.parse_args()
@@ -248,8 +352,9 @@ def main() -> int:
parser.error("--sweep requires --executable and --output")
try:
summary = sweep(args.executable.resolve(), args.output.resolve(), args.shadows,
args.commit or _git_revision(), args.validation, args.driver)
except (OSError, ValueError, RuntimeError) as error:
args.commit or _git_revision(), args.validation, args.driver,
args.source_root)
except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error:
print(f"P3 lighting benchmark failed: {error}", file=sys.stderr)
return 1
print(json.dumps({"summary": str(args.output.resolve() / "summary.json"),