513 lines
28 KiB
Python
513 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Measure real Editor/import/C++ iteration on a disposable copy of a sample project.
|
|
|
|
Numbers are observations, not CI pass/fail thresholds. Linux peak RSS comes from
|
|
GNU time for the command and its waited-for children; other platforms report null.
|
|
"""
|
|
import argparse
|
|
import copy
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import queue
|
|
import random
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
|
|
|
|
def summarize(values):
|
|
"""Median and nearest-rank p95 of positive, finite observations."""
|
|
if not values or any(not isinstance(value, (int, float)) or
|
|
not math.isfinite(value) or value <= 0 for value in values):
|
|
raise ValueError("A distribution requires positive finite samples")
|
|
ordered = sorted(values)
|
|
return {"median": statistics.median(ordered),
|
|
"p95": ordered[math.ceil(.95 * len(ordered)) - 1]}
|
|
|
|
|
|
def parse_peak_rss(text):
|
|
# GNU time prefixes its -o output with a status sentence when the measured
|
|
# command exits nonzero. The JSON format is the final nonempty line.
|
|
for line in reversed(text.splitlines()):
|
|
try:
|
|
value = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(value, dict) and isinstance(value.get("peak_rss_kib"), int):
|
|
return value["peak_rss_kib"]
|
|
raise ValueError("GNU time did not produce a peak RSS record")
|
|
|
|
|
|
def validate_report(report):
|
|
if report.get("format") != "faset.workflow-measurements" or report.get("version") != 2:
|
|
raise ValueError("Expected a version 2 Faset workflow report")
|
|
if not isinstance(report.get("revision"), str) or not report["revision"].strip():
|
|
raise ValueError("A source revision is required")
|
|
hashes = report.get("source_hashes")
|
|
if not isinstance(hashes, dict) or not hashes or any(
|
|
not isinstance(value, str) or len(value) != 64 or
|
|
any(character not in "0123456789abcdef" for character in value)
|
|
for value in hashes.values()):
|
|
raise ValueError("Exact input SHA-256 hashes are required")
|
|
configuration = report.get("build_configuration")
|
|
if configuration not in {"Debug", "Release", "RelWithDebInfo"}:
|
|
raise ValueError("Build configuration is missing or unknown")
|
|
samples = report.get("samples")
|
|
if not isinstance(samples, dict) or not samples:
|
|
raise ValueError("Raw labelled samples are required")
|
|
for label, rows in samples.items():
|
|
if not isinstance(rows, list) or not rows:
|
|
raise ValueError(f"{label} has no raw samples")
|
|
if (label.startswith("warm_") or label.startswith("changed_")) and len(rows) < 5:
|
|
raise ValueError(f"{label} requires at least five measured repetitions")
|
|
for row in rows:
|
|
if row.get("build_configuration") != configuration:
|
|
raise ValueError(f"{label} mixes build configurations")
|
|
if row.get("source_hash") not in hashes.values():
|
|
raise ValueError(f"{label} lacks a declared input source hash")
|
|
summarize([row.get("seconds")])
|
|
if label == "windowed_first_presented_frame" and \
|
|
row.get("presentation_mode") != "windowed":
|
|
raise ValueError("An offscreen frame cannot be labelled windowed presentation")
|
|
if label == "offscreen_first_rendered_frame" and \
|
|
row.get("presentation_mode") != "offscreen":
|
|
raise ValueError("A windowed frame cannot be labelled offscreen")
|
|
|
|
|
|
def generate_scene_fixture(scene, *, seed, count):
|
|
"""Add deterministic visible objects without mutating the checked-in scene."""
|
|
if not isinstance(seed, int) or count < 0 or not scene.get("entities"):
|
|
raise ValueError("A seed, nonnegative count, and source entity are required")
|
|
generated = copy.deepcopy(scene)
|
|
rng = random.Random(seed)
|
|
reference = next((entry for entry in scene["entities"] if any(
|
|
component.get("type") in {"faset.mesh", "faset.sprite"}
|
|
for component in entry.get("components", []))), scene["entities"][0])
|
|
for index in range(count):
|
|
entity = copy.deepcopy(reference)
|
|
entity["id"] = f"measured-{seed}-{index:05d}"
|
|
entity["name"] = f"Measured object {index + 1}"
|
|
entity["parent"] = None
|
|
entity["components"] = [component for component in entity.get("components", [])
|
|
if component.get("type") in {"faset.transform", "faset.mesh",
|
|
"faset.sprite"}]
|
|
for component in entity["components"]:
|
|
component["id"] = f"{entity['id']}-{component['type']}"
|
|
fields = component.get("fields", {})
|
|
if component["type"] == "faset.transform":
|
|
position = list(fields.get("position", [0, 0, 0]))
|
|
position[0] = ((index % 32) - 16) * .45 + rng.uniform(-.05, .05)
|
|
position[2 if scene.get("dimension", 3) == 3 else 1] = \
|
|
(index // 32) * .45 + rng.uniform(-.05, .05)
|
|
fields["position"] = position
|
|
fields["scale"] = [.3, .3, .3]
|
|
elif component["type"] == "faset.sprite":
|
|
fields["size"] = [.3, .3]
|
|
generated["entities"].append(entity)
|
|
return generated
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--editor", type=Path, required=True)
|
|
parser.add_argument("--project", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--lua-project", type=Path,
|
|
default=Path(__file__).resolve().parents[1] / "examples/lua")
|
|
parser.add_argument("--ui-latency-binary", type=Path)
|
|
args = parser.parse_args()
|
|
editor, source, output = args.editor.resolve(), args.project.resolve(), args.output.resolve()
|
|
lua_source = args.lua_project.resolve()
|
|
ui_binary = (args.ui_latency_binary or
|
|
editor.parent / ("faset_editor_ui_latency.exe" if os.name == "nt" else
|
|
"faset_editor_ui_latency")).resolve()
|
|
for file in (editor, ui_binary, source / "project.faset.json",
|
|
lua_source / "project.faset.json"):
|
|
if not file.is_file():
|
|
raise RuntimeError(f"Required measurement input is missing: {file}")
|
|
if output.exists():
|
|
raise RuntimeError("Use a new output directory; existing evidence is never overwritten")
|
|
output.mkdir(parents=True)
|
|
project = output / "project"
|
|
shutil.copytree(source, project, ignore=shutil.ignore_patterns(".faset", "Exports", "*.blend1"))
|
|
lua_project = output / "lua-project"
|
|
shutil.copytree(lua_source, lua_project,
|
|
ignore=shutil.ignore_patterns(".faset", "Exports", "*.blend1"))
|
|
raw = output / "raw"
|
|
raw.mkdir()
|
|
samples = {}
|
|
source_hashes = {}
|
|
counters = {}
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
def remember(path, variant, base=project):
|
|
key = f"{base.name}/{path.relative_to(base).as_posix()}#{variant}"
|
|
value = digest(path)
|
|
source_hashes[key] = value
|
|
return value
|
|
|
|
gameplay = project / "Scripts/Gameplay.cpp"
|
|
header = project / "Scripts/Gameplay.hpp"
|
|
if not gameplay.is_file() or not header.is_file():
|
|
raise RuntimeError("The C++ project needs Scripts/Gameplay.cpp and Gameplay.hpp")
|
|
baseline_hash = remember(gameplay, "baseline")
|
|
project_hashes = {file.relative_to(project).as_posix(): digest(file)
|
|
for file in sorted(project.rglob("*")) if file.is_file()}
|
|
lua_input = json.loads((lua_project / "project.faset.json").read_text(encoding="utf-8"))
|
|
lua_file = lua_project / lua_input["scripting"]["lua"]["scripts"][0]
|
|
lua_baseline_hash = remember(lua_file, "baseline", lua_project)
|
|
|
|
def run(label, arguments, *, source_hash=baseline_hash, timeout=1800,
|
|
expected_failure=False, record=True):
|
|
index = counters.get(label, 0) + 1
|
|
counters[label] = index
|
|
stem = f"{label}-{index:02d}"
|
|
memory = raw / (stem + "-memory.json")
|
|
command = [str(arg) for arg in arguments]
|
|
wrapped = command
|
|
if platform.system() == "Linux" and Path("/usr/bin/time").is_file():
|
|
wrapped = ["/usr/bin/time", "-f", '{"peak_rss_kib":%M}', "-o", str(memory), *command]
|
|
started = time.perf_counter()
|
|
result = subprocess.run(wrapped, cwd=output, capture_output=True, text=True,
|
|
encoding="utf-8", timeout=timeout)
|
|
elapsed = time.perf_counter() - started
|
|
stdout_file, stderr_file = raw / (stem + "-stdout.txt"), raw / (stem + "-stderr.txt")
|
|
stdout_file.write_text(result.stdout, encoding="utf-8")
|
|
stderr_file.write_text(result.stderr, encoding="utf-8")
|
|
sample = {"seconds": elapsed, "returncode": result.returncode,
|
|
"build_configuration": "Debug", "source_hash": source_hash,
|
|
"command": command, "stdout": str(stdout_file.relative_to(output)),
|
|
"stderr": str(stderr_file.relative_to(output)),
|
|
"peak_rss_kib": parse_peak_rss(memory.read_text())
|
|
if memory.exists() else None}
|
|
(raw / (stem + ".json")).write_text(json.dumps(sample, indent=2) + "\n",
|
|
encoding="utf-8")
|
|
if record:
|
|
samples.setdefault(label, []).append(sample)
|
|
(output / "progress.json").write_text(json.dumps(samples, indent=2) + "\n",
|
|
encoding="utf-8")
|
|
if expected_failure and result.returncode == 0:
|
|
raise RuntimeError(f"{label} unexpectedly succeeded")
|
|
if result.returncode and not expected_failure:
|
|
raise RuntimeError(f"{label} failed: {result.stderr[-4000:]} {result.stdout[-2000:]}")
|
|
print(f"{stem}: {elapsed:.3f}s, peak RSS {sample['peak_rss_kib']} KiB", flush=True)
|
|
return result, sample
|
|
|
|
def call(label, name, arguments=None, *, root=project, source_hash=baseline_hash,
|
|
record=True):
|
|
result, sample = run(label, [editor, "--project", root, "--command",
|
|
json.dumps({"name": name, "arguments": arguments or {}}),
|
|
"--wait"], source_hash=source_hash, record=record)
|
|
document = json.loads(result.stdout)
|
|
if "state" in document and document["state"] != "succeeded":
|
|
raise RuntimeError(f"{label} job did not succeed: {document}")
|
|
if name == "faset_build":
|
|
sample["schema_cache_hit"] = document.get("result", {}).get("schema_cache_hit")
|
|
sample["generation_reused"] = document.get("result", {}).get("generation_reused")
|
|
return document, sample
|
|
|
|
for index in range(3):
|
|
call("headless_editor_command_startup", "faset_project")
|
|
# This is window creation plus two frames and shutdown, not first-present latency.
|
|
run("windowed_editor_two_frames", [editor, "--project", project, "--frames", "2"])
|
|
manifest = project / "Assets/exit-arch/manifest.json"
|
|
if manifest.exists():
|
|
call("cold_asset_import", "faset_import", {"path": "Assets/exit-arch/manifest.json"})
|
|
call("cached_asset_import", "faset_import", {"path": "Assets/exit-arch/manifest.json"})
|
|
cold, _ = call("cold_configure_build", "faset_build")
|
|
call("warmup_unchanged_build", "faset_build", record=False)
|
|
for _ in range(5):
|
|
reused, _ = call("warm_unchanged_build", "faset_build")
|
|
if not reused["result"].get("schema_cache_hit"):
|
|
raise RuntimeError("An unchanged build did not hit the verified schema cache")
|
|
|
|
original_cpp = gameplay.read_bytes()
|
|
original_header = header.read_bytes()
|
|
for index in range(6):
|
|
gameplay.write_bytes(original_cpp +
|
|
f"\n// P1 C++ iteration {index}\n".encode("utf-8"))
|
|
source_hash = remember(gameplay, f"cpp-edit-{index}")
|
|
stale, _ = call("schema_stale_check", "faset_schema_status",
|
|
source_hash=source_hash, record=False)
|
|
if not stale["stale"]:
|
|
raise RuntimeError("A C++ source edit did not mark gameplay metadata stale")
|
|
label = "warmup_changed_cpp_build" if index == 0 else "changed_cpp_build"
|
|
call(label, "faset_build", source_hash=source_hash, record=index != 0)
|
|
for index in range(6):
|
|
header.write_bytes(original_header +
|
|
f"\n// P1 header iteration {index}\n".encode("utf-8"))
|
|
source_hash = remember(header, f"header-edit-{index}")
|
|
label = "warmup_changed_header_build" if index == 0 else "changed_header_build"
|
|
call(label, "faset_build", source_hash=source_hash, record=index != 0)
|
|
|
|
working_header = header.read_bytes()
|
|
header.write_bytes(working_header + b"\n#error FASET_P1_EXPECTED_BUILD_FAILURE\n")
|
|
failed_hash = remember(header, "intentional-failure")
|
|
run("build_failure", [editor, "--project", project, "--command",
|
|
json.dumps({"name": "faset_build", "arguments": {}}), "--wait"],
|
|
source_hash=failed_hash, expected_failure=True)
|
|
header.write_bytes(working_header)
|
|
recovery_hash = remember(header, "recovery")
|
|
built, _ = call("build_recovery", "faset_build", source_hash=recovery_hash)
|
|
|
|
settings = json.loads((project / "project.faset.json").read_text(encoding="utf-8"))
|
|
scene = project / settings["start_scene"]
|
|
player = Path(built["result"]["player"])
|
|
player_profiles = []
|
|
for index in range(5):
|
|
profile = raw / f"cpp-first-frame-{index + 1:02d}.json"
|
|
run("player_one_frame_process", [player, "--scene", scene, "--assets",
|
|
project / ".faset/cache", "--headless", "--frames",
|
|
"1", "--profile", profile],
|
|
source_hash=recovery_hash)
|
|
profile_data = json.loads(profile.read_text(encoding="utf-8"))
|
|
if profile_data["presentation_mode"] != "offscreen" or \
|
|
profile_data["completed_frames"] != 1:
|
|
raise RuntimeError("The first-frame profile is not one rendered offscreen frame")
|
|
player_profiles.append(profile_data)
|
|
samples.setdefault("offscreen_first_rendered_frame", []).append({
|
|
"seconds": profile_data["startup_ms"]["main_to_first_frame"] / 1000,
|
|
"build_configuration": "Debug", "source_hash": recovery_hash,
|
|
"presentation_mode": profile_data["presentation_mode"],
|
|
"profile": str(profile.relative_to(output))})
|
|
|
|
# The UI probe measures synthetic keyboard input through snapshot rendering,
|
|
# with no display presentation claim.
|
|
ui_profile = raw / "editor-input-latency.json"
|
|
run("editor_ui_probe_process", [ui_binary, ui_profile], source_hash=recovery_hash)
|
|
ui_data = json.loads(ui_profile.read_text(encoding="utf-8"))
|
|
if ui_data["presentation_mode"] != "offscreen" or \
|
|
ui_data["measured_frames"] != len(ui_data["samples"]):
|
|
raise RuntimeError("UI probe lost raw frames or mislabelled presentation")
|
|
samples["editor_input_visible_frame"] = [
|
|
{"seconds": row["milliseconds"] / 1000, "build_configuration": "Debug",
|
|
"source_hash": recovery_hash, "presentation_mode": "offscreen",
|
|
"input": row["gizmo"], "profile": str(ui_profile.relative_to(output))}
|
|
for row in ui_data["samples"]]
|
|
|
|
original_lua = lua_file.read_bytes()
|
|
lua_build, _ = call("lua_initial_build", "faset_build", root=lua_project,
|
|
source_hash=lua_baseline_hash)
|
|
lua_player = Path(lua_build["result"]["player"])
|
|
lua_scene = lua_project / lua_input["start_scene"]
|
|
control = lua_project / ".faset/measurement-control.json"
|
|
process = subprocess.Popen([str(lua_player), "--scene", str(lua_scene), "--project",
|
|
str(lua_project), "--control", str(control), "--watch-lua",
|
|
"--headless", "--frames", "10000000"], cwd=output,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
text=True, encoding="utf-8")
|
|
events = queue.Queue()
|
|
stderr_lines = []
|
|
|
|
def collect_lua():
|
|
for line in process.stderr:
|
|
event = (time.perf_counter(), line)
|
|
stderr_lines.append(event)
|
|
events.put(event)
|
|
|
|
reader = threading.Thread(target=collect_lua, daemon=True)
|
|
reader.start()
|
|
|
|
def wait_lua(marker, timeout=20):
|
|
deadline = time.perf_counter() + timeout
|
|
while time.perf_counter() < deadline:
|
|
try:
|
|
event = events.get(timeout=min(.2, max(.001, deadline - time.perf_counter())))
|
|
except queue.Empty:
|
|
if process.poll() is not None:
|
|
break
|
|
continue
|
|
if marker in event[1]:
|
|
return event[0]
|
|
raise RuntimeError(f"Lua Player did not report {marker}; see raw/lua-reload-stderr.txt")
|
|
|
|
try:
|
|
# Startup settles before the watch/reload timing begins.
|
|
wait_lua("Lua", timeout=20)
|
|
for index in range(6):
|
|
lua_file.write_bytes(original_lua +
|
|
f"\n-- P1 reload iteration {index}\n".encode("utf-8"))
|
|
started = time.perf_counter()
|
|
source_hash = remember(lua_file, f"reload-{index}", lua_project)
|
|
finished = wait_lua("Lua reloaded:")
|
|
if index:
|
|
samples.setdefault("changed_lua_reload", []).append({
|
|
"seconds": finished - started, "build_configuration": "Debug",
|
|
"source_hash": source_hash, "presentation_mode": "offscreen",
|
|
"player_log": "raw/lua-reload-stderr.txt"})
|
|
control.parent.mkdir(parents=True, exist_ok=True)
|
|
staging = control.with_suffix(".tmp")
|
|
staging.write_text(json.dumps({"sequence": 1, "command": "stop"}),
|
|
encoding="utf-8")
|
|
staging.replace(control)
|
|
process.wait(timeout=20)
|
|
if process.returncode:
|
|
raise RuntimeError("Lua Player failed during reload measurements")
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=3)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait(timeout=3)
|
|
reader.join(timeout=2)
|
|
(raw / "lua-reload-stdout.txt").write_text(process.stdout.read(), encoding="utf-8")
|
|
(raw / "lua-reload-stderr.txt").write_text("".join(line for _, line in stderr_lines),
|
|
encoding="utf-8")
|
|
lua_file.write_bytes(original_lua)
|
|
|
|
lifecycle_profile = raw / "cpp-3000-frame-lifecycle.json"
|
|
run("resource_lifecycle_process", [player, "--scene", scene, "--assets",
|
|
project / ".faset/cache", "--headless", "--frames",
|
|
"3000", "--profile", lifecycle_profile],
|
|
source_hash=recovery_hash)
|
|
lifecycle = json.loads(lifecycle_profile.read_text(encoding="utf-8"))
|
|
frames = lifecycle["samples"]
|
|
if len(frames) != 3000:
|
|
raise RuntimeError("Resource lifecycle did not complete 3,000 frames")
|
|
memory_start = max(row["gpu_allocated_bytes"] for row in frames[100:200])
|
|
memory_end = max(row["gpu_allocated_bytes"] for row in frames[-100:])
|
|
resource_growth = {"reference_frames": [101, 200], "end_frames": [2901, 3000],
|
|
"explicit_gpu_bytes_reference_max": memory_start,
|
|
"explicit_gpu_bytes_end_max": memory_end,
|
|
"explicit_gpu_bytes_growth": memory_end - memory_start,
|
|
"texture_count_reference_max": max(row["texture_count"]
|
|
for row in frames[100:200]),
|
|
"texture_count_end_max": max(row["texture_count"]
|
|
for row in frames[-100:])}
|
|
|
|
generated_scene = generate_scene_fixture(json.loads(scene.read_text(encoding="utf-8")),
|
|
seed=20260924, count=512)
|
|
large_scene = raw / "generated-512.scene.json"
|
|
large_scene.write_text(json.dumps(generated_scene, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8")
|
|
large_hash = remember(large_scene, "generated-512", output)
|
|
large_profile = raw / "generated-512-profile.json"
|
|
run("larger_generated_scene_process", [player, "--scene", large_scene, "--assets",
|
|
project / ".faset/cache", "--headless", "--frames",
|
|
"240", "--profile", large_profile],
|
|
source_hash=large_hash)
|
|
large_data = json.loads(large_profile.read_text(encoding="utf-8"))
|
|
|
|
engine = Path(__file__).resolve().parents[1]
|
|
revision = subprocess.run(["git", "rev-parse", "HEAD"], cwd=engine,
|
|
capture_output=True, text=True, check=True).stdout.strip()
|
|
dirty = subprocess.run(["git", "status", "--porcelain"], cwd=engine,
|
|
capture_output=True, text=True, check=True).stdout.strip()
|
|
cpu = platform.processor()
|
|
if Path("/proc/cpuinfo").exists():
|
|
cpu = next((line.split(":", 1)[1].strip() for line in
|
|
Path("/proc/cpuinfo").read_text().splitlines()
|
|
if line.startswith("model name")), cpu)
|
|
ram_kib = None
|
|
if Path("/proc/meminfo").exists():
|
|
ram_kib = next((int(line.split()[1]) for line in
|
|
Path("/proc/meminfo").read_text().splitlines()
|
|
if line.startswith("MemTotal:")), None)
|
|
|
|
def version(arguments):
|
|
try:
|
|
return subprocess.run(arguments, capture_output=True, text=True, timeout=15,
|
|
encoding="utf-8").stdout.splitlines()[0]
|
|
except (OSError, subprocess.TimeoutExpired, IndexError):
|
|
return None
|
|
|
|
report = {
|
|
"format": "faset.workflow-measurements", "version": 2,
|
|
"recorded_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"revision": revision, "working_tree_dirty": bool(dirty),
|
|
"source_hashes": source_hashes, "project_hashes_at_copy": project_hashes,
|
|
"build_configuration": "Debug", "source_project": str(source),
|
|
"lua_source_project": str(lua_source), "editor": str(editor),
|
|
"host": {"platform": platform.platform(), "cpu": cpu, "ram_kib": ram_kib,
|
|
"logical_cpus": os.cpu_count(), "python": platform.python_version()},
|
|
"toolchain": {"cmake": version(["cmake", "--version"]),
|
|
"cxx": version(["clang++", "--version"]),
|
|
"slang": version(["slangc", "-version"])},
|
|
"gpu": {"player_device": player_profiles[0]["device"],
|
|
"ui_device": ui_data["device"],
|
|
"driver": version(["nvidia-smi", "--query-gpu=driver_version",
|
|
"--format=csv,noheader"])},
|
|
"settings": {"resolution": [player_profiles[0]["width"],
|
|
player_profiles[0]["height"]],
|
|
"validation_enabled": player_profiles[0]["validation_enabled"],
|
|
"readback_enabled": True, "presentation_mode": "offscreen",
|
|
"source_scene_entities": len(json.loads(scene.read_text())["entities"]),
|
|
"generated_scene_entities": len(generated_scene["entities"])},
|
|
"samples": samples,
|
|
"summary_seconds": {label: summarize([row["seconds"] for row in rows])
|
|
for label, rows in samples.items()},
|
|
"resource_lifecycle": resource_growth,
|
|
"large_scene_summary_ms": large_data["summary_ms"],
|
|
"coverage": {"windowed_first_presented_frame":
|
|
"unmeasured; only offscreen rendered frames were sampled",
|
|
"reference_release_budgets":
|
|
"not compared with this Debug development workflow"},
|
|
"method": [
|
|
"Fresh disposable C++ and Lua project copies; checked-in sources never modified.",
|
|
"Cold build means empty project .faset caches, not cold OS cache or dependency download.",
|
|
"Warm/changed C++ and header builds have one warm-up then five measured repetitions.",
|
|
"Lua edit-to-reload uses a persistent watched Player, one warm-up and five edits; "
|
|
"wall time includes up to 500 ms watcher polling and stderr notification.",
|
|
"First frame is Player main-to-first-completed offscreen frame; OS loader excluded.",
|
|
"UI latency uses 10 warm-up and 100 keyboard-input to offscreen-render frames; "
|
|
"there is no window presentation timing claim.",
|
|
"The 3,000-frame lifecycle compares maximum explicit Vulkan allocation and texture "
|
|
"counts in frames 101-200 and 2901-3000; it is not a whole-process leak proof.",
|
|
"Times are observations under ambient host load, not CI pass/fail thresholds."]}
|
|
validate_report(report)
|
|
(output / "report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
print(f"Report: {output / 'report.json'}", flush=True)
|
|
settings = json.loads((project / "project.faset.json").read_text(encoding="utf-8"))
|
|
run("changed-build-player-first-frame", [built["player"], "--scene",
|
|
project / settings["start_scene"], "--assets", project / ".faset/cache",
|
|
"--headless", "--frames", "1", "--profile", profile])
|
|
iteration_seconds = time.perf_counter() - iteration_started
|
|
assert not call("schema-current-after-build", "faset_schema_status")["stale"]
|
|
# Keep original inputs in this disposable project, without implying its last
|
|
# built binary matches the restored source signature.
|
|
gameplay.write_bytes(original)
|
|
engine = Path(__file__).resolve().parents[1]
|
|
revision = subprocess.run(["git", "rev-parse", "HEAD"], cwd=engine,
|
|
capture_output=True, text=True).stdout.strip()
|
|
dirty = subprocess.run(["git", "status", "--porcelain"], cwd=engine,
|
|
capture_output=True, text=True).stdout.strip()
|
|
cpu = platform.processor()
|
|
if Path("/proc/cpuinfo").exists():
|
|
cpu = next((line.split(":", 1)[1].strip() for line in Path("/proc/cpuinfo").read_text().splitlines()
|
|
if line.startswith("model name")), cpu)
|
|
report = {
|
|
"format": "faset.workflow-measurements", "version": 1,
|
|
"recorded_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"revision": revision, "working_tree_dirty": bool(dirty),
|
|
"host": {"platform": platform.platform(), "cpu": cpu,
|
|
"logical_cpus": os.cpu_count(), "python": platform.python_version()},
|
|
"source_project": str(source), "editor": str(editor),
|
|
"build_configuration": "Debug", "samples": samples,
|
|
"changed_build_and_one_frame_process_seconds": iteration_seconds,
|
|
"player": json.loads(profile.read_text()),
|
|
"method": [
|
|
"Fresh disposable project, engine dependency archives and OS file caches already available.",
|
|
"Each Editor command starts a new process and includes shutdown in wall time.",
|
|
"GUI startup sample includes window creation, two frames, and shutdown.",
|
|
"Changed build appends a harmless comment, recompiles Gameplay.cpp and relinks native outputs.",
|
|
"Combined iteration runs the returned Debug Player in a separate process for one offscreen frame.",
|
|
"Peak RSS is GNU time maximum over each command and waited-for children, not a sum.",
|
|
"Numbers include ambient host load; there is no warm-up exclusion or real-time frame guarantee."
|
|
]}
|
|
(output / "report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|