Reuse verified gameplay schema and build generations

include/faset/editor/build_cache.hpp

@@ -25,4 +25,17 @@ BuildInputs capture_build_inputs(const BuildConfig& config, const scripting::Lua
 void ensure_native_toolchain_stamp(const std::filesystem::path& native_directory,
                                    const BuildInputs& inputs);
 
+// Native build completes before package reuse is considered. Hash the actual
+// generated outputs, not their timestamps or just the gameplay sources.
+std::string build_package_key(const BuildInputs& inputs,
+                              const std::filesystem::path& native_directory,
+                              const std::string& configuration,
+                              const std::filesystem::path& player,
+                              const std::filesystem::path& exporter);
+
+// A hit is accepted only if the complete published file set and schema match
+// the manifest. Malformed or old manifests are safe cache misses.
+bool validate_build_generation(const std::filesystem::path& directory,
+                               const std::string& package_key);
+
 } // namespace faset::editor

src/editor/build_cache.cpp

@@ -1,9 +1,12 @@
 #include <faset/editor/build_cache.hpp>
 
+#include <faset/authoring/schema.hpp>
 #include <faset/core/hash.hpp>
 #include <faset/core/io.hpp>
 #include <faset/core/process.hpp>
+#include <cctype>
 #include <cstdlib>
+#include <set>
 #include <stdexcept>
 
 namespace faset::editor {
@@ -151,4 +154,112 @@ void ensure_native_toolchain_stamp(const fs::path& native_directory,
                                   {"toolchain_hash", inputs.toolchain_hash}});
 }
 
+std::string build_package_key(const BuildInputs& inputs, const fs::path& native_directory,
+                              const std::string& configuration, const fs::path& player,
+                              const fs::path& exporter) {
+    Json files = Json::object();
+    for (const auto& [name, path] :
+         {std::pair{"player", player}, std::pair{"exporter", exporter},
+          std::pair{"cmake_cache", native_directory / "CMakeCache.txt"}}) {
+        if (!fs::is_regular_file(path) || fs::is_symlink(path))
+            throw std::runtime_error("Native build artifact is missing: " + path_to_utf8(path));
+        files[name] = sha256_file(path);
+    }
+    const auto shaders = native_directory / "shaders";
+    if (!fs::is_directory(shaders))
+        throw std::runtime_error("Native shader directory is missing");
+    for (const auto& entry : fs::recursive_directory_iterator(shaders)) {
+        if (entry.is_symlink())
+            throw std::runtime_error("Native shader is a symlink");
+        if (entry.is_regular_file())
+            files[generic_path_to_utf8(entry.path().lexically_relative(native_directory))] =
+                sha256_file(entry.path());
+    }
+    // On Windows, a changed runtime DLL must also invalidate a package hit.
+    for (const auto& root : {player.parent_path(), native_directory,
+                             native_directory / configuration}) {
+        if (!fs::is_directory(root))
+            continue;
+        for (const auto& entry : fs::directory_iterator(root)) {
+            auto extension = path_to_utf8(entry.path().extension());
+            for (auto& character : extension)
+                character = static_cast<char>(std::tolower(static_cast<unsigned char>(character)));
+            if (extension != ".dll")
+                continue;
+            if (!entry.is_regular_file() || entry.is_symlink())
+                throw std::runtime_error("Native runtime DLL is not a regular file");
+            files["dll:" + path_to_utf8(entry.path().filename())] = sha256_file(entry.path());
+        }
+    }
+    return normalized_hash({{"format", "faset.package-key.v1"},
+                            {"inputs", inputs.fingerprint()},
+                            {"configuration", configuration},
+                            {"files", files}});
+}
+
+bool validate_build_generation(const fs::path& directory, const std::string& package_key) {
+    try {
+        if (!fs::is_directory(directory) || fs::is_symlink(directory))
+            return false;
+        const auto manifest_file = directory / "manifest.json";
+        if (!fs::is_regular_file(manifest_file) || fs::is_symlink(manifest_file))
+            return false;
+        const auto manifest = read_json(manifest_file);
+        if (manifest.at("format") != "faset.build" || manifest.at("version") != 2 ||
+            manifest.at("package_key") != package_key || !manifest.at("files").is_array())
+            return false;
+        std::set<std::string> expected;
+        for (const auto& record : manifest.at("files")) {
+            const auto name = record.at("path").get<std::string>();
+            const auto relative = path_from_utf8(name);
+            if (relative.empty() || relative.is_absolute() || name == "manifest.json" ||
+                generic_path_to_utf8(relative) != name)
+                return false;
+            for (const auto& component : relative)
+                if (component == "." || component == "..")
+                    return false;
+            if (!expected.insert(name).second)
+                return false;
+            auto file = directory;
+            for (const auto& component : relative) {
+                file /= component;
+                if (fs::is_symlink(file))
+                    return false;
+            }
+            if (!fs::is_regular_file(file) || fs::is_symlink(file) ||
+                sha256_file(file) != record.at("sha256").get<std::string>() ||
+                fs::file_size(file) != record.at("size").get<std::uintmax_t>())
+                return false;
+        }
+        for (const auto& entry : fs::recursive_directory_iterator(directory)) {
+            if (entry.is_symlink())
+                return false;
+            if (entry.is_regular_file()) {
+                const auto name = generic_path_to_utf8(entry.path().lexically_relative(directory));
+                if (name != "manifest.json" && !expected.contains(name))
+                    return false;
+            }
+        }
+        const auto player_name = manifest.at("player").get<std::string>();
+        const auto schema_name = manifest.at("schema").get<std::string>();
+        if (!expected.contains(player_name) || !expected.contains(schema_name) ||
+            !expected.contains("faset_schema_exporter" + fs::path(player_name).extension().string()))
+            return false;
+        for (const auto* shader : {"vertexMain", "fragmentMain", "shadowMain",
+                                   "gpuVertexMain", "gpuShadowMain", "gpuCullMain",
+                                   "gpuHzbMain", "gpuPostCullMain"})
+            for (const auto* extension : {".spv", ".reflection.json"})
+                if (!expected.contains("shaders/" + std::string(shader) + extension))
+                    return false;
+        const auto schema = read_json(directory / schema_name);
+        if (schema.at("format") != "faset.schema" || schema.at("version") != 1 ||
+            schema.at("build_fingerprint") != manifest.at("fingerprint"))
+            return false;
+        (void)authoring::gameplay_schemas(schema);
+        return true;
+    } catch (const std::exception&) {
+        return false;
+    }
+}
+
 } // namespace faset::editor

src/editor/build_service.cpp

@@ -241,6 +241,10 @@ struct BuildService::Impl {
         }
     }
     Json build(Job& job, bool exporting = false) {
+        const auto started = std::chrono::steady_clock::now();
+        const auto milliseconds = [](auto from, auto to) {
+            return std::chrono::duration_cast<std::chrono::milliseconds>(to - from).count();
+        };
         const auto& configuration = exporting ? config.export_configuration : config.configuration;
         const auto native_directory = config.build_directory / configuration;
         checkpoint(job, "Configuring gameplay", .05);
@@ -252,8 +256,6 @@ struct BuildService::Impl {
         if (has_cpp != has_hpp || (!has_cpp && !job.lua.enabled()))
             throw std::runtime_error("Project requires Scripts/Gameplay.cpp and Gameplay.hpp, "
                                      "or Lua entry scripts declared in project.faset.json");
-        const auto cpp_source = has_cpp ? read_text(cpp) : std::string{};
-        const auto hpp_source = has_hpp ? read_text(hpp) : std::string{};
         ensure_native_toolchain_stamp(native_directory, inputs);
         std::vector<std::string> arguments = {config.cmake,
                                               "-S",
@@ -292,14 +294,62 @@ struct BuildService::Impl {
         arguments.push_back(std::string("-DFASET_ENABLE_LUA=") +
                             (job.lua.enabled() ? "ON" : "OFF"));
         run(job, std::move(arguments), config.project_root);
+        const auto configured = std::chrono::steady_clock::now();
         checkpoint(job, "Compiling and linking Player", .25);
         run(job,
             {config.cmake, "--build", path_to_utf8(native_directory), "--config", configuration,
              "--parallel", "4", "--target", "faset_player", "faset_schema_exporter"},
             config.project_root);
-        checkpoint(job, "Exporting gameplay schema", .58);
+        const auto compiled = std::chrono::steady_clock::now();
         auto player = build_executable(native_directory, configuration, "faset_player");
         auto exporter = build_executable(native_directory, configuration, "faset_schema_exporter");
+        const auto package_key =
+            build_package_key(inputs, native_directory, configuration, player, exporter);
+        const auto source_unchanged = [&] {
+            if (gameplay_source_hash(config.project_root,
+                                     scripting::loadLuaProject(config.project_root)) !=
+                inputs.source_hash)
+                throw std::runtime_error("Gameplay sources changed during the build; build again");
+        };
+        const auto result_for = [&](const std::string& id, const std::string& fingerprint,
+                                    bool reused) -> Json {
+            const auto directory = config.cache_root / "builds" / id;
+            const auto finished = std::chrono::steady_clock::now();
+            return {{"generation", id},
+                    {"directory", path_to_utf8(directory)},
+                    {"build_directory", path_to_utf8(native_directory)},
+                    {"configuration", configuration},
+                    {"player", path_to_utf8(directory / ("faset_player" + executable_suffix()))},
+                    {"schema", path_to_utf8(directory / "schema.json")},
+                    {"lua_enabled", job.lua.enabled()},
+                    {"lua_fingerprint", job.lua.fingerprint},
+                    {"fingerprint", fingerprint},
+                    {"schema_cache_hit", reused},
+                    {"generation_reused", reused},
+                    {"phase_times_ms",
+                     {{"configure", milliseconds(started, configured)},
+                      {"native_build", milliseconds(configured, compiled)},
+                      {"schema_package", reused ? 0 : milliseconds(compiled, finished)},
+                      {"total", milliseconds(started, finished)}}}};
+        };
+        const auto pointer_file = config.cache_root / "last_build.json";
+        if (fs::is_regular_file(pointer_file))
+            try {
+                const auto pointer = read_json(pointer_file);
+                const auto id = pointer.at("generation").get<std::string>();
+                const auto candidate =
+                    project_path(config.cache_root, fs::path("builds") / id);
+                if (validate_build_generation(candidate, package_key)) {
+                    source_unchanged();
+                    checkpoint(job, "Reusing verified build generation", .68);
+                    const auto manifest = read_json(candidate / "manifest.json");
+                    log(job, "Verified schema/package cache hit: " + id + "\n");
+                    return result_for(id, manifest.at("fingerprint").get<std::string>(), true);
+                }
+            } catch (const std::exception&) {
+                // A bad pointer or old/corrupt generation is a cache miss.
+            }
+        checkpoint(job, "Exporting gameplay schema", .58);
         const auto staging = config.cache_root / "builds" / (".staging-" + job.status.id);
         const auto generation = config.cache_root / "builds" / job.status.id;
         fs::create_directories(staging);
@@ -324,10 +374,7 @@ struct BuildService::Impl {
             // Validate the complete metadata before publishing either the schema or
             // its Player generation. Session uses this same authoring contract.
             (void)authoring::gameplay_schemas(schema);
-            std::string fingerprint = sha256_file(player) + sha256_file(exporter) +
-                                      read_text(native_directory / "CMakeCache.txt");
-            fingerprint += cpp_source + hpp_source + job.lua.fingerprint;
-            fingerprint = sha256(fingerprint);
+            const auto fingerprint = package_key;
             schema["build_fingerprint"] = fingerprint;
             if (job.lua.enabled())
                 schema["lua_fingerprint"] = job.lua.fingerprint;
@@ -343,36 +390,39 @@ struct BuildService::Impl {
                                      "gpuHzbMain.reflection.json", "gpuPostCullMain.reflection.json"})
                 copy_required_file(native_directory / "shaders" / file, staging / "shaders" / file);
             copy_runtime_libraries(job, player, staging, native_directory, configuration);
+            Json files = Json::array();
+            for (const auto& entry : fs::recursive_directory_iterator(staging)) {
+                if (entry.is_symlink())
+                    throw std::runtime_error("Build generation contains a symlink");
+                if (entry.is_regular_file())
+                    files.push_back(
+                        {{"path", generic_path_to_utf8(entry.path().lexically_relative(staging))},
+                         {"sha256", sha256_file(entry.path())},
+                         {"size", entry.file_size()}});
+            }
             Json manifest{{"format", "faset.build"},
-                          {"version", 1},
+                          {"version", 2},
                           {"id", job.status.id},
                           {"fingerprint", fingerprint},
+                          {"package_key", package_key},
                           {"configuration", configuration},
                           {"lua_enabled", job.lua.enabled()},
                           {"lua_fingerprint", job.lua.fingerprint},
                           {"player", "faset_player" + executable_suffix()},
-                          {"schema", "schema.json"}};
+                          {"schema", "schema.json"},
+                          {"files", files}};
             atomic_write_json(staging / "manifest.json", manifest);
             checkpoint(job, "Publishing build generation", .68);
             if (job.lua.enabled() &&
                 scripting::loadLuaProject(staging).fingerprint != job.lua.fingerprint)
                 throw std::runtime_error("Lua build snapshot changed during schema export");
-            if (gameplay_source_hash(config.project_root,
-                                     scripting::loadLuaProject(config.project_root)) !=
-                inputs.source_hash)
-                throw std::runtime_error("Gameplay sources changed during the build; build again");
+            source_unchanged();
+            if (!validate_build_generation(staging, package_key))
+                throw std::runtime_error("Build generation failed final integrity validation");
             fs::rename(staging, generation);
             atomic_write_json(config.cache_root / "last_build.json",
                               {{"generation", job.status.id}, {"fingerprint", fingerprint}});
-            return {{"generation", job.status.id},
-                    {"directory", path_to_utf8(generation)},
-                    {"build_directory", path_to_utf8(native_directory)},
-                    {"configuration", configuration},
-                    {"player", path_to_utf8(generation / ("faset_player" + executable_suffix()))},
-                    {"schema", path_to_utf8(generation / "schema.json")},
-                    {"lua_enabled", job.lua.enabled()},
-                    {"lua_fingerprint", job.lua.fingerprint},
-                    {"fingerprint", fingerprint}};
+            return result_for(job.status.id, fingerprint, false);
         } catch (...) {
             std::error_code error;
             fs::remove_all(staging, error);

tests/build_cache_tests.cpp

@@ -72,6 +72,31 @@ void run(const fs::path& root) {
     check(!fs::exists(native / "sentinel"),
           "Changed toolchain invalidates only the generated native tree");
 
+    atomic_write(native / "faset_player", "player-v1\n");
+    atomic_write(native / "faset_schema_exporter", "exporter-v1\n");
+    atomic_write(native / "CMakeCache.txt", "recipe-v1\n");
+    atomic_write(native / "shaders/vertexMain.spv", "shader-v1\n");
+    atomic_write(native / "faset_runtime.dll", "runtime-v1\n");
+    auto package_key = [&] (const editor::BuildInputs& inputs) {
+        return editor::build_package_key(inputs, native, "Debug", native / "faset_player",
+                                         native / "faset_schema_exporter");
+    };
+    const auto first_key = package_key(changed_tool);
+    check(first_key != package_key(changed_declaration) &&
+              first_key != package_key(with_lua),
+          "Toolchain and Lua source changes invalidate package identity");
+    auto option_config = config;
+    option_config.configure_arguments.push_back("-DFASET_TEST_OPTION=ON");
+    check(first_key != package_key(editor::capture_build_inputs(option_config, lua)),
+          "Configure option changes invalidate package identity");
+    atomic_write(native / "shaders/vertexMain.spv", "shader-v2\n");
+    check(first_key != package_key(changed_tool),
+          "Shader bytes invalidate package identity");
+    atomic_write(native / "shaders/vertexMain.spv", "shader-v1\n");
+    atomic_write(native / "faset_runtime.dll", "runtime-v2\n");
+    check(first_key != package_key(changed_tool),
+          "Runtime DLL bytes invalidate package identity");
+
     fs::create_directories(root / "outside");
     std::error_code link_error;
     fs::create_directory_symlink(root / "outside", scripts / "linked", link_error);
@@ -92,7 +117,7 @@ int main() {
     try {
         run(root);
         fs::remove_all(root);
-        std::cout << "Complete source and toolchain snapshot contracts passed\n";
+        std::cout << "Source, toolchain and native package identity contracts passed\n";
         return 0;
     } catch (const std::exception& error) {
         std::cerr << error.what() << '\n';

tests/build_schema_tests.cpp

@@ -147,8 +147,27 @@ int test_main(int argc, char** argv) {
         authoring::AuthoringService authoring(config.project_root, authoring::builtin_schemas());
         const auto valid = manifest();
         atomic_write_json(config.project_root / "schema-fixture.json", valid);
-        const auto first = builds.wait(builds.start_build());
+        auto first = builds.wait(builds.start_build());
         check(first.state == "succeeded", "Valid custom schema v2 publishes: " + first.error);
+        const auto repeated = builds.wait(builds.start_build());
+        check(repeated.state == "succeeded" &&
+                  repeated.result.at("generation") == first.result.at("generation") &&
+                  repeated.result.at("schema_cache_hit") == true &&
+                  repeated.result.at("generation_reused") == true &&
+                  read_text(config.project_root / "schema-export-count.txt") == "1",
+              "Unchanged native build reuses a verified schema generation");
+        atomic_write(path_from_utf8(first.result.at("schema").get<std::string>()), "truncated");
+        first = builds.wait(builds.start_build());
+        check(first.state == "succeeded" && first.result.at("schema_cache_hit") == false &&
+                  read_text(config.project_root / "schema-export-count.txt") == "2",
+              "Corrupt schema cannot be a cache hit");
+        atomic_write(path_from_utf8(first.result.at("directory").get<std::string>()) /
+                         "shaders/vertexMain.spv",
+                     "corrupt");
+        first = builds.wait(builds.start_build());
+        check(first.state == "succeeded" && first.result.at("schema_cache_hit") == false &&
+                  read_text(config.project_root / "schema-export-count.txt") == "3",
+              "Corrupt shader cannot be a cache hit");
         const auto directory = path_from_utf8(first.result.at("directory").get<std::string>());
         for (const auto* entry : {"gpuVertexMain", "gpuShadowMain", "gpuCullMain",
                                   "gpuHzbMain", "gpuPostCullMain"})
@@ -163,6 +182,8 @@ int test_main(int argc, char** argv) {
         const auto previous_player = sha256_file(player);
         const auto previous_schema = read_text(schema);
         const auto previous_manifest = read_text(directory / "manifest.json");
+        atomic_write(config.project_root / "Scripts/Extensions/BuildOnly.hpp",
+                     "#define BUILD_ONLY 3\n");
         atomic_write(config.project_root / "mutate-cpp-header-during-build", "fixture\n");
         const auto raced_header = builds.wait(builds.start_build());
         check(raced_header.state == "failed" && read_text(last_build) == previous_pointer,
@@ -178,6 +199,11 @@ int test_main(int argc, char** argv) {
               "C++-only build explicitly disables the Lua VM in CMake");
         authoring.replace_external_schemas(read_json(schema));
         const auto previous_registry = authoring.schemas().manifest();
+        std::size_t published_generations{};
+        for (const auto& entry : fs::directory_iterator(directory.parent_path())) {
+            (void)entry;
+            ++published_generations;
+        }
         check(authoring.schemas().schema("game.mover").at("version") == 2,
               "Matching custom v2 metadata reaches authoring");
         migration_contracts(config.project_root / "migration-contracts", read_json(schema));
@@ -261,9 +287,11 @@ int test_main(int argc, char** argv) {
             std::size_t generations{};
             for (const auto& entry : fs::directory_iterator(directory.parent_path())) {
                 ++generations;
-                check(entry.path() == directory, "Invalid build leaves no staging or generation");
+                check(!entry.path().filename().string().starts_with(".staging-"),
+                      "Invalid build leaves no staging generation");
             }
-            check(generations == 1, "Only the validated build generation remains");
+            check(generations == published_generations,
+                  "Invalid build publishes no new generation");
             bool rejected{};
             try {
                 authoring.replace_external_schemas(invalid[index]);
@@ -314,6 +342,8 @@ int test_main(int argc, char** argv) {
               "Later edits never mutate an already published Lua generation");
         const auto lua_pointer = read_text(last_build);
         for (const auto* marker : {"mutate-lua-during-build", "mutate-lua-snapshot"}) {
+            atomic_write(config.project_root / "Scripts/main.lua",
+                         std::string(lua_source) + "-- force a schema export\n");
             atomic_write(config.project_root / marker, "fixture\n");
             const auto raced = builds.wait(builds.start_build());
             check(raced.state == "failed" && read_text(last_build) == lua_pointer,

tests/build_schema_tool.cpp

@@ -8,6 +8,9 @@ using namespace faset;
 int tool_main(int argc, char** argv) {
     try {
         if (argc >= 3 && std::string_view(argv[1]) == "--output") {
+            const auto count_file = fs::path("schema-export-count.txt");
+            const auto count = fs::exists(count_file) ? std::stoi(read_text(count_file)) : 0;
+            atomic_write(count_file, std::to_string(count + 1));
             if (argc == 5 && std::string_view(argv[3]) == "--project") {
                 const auto snapshot = path_from_utf8(argv[4]);
                 const auto project = read_json(snapshot / "project.faset.json");