feat: world state, multiple lights, textures, stdio MCP, Claude config
- Add get_world_state AI command that dumps ECS entities with Transform, Camera, Material, Light, and Mesh summaries. - Add set_material AI command to update albedo/roughness/metallic/texture. - Expose both commands as MCP HTTP tools and stdio tools. - Add Light component and support up to 4 directional lights via a Vulkan uniform buffer (descriptor set 0) with std140 layout. - Move per-frame lighting/camera data into the uniform buffer; push constants now carry only MVP + material properties (96 bytes). - Add Texture class for PNG loading and Vulkan image/view/sampler creation. - Add per-entity combined image sampler descriptor set (set 1) and use it for albedo texture sampling in the fragment shader. - Generate a checkerboard floor texture in Program.cs. - Add McpStdioServer for headless stdio MCP (Claude Desktop compatible). - Add claude_desktop_config.json and scripts/start_mcp_engine.sh. - Update CORTEX_ENGINE_ARCHITECTURE.md with runtime notes, CLI arguments, Vulkan pipeline details, and MCP client configuration. - All configs (Debug/Release/ReleaseAOT) build successfully.
This commit is contained in:
emil28092005
@@ -1,4 +1,5 @@
|
||||
using System.Numerics;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Engine.AI.Commands;
|
||||
@@ -55,6 +56,8 @@ public sealed class AiCommandProcessor
|
||||
DeleteEntityCommand c => DeleteEntity(c),
|
||||
ListEntitiesCommand => ListEntities(),
|
||||
CaptureScreenshotCommand c => CaptureScreenshot(c),
|
||||
GetWorldStateCommand => GetWorldState(),
|
||||
SetMaterialCommand c => SetMaterial(c),
|
||||
_ => AiCommandResult.Error($"Unknown command type: {command.Type}")
|
||||
};
|
||||
}
|
||||
@@ -132,4 +135,128 @@ public sealed class AiCommandProcessor
|
||||
_requestScreenshot(path);
|
||||
return AiCommandResult.Ok($"Screenshot requested: {path}");
|
||||
}
|
||||
|
||||
private AiCommandResult SetMaterial(SetMaterialCommand command)
|
||||
{
|
||||
var entity = _world.Lookup(command.Name);
|
||||
if ((ulong)entity.Id == 0)
|
||||
return AiCommandResult.Error($"Entity '{command.Name}' not found.");
|
||||
|
||||
ref var material = ref entity.Ensure<Material>();
|
||||
|
||||
if (command.Albedo.HasValue)
|
||||
material.Albedo = command.Albedo.Value;
|
||||
if (command.Roughness.HasValue)
|
||||
material.Roughness = command.Roughness.Value;
|
||||
if (command.Metallic.HasValue)
|
||||
material.Metallic = command.Metallic.Value;
|
||||
if (command.TexturePath is not null)
|
||||
material.TexturePath = command.TexturePath;
|
||||
|
||||
return AiCommandResult.Ok($"Updated material for entity '{command.Name}'.");
|
||||
}
|
||||
|
||||
private AiCommandResult GetWorldState()
|
||||
{
|
||||
using var stream = new MemoryStream();
|
||||
using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = false }))
|
||||
{
|
||||
writer.WriteStartArray();
|
||||
|
||||
_world.Each((Entity e, ref Transform _) =>
|
||||
{
|
||||
var name = e.Name();
|
||||
if (string.IsNullOrEmpty(name))
|
||||
return;
|
||||
|
||||
writer.WriteStartObject();
|
||||
writer.WriteString("name", name);
|
||||
writer.WriteNumber("id", (ulong)e.Id);
|
||||
|
||||
writer.WriteStartObject("components");
|
||||
|
||||
if (e.Has<Transform>())
|
||||
{
|
||||
ref var transform = ref e.Ensure<Transform>();
|
||||
writer.WriteStartObject("Transform");
|
||||
WriteVector3(writer, "position", transform.Position);
|
||||
WriteQuaternion(writer, "rotation", transform.Rotation);
|
||||
WriteVector3(writer, "scale", transform.Scale);
|
||||
writer.WriteEndObject();
|
||||
}
|
||||
|
||||
if (e.Has<Camera>())
|
||||
{
|
||||
ref var camera = ref e.Ensure<Camera>();
|
||||
writer.WriteStartObject("Camera");
|
||||
writer.WriteNumber("fieldOfView", camera.FieldOfView);
|
||||
writer.WriteNumber("aspectRatio", camera.AspectRatio);
|
||||
writer.WriteNumber("nearPlane", camera.NearPlane);
|
||||
writer.WriteNumber("farPlane", camera.FarPlane);
|
||||
WriteVector3(writer, "position", camera.Position);
|
||||
WriteVector3(writer, "target", camera.Target);
|
||||
WriteVector3(writer, "up", camera.Up);
|
||||
writer.WriteEndObject();
|
||||
}
|
||||
|
||||
if (e.Has<Material>())
|
||||
{
|
||||
ref var material = ref e.Ensure<Material>();
|
||||
writer.WriteStartObject("Material");
|
||||
WriteVector3(writer, "albedo", material.Albedo);
|
||||
writer.WriteNumber("roughness", material.Roughness);
|
||||
writer.WriteNumber("metallic", material.Metallic);
|
||||
if (material.HasTexture)
|
||||
writer.WriteString("texturePath", material.TexturePath);
|
||||
writer.WriteEndObject();
|
||||
}
|
||||
|
||||
if (e.Has<Mesh>())
|
||||
{
|
||||
ref var mesh = ref e.Ensure<Mesh>();
|
||||
writer.WriteStartObject("Mesh");
|
||||
writer.WriteNumber("vertexCount", mesh.Vertices.Length);
|
||||
writer.WriteNumber("indexCount", mesh.Indices.Length);
|
||||
writer.WriteEndObject();
|
||||
}
|
||||
|
||||
if (e.Has<Light>())
|
||||
{
|
||||
ref var light = ref e.Ensure<Light>();
|
||||
writer.WriteStartObject("Light");
|
||||
WriteVector3(writer, "direction", light.Direction);
|
||||
WriteVector3(writer, "color", light.Color);
|
||||
writer.WriteNumber("intensity", light.Intensity);
|
||||
writer.WriteEndObject();
|
||||
}
|
||||
|
||||
writer.WriteEndObject();
|
||||
writer.WriteEndObject();
|
||||
});
|
||||
|
||||
writer.WriteEndArray();
|
||||
}
|
||||
|
||||
var json = System.Text.Encoding.UTF8.GetString(stream.ToArray());
|
||||
return AiCommandResult.Ok(json);
|
||||
}
|
||||
|
||||
private static void WriteVector3(Utf8JsonWriter writer, string propertyName, Vector3 value)
|
||||
{
|
||||
writer.WriteStartArray(propertyName);
|
||||
writer.WriteNumberValue(value.X);
|
||||
writer.WriteNumberValue(value.Y);
|
||||
writer.WriteNumberValue(value.Z);
|
||||
writer.WriteEndArray();
|
||||
}
|
||||
|
||||
private static void WriteQuaternion(Utf8JsonWriter writer, string propertyName, Quaternion value)
|
||||
{
|
||||
writer.WriteStartArray(propertyName);
|
||||
writer.WriteNumberValue(value.X);
|
||||
writer.WriteNumberValue(value.Y);
|
||||
writer.WriteNumberValue(value.Z);
|
||||
writer.WriteNumberValue(value.W);
|
||||
writer.WriteEndArray();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@ namespace Engine.AI.Commands;
|
||||
[JsonDerivedType(typeof(DeleteEntityCommand), "delete_entity")]
|
||||
[JsonDerivedType(typeof(ListEntitiesCommand), "list_entities")]
|
||||
[JsonDerivedType(typeof(CaptureScreenshotCommand), "capture_screenshot")]
|
||||
[JsonDerivedType(typeof(GetWorldStateCommand), "get_world_state")]
|
||||
[JsonDerivedType(typeof(SetMaterialCommand), "set_material")]
|
||||
public abstract record AiCommand
|
||||
{
|
||||
public string Type => GetType().Name.Replace("Command", "").ToLowerInvariant();
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Engine.AI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Dump the current ECS world state as JSON for AI analysis.
|
||||
/// </summary>
|
||||
public sealed record GetWorldStateCommand : AiCommand;
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Numerics;
|
||||
|
||||
namespace Engine.AI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Update the Material component of an existing entity by name.
|
||||
/// </summary>
|
||||
public sealed record SetMaterialCommand : AiCommand
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public Vector3? Albedo { get; init; }
|
||||
public float? Roughness { get; init; }
|
||||
public float? Metallic { get; init; }
|
||||
public string? TexturePath { get; init; }
|
||||
}
|
||||
@@ -77,6 +77,33 @@ public sealed class EngineMcpTools
|
||||
return EnqueueAndReturnMessage(cmd);
|
||||
}
|
||||
|
||||
[McpServerTool, Description("Dump the current ECS world state as JSON, including Transform, Camera, Material, Light, and Mesh component summaries.")]
|
||||
public Task<string> GetWorldState()
|
||||
{
|
||||
var cmd = new GetWorldStateCommand();
|
||||
return EnqueueAndReturnMessage(cmd);
|
||||
}
|
||||
|
||||
[McpServerTool, Description("Update the material of an existing entity by name.")]
|
||||
public Task<string> SetMaterial(
|
||||
string name,
|
||||
[Description("Optional albedo color as [r, g, b] (0-1)")] IReadOnlyList<double>? albedo = null,
|
||||
[Description("Optional roughness value (0-1)")] float? roughness = null,
|
||||
[Description("Optional metallic value (0-1)")] float? metallic = null,
|
||||
[Description("Optional path to a PNG texture file")] string? texturePath = null)
|
||||
{
|
||||
var cmd = new SetMaterialCommand
|
||||
{
|
||||
Name = name,
|
||||
Albedo = ToVector3(albedo),
|
||||
Roughness = roughness,
|
||||
Metallic = metallic,
|
||||
TexturePath = texturePath
|
||||
};
|
||||
|
||||
return EnqueueAndReturnMessage(cmd);
|
||||
}
|
||||
|
||||
private async Task<string> EnqueueAndReturnMessage(AiCommand command)
|
||||
{
|
||||
var result = await _queue.EnqueueAsync(command).ConfigureAwait(false);
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Engine.AI.Commands;
|
||||
|
||||
namespace Engine.AI.Stdio;
|
||||
|
||||
/// <summary>
|
||||
/// A minimal stdio MCP server that routes JSON-RPC requests to an <see cref="AiCommandProcessor"/>.
|
||||
/// This is suitable for clients like Claude Desktop that speak MCP over stdio.
|
||||
/// </summary>
|
||||
public sealed class McpStdioServer
|
||||
{
|
||||
private readonly AiCommandProcessor _processor;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
private readonly Dictionary<string, ToolDefinition> _tools;
|
||||
|
||||
public McpStdioServer(AiCommandProcessor processor)
|
||||
{
|
||||
_processor = processor;
|
||||
_jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
|
||||
_tools = new Dictionary<string, ToolDefinition>
|
||||
{
|
||||
["SpawnModel"] = new(
|
||||
"Spawn a 3D model entity in the engine world.",
|
||||
new JsonSchemaBuilder()
|
||||
.AddRequiredString("name")
|
||||
.AddRequiredString("modelPath")
|
||||
.AddOptionalArray("position", "number", 3)
|
||||
.AddOptionalArray("rotation", "number", 4)
|
||||
.AddOptionalArray("scale", "number", 3)
|
||||
.Build()),
|
||||
["SetTransform"] = new(
|
||||
"Update the transform of an existing entity by name.",
|
||||
new JsonSchemaBuilder()
|
||||
.AddRequiredString("name")
|
||||
.AddOptionalArray("position", "number", 3)
|
||||
.AddOptionalArray("rotation", "number", 4)
|
||||
.AddOptionalArray("scale", "number", 3)
|
||||
.Build()),
|
||||
["SetMaterial"] = new(
|
||||
"Update the material of an existing entity by name.",
|
||||
new JsonSchemaBuilder()
|
||||
.AddRequiredString("name")
|
||||
.AddOptionalArray("albedo", "number", 3)
|
||||
.AddOptionalNumber("roughness")
|
||||
.AddOptionalNumber("metallic")
|
||||
.AddOptionalString("texturePath")
|
||||
.Build()),
|
||||
["DeleteEntity"] = new(
|
||||
"Delete an entity by name.",
|
||||
new JsonSchemaBuilder()
|
||||
.AddRequiredString("name")
|
||||
.Build()),
|
||||
["ListEntities"] = new(
|
||||
"List all named entities in the ECS world.",
|
||||
new JsonSchemaBuilder().Build()),
|
||||
["CaptureScreenshot"] = new(
|
||||
"Capture a screenshot of the current rendered frame and save it to disk.",
|
||||
new JsonSchemaBuilder()
|
||||
.AddOptionalString("outputPath")
|
||||
.Build()),
|
||||
["GetWorldState"] = new(
|
||||
"Dump the current ECS world state as JSON.",
|
||||
new JsonSchemaBuilder().Build())
|
||||
};
|
||||
}
|
||||
|
||||
public void Run()
|
||||
{
|
||||
var input = Console.OpenStandardInput();
|
||||
var output = Console.OpenStandardOutput();
|
||||
using var reader = new StreamReader(input, Encoding.UTF8);
|
||||
using var writer = new StreamWriter(output, Encoding.UTF8) { AutoFlush = true };
|
||||
|
||||
while (true)
|
||||
{
|
||||
var line = reader.ReadLine();
|
||||
if (line == null)
|
||||
break;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
continue;
|
||||
|
||||
var response = HandleMessage(line);
|
||||
if (response != null)
|
||||
{
|
||||
writer.WriteLine(JsonSerializer.Serialize(response, _jsonOptions));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private JsonElement? HandleMessage(string line)
|
||||
{
|
||||
using var document = JsonDocument.Parse(line);
|
||||
var root = document.RootElement;
|
||||
|
||||
var method = root.GetProperty("method").GetString();
|
||||
var id = root.TryGetProperty("id", out var idProp) ? (JsonElement?)idProp : null;
|
||||
|
||||
switch (method)
|
||||
{
|
||||
case "initialize":
|
||||
return MakeResponse(id, new
|
||||
{
|
||||
protocolVersion = "2024-11-05",
|
||||
capabilities = new { tools = new { } },
|
||||
serverInfo = new { name = "CortexEngine", version = "0.1.0" }
|
||||
});
|
||||
|
||||
case "notifications/initialized":
|
||||
return null;
|
||||
|
||||
case "tools/list":
|
||||
return MakeResponse(id, new { tools = _tools.Select(t => new { type = "function", function = new { name = t.Key, description = t.Value.Description, parameters = t.Value.Parameters } }).ToList() });
|
||||
|
||||
case "tools/call":
|
||||
return HandleToolCall(id, root.GetProperty("params"));
|
||||
|
||||
case "ping":
|
||||
return MakeResponse(id, new { });
|
||||
|
||||
default:
|
||||
return MakeError(id, -32601, $"Method not found: {method}");
|
||||
}
|
||||
}
|
||||
|
||||
private JsonElement? HandleToolCall(JsonElement? id, JsonElement paramsElement)
|
||||
{
|
||||
var name = paramsElement.GetProperty("name").GetString();
|
||||
var arguments = paramsElement.GetProperty("arguments");
|
||||
|
||||
if (!_tools.TryGetValue(name ?? string.Empty, out _))
|
||||
return MakeError(id, -32601, $"Tool not found: {name}");
|
||||
|
||||
var command = BuildCommand(name!, arguments);
|
||||
var result = _processor.Process(command);
|
||||
|
||||
return MakeResponse(id, new { content = new[] { new { type = "text", text = result.Message } }, isError = !result.Success });
|
||||
}
|
||||
|
||||
private string BuildCommand(string toolName, JsonElement arguments)
|
||||
{
|
||||
var type = toolName switch
|
||||
{
|
||||
"SpawnModel" => "spawn_model",
|
||||
"SetTransform" => "set_transform",
|
||||
"SetMaterial" => "set_material",
|
||||
"DeleteEntity" => "delete_entity",
|
||||
"ListEntities" => "list_entities",
|
||||
"CaptureScreenshot" => "capture_screenshot",
|
||||
"GetWorldState" => "get_world_state",
|
||||
_ => toolName.ToLowerInvariant()
|
||||
};
|
||||
|
||||
var dict = new Dictionary<string, object> { ["type"] = type };
|
||||
foreach (var property in arguments.EnumerateObject())
|
||||
dict[property.Name] = ConvertArgument(property.Value);
|
||||
|
||||
return JsonSerializer.Serialize(dict, _jsonOptions);
|
||||
}
|
||||
|
||||
private static object ConvertArgument(JsonElement element)
|
||||
{
|
||||
return element.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => element.GetString()!,
|
||||
JsonValueKind.Number => element.GetDouble(),
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
JsonValueKind.Array => element.EnumerateArray().Select(ConvertArgument).ToList(),
|
||||
JsonValueKind.Object => element.EnumerateObject().ToDictionary(p => p.Name, p => ConvertArgument(p.Value)),
|
||||
_ => element.GetRawText()
|
||||
};
|
||||
}
|
||||
|
||||
private JsonElement? MakeResponse(JsonElement? id, object result)
|
||||
{
|
||||
if (id == null)
|
||||
return null;
|
||||
|
||||
var json = JsonSerializer.Serialize(new { jsonrpc = "2.0", result, id = id.Value }, _jsonOptions);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
return doc.RootElement.Clone();
|
||||
}
|
||||
|
||||
private JsonElement? MakeError(JsonElement? id, int code, string message)
|
||||
{
|
||||
if (id == null)
|
||||
return null;
|
||||
|
||||
var json = JsonSerializer.Serialize(new { jsonrpc = "2.0", error = new { code, message }, id = id.Value }, _jsonOptions);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
return doc.RootElement.Clone();
|
||||
}
|
||||
|
||||
private sealed record ToolDefinition(string Description, JsonElement Parameters);
|
||||
|
||||
private sealed class JsonSchemaBuilder
|
||||
{
|
||||
private readonly Dictionary<string, object> _properties = new();
|
||||
private readonly List<string> _required = new();
|
||||
private readonly string _type = "object";
|
||||
|
||||
public JsonSchemaBuilder AddRequiredString(string name)
|
||||
{
|
||||
_properties[name] = new { type = "string" };
|
||||
_required.Add(name);
|
||||
return this;
|
||||
}
|
||||
|
||||
public JsonSchemaBuilder AddOptionalString(string name)
|
||||
{
|
||||
_properties[name] = new { type = "string" };
|
||||
return this;
|
||||
}
|
||||
|
||||
public JsonSchemaBuilder AddOptionalNumber(string name)
|
||||
{
|
||||
_properties[name] = new { type = "number" };
|
||||
return this;
|
||||
}
|
||||
|
||||
public JsonSchemaBuilder AddOptionalArray(string name, string itemType, int? minItems = null)
|
||||
{
|
||||
_properties[name] = new { type = "array", items = new { type = itemType }, minItems };
|
||||
return this;
|
||||
}
|
||||
|
||||
public JsonElement Build()
|
||||
{
|
||||
var dict = new Dictionary<string, object>
|
||||
{
|
||||
["type"] = _type,
|
||||
["properties"] = _properties,
|
||||
["required"] = _required
|
||||
};
|
||||
var json = JsonSerializer.Serialize(dict);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
return doc.RootElement.Clone();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user