feat: Step 6 MCP server bridge for AI agents

- Add ModelContextProtocol + ModelContextProtocol.AspNetCore 1.4.0 to Engine.AI.
- Expose AI commands as MCP tools: spawn_model, set_transform, delete_entity, list_entities.
- Add AiCommandQueue to marshal commands from the MCP server thread to the main thread.
- Start in-process HTTP MCP server in Program.cs (Debug/Release only; excluded in ReleaseAOT).
- Add --mcp-port CLI argument to configure the MCP server port.
- Fix Flecs.NET package conditions to include ReleaseAOT config.
This commit is contained in:
emil28092005
2026-06-16 19:59:19 +03:00
parent 2f21ffada4
commit a9c783d204
8 changed files with 314 additions and 20 deletions
+71 -14
View File
@@ -2,6 +2,9 @@ using System;
using System.IO;
using System.Numerics;
using Engine.AI;
#if !RELEASE_AOT
using Engine.AI.Mcp;
#endif
using Engine.Core;
using Engine.Core.Components;
using Engine.Graphics;
@@ -12,23 +15,24 @@ namespace CortexEngine.App;
class Program
{
static void Main(string[] args)
static async Task Main(string[] args)
{
Console.WriteLine("Cortex Engine Step 5Starting up...");
Console.WriteLine("Cortex Engine Step 6MCP integration...");
try
{
using var world = World.Create();
using var window = new Sdl3Window("Cortex Engine — Step 5", 1280, 720);
using var window = new Sdl3Window("Cortex Engine — Step 6", 1280, 720);
var timing = new Timing();
var input = new InputMapping();
using var vulkan = new VulkanContext(window, enableValidation: false);
using var swapchain = new Swapchain(vulkan);
using var renderer = new MeshRenderer(vulkan, swapchain);
var ai = new AiCommandProcessor(world, LoadModel);
var processor = new AiCommandProcessor(world, LoadModel);
var queue = new AiCommandQueue(processor);
var modelPath = FindModelPath(args);
var (modelPath, mcpPort) = ParseArgs(args);
var mesh = LoadModel(modelPath);
var model = world.Entity("Model")
@@ -45,12 +49,28 @@ class Program
0.1f,
100.0f));
// Demo: AI agent commands.
// Demo: local AI commands processed on the main thread.
Console.WriteLine("AI demo commands:");
Console.WriteLine(ai.Process("""{ "type": "list_entities" }""").Message);
Console.WriteLine(ai.Process("""{ "type": "spawn_model", "name": "SecondCube", "modelPath": "Content/cube.obj", "position": [0.8, 0, 0], "scale": [0.3, 0.3, 0.3] }""").Message);
Console.WriteLine(ai.Process("""{ "type": "list_entities" }""").Message);
Console.WriteLine(ai.Process("""{ "type": "set_transform", "name": "SecondCube", "position": [0.8, 0.5, 0], "rotation": [0, 0, 0, 1], "scale": [0.3, 0.3, 0.3] }""").Message);
Console.WriteLine(processor.Process("""{ "type": "list_entities" }""").Message);
Console.WriteLine(processor.Process("""{ "type": "spawn_model", "name": "SecondCube", "modelPath": "Content/cube.obj", "position": [0.8, 0, 0], "scale": [0.3, 0.3, 0.3] }""").Message);
Console.WriteLine(processor.Process("""{ "type": "list_entities" }""").Message);
Console.WriteLine(processor.Process("""{ "type": "set_transform", "name": "SecondCube", "position": [0.8, 0.5, 0], "rotation": [0, 0, 0, 1], "scale": [0.3, 0.3, 0.3] }""").Message);
#if !RELEASE_AOT
// Start the MCP server in the background so AI agents can connect via HTTP.
var mcpApp = McpEngineServerHost.Create(args, queue, port: mcpPort);
var mcpTask = mcpApp.RunAsync();
_ = mcpTask.ContinueWith(t =>
{
if (t.IsFaulted)
Console.WriteLine($"MCP server error: {t.Exception?.GetBaseException().Message}");
else if (t.IsCanceled)
Console.WriteLine("MCP server canceled.");
else
Console.WriteLine("MCP server stopped.");
}, TaskScheduler.Default);
Console.WriteLine($"MCP server starting on http://localhost:{mcpPort}");
#endif
var frames = 0;
var lastFpsTime = 0.0;
@@ -62,8 +82,11 @@ class Program
timing.Tick();
window.PumpEvents();
input.BeginFrame();
// Note: SDL events are already polled in PumpEvents.
// In a real engine, the window would expose an event iterator.
// Drain any commands that arrived from the MCP server.
var processed = queue.ProcessPending();
if (processed > 0)
Console.WriteLine($"Processed {processed} AI command(s)");
if (window.Width != lastWidth || window.Height != lastHeight)
{
@@ -89,6 +112,12 @@ class Program
}
Console.WriteLine("Shutting down...");
#if !RELEASE_AOT
await mcpApp.StopAsync();
await mcpTask;
#else
await Task.CompletedTask;
#endif
}
catch (Exception ex)
{
@@ -105,10 +134,38 @@ class Program
: ObjLoader.Load(path, new Vector3(0.7f, 0.6f, 0.5f));
}
private static (string modelPath, int mcpPort) ParseArgs(string[] args)
{
var modelPath = FindModelPath(args);
var mcpPort = 5000;
for (var i = 0; i < args.Length; i++)
{
if (args[i] == "--mcp-port" && i + 1 < args.Length && int.TryParse(args[i + 1], out var port))
{
mcpPort = port;
break;
}
}
return (modelPath, mcpPort);
}
private static string FindModelPath(string[] args)
{
if (args.Length > 0 && File.Exists(args[0]))
return args[0];
// Skip recognized flags so they are not treated as a model path.
for (var i = 0; i < args.Length; i++)
{
var arg = args[i];
if (arg == "--mcp-port")
{
i++; // skip the value
continue;
}
if (File.Exists(arg))
return arg;
}
var candidates = new[]
{
+5 -4
View File
@@ -16,13 +16,14 @@ public sealed class AiCommandProcessor
{
private readonly World _world;
private readonly Func<string, Mesh> _modelLoader;
private readonly JsonSerializerOptions _jsonOptions;
public JsonSerializerOptions JsonOptions { get; }
public AiCommandProcessor(World world, Func<string, Mesh> modelLoader)
{
_world = world;
_modelLoader = modelLoader;
_jsonOptions = new JsonSerializerOptions
JsonOptions = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
Converters =
@@ -41,7 +42,7 @@ public sealed class AiCommandProcessor
{
try
{
var command = JsonSerializer.Deserialize<AiCommand>(json, _jsonOptions);
var command = JsonSerializer.Deserialize<AiCommand>(json, JsonOptions);
if (command == null)
return AiCommandResult.Error("Failed to parse command.");
@@ -118,7 +119,7 @@ public sealed class AiCommandProcessor
names.Add(name);
});
var json = JsonSerializer.Serialize(names, _jsonOptions);
var json = JsonSerializer.Serialize(names, JsonOptions);
return AiCommandResult.Ok(json);
}
}
+63
View File
@@ -0,0 +1,63 @@
using System.Collections.Concurrent;
using System.Text.Json;
using Engine.AI.Commands;
namespace Engine.AI;
/// <summary>
/// Thread-safe queue that marshals AI commands from the MCP server thread
/// to the main engine thread where the Flecs world is touched.
/// </summary>
public sealed class AiCommandQueue
{
private readonly ConcurrentQueue<(string commandJson, TaskCompletionSource<AiCommandResult> tcs)> _queue = new();
private readonly AiCommandProcessor _processor;
private readonly JsonSerializerOptions _jsonOptions;
public AiCommandQueue(AiCommandProcessor processor)
{
_processor = processor;
_jsonOptions = processor.JsonOptions;
}
/// <summary>
/// Enqueue a command object. The returned task completes on the main thread
/// when the command has been processed.
/// </summary>
public Task<AiCommandResult> EnqueueAsync(AiCommand command)
{
var json = JsonSerializer.Serialize<AiCommand>(command, _jsonOptions);
return EnqueueJsonAsync(json);
}
/// <summary>
/// Enqueue a raw JSON command string.
/// </summary>
public Task<AiCommandResult> EnqueueJsonAsync(string commandJson)
{
var tcs = new TaskCompletionSource<AiCommandResult>(TaskCreationOptions.RunContinuationsAsynchronously);
_queue.Enqueue((commandJson, tcs));
return tcs.Task;
}
/// <summary>
/// Process all pending commands. Must be called on the main engine thread.
/// Returns the number of processed commands.
/// </summary>
public int ProcessPending()
{
int processed = 0;
while (_queue.TryDequeue(out var item))
{
var result = _processor.Process(item.commandJson);
item.tcs.TrySetResult(result);
processed++;
}
return processed;
}
/// <summary>
/// Number of commands waiting to be processed.
/// </summary>
public int PendingCount => _queue.Count;
}
+14
View File
@@ -7,8 +7,22 @@
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)' == 'ReleaseAOT'">
<DefineConstants>RELEASE_AOT</DefineConstants>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Engine.Core\Engine.Core.csproj" />
</ItemGroup>
<ItemGroup Condition="'$(Configuration)' != 'ReleaseAOT'">
<PackageReference Include="ModelContextProtocol" Version="1.4.0" />
<PackageReference Include="ModelContextProtocol.AspNetCore" Version="1.4.0" />
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<ItemGroup Condition="'$(Configuration)' == 'ReleaseAOT'">
<Compile Remove="Mcp\*.cs" />
</ItemGroup>
</Project>
+124
View File
@@ -0,0 +1,124 @@
using System.ComponentModel;
using System.Numerics;
using Engine.AI.Commands;
using ModelContextProtocol.Server;
namespace Engine.AI.Mcp;
/// <summary>
/// MCP tools that expose engine commands to AI agents.
/// </summary>
[McpServerToolType]
public sealed class EngineMcpTools
{
private readonly AiCommandQueue _queue;
public EngineMcpTools(AiCommandQueue queue)
{
_queue = queue;
}
[McpServerTool, Description("Spawn a 3D model entity in the engine world.")]
public Task<string> SpawnModel(
string name,
string modelPath,
[Description("Optional position as [x, y, z] (default: 0,0,0)")] IReadOnlyList<double>? position = null,
[Description("Optional rotation as [x, y, z, w] quaternion (default: identity)")] IReadOnlyList<double>? rotation = null,
[Description("Optional scale as [x, y, z] (default: 1,1,1)")] IReadOnlyList<double>? scale = null)
{
var cmd = new SpawnModelCommand
{
Name = name,
ModelPath = modelPath,
Position = ToVector3(position, Vector3.Zero),
Rotation = ToQuaternion(rotation, Quaternion.Identity),
Scale = ToVector3(scale, Vector3.One)
};
return EnqueueAndReturnMessage(cmd);
}
[McpServerTool, Description("Update the transform of an existing entity by name.")]
public Task<string> SetTransform(
string name,
[Description("Optional position as [x, y, z]")] IReadOnlyList<double>? position = null,
[Description("Optional rotation as [x, y, z, w] quaternion")] IReadOnlyList<double>? rotation = null,
[Description("Optional scale as [x, y, z]")] IReadOnlyList<double>? scale = null)
{
var cmd = new SetTransformCommand
{
Name = name,
Position = ToVector3(position),
Rotation = ToQuaternion(rotation),
Scale = ToVector3(scale)
};
return EnqueueAndReturnMessage(cmd);
}
[McpServerTool, Description("Delete an entity by name.")]
public Task<string> DeleteEntity(string name)
{
var cmd = new DeleteEntityCommand { Name = name };
return EnqueueAndReturnMessage(cmd);
}
[McpServerTool, Description("List all named entities in the ECS world.")]
public Task<string> ListEntities()
{
var cmd = new ListEntitiesCommand();
return EnqueueAndReturnMessage(cmd);
}
private async Task<string> EnqueueAndReturnMessage(AiCommand command)
{
var result = await _queue.EnqueueAsync(command).ConfigureAwait(false);
return result.Success
? result.Message
: throw new InvalidOperationException(result.Message);
}
private static Vector3 ToVector3(IReadOnlyList<double>? values, Vector3 defaultValue)
{
if (values == null || values.Count == 0)
return defaultValue;
if (values.Count != 3)
throw new ArgumentException("Expected 3 values for Vector3.", nameof(values));
return new Vector3((float)values[0], (float)values[1], (float)values[2]);
}
private static Vector3? ToVector3(IReadOnlyList<double>? values)
{
if (values == null || values.Count == 0)
return null;
if (values.Count != 3)
throw new ArgumentException("Expected 3 values for Vector3.", nameof(values));
return new Vector3((float)values[0], (float)values[1], (float)values[2]);
}
private static Quaternion ToQuaternion(IReadOnlyList<double>? values, Quaternion defaultValue)
{
if (values == null || values.Count == 0)
return defaultValue;
if (values.Count != 4)
throw new ArgumentException("Expected 4 values for Quaternion.", nameof(values));
return new Quaternion((float)values[0], (float)values[1], (float)values[2], (float)values[3]);
}
private static Quaternion? ToQuaternion(IReadOnlyList<double>? values)
{
if (values == null || values.Count == 0)
return null;
if (values.Count != 4)
throw new ArgumentException("Expected 4 values for Quaternion.", nameof(values));
return new Quaternion((float)values[0], (float)values[1], (float)values[2], (float)values[3]);
}
}
+35
View File
@@ -0,0 +1,35 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using ModelContextProtocol.AspNetCore;
using ModelContextProtocol.Server;
namespace Engine.AI.Mcp;
/// <summary>
/// Factory for the in-process MCP HTTP server that exposes engine commands as AI tools.
/// </summary>
public static class McpEngineServerHost
{
/// <summary>
/// Create a web application that serves the MCP protocol over HTTP.
/// Call <c>RunAsync()</c> on the returned application to start the server.
/// </summary>
public static WebApplication Create(string[] args, AiCommandQueue queue, int port = 5000)
{
var builder = WebApplication.CreateBuilder(args);
builder.WebHost.ConfigureKestrel(options => options.ListenLocalhost(port));
builder.Services.AddSingleton(queue);
builder.Services.AddMcpServer()
.WithHttpTransport(options =>
{
options.Stateless = true;
})
.WithTools<EngineMcpTools>();
var app = builder.Build();
app.MapMcp();
return app;
}
}
+1 -1
View File
@@ -20,7 +20,7 @@
<ItemGroup>
<PackageReference Include="ppy.SDL3-CS" Version="2026.520.0" />
<PackageReference Include="Flecs.NET.Debug" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Debug'" />
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release'" />
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release' OR '$(Configuration)' == 'ReleaseAOT'" />
</ItemGroup>
</Project>
+1 -1
View File
@@ -21,7 +21,7 @@
<PackageReference Include="Silk.NET.Vulkan" Version="2.21.0" />
<PackageReference Include="Silk.NET.Vulkan.Extensions.KHR" Version="2.21.0" />
<PackageReference Include="Flecs.NET.Debug" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Debug'" />
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release'" />
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release' OR '$(Configuration)' == 'ReleaseAOT'" />
<PackageReference Include="SharpGLTF.Core" Version="1.0.6" />
</ItemGroup>