feat: Step 6 MCP server bridge for AI agents
- Add ModelContextProtocol + ModelContextProtocol.AspNetCore 1.4.0 to Engine.AI. - Expose AI commands as MCP tools: spawn_model, set_transform, delete_entity, list_entities. - Add AiCommandQueue to marshal commands from the MCP server thread to the main thread. - Start in-process HTTP MCP server in Program.cs (Debug/Release only; excluded in ReleaseAOT). - Add --mcp-port CLI argument to configure the MCP server port. - Fix Flecs.NET package conditions to include ReleaseAOT config.
This commit is contained in:
@@ -2,6 +2,9 @@ using System;
|
||||
using System.IO;
|
||||
using System.Numerics;
|
||||
using Engine.AI;
|
||||
#if !RELEASE_AOT
|
||||
using Engine.AI.Mcp;
|
||||
#endif
|
||||
using Engine.Core;
|
||||
using Engine.Core.Components;
|
||||
using Engine.Graphics;
|
||||
@@ -12,23 +15,24 @@ namespace CortexEngine.App;
|
||||
|
||||
class Program
|
||||
{
|
||||
static void Main(string[] args)
|
||||
static async Task Main(string[] args)
|
||||
{
|
||||
Console.WriteLine("Cortex Engine Step 5 — Starting up...");
|
||||
Console.WriteLine("Cortex Engine Step 6 — MCP integration...");
|
||||
|
||||
try
|
||||
{
|
||||
using var world = World.Create();
|
||||
using var window = new Sdl3Window("Cortex Engine — Step 5", 1280, 720);
|
||||
using var window = new Sdl3Window("Cortex Engine — Step 6", 1280, 720);
|
||||
var timing = new Timing();
|
||||
var input = new InputMapping();
|
||||
using var vulkan = new VulkanContext(window, enableValidation: false);
|
||||
using var swapchain = new Swapchain(vulkan);
|
||||
using var renderer = new MeshRenderer(vulkan, swapchain);
|
||||
|
||||
var ai = new AiCommandProcessor(world, LoadModel);
|
||||
var processor = new AiCommandProcessor(world, LoadModel);
|
||||
var queue = new AiCommandQueue(processor);
|
||||
|
||||
var modelPath = FindModelPath(args);
|
||||
var (modelPath, mcpPort) = ParseArgs(args);
|
||||
var mesh = LoadModel(modelPath);
|
||||
|
||||
var model = world.Entity("Model")
|
||||
@@ -45,12 +49,28 @@ class Program
|
||||
0.1f,
|
||||
100.0f));
|
||||
|
||||
// Demo: AI agent commands.
|
||||
// Demo: local AI commands processed on the main thread.
|
||||
Console.WriteLine("AI demo commands:");
|
||||
Console.WriteLine(ai.Process("""{ "type": "list_entities" }""").Message);
|
||||
Console.WriteLine(ai.Process("""{ "type": "spawn_model", "name": "SecondCube", "modelPath": "Content/cube.obj", "position": [0.8, 0, 0], "scale": [0.3, 0.3, 0.3] }""").Message);
|
||||
Console.WriteLine(ai.Process("""{ "type": "list_entities" }""").Message);
|
||||
Console.WriteLine(ai.Process("""{ "type": "set_transform", "name": "SecondCube", "position": [0.8, 0.5, 0], "rotation": [0, 0, 0, 1], "scale": [0.3, 0.3, 0.3] }""").Message);
|
||||
Console.WriteLine(processor.Process("""{ "type": "list_entities" }""").Message);
|
||||
Console.WriteLine(processor.Process("""{ "type": "spawn_model", "name": "SecondCube", "modelPath": "Content/cube.obj", "position": [0.8, 0, 0], "scale": [0.3, 0.3, 0.3] }""").Message);
|
||||
Console.WriteLine(processor.Process("""{ "type": "list_entities" }""").Message);
|
||||
Console.WriteLine(processor.Process("""{ "type": "set_transform", "name": "SecondCube", "position": [0.8, 0.5, 0], "rotation": [0, 0, 0, 1], "scale": [0.3, 0.3, 0.3] }""").Message);
|
||||
|
||||
#if !RELEASE_AOT
|
||||
// Start the MCP server in the background so AI agents can connect via HTTP.
|
||||
var mcpApp = McpEngineServerHost.Create(args, queue, port: mcpPort);
|
||||
var mcpTask = mcpApp.RunAsync();
|
||||
_ = mcpTask.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
Console.WriteLine($"MCP server error: {t.Exception?.GetBaseException().Message}");
|
||||
else if (t.IsCanceled)
|
||||
Console.WriteLine("MCP server canceled.");
|
||||
else
|
||||
Console.WriteLine("MCP server stopped.");
|
||||
}, TaskScheduler.Default);
|
||||
Console.WriteLine($"MCP server starting on http://localhost:{mcpPort}");
|
||||
#endif
|
||||
|
||||
var frames = 0;
|
||||
var lastFpsTime = 0.0;
|
||||
@@ -62,8 +82,11 @@ class Program
|
||||
timing.Tick();
|
||||
window.PumpEvents();
|
||||
input.BeginFrame();
|
||||
// Note: SDL events are already polled in PumpEvents.
|
||||
// In a real engine, the window would expose an event iterator.
|
||||
|
||||
// Drain any commands that arrived from the MCP server.
|
||||
var processed = queue.ProcessPending();
|
||||
if (processed > 0)
|
||||
Console.WriteLine($"Processed {processed} AI command(s)");
|
||||
|
||||
if (window.Width != lastWidth || window.Height != lastHeight)
|
||||
{
|
||||
@@ -89,6 +112,12 @@ class Program
|
||||
}
|
||||
|
||||
Console.WriteLine("Shutting down...");
|
||||
#if !RELEASE_AOT
|
||||
await mcpApp.StopAsync();
|
||||
await mcpTask;
|
||||
#else
|
||||
await Task.CompletedTask;
|
||||
#endif
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -105,10 +134,38 @@ class Program
|
||||
: ObjLoader.Load(path, new Vector3(0.7f, 0.6f, 0.5f));
|
||||
}
|
||||
|
||||
private static (string modelPath, int mcpPort) ParseArgs(string[] args)
|
||||
{
|
||||
var modelPath = FindModelPath(args);
|
||||
var mcpPort = 5000;
|
||||
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
if (args[i] == "--mcp-port" && i + 1 < args.Length && int.TryParse(args[i + 1], out var port))
|
||||
{
|
||||
mcpPort = port;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return (modelPath, mcpPort);
|
||||
}
|
||||
|
||||
private static string FindModelPath(string[] args)
|
||||
{
|
||||
if (args.Length > 0 && File.Exists(args[0]))
|
||||
return args[0];
|
||||
// Skip recognized flags so they are not treated as a model path.
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
var arg = args[i];
|
||||
if (arg == "--mcp-port")
|
||||
{
|
||||
i++; // skip the value
|
||||
continue;
|
||||
}
|
||||
|
||||
if (File.Exists(arg))
|
||||
return arg;
|
||||
}
|
||||
|
||||
var candidates = new[]
|
||||
{
|
||||
|
||||
@@ -16,13 +16,14 @@ public sealed class AiCommandProcessor
|
||||
{
|
||||
private readonly World _world;
|
||||
private readonly Func<string, Mesh> _modelLoader;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
|
||||
public JsonSerializerOptions JsonOptions { get; }
|
||||
|
||||
public AiCommandProcessor(World world, Func<string, Mesh> modelLoader)
|
||||
{
|
||||
_world = world;
|
||||
_modelLoader = modelLoader;
|
||||
_jsonOptions = new JsonSerializerOptions
|
||||
JsonOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
Converters =
|
||||
@@ -41,7 +42,7 @@ public sealed class AiCommandProcessor
|
||||
{
|
||||
try
|
||||
{
|
||||
var command = JsonSerializer.Deserialize<AiCommand>(json, _jsonOptions);
|
||||
var command = JsonSerializer.Deserialize<AiCommand>(json, JsonOptions);
|
||||
if (command == null)
|
||||
return AiCommandResult.Error("Failed to parse command.");
|
||||
|
||||
@@ -118,7 +119,7 @@ public sealed class AiCommandProcessor
|
||||
names.Add(name);
|
||||
});
|
||||
|
||||
var json = JsonSerializer.Serialize(names, _jsonOptions);
|
||||
var json = JsonSerializer.Serialize(names, JsonOptions);
|
||||
return AiCommandResult.Ok(json);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.Json;
|
||||
using Engine.AI.Commands;
|
||||
|
||||
namespace Engine.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe queue that marshals AI commands from the MCP server thread
|
||||
/// to the main engine thread where the Flecs world is touched.
|
||||
/// </summary>
|
||||
public sealed class AiCommandQueue
|
||||
{
|
||||
private readonly ConcurrentQueue<(string commandJson, TaskCompletionSource<AiCommandResult> tcs)> _queue = new();
|
||||
private readonly AiCommandProcessor _processor;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
|
||||
public AiCommandQueue(AiCommandProcessor processor)
|
||||
{
|
||||
_processor = processor;
|
||||
_jsonOptions = processor.JsonOptions;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enqueue a command object. The returned task completes on the main thread
|
||||
/// when the command has been processed.
|
||||
/// </summary>
|
||||
public Task<AiCommandResult> EnqueueAsync(AiCommand command)
|
||||
{
|
||||
var json = JsonSerializer.Serialize<AiCommand>(command, _jsonOptions);
|
||||
return EnqueueJsonAsync(json);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enqueue a raw JSON command string.
|
||||
/// </summary>
|
||||
public Task<AiCommandResult> EnqueueJsonAsync(string commandJson)
|
||||
{
|
||||
var tcs = new TaskCompletionSource<AiCommandResult>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_queue.Enqueue((commandJson, tcs));
|
||||
return tcs.Task;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process all pending commands. Must be called on the main engine thread.
|
||||
/// Returns the number of processed commands.
|
||||
/// </summary>
|
||||
public int ProcessPending()
|
||||
{
|
||||
int processed = 0;
|
||||
while (_queue.TryDequeue(out var item))
|
||||
{
|
||||
var result = _processor.Process(item.commandJson);
|
||||
item.tcs.TrySetResult(result);
|
||||
processed++;
|
||||
}
|
||||
return processed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Number of commands waiting to be processed.
|
||||
/// </summary>
|
||||
public int PendingCount => _queue.Count;
|
||||
}
|
||||
@@ -7,8 +7,22 @@
|
||||
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'ReleaseAOT'">
|
||||
<DefineConstants>RELEASE_AOT</DefineConstants>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Engine.Core\Engine.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="'$(Configuration)' != 'ReleaseAOT'">
|
||||
<PackageReference Include="ModelContextProtocol" Version="1.4.0" />
|
||||
<PackageReference Include="ModelContextProtocol.AspNetCore" Version="1.4.0" />
|
||||
<FrameworkReference Include="Microsoft.AspNetCore.App" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="'$(Configuration)' == 'ReleaseAOT'">
|
||||
<Compile Remove="Mcp\*.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
using System.ComponentModel;
|
||||
using System.Numerics;
|
||||
using Engine.AI.Commands;
|
||||
using ModelContextProtocol.Server;
|
||||
|
||||
namespace Engine.AI.Mcp;
|
||||
|
||||
/// <summary>
|
||||
/// MCP tools that expose engine commands to AI agents.
|
||||
/// </summary>
|
||||
[McpServerToolType]
|
||||
public sealed class EngineMcpTools
|
||||
{
|
||||
private readonly AiCommandQueue _queue;
|
||||
|
||||
public EngineMcpTools(AiCommandQueue queue)
|
||||
{
|
||||
_queue = queue;
|
||||
}
|
||||
|
||||
[McpServerTool, Description("Spawn a 3D model entity in the engine world.")]
|
||||
public Task<string> SpawnModel(
|
||||
string name,
|
||||
string modelPath,
|
||||
[Description("Optional position as [x, y, z] (default: 0,0,0)")] IReadOnlyList<double>? position = null,
|
||||
[Description("Optional rotation as [x, y, z, w] quaternion (default: identity)")] IReadOnlyList<double>? rotation = null,
|
||||
[Description("Optional scale as [x, y, z] (default: 1,1,1)")] IReadOnlyList<double>? scale = null)
|
||||
{
|
||||
var cmd = new SpawnModelCommand
|
||||
{
|
||||
Name = name,
|
||||
ModelPath = modelPath,
|
||||
Position = ToVector3(position, Vector3.Zero),
|
||||
Rotation = ToQuaternion(rotation, Quaternion.Identity),
|
||||
Scale = ToVector3(scale, Vector3.One)
|
||||
};
|
||||
|
||||
return EnqueueAndReturnMessage(cmd);
|
||||
}
|
||||
|
||||
[McpServerTool, Description("Update the transform of an existing entity by name.")]
|
||||
public Task<string> SetTransform(
|
||||
string name,
|
||||
[Description("Optional position as [x, y, z]")] IReadOnlyList<double>? position = null,
|
||||
[Description("Optional rotation as [x, y, z, w] quaternion")] IReadOnlyList<double>? rotation = null,
|
||||
[Description("Optional scale as [x, y, z]")] IReadOnlyList<double>? scale = null)
|
||||
{
|
||||
var cmd = new SetTransformCommand
|
||||
{
|
||||
Name = name,
|
||||
Position = ToVector3(position),
|
||||
Rotation = ToQuaternion(rotation),
|
||||
Scale = ToVector3(scale)
|
||||
};
|
||||
|
||||
return EnqueueAndReturnMessage(cmd);
|
||||
}
|
||||
|
||||
[McpServerTool, Description("Delete an entity by name.")]
|
||||
public Task<string> DeleteEntity(string name)
|
||||
{
|
||||
var cmd = new DeleteEntityCommand { Name = name };
|
||||
return EnqueueAndReturnMessage(cmd);
|
||||
}
|
||||
|
||||
[McpServerTool, Description("List all named entities in the ECS world.")]
|
||||
public Task<string> ListEntities()
|
||||
{
|
||||
var cmd = new ListEntitiesCommand();
|
||||
return EnqueueAndReturnMessage(cmd);
|
||||
}
|
||||
|
||||
private async Task<string> EnqueueAndReturnMessage(AiCommand command)
|
||||
{
|
||||
var result = await _queue.EnqueueAsync(command).ConfigureAwait(false);
|
||||
return result.Success
|
||||
? result.Message
|
||||
: throw new InvalidOperationException(result.Message);
|
||||
}
|
||||
|
||||
private static Vector3 ToVector3(IReadOnlyList<double>? values, Vector3 defaultValue)
|
||||
{
|
||||
if (values == null || values.Count == 0)
|
||||
return defaultValue;
|
||||
|
||||
if (values.Count != 3)
|
||||
throw new ArgumentException("Expected 3 values for Vector3.", nameof(values));
|
||||
|
||||
return new Vector3((float)values[0], (float)values[1], (float)values[2]);
|
||||
}
|
||||
|
||||
private static Vector3? ToVector3(IReadOnlyList<double>? values)
|
||||
{
|
||||
if (values == null || values.Count == 0)
|
||||
return null;
|
||||
|
||||
if (values.Count != 3)
|
||||
throw new ArgumentException("Expected 3 values for Vector3.", nameof(values));
|
||||
|
||||
return new Vector3((float)values[0], (float)values[1], (float)values[2]);
|
||||
}
|
||||
|
||||
private static Quaternion ToQuaternion(IReadOnlyList<double>? values, Quaternion defaultValue)
|
||||
{
|
||||
if (values == null || values.Count == 0)
|
||||
return defaultValue;
|
||||
|
||||
if (values.Count != 4)
|
||||
throw new ArgumentException("Expected 4 values for Quaternion.", nameof(values));
|
||||
|
||||
return new Quaternion((float)values[0], (float)values[1], (float)values[2], (float)values[3]);
|
||||
}
|
||||
|
||||
private static Quaternion? ToQuaternion(IReadOnlyList<double>? values)
|
||||
{
|
||||
if (values == null || values.Count == 0)
|
||||
return null;
|
||||
|
||||
if (values.Count != 4)
|
||||
throw new ArgumentException("Expected 4 values for Quaternion.", nameof(values));
|
||||
|
||||
return new Quaternion((float)values[0], (float)values[1], (float)values[2], (float)values[3]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ModelContextProtocol.AspNetCore;
|
||||
using ModelContextProtocol.Server;
|
||||
|
||||
namespace Engine.AI.Mcp;
|
||||
|
||||
/// <summary>
|
||||
/// Factory for the in-process MCP HTTP server that exposes engine commands as AI tools.
|
||||
/// </summary>
|
||||
public static class McpEngineServerHost
|
||||
{
|
||||
/// <summary>
|
||||
/// Create a web application that serves the MCP protocol over HTTP.
|
||||
/// Call <c>RunAsync()</c> on the returned application to start the server.
|
||||
/// </summary>
|
||||
public static WebApplication Create(string[] args, AiCommandQueue queue, int port = 5000)
|
||||
{
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
builder.WebHost.ConfigureKestrel(options => options.ListenLocalhost(port));
|
||||
builder.Services.AddSingleton(queue);
|
||||
builder.Services.AddMcpServer()
|
||||
.WithHttpTransport(options =>
|
||||
{
|
||||
options.Stateless = true;
|
||||
})
|
||||
.WithTools<EngineMcpTools>();
|
||||
|
||||
var app = builder.Build();
|
||||
app.MapMcp();
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="ppy.SDL3-CS" Version="2026.520.0" />
|
||||
<PackageReference Include="Flecs.NET.Debug" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Debug'" />
|
||||
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release'" />
|
||||
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release' OR '$(Configuration)' == 'ReleaseAOT'" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<PackageReference Include="Silk.NET.Vulkan" Version="2.21.0" />
|
||||
<PackageReference Include="Silk.NET.Vulkan.Extensions.KHR" Version="2.21.0" />
|
||||
<PackageReference Include="Flecs.NET.Debug" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Debug'" />
|
||||
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release'" />
|
||||
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release' OR '$(Configuration)' == 'ReleaseAOT'" />
|
||||
<PackageReference Include="SharpGLTF.Core" Version="1.0.6" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user