feat: Step 5 AI agent command bridge
- Add new Engine.AI project referenced by the App. - Define JSON command model: spawn_model, set_transform, delete_entity, list_entities. - Implement AiCommandProcessor with System.Text.Json and custom Vector3/Quaternion converters. - Wire processor into Program.cs; demo commands spawn and move a second cube. - AI commands are executed against the live Flecs world and visible in the next frame.
This commit is contained in:
@@ -8,6 +8,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Engine.Graphics", "src\Engi
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CortexEngine.App", "src\CortexEngine.App\CortexEngine.App.csproj", "{33333333-3333-3333-3333-333333333333}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CortexEngine.App", "src\CortexEngine.App\CortexEngine.App.csproj", "{33333333-3333-3333-3333-333333333333}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Engine.AI", "src\Engine.AI\Engine.AI.csproj", "{44444444-4444-4444-4444-444444444444}"
|
||||||
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
@@ -33,5 +35,11 @@ Global
|
|||||||
{33333333-3333-3333-3333-333333333333}.Release|Any CPU.Build.0 = Release|Any CPU
|
{33333333-3333-3333-3333-333333333333}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
{33333333-3333-3333-3333-333333333333}.ReleaseAOT|Any CPU.ActiveCfg = ReleaseAOT|Any CPU
|
{33333333-3333-3333-3333-333333333333}.ReleaseAOT|Any CPU.ActiveCfg = ReleaseAOT|Any CPU
|
||||||
{33333333-3333-3333-3333-333333333333}.ReleaseAOT|Any CPU.Build.0 = ReleaseAOT|Any CPU
|
{33333333-3333-3333-3333-333333333333}.ReleaseAOT|Any CPU.Build.0 = ReleaseAOT|Any CPU
|
||||||
|
{44444444-4444-4444-4444-444444444444}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{44444444-4444-4444-4444-444444444444}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{44444444-4444-4444-4444-444444444444}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{44444444-4444-4444-4444-444444444444}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{44444444-4444-4444-4444-444444444444}.ReleaseAOT|Any CPU.ActiveCfg = ReleaseAOT|Any CPU
|
||||||
|
{44444444-4444-4444-4444-444444444444}.ReleaseAOT|Any CPU.Build.0 = ReleaseAOT|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
EndGlobal
|
EndGlobal
|
||||||
|
|||||||
@@ -22,6 +22,7 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\Engine.Core\Engine.Core.csproj" />
|
<ProjectReference Include="..\Engine.Core\Engine.Core.csproj" />
|
||||||
<ProjectReference Include="..\Engine.Graphics\Engine.Graphics.csproj" />
|
<ProjectReference Include="..\Engine.Graphics\Engine.Graphics.csproj" />
|
||||||
|
<ProjectReference Include="..\Engine.AI\Engine.AI.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Numerics;
|
using System.Numerics;
|
||||||
|
using Engine.AI;
|
||||||
using Engine.Core;
|
using Engine.Core;
|
||||||
using Engine.Core.Components;
|
using Engine.Core.Components;
|
||||||
using Engine.Graphics;
|
using Engine.Graphics;
|
||||||
@@ -13,23 +14,22 @@ class Program
|
|||||||
{
|
{
|
||||||
static void Main(string[] args)
|
static void Main(string[] args)
|
||||||
{
|
{
|
||||||
Console.WriteLine("Cortex Engine Step 4 — Starting up...");
|
Console.WriteLine("Cortex Engine Step 5 — Starting up...");
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
using var world = World.Create();
|
using var world = World.Create();
|
||||||
using var window = new Sdl3Window("Cortex Engine — Step 4", 1280, 720);
|
using var window = new Sdl3Window("Cortex Engine — Step 5", 1280, 720);
|
||||||
var timing = new Timing();
|
var timing = new Timing();
|
||||||
var input = new InputMapping();
|
var input = new InputMapping();
|
||||||
using var vulkan = new VulkanContext(window, enableValidation: false);
|
using var vulkan = new VulkanContext(window, enableValidation: false);
|
||||||
using var swapchain = new Swapchain(vulkan);
|
using var swapchain = new Swapchain(vulkan);
|
||||||
using var renderer = new MeshRenderer(vulkan, swapchain);
|
using var renderer = new MeshRenderer(vulkan, swapchain);
|
||||||
|
|
||||||
|
var ai = new AiCommandProcessor(world, LoadModel);
|
||||||
|
|
||||||
var modelPath = FindModelPath(args);
|
var modelPath = FindModelPath(args);
|
||||||
var mesh = modelPath.EndsWith(".gltf", StringComparison.OrdinalIgnoreCase)
|
var mesh = LoadModel(modelPath);
|
||||||
|| modelPath.EndsWith(".glb", StringComparison.OrdinalIgnoreCase)
|
|
||||||
? GltfLoader.Load(modelPath, new Vector3(0.7f, 0.6f, 0.5f))
|
|
||||||
: ObjLoader.Load(modelPath, new Vector3(0.7f, 0.6f, 0.5f));
|
|
||||||
|
|
||||||
var model = world.Entity("Model")
|
var model = world.Entity("Model")
|
||||||
.Set(new Transform(new Vector3(0.0f, 0.0f, 0.0f), Quaternion.Identity, new Vector3(0.5f)))
|
.Set(new Transform(new Vector3(0.0f, 0.0f, 0.0f), Quaternion.Identity, new Vector3(0.5f)))
|
||||||
@@ -45,6 +45,13 @@ class Program
|
|||||||
0.1f,
|
0.1f,
|
||||||
100.0f));
|
100.0f));
|
||||||
|
|
||||||
|
// Demo: AI agent commands.
|
||||||
|
Console.WriteLine("AI demo commands:");
|
||||||
|
Console.WriteLine(ai.Process("""{ "type": "list_entities" }""").Message);
|
||||||
|
Console.WriteLine(ai.Process("""{ "type": "spawn_model", "name": "SecondCube", "modelPath": "Content/cube.obj", "position": [0.8, 0, 0], "scale": [0.3, 0.3, 0.3] }""").Message);
|
||||||
|
Console.WriteLine(ai.Process("""{ "type": "list_entities" }""").Message);
|
||||||
|
Console.WriteLine(ai.Process("""{ "type": "set_transform", "name": "SecondCube", "position": [0.8, 0.5, 0], "rotation": [0, 0, 0, 1], "scale": [0.3, 0.3, 0.3] }""").Message);
|
||||||
|
|
||||||
var frames = 0;
|
var frames = 0;
|
||||||
var lastFpsTime = 0.0;
|
var lastFpsTime = 0.0;
|
||||||
var lastWidth = window.Width;
|
var lastWidth = window.Width;
|
||||||
@@ -90,6 +97,14 @@ class Program
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static Mesh LoadModel(string path)
|
||||||
|
{
|
||||||
|
return path.EndsWith(".gltf", StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| path.EndsWith(".glb", StringComparison.OrdinalIgnoreCase)
|
||||||
|
? GltfLoader.Load(path, new Vector3(0.7f, 0.6f, 0.5f))
|
||||||
|
: ObjLoader.Load(path, new Vector3(0.7f, 0.6f, 0.5f));
|
||||||
|
}
|
||||||
|
|
||||||
private static string FindModelPath(string[] args)
|
private static string FindModelPath(string[] args)
|
||||||
{
|
{
|
||||||
if (args.Length > 0 && File.Exists(args[0]))
|
if (args.Length > 0 && File.Exists(args[0]))
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
using System.Numerics;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
using Engine.AI.Commands;
|
||||||
|
using Engine.AI.Serialization;
|
||||||
|
using Engine.Core;
|
||||||
|
using Engine.Core.Components;
|
||||||
|
using Flecs.NET.Core;
|
||||||
|
|
||||||
|
namespace Engine.AI;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Parses and executes JSON commands from an AI agent.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class AiCommandProcessor
|
||||||
|
{
|
||||||
|
private readonly World _world;
|
||||||
|
private readonly Func<string, Mesh> _modelLoader;
|
||||||
|
private readonly JsonSerializerOptions _jsonOptions;
|
||||||
|
|
||||||
|
public AiCommandProcessor(World world, Func<string, Mesh> modelLoader)
|
||||||
|
{
|
||||||
|
_world = world;
|
||||||
|
_modelLoader = modelLoader;
|
||||||
|
_jsonOptions = new JsonSerializerOptions
|
||||||
|
{
|
||||||
|
PropertyNameCaseInsensitive = true,
|
||||||
|
Converters =
|
||||||
|
{
|
||||||
|
new JsonStringEnumConverter(),
|
||||||
|
new Vector3JsonConverter(),
|
||||||
|
new QuaternionJsonConverter()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Process a single JSON command string.
|
||||||
|
/// </summary>
|
||||||
|
public AiCommandResult Process(string json)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var command = JsonSerializer.Deserialize<AiCommand>(json, _jsonOptions);
|
||||||
|
if (command == null)
|
||||||
|
return AiCommandResult.Error("Failed to parse command.");
|
||||||
|
|
||||||
|
return command switch
|
||||||
|
{
|
||||||
|
SpawnModelCommand c => SpawnModel(c),
|
||||||
|
SetTransformCommand c => SetTransform(c),
|
||||||
|
DeleteEntityCommand c => DeleteEntity(c),
|
||||||
|
ListEntitiesCommand => ListEntities(),
|
||||||
|
_ => AiCommandResult.Error($"Unknown command type: {command.Type}")
|
||||||
|
};
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
return AiCommandResult.Error($"Command execution failed: {ex.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Process a batch of JSON commands separated by newlines.
|
||||||
|
/// </summary>
|
||||||
|
public AiCommandResult[] ProcessBatch(string jsonBatch)
|
||||||
|
{
|
||||||
|
var lines = jsonBatch.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
|
||||||
|
return lines.Select(Process).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private AiCommandResult SpawnModel(SpawnModelCommand command)
|
||||||
|
{
|
||||||
|
var mesh = _modelLoader(command.ModelPath);
|
||||||
|
var entity = _world.Entity(command.Name)
|
||||||
|
.Set(new Transform(command.Position, command.Rotation, command.Scale))
|
||||||
|
.Set(mesh);
|
||||||
|
|
||||||
|
return AiCommandResult.Ok($"Spawned entity '{command.Name}' with model '{command.ModelPath}' (id {(ulong)entity.Id}).");
|
||||||
|
}
|
||||||
|
|
||||||
|
private AiCommandResult SetTransform(SetTransformCommand command)
|
||||||
|
{
|
||||||
|
var entity = _world.Lookup(command.Name);
|
||||||
|
if ((ulong)entity.Id == 0)
|
||||||
|
return AiCommandResult.Error($"Entity '{command.Name}' not found.");
|
||||||
|
|
||||||
|
ref var transform = ref entity.Ensure<Transform>();
|
||||||
|
|
||||||
|
if (command.Position.HasValue)
|
||||||
|
transform.Position = command.Position.Value;
|
||||||
|
if (command.Rotation.HasValue)
|
||||||
|
transform.Rotation = command.Rotation.Value;
|
||||||
|
if (command.Scale.HasValue)
|
||||||
|
transform.Scale = command.Scale.Value;
|
||||||
|
|
||||||
|
return AiCommandResult.Ok($"Updated transform for entity '{command.Name}'.");
|
||||||
|
}
|
||||||
|
|
||||||
|
private AiCommandResult DeleteEntity(DeleteEntityCommand command)
|
||||||
|
{
|
||||||
|
var entity = _world.Lookup(command.Name);
|
||||||
|
if ((ulong)entity.Id == 0)
|
||||||
|
return AiCommandResult.Error($"Entity '{command.Name}' not found.");
|
||||||
|
|
||||||
|
entity.Destruct();
|
||||||
|
|
||||||
|
return AiCommandResult.Ok($"Deleted entity '{command.Name}'.");
|
||||||
|
}
|
||||||
|
|
||||||
|
private AiCommandResult ListEntities()
|
||||||
|
{
|
||||||
|
var names = new List<string>();
|
||||||
|
_world.Each((Entity e, ref Transform t) =>
|
||||||
|
{
|
||||||
|
var name = e.Name();
|
||||||
|
if (!string.IsNullOrEmpty(name))
|
||||||
|
names.Add(name);
|
||||||
|
});
|
||||||
|
|
||||||
|
var json = JsonSerializer.Serialize(names, _jsonOptions);
|
||||||
|
return AiCommandResult.Ok(json);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
namespace Engine.AI;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Result returned by the AI command processor.
|
||||||
|
/// </summary>
|
||||||
|
public sealed record AiCommandResult(bool Success, string Message)
|
||||||
|
{
|
||||||
|
public static AiCommandResult Ok(string message) => new(true, message);
|
||||||
|
public static AiCommandResult Error(string message) => new(false, message);
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace Engine.AI.Commands;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Base class for AI commands sent to the engine as JSON.
|
||||||
|
/// The discriminator is the <c>type</c> property.
|
||||||
|
/// </summary>
|
||||||
|
[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")]
|
||||||
|
[JsonDerivedType(typeof(SpawnModelCommand), "spawn_model")]
|
||||||
|
[JsonDerivedType(typeof(SetTransformCommand), "set_transform")]
|
||||||
|
[JsonDerivedType(typeof(DeleteEntityCommand), "delete_entity")]
|
||||||
|
[JsonDerivedType(typeof(ListEntitiesCommand), "list_entities")]
|
||||||
|
public abstract record AiCommand
|
||||||
|
{
|
||||||
|
public string Type => GetType().Name.Replace("Command", "").ToLowerInvariant();
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
namespace Engine.AI.Commands;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Delete an entity by name.
|
||||||
|
/// </summary>
|
||||||
|
public sealed record DeleteEntityCommand : AiCommand
|
||||||
|
{
|
||||||
|
public required string Name { get; init; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
namespace Engine.AI.Commands;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// List all named entities in the ECS world.
|
||||||
|
/// </summary>
|
||||||
|
public sealed record ListEntitiesCommand : AiCommand;
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
using System.Numerics;
|
||||||
|
|
||||||
|
namespace Engine.AI.Commands;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Update the Transform component of an existing entity by name.
|
||||||
|
/// </summary>
|
||||||
|
public sealed record SetTransformCommand : AiCommand
|
||||||
|
{
|
||||||
|
public required string Name { get; init; }
|
||||||
|
public Vector3? Position { get; init; }
|
||||||
|
public Quaternion? Rotation { get; init; }
|
||||||
|
public Vector3? Scale { get; init; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
using System.Numerics;
|
||||||
|
|
||||||
|
namespace Engine.AI.Commands;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Spawn a named entity with a Mesh loaded from a model file and a Transform.
|
||||||
|
/// </summary>
|
||||||
|
public sealed record SpawnModelCommand : AiCommand
|
||||||
|
{
|
||||||
|
public required string Name { get; init; }
|
||||||
|
public required string ModelPath { get; init; }
|
||||||
|
public Vector3 Position { get; init; } = Vector3.Zero;
|
||||||
|
public Quaternion Rotation { get; init; } = Quaternion.Identity;
|
||||||
|
public Vector3 Scale { get; init; } = Vector3.One;
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\Engine.Core\Engine.Core.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
using System.Numerics;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace Engine.AI.Serialization;
|
||||||
|
|
||||||
|
public sealed class QuaternionJsonConverter : JsonConverter<Quaternion>
|
||||||
|
{
|
||||||
|
public override Quaternion Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||||
|
{
|
||||||
|
if (reader.TokenType != JsonTokenType.StartArray)
|
||||||
|
throw new JsonException("Expected array for Quaternion.");
|
||||||
|
|
||||||
|
reader.Read();
|
||||||
|
var x = reader.GetSingle();
|
||||||
|
reader.Read();
|
||||||
|
var y = reader.GetSingle();
|
||||||
|
reader.Read();
|
||||||
|
var z = reader.GetSingle();
|
||||||
|
reader.Read();
|
||||||
|
var w = reader.GetSingle();
|
||||||
|
reader.Read();
|
||||||
|
|
||||||
|
if (reader.TokenType != JsonTokenType.EndArray)
|
||||||
|
throw new JsonException("Expected 4 elements for Quaternion.");
|
||||||
|
|
||||||
|
return new Quaternion(x, y, z, w);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Write(Utf8JsonWriter writer, Quaternion value, JsonSerializerOptions options)
|
||||||
|
{
|
||||||
|
writer.WriteStartArray();
|
||||||
|
writer.WriteNumberValue(value.X);
|
||||||
|
writer.WriteNumberValue(value.Y);
|
||||||
|
writer.WriteNumberValue(value.Z);
|
||||||
|
writer.WriteNumberValue(value.W);
|
||||||
|
writer.WriteEndArray();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
using System.Numerics;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace Engine.AI.Serialization;
|
||||||
|
|
||||||
|
public sealed class Vector3JsonConverter : JsonConverter<Vector3>
|
||||||
|
{
|
||||||
|
public override Vector3 Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||||
|
{
|
||||||
|
if (reader.TokenType != JsonTokenType.StartArray)
|
||||||
|
throw new JsonException("Expected array for Vector3.");
|
||||||
|
|
||||||
|
reader.Read();
|
||||||
|
var x = reader.GetSingle();
|
||||||
|
reader.Read();
|
||||||
|
var y = reader.GetSingle();
|
||||||
|
reader.Read();
|
||||||
|
var z = reader.GetSingle();
|
||||||
|
reader.Read();
|
||||||
|
|
||||||
|
if (reader.TokenType != JsonTokenType.EndArray)
|
||||||
|
throw new JsonException("Expected 3 elements for Vector3.");
|
||||||
|
|
||||||
|
return new Vector3(x, y, z);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Write(Utf8JsonWriter writer, Vector3 value, JsonSerializerOptions options)
|
||||||
|
{
|
||||||
|
writer.WriteStartArray();
|
||||||
|
writer.WriteNumberValue(value.X);
|
||||||
|
writer.WriteNumberValue(value.Y);
|
||||||
|
writer.WriteNumberValue(value.Z);
|
||||||
|
writer.WriteEndArray();
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user