feat: Step 5 AI agent command bridge
- Add new Engine.AI project referenced by the App. - Define JSON command model: spawn_model, set_transform, delete_entity, list_entities. - Implement AiCommandProcessor with System.Text.Json and custom Vector3/Quaternion converters. - Wire processor into Program.cs; demo commands spawn and move a second cube. - AI commands are executed against the live Flecs world and visible in the next frame.
This commit is contained in:
@@ -22,6 +22,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Engine.Core\Engine.Core.csproj" />
|
||||
<ProjectReference Include="..\Engine.Graphics\Engine.Graphics.csproj" />
|
||||
<ProjectReference Include="..\Engine.AI\Engine.AI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Numerics;
|
||||
using Engine.AI;
|
||||
using Engine.Core;
|
||||
using Engine.Core.Components;
|
||||
using Engine.Graphics;
|
||||
@@ -13,23 +14,22 @@ class Program
|
||||
{
|
||||
static void Main(string[] args)
|
||||
{
|
||||
Console.WriteLine("Cortex Engine Step 4 — Starting up...");
|
||||
Console.WriteLine("Cortex Engine Step 5 — Starting up...");
|
||||
|
||||
try
|
||||
{
|
||||
using var world = World.Create();
|
||||
using var window = new Sdl3Window("Cortex Engine — Step 4", 1280, 720);
|
||||
using var window = new Sdl3Window("Cortex Engine — Step 5", 1280, 720);
|
||||
var timing = new Timing();
|
||||
var input = new InputMapping();
|
||||
using var vulkan = new VulkanContext(window, enableValidation: false);
|
||||
using var swapchain = new Swapchain(vulkan);
|
||||
using var renderer = new MeshRenderer(vulkan, swapchain);
|
||||
|
||||
var ai = new AiCommandProcessor(world, LoadModel);
|
||||
|
||||
var modelPath = FindModelPath(args);
|
||||
var mesh = modelPath.EndsWith(".gltf", StringComparison.OrdinalIgnoreCase)
|
||||
|| modelPath.EndsWith(".glb", StringComparison.OrdinalIgnoreCase)
|
||||
? GltfLoader.Load(modelPath, new Vector3(0.7f, 0.6f, 0.5f))
|
||||
: ObjLoader.Load(modelPath, new Vector3(0.7f, 0.6f, 0.5f));
|
||||
var mesh = LoadModel(modelPath);
|
||||
|
||||
var model = world.Entity("Model")
|
||||
.Set(new Transform(new Vector3(0.0f, 0.0f, 0.0f), Quaternion.Identity, new Vector3(0.5f)))
|
||||
@@ -45,6 +45,13 @@ class Program
|
||||
0.1f,
|
||||
100.0f));
|
||||
|
||||
// Demo: AI agent commands.
|
||||
Console.WriteLine("AI demo commands:");
|
||||
Console.WriteLine(ai.Process("""{ "type": "list_entities" }""").Message);
|
||||
Console.WriteLine(ai.Process("""{ "type": "spawn_model", "name": "SecondCube", "modelPath": "Content/cube.obj", "position": [0.8, 0, 0], "scale": [0.3, 0.3, 0.3] }""").Message);
|
||||
Console.WriteLine(ai.Process("""{ "type": "list_entities" }""").Message);
|
||||
Console.WriteLine(ai.Process("""{ "type": "set_transform", "name": "SecondCube", "position": [0.8, 0.5, 0], "rotation": [0, 0, 0, 1], "scale": [0.3, 0.3, 0.3] }""").Message);
|
||||
|
||||
var frames = 0;
|
||||
var lastFpsTime = 0.0;
|
||||
var lastWidth = window.Width;
|
||||
@@ -90,6 +97,14 @@ class Program
|
||||
}
|
||||
}
|
||||
|
||||
private static Mesh LoadModel(string path)
|
||||
{
|
||||
return path.EndsWith(".gltf", StringComparison.OrdinalIgnoreCase)
|
||||
|| path.EndsWith(".glb", StringComparison.OrdinalIgnoreCase)
|
||||
? GltfLoader.Load(path, new Vector3(0.7f, 0.6f, 0.5f))
|
||||
: ObjLoader.Load(path, new Vector3(0.7f, 0.6f, 0.5f));
|
||||
}
|
||||
|
||||
private static string FindModelPath(string[] args)
|
||||
{
|
||||
if (args.Length > 0 && File.Exists(args[0]))
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
using System.Numerics;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Engine.AI.Commands;
|
||||
using Engine.AI.Serialization;
|
||||
using Engine.Core;
|
||||
using Engine.Core.Components;
|
||||
using Flecs.NET.Core;
|
||||
|
||||
namespace Engine.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Parses and executes JSON commands from an AI agent.
|
||||
/// </summary>
|
||||
public sealed class AiCommandProcessor
|
||||
{
|
||||
private readonly World _world;
|
||||
private readonly Func<string, Mesh> _modelLoader;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
|
||||
public AiCommandProcessor(World world, Func<string, Mesh> modelLoader)
|
||||
{
|
||||
_world = world;
|
||||
_modelLoader = modelLoader;
|
||||
_jsonOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
Converters =
|
||||
{
|
||||
new JsonStringEnumConverter(),
|
||||
new Vector3JsonConverter(),
|
||||
new QuaternionJsonConverter()
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process a single JSON command string.
|
||||
/// </summary>
|
||||
public AiCommandResult Process(string json)
|
||||
{
|
||||
try
|
||||
{
|
||||
var command = JsonSerializer.Deserialize<AiCommand>(json, _jsonOptions);
|
||||
if (command == null)
|
||||
return AiCommandResult.Error("Failed to parse command.");
|
||||
|
||||
return command switch
|
||||
{
|
||||
SpawnModelCommand c => SpawnModel(c),
|
||||
SetTransformCommand c => SetTransform(c),
|
||||
DeleteEntityCommand c => DeleteEntity(c),
|
||||
ListEntitiesCommand => ListEntities(),
|
||||
_ => AiCommandResult.Error($"Unknown command type: {command.Type}")
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return AiCommandResult.Error($"Command execution failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process a batch of JSON commands separated by newlines.
|
||||
/// </summary>
|
||||
public AiCommandResult[] ProcessBatch(string jsonBatch)
|
||||
{
|
||||
var lines = jsonBatch.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
return lines.Select(Process).ToArray();
|
||||
}
|
||||
|
||||
private AiCommandResult SpawnModel(SpawnModelCommand command)
|
||||
{
|
||||
var mesh = _modelLoader(command.ModelPath);
|
||||
var entity = _world.Entity(command.Name)
|
||||
.Set(new Transform(command.Position, command.Rotation, command.Scale))
|
||||
.Set(mesh);
|
||||
|
||||
return AiCommandResult.Ok($"Spawned entity '{command.Name}' with model '{command.ModelPath}' (id {(ulong)entity.Id}).");
|
||||
}
|
||||
|
||||
private AiCommandResult SetTransform(SetTransformCommand command)
|
||||
{
|
||||
var entity = _world.Lookup(command.Name);
|
||||
if ((ulong)entity.Id == 0)
|
||||
return AiCommandResult.Error($"Entity '{command.Name}' not found.");
|
||||
|
||||
ref var transform = ref entity.Ensure<Transform>();
|
||||
|
||||
if (command.Position.HasValue)
|
||||
transform.Position = command.Position.Value;
|
||||
if (command.Rotation.HasValue)
|
||||
transform.Rotation = command.Rotation.Value;
|
||||
if (command.Scale.HasValue)
|
||||
transform.Scale = command.Scale.Value;
|
||||
|
||||
return AiCommandResult.Ok($"Updated transform for entity '{command.Name}'.");
|
||||
}
|
||||
|
||||
private AiCommandResult DeleteEntity(DeleteEntityCommand command)
|
||||
{
|
||||
var entity = _world.Lookup(command.Name);
|
||||
if ((ulong)entity.Id == 0)
|
||||
return AiCommandResult.Error($"Entity '{command.Name}' not found.");
|
||||
|
||||
entity.Destruct();
|
||||
|
||||
return AiCommandResult.Ok($"Deleted entity '{command.Name}'.");
|
||||
}
|
||||
|
||||
private AiCommandResult ListEntities()
|
||||
{
|
||||
var names = new List<string>();
|
||||
_world.Each((Entity e, ref Transform t) =>
|
||||
{
|
||||
var name = e.Name();
|
||||
if (!string.IsNullOrEmpty(name))
|
||||
names.Add(name);
|
||||
});
|
||||
|
||||
var json = JsonSerializer.Serialize(names, _jsonOptions);
|
||||
return AiCommandResult.Ok(json);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace Engine.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Result returned by the AI command processor.
|
||||
/// </summary>
|
||||
public sealed record AiCommandResult(bool Success, string Message)
|
||||
{
|
||||
public static AiCommandResult Ok(string message) => new(true, message);
|
||||
public static AiCommandResult Error(string message) => new(false, message);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Engine.AI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for AI commands sent to the engine as JSON.
|
||||
/// The discriminator is the <c>type</c> property.
|
||||
/// </summary>
|
||||
[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")]
|
||||
[JsonDerivedType(typeof(SpawnModelCommand), "spawn_model")]
|
||||
[JsonDerivedType(typeof(SetTransformCommand), "set_transform")]
|
||||
[JsonDerivedType(typeof(DeleteEntityCommand), "delete_entity")]
|
||||
[JsonDerivedType(typeof(ListEntitiesCommand), "list_entities")]
|
||||
public abstract record AiCommand
|
||||
{
|
||||
public string Type => GetType().Name.Replace("Command", "").ToLowerInvariant();
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace Engine.AI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Delete an entity by name.
|
||||
/// </summary>
|
||||
public sealed record DeleteEntityCommand : AiCommand
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Engine.AI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// List all named entities in the ECS world.
|
||||
/// </summary>
|
||||
public sealed record ListEntitiesCommand : AiCommand;
|
||||
@@ -0,0 +1,14 @@
|
||||
using System.Numerics;
|
||||
|
||||
namespace Engine.AI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Update the Transform component of an existing entity by name.
|
||||
/// </summary>
|
||||
public sealed record SetTransformCommand : AiCommand
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public Vector3? Position { get; init; }
|
||||
public Quaternion? Rotation { get; init; }
|
||||
public Vector3? Scale { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Numerics;
|
||||
|
||||
namespace Engine.AI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Spawn a named entity with a Mesh loaded from a model file and a Transform.
|
||||
/// </summary>
|
||||
public sealed record SpawnModelCommand : AiCommand
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public required string ModelPath { get; init; }
|
||||
public Vector3 Position { get; init; } = Vector3.Zero;
|
||||
public Quaternion Rotation { get; init; } = Quaternion.Identity;
|
||||
public Vector3 Scale { get; init; } = Vector3.One;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Engine.Core\Engine.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,39 @@
|
||||
using System.Numerics;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Engine.AI.Serialization;
|
||||
|
||||
public sealed class QuaternionJsonConverter : JsonConverter<Quaternion>
|
||||
{
|
||||
public override Quaternion Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||
{
|
||||
if (reader.TokenType != JsonTokenType.StartArray)
|
||||
throw new JsonException("Expected array for Quaternion.");
|
||||
|
||||
reader.Read();
|
||||
var x = reader.GetSingle();
|
||||
reader.Read();
|
||||
var y = reader.GetSingle();
|
||||
reader.Read();
|
||||
var z = reader.GetSingle();
|
||||
reader.Read();
|
||||
var w = reader.GetSingle();
|
||||
reader.Read();
|
||||
|
||||
if (reader.TokenType != JsonTokenType.EndArray)
|
||||
throw new JsonException("Expected 4 elements for Quaternion.");
|
||||
|
||||
return new Quaternion(x, y, z, w);
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, Quaternion value, JsonSerializerOptions options)
|
||||
{
|
||||
writer.WriteStartArray();
|
||||
writer.WriteNumberValue(value.X);
|
||||
writer.WriteNumberValue(value.Y);
|
||||
writer.WriteNumberValue(value.Z);
|
||||
writer.WriteNumberValue(value.W);
|
||||
writer.WriteEndArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using System.Numerics;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Engine.AI.Serialization;
|
||||
|
||||
public sealed class Vector3JsonConverter : JsonConverter<Vector3>
|
||||
{
|
||||
public override Vector3 Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||
{
|
||||
if (reader.TokenType != JsonTokenType.StartArray)
|
||||
throw new JsonException("Expected array for Vector3.");
|
||||
|
||||
reader.Read();
|
||||
var x = reader.GetSingle();
|
||||
reader.Read();
|
||||
var y = reader.GetSingle();
|
||||
reader.Read();
|
||||
var z = reader.GetSingle();
|
||||
reader.Read();
|
||||
|
||||
if (reader.TokenType != JsonTokenType.EndArray)
|
||||
throw new JsonException("Expected 3 elements for Vector3.");
|
||||
|
||||
return new Vector3(x, y, z);
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, Vector3 value, JsonSerializerOptions options)
|
||||
{
|
||||
writer.WriteStartArray();
|
||||
writer.WriteNumberValue(value.X);
|
||||
writer.WriteNumberValue(value.Y);
|
||||
writer.WriteNumberValue(value.Z);
|
||||
writer.WriteEndArray();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user