Files
Cortex_Engine/src/Engine.AI/AiCommandProcessor.cs
T
emil28092005 9312810f0c feat: AI screenshot capture for visual analysis
- Add ScreenshotCapture class to Engine.Graphics using Vulkan image readback.
- Integrate screenshot capture into MeshRenderer; request triggers readback on next frame.
- Add SixLabors.ImageSharp 3.1.11 for PNG encoding (patched for CVE-2025-54575).
- Add capture_screenshot AI command and MCP tool.
- Wire screenshot request into Program.cs with demo command.
- Expose swapchain surface format and image accessor for readback.
- Add Screenshots/ to .gitignore.
- Update CORTEX_ENGINE_ARCHITECTURE.md with MCP, Silk.NET.Vulkan, ImageSharp, and screenshot capture.
2026-06-16 20:09:57 +03:00

136 lines
4.4 KiB
C#

using System.Numerics;
using System.Text.Json;
using System.Text.Json.Serialization;
using Engine.AI.Commands;
using Engine.AI.Serialization;
using Engine.Core;
using Engine.Core.Components;
using Flecs.NET.Core;
namespace Engine.AI;
/// <summary>
/// Parses and executes JSON commands from an AI agent.
/// </summary>
public sealed class AiCommandProcessor
{
private readonly World _world;
private readonly Func<string, Mesh> _modelLoader;
private readonly Action<string> _requestScreenshot;
public JsonSerializerOptions JsonOptions { get; }
public AiCommandProcessor(World world, Func<string, Mesh> modelLoader, Action<string> requestScreenshot)
{
_world = world;
_modelLoader = modelLoader;
_requestScreenshot = requestScreenshot;
JsonOptions = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
Converters =
{
new JsonStringEnumConverter(),
new Vector3JsonConverter(),
new QuaternionJsonConverter()
}
};
}
/// <summary>
/// Process a single JSON command string.
/// </summary>
public AiCommandResult Process(string json)
{
try
{
var command = JsonSerializer.Deserialize<AiCommand>(json, JsonOptions);
if (command == null)
return AiCommandResult.Error("Failed to parse command.");
return command switch
{
SpawnModelCommand c => SpawnModel(c),
SetTransformCommand c => SetTransform(c),
DeleteEntityCommand c => DeleteEntity(c),
ListEntitiesCommand => ListEntities(),
CaptureScreenshotCommand c => CaptureScreenshot(c),
_ => AiCommandResult.Error($"Unknown command type: {command.Type}")
};
}
catch (Exception ex)
{
return AiCommandResult.Error($"Command execution failed: {ex.Message}");
}
}
/// <summary>
/// Process a batch of JSON commands separated by newlines.
/// </summary>
public AiCommandResult[] ProcessBatch(string jsonBatch)
{
var lines = jsonBatch.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
return lines.Select(Process).ToArray();
}
private AiCommandResult SpawnModel(SpawnModelCommand command)
{
var mesh = _modelLoader(command.ModelPath);
var entity = _world.Entity(command.Name)
.Set(new Transform(command.Position, command.Rotation, command.Scale))
.Set(mesh);
return AiCommandResult.Ok($"Spawned entity '{command.Name}' with model '{command.ModelPath}' (id {(ulong)entity.Id}).");
}
private AiCommandResult SetTransform(SetTransformCommand command)
{
var entity = _world.Lookup(command.Name);
if ((ulong)entity.Id == 0)
return AiCommandResult.Error($"Entity '{command.Name}' not found.");
ref var transform = ref entity.Ensure<Transform>();
if (command.Position.HasValue)
transform.Position = command.Position.Value;
if (command.Rotation.HasValue)
transform.Rotation = command.Rotation.Value;
if (command.Scale.HasValue)
transform.Scale = command.Scale.Value;
return AiCommandResult.Ok($"Updated transform for entity '{command.Name}'.");
}
private AiCommandResult DeleteEntity(DeleteEntityCommand command)
{
var entity = _world.Lookup(command.Name);
if ((ulong)entity.Id == 0)
return AiCommandResult.Error($"Entity '{command.Name}' not found.");
entity.Destruct();
return AiCommandResult.Ok($"Deleted entity '{command.Name}'.");
}
private AiCommandResult ListEntities()
{
var names = new List<string>();
_world.Each((Entity e, ref Transform t) =>
{
var name = e.Name();
if (!string.IsNullOrEmpty(name))
names.Add(name);
});
var json = JsonSerializer.Serialize(names, JsonOptions);
return AiCommandResult.Ok(json);
}
private AiCommandResult CaptureScreenshot(CaptureScreenshotCommand command)
{
var path = command.OutputPath ?? $"screenshot_{DateTime.UtcNow:yyyyMMdd_HHmmss_fff}.png";
_requestScreenshot(path);
return AiCommandResult.Ok($"Screenshot requested: {path}");
}
}