feat: AI screenshot capture for visual analysis

- Add ScreenshotCapture class to Engine.Graphics using Vulkan image readback.
- Integrate screenshot capture into MeshRenderer; request triggers readback on next frame.
- Add SixLabors.ImageSharp 3.1.11 for PNG encoding (patched for CVE-2025-54575).
- Add capture_screenshot AI command and MCP tool.
- Wire screenshot request into Program.cs with demo command.
- Expose swapchain surface format and image accessor for readback.
- Add Screenshots/ to .gitignore.
- Update CORTEX_ENGINE_ARCHITECTURE.md with MCP, Silk.NET.Vulkan, ImageSharp, and screenshot capture.
This commit is contained in:
emil28092005
2026-06-16 20:09:57 +03:00
parent a9c783d204
commit 9312810f0c
11 changed files with 405 additions and 37 deletions
+4 -3
View File
@@ -29,12 +29,12 @@ class Program
using var swapchain = new Swapchain(vulkan);
using var renderer = new MeshRenderer(vulkan, swapchain);
var processor = new AiCommandProcessor(world, LoadModel);
var queue = new AiCommandQueue(processor);
var (modelPath, mcpPort) = ParseArgs(args);
var mesh = LoadModel(modelPath);
var processor = new AiCommandProcessor(world, LoadModel, path => renderer.RequestScreenshot(path));
var queue = new AiCommandQueue(processor);
var model = world.Entity("Model")
.Set(new Transform(new Vector3(0.0f, 0.0f, 0.0f), Quaternion.Identity, new Vector3(0.5f)))
.Set(mesh);
@@ -55,6 +55,7 @@ class Program
Console.WriteLine(processor.Process("""{ "type": "spawn_model", "name": "SecondCube", "modelPath": "Content/cube.obj", "position": [0.8, 0, 0], "scale": [0.3, 0.3, 0.3] }""").Message);
Console.WriteLine(processor.Process("""{ "type": "list_entities" }""").Message);
Console.WriteLine(processor.Process("""{ "type": "set_transform", "name": "SecondCube", "position": [0.8, 0.5, 0], "rotation": [0, 0, 0, 1], "scale": [0.3, 0.3, 0.3] }""").Message);
Console.WriteLine(processor.Process("""{ "type": "capture_screenshot", "outputPath": "Screenshots/demo.png" }""").Message);
#if !RELEASE_AOT
// Start the MCP server in the background so AI agents can connect via HTTP.
+11 -1
View File
@@ -16,13 +16,15 @@ public sealed class AiCommandProcessor
{
private readonly World _world;
private readonly Func<string, Mesh> _modelLoader;
private readonly Action<string> _requestScreenshot;
public JsonSerializerOptions JsonOptions { get; }
public AiCommandProcessor(World world, Func<string, Mesh> modelLoader)
public AiCommandProcessor(World world, Func<string, Mesh> modelLoader, Action<string> requestScreenshot)
{
_world = world;
_modelLoader = modelLoader;
_requestScreenshot = requestScreenshot;
JsonOptions = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
@@ -52,6 +54,7 @@ public sealed class AiCommandProcessor
SetTransformCommand c => SetTransform(c),
DeleteEntityCommand c => DeleteEntity(c),
ListEntitiesCommand => ListEntities(),
CaptureScreenshotCommand c => CaptureScreenshot(c),
_ => AiCommandResult.Error($"Unknown command type: {command.Type}")
};
}
@@ -122,4 +125,11 @@ public sealed class AiCommandProcessor
var json = JsonSerializer.Serialize(names, JsonOptions);
return AiCommandResult.Ok(json);
}
private AiCommandResult CaptureScreenshot(CaptureScreenshotCommand command)
{
var path = command.OutputPath ?? $"screenshot_{DateTime.UtcNow:yyyyMMdd_HHmmss_fff}.png";
_requestScreenshot(path);
return AiCommandResult.Ok($"Screenshot requested: {path}");
}
}
+1
View File
@@ -11,6 +11,7 @@ namespace Engine.AI.Commands;
[JsonDerivedType(typeof(SetTransformCommand), "set_transform")]
[JsonDerivedType(typeof(DeleteEntityCommand), "delete_entity")]
[JsonDerivedType(typeof(ListEntitiesCommand), "list_entities")]
[JsonDerivedType(typeof(CaptureScreenshotCommand), "capture_screenshot")]
public abstract record AiCommand
{
public string Type => GetType().Name.Replace("Command", "").ToLowerInvariant();
@@ -0,0 +1,12 @@
namespace Engine.AI.Commands;
/// <summary>
/// Request a screenshot of the current rendered frame for AI visual analysis.
/// </summary>
public sealed record CaptureScreenshotCommand : AiCommand
{
/// <summary>
/// Output file path. If omitted, the engine chooses a default path.
/// </summary>
public string? OutputPath { get; init; }
}
+7
View File
@@ -70,6 +70,13 @@ public sealed class EngineMcpTools
return EnqueueAndReturnMessage(cmd);
}
[McpServerTool, Description("Capture a screenshot of the current rendered frame and save it to disk.")]
public Task<string> CaptureScreenshot([Description("Optional output file path (default: screenshot_<timestamp>.png)")] string? outputPath = null)
{
var cmd = new CaptureScreenshotCommand { OutputPath = outputPath };
return EnqueueAndReturnMessage(cmd);
}
private async Task<string> EnqueueAndReturnMessage(AiCommand command)
{
var result = await _queue.EnqueueAsync(command).ConfigureAwait(false);
@@ -23,6 +23,7 @@
<PackageReference Include="Flecs.NET.Debug" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Debug'" />
<PackageReference Include="Flecs.NET.Release" Version="4.0.4-build.546" Condition="'$(Configuration)' == 'Release' OR '$(Configuration)' == 'ReleaseAOT'" />
<PackageReference Include="SharpGLTF.Core" Version="1.0.6" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
</ItemGroup>
<ItemGroup>
+20
View File
@@ -18,6 +18,7 @@ public sealed unsafe class MeshRenderer : IDisposable
private readonly VulkanContext _context;
private readonly Swapchain _swapchain;
private readonly VulkanPipeline _pipeline;
private readonly ScreenshotCapture _screenshot;
private readonly Dictionary<Entity, MeshBuffers> _buffers = new();
private CommandPool _commandPool;
private CommandBuffer[] _commandBuffers = null!;
@@ -48,6 +49,7 @@ public sealed unsafe class MeshRenderer : IDisposable
{
_context = context;
_swapchain = swapchain;
_screenshot = new ScreenshotCapture(context, swapchain);
_pipeline = new VulkanPipeline(context, swapchain);
CreateCommandPool();
@@ -118,6 +120,10 @@ public sealed unsafe class MeshRenderer : IDisposable
}
}
public void RequestScreenshot(string outputPath) => _screenshot.Request(outputPath);
public bool IsScreenshotRequested => _screenshot.IsRequested;
public void RenderWorld(World world)
{
var frame = _currentFrame % 2;
@@ -198,6 +204,10 @@ public sealed unsafe class MeshRenderer : IDisposable
});
_context.Vk.CmdEndRenderPass(cmd);
var swapchainImage = _swapchain.GetImage(imageIndex);
_screenshot.RecordReadback(cmd, swapchainImage, _swapchain.Extent.Width, _swapchain.Extent.Height, _swapchain.SurfaceFormat);
_context.Vk.EndCommandBuffer(cmd);
var waitSemaphore = _imageAvailableSemaphores[frame];
@@ -229,6 +239,14 @@ public sealed unsafe class MeshRenderer : IDisposable
};
_context.KhrSwapchain!.QueuePresent(_context.PresentQueue, &presentInfo);
// If a screenshot was requested, wait for the GPU to finish the readback and save the file.
if (_screenshot.IsRequested)
{
_context.Vk.WaitForFences(_context.Device, 1, &fence, true, ulong.MaxValue);
_screenshot.Save(_swapchain.Extent.Width, _swapchain.Extent.Height, _swapchain.SurfaceFormat);
}
_currentFrame++;
}
@@ -308,6 +326,8 @@ public sealed unsafe class MeshRenderer : IDisposable
{
_context.Vk.DeviceWaitIdle(_context.Device);
_screenshot.Dispose();
foreach (var buffers in _buffers.Values)
buffers.Dispose();
_buffers.Clear();
+287
View File
@@ -0,0 +1,287 @@
using System;
using System.IO;
using Silk.NET.Core;
using Silk.NET.Vulkan;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
namespace Engine.Graphics;
/// <summary>
/// Captures the current swapchain image to a PNG file on disk.
/// Used by AI agents to visually inspect the running engine.
/// </summary>
public sealed unsafe class ScreenshotCapture : IDisposable
{
private readonly VulkanContext _context;
private readonly Swapchain _swapchain;
private Silk.NET.Vulkan.Buffer _stagingBuffer;
private DeviceMemory _stagingMemory;
private ulong _stagingSize;
private bool _requested;
private string _outputPath = string.Empty;
private bool _ready;
public ScreenshotCapture(VulkanContext context, Swapchain swapchain)
{
_context = context;
_swapchain = swapchain;
}
/// <summary>
/// Request a screenshot to be captured on the next frame.
/// </summary>
public void Request(string outputPath)
{
_outputPath = outputPath;
_requested = true;
_ready = false;
}
/// <summary>
/// True if a screenshot has been requested but not yet saved.
/// </summary>
public bool IsRequested => _requested;
/// <summary>
/// Records the image readback commands into the given command buffer.
/// Must be called after the render pass has ended and before the image is presented.
/// </summary>
public void RecordReadback(CommandBuffer cmd, Silk.NET.Vulkan.Image sourceImage, uint width, uint height, Format format)
{
if (!_requested)
return;
var pixelSize = GetPixelSize(format);
var rowPitch = width * pixelSize;
var imageSize = rowPitch * height;
EnsureStagingBuffer(imageSize);
// Transition from present layout to transfer source.
var barrier = new ImageMemoryBarrier
{
SType = StructureType.ImageMemoryBarrier,
OldLayout = ImageLayout.PresentSrcKhr,
NewLayout = ImageLayout.TransferSrcOptimal,
SrcAccessMask = AccessFlags.None,
DstAccessMask = AccessFlags.TransferReadBit,
Image = sourceImage,
SubresourceRange = new ImageSubresourceRange(ImageAspectFlags.ColorBit, 0, 1, 0, 1)
};
_context.Vk.CmdPipelineBarrier(cmd, PipelineStageFlags.TransferBit, PipelineStageFlags.TransferBit, 0, 0, null, 0, null, 1, &barrier);
var copyRegion = new BufferImageCopy
{
BufferOffset = 0,
BufferRowLength = 0,
BufferImageHeight = 0,
ImageSubresource = new ImageSubresourceLayers(ImageAspectFlags.ColorBit, 0, 0, 1),
ImageOffset = new Offset3D(0, 0, 0),
ImageExtent = new Extent3D(width, height, 1)
};
_context.Vk.CmdCopyImageToBuffer(cmd, sourceImage, ImageLayout.TransferSrcOptimal, _stagingBuffer, 1, &copyRegion);
// Transition back to present layout.
barrier.OldLayout = ImageLayout.TransferSrcOptimal;
barrier.NewLayout = ImageLayout.PresentSrcKhr;
barrier.SrcAccessMask = AccessFlags.TransferReadBit;
barrier.DstAccessMask = AccessFlags.None;
_context.Vk.CmdPipelineBarrier(cmd, PipelineStageFlags.TransferBit, PipelineStageFlags.TransferBit, 0, 0, null, 0, null, 1, &barrier);
_ready = true;
}
/// <summary>
/// Save the captured pixels to disk. Must be called after the command buffer containing the readback has finished.
/// </summary>
public void Save(uint width, uint height, Format format)
{
if (!_ready)
return;
var pixelSize = GetPixelSize(format);
var rowPitch = width * pixelSize;
var imageSize = rowPitch * height;
void* mappedData;
var result = _context.Vk.MapMemory(_context.Device, _stagingMemory, 0, imageSize, MemoryMapFlags.None, &mappedData);
if (result != Result.Success)
throw new InvalidOperationException($"vkMapMemory failed: {result}");
try
{
var directory = Path.GetDirectoryName(_outputPath);
if (!string.IsNullOrEmpty(directory))
Directory.CreateDirectory(directory);
SavePixels(mappedData, width, height, rowPitch, format);
}
finally
{
_context.Vk.UnmapMemory(_context.Device, _stagingMemory);
}
Console.WriteLine($"Screenshot saved: {_outputPath}");
_requested = false;
_ready = false;
}
private void SavePixels(void* mappedData, uint width, uint height, uint rowPitch, Format format)
{
if (format == Format.B8G8R8A8Unorm || format == Format.B8G8R8A8Srgb)
{
SaveBgra(mappedData, width, height, rowPitch);
return;
}
if (format == Format.R8G8B8A8Unorm || format == Format.R8G8B8A8Srgb)
{
SaveRgba(mappedData, width, height, rowPitch);
return;
}
throw new NotSupportedException($"Screenshot format {format} is not supported.");
}
private void SaveBgra(void* mappedData, uint width, uint height, uint rowPitch)
{
using var image = new SixLabors.ImageSharp.Image<Rgba32>((int)width, (int)height);
var src = (byte*)mappedData;
for (var y = 0; y < height; y++)
{
var rowStart = src + y * rowPitch;
for (var x = 0; x < width; x++)
{
var b = rowStart[x * 4 + 0];
var g = rowStart[x * 4 + 1];
var r = rowStart[x * 4 + 2];
var a = rowStart[x * 4 + 3];
image[x, y] = new Rgba32(r, g, b, a);
}
}
image.SaveAsPng(_outputPath);
}
private void SaveRgba(void* mappedData, uint width, uint height, uint rowPitch)
{
using var image = new SixLabors.ImageSharp.Image<Rgba32>((int)width, (int)height);
var src = (byte*)mappedData;
for (var y = 0; y < height; y++)
{
var rowStart = src + y * rowPitch;
for (var x = 0; x < width; x++)
{
var r = rowStart[x * 4 + 0];
var g = rowStart[x * 4 + 1];
var b = rowStart[x * 4 + 2];
var a = rowStart[x * 4 + 3];
image[x, y] = new Rgba32(r, g, b, a);
}
}
image.SaveAsPng(_outputPath);
}
private void EnsureStagingBuffer(ulong size)
{
if (_stagingSize >= size)
return;
if (_stagingBuffer.Handle != 0)
{
_context.Vk.DestroyBuffer(_context.Device, _stagingBuffer, null);
_context.Vk.FreeMemory(_context.Device, _stagingMemory, null);
}
_stagingSize = size;
_stagingBuffer = CreateBuffer(size, BufferUsageFlags.TransferDstBit);
var requirements = GetMemoryRequirements(_stagingBuffer);
_stagingMemory = AllocateMemory(requirements, MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit);
var result = _context.Vk.BindBufferMemory(_context.Device, _stagingBuffer, _stagingMemory, 0);
if (result != Result.Success)
throw new InvalidOperationException($"vkBindBufferMemory failed: {result}");
}
private Silk.NET.Vulkan.Buffer CreateBuffer(ulong size, BufferUsageFlags usage)
{
var createInfo = new BufferCreateInfo
{
SType = StructureType.BufferCreateInfo,
Size = size,
Usage = usage,
SharingMode = SharingMode.Exclusive
};
Silk.NET.Vulkan.Buffer buffer;
var result = _context.Vk.CreateBuffer(_context.Device, &createInfo, null, &buffer);
if (result != Result.Success)
throw new InvalidOperationException($"vkCreateBuffer failed: {result}");
return buffer;
}
private MemoryRequirements GetMemoryRequirements(Silk.NET.Vulkan.Buffer buffer)
{
MemoryRequirements requirements;
_context.Vk.GetBufferMemoryRequirements(_context.Device, buffer, &requirements);
return requirements;
}
private DeviceMemory AllocateMemory(MemoryRequirements requirements, MemoryPropertyFlags properties)
{
var memoryTypeIndex = FindMemoryType(requirements.MemoryTypeBits, properties);
var allocInfo = new MemoryAllocateInfo
{
SType = StructureType.MemoryAllocateInfo,
AllocationSize = requirements.Size,
MemoryTypeIndex = memoryTypeIndex
};
DeviceMemory memory;
var result = _context.Vk.AllocateMemory(_context.Device, &allocInfo, null, &memory);
if (result != Result.Success)
throw new InvalidOperationException($"vkAllocateMemory failed: {result}");
return memory;
}
private uint FindMemoryType(uint typeFilter, MemoryPropertyFlags properties)
{
PhysicalDeviceMemoryProperties memoryProperties;
_context.Vk.GetPhysicalDeviceMemoryProperties(_context.PhysicalDevice, &memoryProperties);
for (var i = 0; i < memoryProperties.MemoryTypeCount; i++)
{
if ((typeFilter & (1u << i)) != 0 &&
(memoryProperties.MemoryTypes[i].PropertyFlags & properties) == properties)
{
return (uint)i;
}
}
throw new InvalidOperationException("Failed to find suitable memory type.");
}
private static uint GetPixelSize(Format format)
{
return format switch
{
Format.B8G8R8A8Unorm or Format.B8G8R8A8Srgb or Format.R8G8B8A8Unorm or Format.R8G8B8A8Srgb => 4,
_ => throw new NotSupportedException($"Format {format} is not supported for screenshots.")
};
}
public void Dispose()
{
if (_stagingBuffer.Handle != 0)
{
_context.Vk.DeviceWaitIdle(_context.Device);
_context.Vk.DestroyBuffer(_context.Device, _stagingBuffer, null);
_context.Vk.FreeMemory(_context.Device, _stagingMemory, null);
}
}
}
+3
View File
@@ -30,6 +30,9 @@ public sealed unsafe class Swapchain : IDisposable
public Extent2D Extent => _extent;
public SwapchainKHR Handle => _swapchain;
public uint ImageCount => (uint)_images.Length;
public Format SurfaceFormat => _surfaceFormat.Format;
public Image GetImage(uint index) => _images[index];
public Swapchain(VulkanContext context)
{