feat: video recording via FFmpeg pipe

- vkCmdCopyImageToBuffer added to Vk.cs - VulkanRenderer.CaptureFrame: copies swapchain image to staging buffer, converts BGRA→RGBA, returns byte[] - ImGui 'Video Recording' panel with Start/Stop buttons - Start: launches FFmpeg process (rawvideo rgba → libx264 mp4) - Each 2nd frame: writes RGBA pixels to FFmpeg stdin pipe - Output: Videos/cortex_<timestamp>.mp4 at 30fps - Stop: closes stdin, waits for FFmpeg to finish - Requires ffmpeg installed (already available on system) - Captures at 1280x720 resolution
2026-06-19 08:37:37 +03:00
parent 4955a042e9
commit ace815c77d
3 changed files with 211 additions and 0 deletions
@@ -1,3 +1,4 @@
+using System.Diagnostics;
 using System.Numerics;
 using Engine.AI;
 using Engine.Core;
@@ -73,6 +74,12 @@ class Program
            var timing = new Timing();
            var totalTime = 0.0f;

+            bool isRecording = false;
+            Process? ffmpegProcess = null;
+            string? recordingPath = null;
+            int recordedFrames = 0;
+            int skipFrames = 0;
+
            while (!window.ShouldClose)
            {
                timing.Tick();
@@ -307,12 +314,105 @@ class Program

                    ImGui.End();

+                    // Video recording panel
+                    ImGui.Begin("Video Recording");
+
+                    if (!isRecording)
+                    {
+                        if (ImGui.Button("Start Recording"))
+                        {
+                            var timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
+                            recordingPath = $"Videos/cortex_{timestamp}.mp4";
+                            Directory.CreateDirectory("Videos");
+
+                            var w = window.Width;
+                            var h = window.Height;
+                            var ffmpegArgs = $"-y -f rawvideo -pixel_format rgba -video_size {w}x{h} -framerate 30 -i - -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p {recordingPath}";
+
+                            var psi = new ProcessStartInfo
+                            {
+                                FileName = "ffmpeg",
+                                Arguments = ffmpegArgs,
+                                UseShellExecute = false,
+                                RedirectStandardInput = true,
+                                RedirectStandardOutput = true,
+                                RedirectStandardError = true,
+                                CreateNoWindow = true,
+                            };
+
+                            try
+                            {
+                                ffmpegProcess = Process.Start(psi);
+                                isRecording = true;
+                                recordedFrames = 0;
+                                skipFrames = 0;
+                                Console.WriteLine($"[App] Recording started: {recordingPath}");
+                            }
+                            catch (Exception ex)
+                            {
+                                Console.WriteLine($"[App] Failed to start FFmpeg: {ex.Message}");
+                                Console.WriteLine("[App] Install FFmpeg: sudo apt install ffmpeg");
+                            }
+                        }
+                    }
+                    else
+                    {
+                        if (ImGui.Button("Stop Recording"))
+                        {
+                            isRecording = false;
+                            if (ffmpegProcess != null && !ffmpegProcess.HasExited)
+                            {
+                                ffmpegProcess.StandardInput.BaseStream.Flush();
+                                ffmpegProcess.StandardInput.BaseStream.Close();
+                                ffmpegProcess.WaitForExit(5000);
+                                if (!ffmpegProcess.HasExited)
+                                    ffmpegProcess.Kill();
+                            }
+                            ffmpegProcess?.Dispose();
+                            ffmpegProcess = null;
+                            Console.WriteLine($"[App] Recording stopped: {recordingPath} ({recordedFrames} frames)");
+                        }
+
+                        ImGui.SameLine();
+                        ImGui.Text($"Recording... {recordedFrames} frames");
+                    }
+
+                    ImGui.End();
+
                    renderer.EndImGuiFrame();
                }

+                // Set recording flag before render so renderer can capture
+                vkRenderer!.IsRecording = isRecording;
+
                renderer.RenderWorld(world);
                queue.CompletePendingScreenshots();

+                // If recording, get captured frame and write to FFmpeg
+                if (isRecording && ffmpegProcess != null && !ffmpegProcess.HasExited)
+                {
+                    skipFrames++;
+                    if (skipFrames >= 2) // Capture every 2nd frame for 30fps at ~60fps
+                    {
+                        skipFrames = 0;
+                        var frameData = vkRenderer.CapturedFrame;
+                        if (frameData != null && frameData.Length > 0)
+                        {
+                            try
+                            {
+                                ffmpegProcess.StandardInput.BaseStream.Write(frameData, 0, frameData.Length);
+                                ffmpegProcess.StandardInput.BaseStream.Flush();
+                                recordedFrames++;
+                            }
+                            catch (Exception ex)
+                            {
+                                Console.WriteLine($"[App] FFmpeg write error: {ex.Message}");
+                                isRecording = false;
+                            }
+                        }
+                    }
+                }
+
                frames++;
                if (timing.TotalTime - lastFpsTime >= 1.0)
                {
@@ -70,6 +70,7 @@ internal static unsafe class Vk
    public delegate void VkCmdPipelineBarrier2(VkCommandBuffer commandBuffer, VkDependencyInfo* pDependencyInfo);
    public delegate void VkCmdCopyBuffer(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkBuffer dstBuffer, uint regionCount, VkBufferCopy* pRegions);
    public delegate void VkCmdCopyBufferToImage(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkImage dstImage, VkImageLayout dstImageLayout, uint regionCount, VkBufferImageCopyRegion* pRegions);
+    public delegate void VkCmdCopyImageToBuffer(VkCommandBuffer commandBuffer, VkImage srcImage, VkImageLayout srcImageLayout, VkBuffer dstBuffer, uint regionCount, VkBufferImageCopyRegion* pRegions);
    public delegate void VkCmdPushConstants(VkCommandBuffer commandBuffer, VkPipelineLayout layout, VkShaderStageFlags stageFlags, uint offset, uint size, void* pValues);
    public delegate void VkCmdBindDescriptorSets(VkCommandBuffer commandBuffer, VkPipelineBindPoint pipelineBindPoint, VkPipelineLayout layout, uint firstSet, uint descriptorSetCount, VkDescriptorSet* pDescriptorSets, uint dynamicOffsetCount, uint* pDynamicOffsets);
    public delegate VkResult VkCreateDescriptorSetLayout(VkDevice device, VkDescriptorSetLayoutCreateInfo* pCreateInfo, nint pAllocator, VkDescriptorSetLayout* pSetLayout);
@@ -155,6 +156,7 @@ internal static unsafe class Vk
    public static VkCmdPipelineBarrier2 vkCmdPipelineBarrier2;
    public static VkCmdCopyBuffer vkCmdCopyBuffer;
    public static VkCmdCopyBufferToImage vkCmdCopyBufferToImage;
+    public static VkCmdCopyImageToBuffer vkCmdCopyImageToBuffer;
    public static VkCmdPushConstants vkCmdPushConstants;
    public static VkCmdBindDescriptorSets vkCmdBindDescriptorSets;
    public static VkCreateDescriptorSetLayout vkCreateDescriptorSetLayout;
@@ -247,6 +249,7 @@ internal static unsafe class Vk
        vkCmdPipelineBarrier2 = LoadDev<VkCmdPipelineBarrier2>(p, "vkCmdPipelineBarrier2");
        vkCmdCopyBuffer = LoadDev<VkCmdCopyBuffer>(p, "vkCmdCopyBuffer");
        vkCmdCopyBufferToImage = LoadDev<VkCmdCopyBufferToImage>(p, "vkCmdCopyBufferToImage");
+        vkCmdCopyImageToBuffer = LoadDev<VkCmdCopyImageToBuffer>(p, "vkCmdCopyImageToBuffer");
        vkCmdPushConstants = LoadDev<VkCmdPushConstants>(p, "vkCmdPushConstants");
        vkCmdBindDescriptorSets = LoadDev<VkCmdBindDescriptorSets>(p, "vkCmdBindDescriptorSets");
        vkCreateDescriptorSetLayout = LoadDev<VkCreateDescriptorSetLayout>(p, "vkCreateDescriptorSetLayout");
@@ -18,6 +18,11 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh

    private readonly Dictionary<ulong, (VulkanVertexBuffer vb, VulkanIndexBuffer ib, uint indexCount)> _meshCache = new();

+    private VkBuffer _screenshotBuffer;
+    private VkDeviceMemory _screenshotMemory;
+    private ulong _screenshotBufferSize;
+    private bool _screenshotInitialized;
+
    private int _frameIndex;
    private bool _disposed;
    private bool _screenshotRequested;
@@ -28,6 +33,9 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh
    public float ShadowSampleRadius { get; set; } = 0.015f;
    public float ShadowFarPlane { get; set; } = 60.0f;

+    public bool IsRecording { get; set; }
+    public byte[]? CapturedFrame { get; private set; }
+
    public bool IsScreenshotRequested => _screenshotRequested;
    public IScreenshotProvider ScreenshotProvider => this;

@@ -54,6 +62,98 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh

    internal VulkanImGui? ImGuiLayer => _imGui;

+    private void InitScreenshotBuffer()
+    {
+        if (_screenshotInitialized) return;
+        _screenshotInitialized = true;
+
+        var w = _swapchain.Extent.Width;
+        var h = _swapchain.Extent.Height;
+        _screenshotBufferSize = (ulong)(w * h * 4);
+
+        var info = new VkBufferCreateInfo
+        {
+            sType = VkStructureType.BufferCreateInfo,
+            size = _screenshotBufferSize,
+            usage = VkBufferUsageFlags.TransferDst,
+            sharingMode = VkSharingMode.Exclusive,
+        };
+
+        var buf = VkBuffer.Null;
+        Vk.vkCreateBuffer(_ctx.Device, &info, 0, &buf);
+        _screenshotBuffer = buf;
+
+        var reqs = new VkMemoryRequirements();
+        Vk.vkGetBufferMemoryRequirements(_ctx.Device, _screenshotBuffer, &reqs);
+        var memType = _ctx.FindMemoryType(reqs.memoryTypeBits, VkMemoryPropertyFlags.HostVisible | VkMemoryPropertyFlags.HostCoherent);
+
+        var allocInfo = new VkMemoryAllocateInfo
+        {
+            sType = VkStructureType.MemoryAllocateInfo,
+            allocationSize = reqs.size,
+            memoryTypeIndex = memType,
+        };
+
+        var mem = VkDeviceMemory.Null;
+        Vk.vkAllocateMemory(_ctx.Device, &allocInfo, 0, &mem);
+        _screenshotMemory = mem;
+        Vk.vkBindBufferMemory(_ctx.Device, _screenshotBuffer, _screenshotMemory, 0);
+    }
+
+    public byte[]? CaptureFrame(VkCommandBuffer cmd, uint imageIndex)
+    {
+        InitScreenshotBuffer();
+
+        var w = _swapchain.Extent.Width;
+        var h = _swapchain.Extent.Height;
+
+        // Transition swapchain image: PresentSrc → TransferSrc
+        TransitionImageLayout(cmd, _swapchain.Images[imageIndex],
+            VkImageLayout.PresentSrcKHR, VkImageLayout.TransferSrcOptimal,
+            0x8000, 0, 0x10000, 0x2000);
+
+        // Copy image to buffer
+        var region = new VkBufferImageCopyRegion
+        {
+            bufferOffset = 0,
+            bufferRowLength = w,
+            bufferImageHeight = h,
+            imageSubresource = new VkImageSubresourceLayers
+            {
+                aspectMask = VkImageAspectFlags.Color,
+                mipLevel = 0,
+                baseArrayLayer = 0,
+                layerCount = 1,
+            },
+            imageOffset = new VkOffset3D { X = 0, Y = 0, Z = 0 },
+            imageExtent = new VkExtent3D { Width = w, Height = h, Depth = 1 },
+        };
+
+        Vk.vkCmdCopyImageToBuffer(cmd, _swapchain.Images[imageIndex], VkImageLayout.TransferSrcOptimal,
+            _screenshotBuffer, 1, &region);
+
+        // Transition back: TransferSrc → PresentSrc
+        TransitionImageLayout(cmd, _swapchain.Images[imageIndex],
+            VkImageLayout.TransferSrcOptimal, VkImageLayout.PresentSrcKHR,
+            0x10000, 0x2000, 0x8000, 0);
+
+        // Map and read
+        void* pData = null;
+        Vk.vkMapMemory(_ctx.Device, _screenshotMemory, 0, _screenshotBufferSize, 0, &pData);
+
+        var result = new byte[(int)_screenshotBufferSize];
+        Marshal.Copy((nint)pData, result, 0, (int)_screenshotBufferSize);
+        Vk.vkUnmapMemory(_ctx.Device, _screenshotMemory);
+
+        // Vulkan data is BGRA, convert to RGBA
+        for (int i = 0; i < result.Length; i += 4)
+        {
+            (result[i], result[i + 2]) = (result[i + 2], result[i]);
+        }
+
+        return result;
+    }
+
    public void BeginImGuiFrame()
    {
        _imGui?.NewFrame();
@@ -409,6 +509,12 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh

        Vk.vkCmdEndRendering(cmd);

+        // Capture frame for video recording if enabled
+        if (IsRecording)
+        {
+            CapturedFrame = CaptureFrame(cmd, imageIndex);
+        }
+
        TransitionImageLayout(cmd, _swapchain.Images[imageIndex],
            VkImageLayout.ColorAttachmentOptimal, VkImageLayout.PresentSrcKHR,
            0x400, 0x100,
@@ -591,6 +697,8 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh
        }
        _meshCache.Clear();

+        if (_screenshotBuffer.Handle != 0) Vk.vkDestroyBuffer(_ctx.Device, _screenshotBuffer, 0);
+        if (_screenshotMemory.Handle != 0) Vk.vkFreeMemory(_ctx.Device, _screenshotMemory, 0);
        _shadowMap?.Dispose();
        _imGui?.Dispose();
        _frameResources?.Dispose();