From ace815c77dd579cf5cbd4fc5b2af441409523745 Mon Sep 17 00:00:00 2001 From: emil28092005 Date: Fri, 19 Jun 2026 08:37:37 +0300 Subject: [PATCH] feat: video recording via FFmpeg pipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - vkCmdCopyImageToBuffer added to Vk.cs - VulkanRenderer.CaptureFrame: copies swapchain image to staging buffer, converts BGRA→RGBA, returns byte[] - ImGui 'Video Recording' panel with Start/Stop buttons - Start: launches FFmpeg process (rawvideo rgba → libx264 mp4) - Each 2nd frame: writes RGBA pixels to FFmpeg stdin pipe - Output: Videos/cortex_.mp4 at 30fps - Stop: closes stdin, waits for FFmpeg to finish - Requires ffmpeg installed (already available on system) - Captures at 1280x720 resolution --- src/CortexEngine.App/Program.cs | 100 +++++++++++++++++ src/Engine.Graphics.Vulkan/Vk.cs | 3 + src/Engine.Graphics.Vulkan/VulkanRenderer.cs | 108 +++++++++++++++++++ 3 files changed, 211 insertions(+) diff --git a/src/CortexEngine.App/Program.cs b/src/CortexEngine.App/Program.cs index 02226a8..e31682d 100644 --- a/src/CortexEngine.App/Program.cs +++ b/src/CortexEngine.App/Program.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Numerics; using Engine.AI; using Engine.Core; @@ -73,6 +74,12 @@ class Program var timing = new Timing(); var totalTime = 0.0f; + bool isRecording = false; + Process? ffmpegProcess = null; + string? recordingPath = null; + int recordedFrames = 0; + int skipFrames = 0; + while (!window.ShouldClose) { timing.Tick(); @@ -307,12 +314,105 @@ class Program ImGui.End(); + // Video recording panel + ImGui.Begin("Video Recording"); + + if (!isRecording) + { + if (ImGui.Button("Start Recording")) + { + var timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss"); + recordingPath = $"Videos/cortex_{timestamp}.mp4"; + Directory.CreateDirectory("Videos"); + + var w = window.Width; + var h = window.Height; + var ffmpegArgs = $"-y -f rawvideo -pixel_format rgba -video_size {w}x{h} -framerate 30 -i - -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p {recordingPath}"; + + var psi = new ProcessStartInfo + { + FileName = "ffmpeg", + Arguments = ffmpegArgs, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + + try + { + ffmpegProcess = Process.Start(psi); + isRecording = true; + recordedFrames = 0; + skipFrames = 0; + Console.WriteLine($"[App] Recording started: {recordingPath}"); + } + catch (Exception ex) + { + Console.WriteLine($"[App] Failed to start FFmpeg: {ex.Message}"); + Console.WriteLine("[App] Install FFmpeg: sudo apt install ffmpeg"); + } + } + } + else + { + if (ImGui.Button("Stop Recording")) + { + isRecording = false; + if (ffmpegProcess != null && !ffmpegProcess.HasExited) + { + ffmpegProcess.StandardInput.BaseStream.Flush(); + ffmpegProcess.StandardInput.BaseStream.Close(); + ffmpegProcess.WaitForExit(5000); + if (!ffmpegProcess.HasExited) + ffmpegProcess.Kill(); + } + ffmpegProcess?.Dispose(); + ffmpegProcess = null; + Console.WriteLine($"[App] Recording stopped: {recordingPath} ({recordedFrames} frames)"); + } + + ImGui.SameLine(); + ImGui.Text($"Recording... {recordedFrames} frames"); + } + + ImGui.End(); + renderer.EndImGuiFrame(); } + // Set recording flag before render so renderer can capture + vkRenderer!.IsRecording = isRecording; + renderer.RenderWorld(world); queue.CompletePendingScreenshots(); + // If recording, get captured frame and write to FFmpeg + if (isRecording && ffmpegProcess != null && !ffmpegProcess.HasExited) + { + skipFrames++; + if (skipFrames >= 2) // Capture every 2nd frame for 30fps at ~60fps + { + skipFrames = 0; + var frameData = vkRenderer.CapturedFrame; + if (frameData != null && frameData.Length > 0) + { + try + { + ffmpegProcess.StandardInput.BaseStream.Write(frameData, 0, frameData.Length); + ffmpegProcess.StandardInput.BaseStream.Flush(); + recordedFrames++; + } + catch (Exception ex) + { + Console.WriteLine($"[App] FFmpeg write error: {ex.Message}"); + isRecording = false; + } + } + } + } + frames++; if (timing.TotalTime - lastFpsTime >= 1.0) { diff --git a/src/Engine.Graphics.Vulkan/Vk.cs b/src/Engine.Graphics.Vulkan/Vk.cs index 2882df2..02f22c5 100644 --- a/src/Engine.Graphics.Vulkan/Vk.cs +++ b/src/Engine.Graphics.Vulkan/Vk.cs @@ -70,6 +70,7 @@ internal static unsafe class Vk public delegate void VkCmdPipelineBarrier2(VkCommandBuffer commandBuffer, VkDependencyInfo* pDependencyInfo); public delegate void VkCmdCopyBuffer(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkBuffer dstBuffer, uint regionCount, VkBufferCopy* pRegions); public delegate void VkCmdCopyBufferToImage(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkImage dstImage, VkImageLayout dstImageLayout, uint regionCount, VkBufferImageCopyRegion* pRegions); + public delegate void VkCmdCopyImageToBuffer(VkCommandBuffer commandBuffer, VkImage srcImage, VkImageLayout srcImageLayout, VkBuffer dstBuffer, uint regionCount, VkBufferImageCopyRegion* pRegions); public delegate void VkCmdPushConstants(VkCommandBuffer commandBuffer, VkPipelineLayout layout, VkShaderStageFlags stageFlags, uint offset, uint size, void* pValues); public delegate void VkCmdBindDescriptorSets(VkCommandBuffer commandBuffer, VkPipelineBindPoint pipelineBindPoint, VkPipelineLayout layout, uint firstSet, uint descriptorSetCount, VkDescriptorSet* pDescriptorSets, uint dynamicOffsetCount, uint* pDynamicOffsets); public delegate VkResult VkCreateDescriptorSetLayout(VkDevice device, VkDescriptorSetLayoutCreateInfo* pCreateInfo, nint pAllocator, VkDescriptorSetLayout* pSetLayout); @@ -155,6 +156,7 @@ internal static unsafe class Vk public static VkCmdPipelineBarrier2 vkCmdPipelineBarrier2; public static VkCmdCopyBuffer vkCmdCopyBuffer; public static VkCmdCopyBufferToImage vkCmdCopyBufferToImage; + public static VkCmdCopyImageToBuffer vkCmdCopyImageToBuffer; public static VkCmdPushConstants vkCmdPushConstants; public static VkCmdBindDescriptorSets vkCmdBindDescriptorSets; public static VkCreateDescriptorSetLayout vkCreateDescriptorSetLayout; @@ -247,6 +249,7 @@ internal static unsafe class Vk vkCmdPipelineBarrier2 = LoadDev(p, "vkCmdPipelineBarrier2"); vkCmdCopyBuffer = LoadDev(p, "vkCmdCopyBuffer"); vkCmdCopyBufferToImage = LoadDev(p, "vkCmdCopyBufferToImage"); + vkCmdCopyImageToBuffer = LoadDev(p, "vkCmdCopyImageToBuffer"); vkCmdPushConstants = LoadDev(p, "vkCmdPushConstants"); vkCmdBindDescriptorSets = LoadDev(p, "vkCmdBindDescriptorSets"); vkCreateDescriptorSetLayout = LoadDev(p, "vkCreateDescriptorSetLayout"); diff --git a/src/Engine.Graphics.Vulkan/VulkanRenderer.cs b/src/Engine.Graphics.Vulkan/VulkanRenderer.cs index f464712..628c444 100644 --- a/src/Engine.Graphics.Vulkan/VulkanRenderer.cs +++ b/src/Engine.Graphics.Vulkan/VulkanRenderer.cs @@ -18,6 +18,11 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh private readonly Dictionary _meshCache = new(); + private VkBuffer _screenshotBuffer; + private VkDeviceMemory _screenshotMemory; + private ulong _screenshotBufferSize; + private bool _screenshotInitialized; + private int _frameIndex; private bool _disposed; private bool _screenshotRequested; @@ -28,6 +33,9 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh public float ShadowSampleRadius { get; set; } = 0.015f; public float ShadowFarPlane { get; set; } = 60.0f; + public bool IsRecording { get; set; } + public byte[]? CapturedFrame { get; private set; } + public bool IsScreenshotRequested => _screenshotRequested; public IScreenshotProvider ScreenshotProvider => this; @@ -54,6 +62,98 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh internal VulkanImGui? ImGuiLayer => _imGui; + private void InitScreenshotBuffer() + { + if (_screenshotInitialized) return; + _screenshotInitialized = true; + + var w = _swapchain.Extent.Width; + var h = _swapchain.Extent.Height; + _screenshotBufferSize = (ulong)(w * h * 4); + + var info = new VkBufferCreateInfo + { + sType = VkStructureType.BufferCreateInfo, + size = _screenshotBufferSize, + usage = VkBufferUsageFlags.TransferDst, + sharingMode = VkSharingMode.Exclusive, + }; + + var buf = VkBuffer.Null; + Vk.vkCreateBuffer(_ctx.Device, &info, 0, &buf); + _screenshotBuffer = buf; + + var reqs = new VkMemoryRequirements(); + Vk.vkGetBufferMemoryRequirements(_ctx.Device, _screenshotBuffer, &reqs); + var memType = _ctx.FindMemoryType(reqs.memoryTypeBits, VkMemoryPropertyFlags.HostVisible | VkMemoryPropertyFlags.HostCoherent); + + var allocInfo = new VkMemoryAllocateInfo + { + sType = VkStructureType.MemoryAllocateInfo, + allocationSize = reqs.size, + memoryTypeIndex = memType, + }; + + var mem = VkDeviceMemory.Null; + Vk.vkAllocateMemory(_ctx.Device, &allocInfo, 0, &mem); + _screenshotMemory = mem; + Vk.vkBindBufferMemory(_ctx.Device, _screenshotBuffer, _screenshotMemory, 0); + } + + public byte[]? CaptureFrame(VkCommandBuffer cmd, uint imageIndex) + { + InitScreenshotBuffer(); + + var w = _swapchain.Extent.Width; + var h = _swapchain.Extent.Height; + + // Transition swapchain image: PresentSrc → TransferSrc + TransitionImageLayout(cmd, _swapchain.Images[imageIndex], + VkImageLayout.PresentSrcKHR, VkImageLayout.TransferSrcOptimal, + 0x8000, 0, 0x10000, 0x2000); + + // Copy image to buffer + var region = new VkBufferImageCopyRegion + { + bufferOffset = 0, + bufferRowLength = w, + bufferImageHeight = h, + imageSubresource = new VkImageSubresourceLayers + { + aspectMask = VkImageAspectFlags.Color, + mipLevel = 0, + baseArrayLayer = 0, + layerCount = 1, + }, + imageOffset = new VkOffset3D { X = 0, Y = 0, Z = 0 }, + imageExtent = new VkExtent3D { Width = w, Height = h, Depth = 1 }, + }; + + Vk.vkCmdCopyImageToBuffer(cmd, _swapchain.Images[imageIndex], VkImageLayout.TransferSrcOptimal, + _screenshotBuffer, 1, ®ion); + + // Transition back: TransferSrc → PresentSrc + TransitionImageLayout(cmd, _swapchain.Images[imageIndex], + VkImageLayout.TransferSrcOptimal, VkImageLayout.PresentSrcKHR, + 0x10000, 0x2000, 0x8000, 0); + + // Map and read + void* pData = null; + Vk.vkMapMemory(_ctx.Device, _screenshotMemory, 0, _screenshotBufferSize, 0, &pData); + + var result = new byte[(int)_screenshotBufferSize]; + Marshal.Copy((nint)pData, result, 0, (int)_screenshotBufferSize); + Vk.vkUnmapMemory(_ctx.Device, _screenshotMemory); + + // Vulkan data is BGRA, convert to RGBA + for (int i = 0; i < result.Length; i += 4) + { + (result[i], result[i + 2]) = (result[i + 2], result[i]); + } + + return result; + } + public void BeginImGuiFrame() { _imGui?.NewFrame(); @@ -409,6 +509,12 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh Vk.vkCmdEndRendering(cmd); + // Capture frame for video recording if enabled + if (IsRecording) + { + CapturedFrame = CaptureFrame(cmd, imageIndex); + } + TransitionImageLayout(cmd, _swapchain.Images[imageIndex], VkImageLayout.ColorAttachmentOptimal, VkImageLayout.PresentSrcKHR, 0x400, 0x100, @@ -591,6 +697,8 @@ public sealed unsafe class VulkanRenderer : IRenderer, Engine.Graphics.IScreensh } _meshCache.Clear(); + if (_screenshotBuffer.Handle != 0) Vk.vkDestroyBuffer(_ctx.Device, _screenshotBuffer, 0); + if (_screenshotMemory.Handle != 0) Vk.vkFreeMemory(_ctx.Device, _screenshotMemory, 0); _shadowMap?.Dispose(); _imGui?.Dispose(); _frameResources?.Dispose();