From dab0f520ca2fe50e64f8f07ad5210f396f6ebb76 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 27 May 2026 17:22:30 +0100 Subject: [PATCH] Signal MF GPU camera producer fences --- .../camera_capture/mimir_mf_gpu_capture.cpp | 55 ++++++++++++++++++- src/Mimir.BufferSmoke/Program.cs | 6 +- .../MimirFensalirBridgeContracts.cs | 3 + .../MimirFensalirFieldLowering.cs | 8 ++- ...mirMediaFoundationGpuVideoCaptureDriver.cs | 13 +++-- .../MimirVideoFrameDescriptor.cs | 1 + state/evidence.jsonl | 1 + state/map.yaml | 9 ++- 8 files changed, 83 insertions(+), 13 deletions(-) diff --git a/native/camera_capture/mimir_mf_gpu_capture.cpp b/native/camera_capture/mimir_mf_gpu_capture.cpp index b5002c1..4dbff0d 100644 --- a/native/camera_capture/mimir_mf_gpu_capture.cpp +++ b/native/camera_capture/mimir_mf_gpu_capture.cpp @@ -27,7 +27,10 @@ struct MimirMfGpuCapture ComPtr context; ComPtr deviceManager; ComPtr sharedTexture; + ComPtr producerFence; + ComPtr fenceContext; HANDLE sharedHandle = nullptr; + HANDLE producerFenceHandle = nullptr; int width = 0; int height = 0; DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN; @@ -339,6 +342,38 @@ bool createSharedTexture(MimirMfGpuCapture& capture, const GUID& outputSubtype) return true; } +bool createProducerFence(MimirMfGpuCapture& capture) +{ + ComPtr device5; + if (FAILED(capture.device.As(&device5)) || + FAILED(capture.context.As(&capture.fenceContext))) + { + debugLog("D3D11 fence interfaces unavailable; shared texture will be unsynchronized"); + return true; + } + + const HRESULT fenceHr = device5->CreateFence( + 0, + D3D11_FENCE_FLAG_SHARED, + IID_PPV_ARGS(&capture.producerFence)); + if (FAILED(fenceHr)) + { + debugLog("CreateFence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast(fenceHr)); + capture.fenceContext.Reset(); + return true; + } + + const HRESULT handleHr = capture.producerFence->CreateSharedHandle(nullptr, GENERIC_ALL, nullptr, &capture.producerFenceHandle); + if (FAILED(handleHr)) + { + debugLog("CreateSharedHandle fence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast(handleHr)); + capture.producerFence.Reset(); + capture.fenceContext.Reset(); + } + + return true; +} + } // namespace extern "C" @@ -430,7 +465,8 @@ __declspec(dllexport) MimirMfGpuCapture* mimir_mf_gpu_create( outputSubtype == GUID_NULL || !setInputType(capture->reader.Get(), width, height, inputSubtype, minFps) || !setOutputType(capture->reader.Get(), outputSubtype) || - !createSharedTexture(*capture, outputSubtype)) + !createSharedTexture(*capture, outputSubtype) || + !createProducerFence(*capture)) { debugLog("create failed input=%s output=%s", inputSubtypeText ? inputSubtypeText : "", outputSubtypeText ? outputSubtypeText : ""); delete capture; @@ -448,7 +484,8 @@ __declspec(dllexport) int mimir_mf_gpu_read( char* format, int formatCapacity, std::int64_t* timestampNs, - std::uint64_t* sequence) + std::uint64_t* sequence, + HANDLE* producerFenceHandle) { if (!capture || !capture->reader || !capture->sharedTexture || !sharedHandle) { @@ -483,7 +520,13 @@ __declspec(dllexport) int mimir_mf_gpu_read( return 0; } + const auto nextSequence = capture->sequence + 1; capture->context->CopySubresourceRegion(capture->sharedTexture.Get(), 0, 0, 0, 0, decodedTexture.Get(), subresource, nullptr); + if (capture->producerFence && capture->fenceContext) + { + capture->fenceContext->Signal(capture->producerFence.Get(), nextSequence); + } + capture->context->Flush(); *sharedHandle = capture->sharedHandle; if (width) *width = capture->width; @@ -498,7 +541,9 @@ __declspec(dllexport) int mimir_mf_gpu_read( std::snprintf(format, static_cast(formatCapacity), "%s", text); } if (timestampNs) *timestampNs = timestamp * 100; - if (sequence) *sequence = ++capture->sequence; + capture->sequence = nextSequence; + if (sequence) *sequence = capture->sequence; + if (producerFenceHandle) *producerFenceHandle = capture->producerFenceHandle; return 1; } @@ -513,6 +558,10 @@ __declspec(dllexport) void mimir_mf_gpu_destroy(MimirMfGpuCapture* capture) { CloseHandle(capture->sharedHandle); } + if (capture->producerFenceHandle) + { + CloseHandle(capture->producerFenceHandle); + } if (capture->source) { capture->source->Shutdown(); diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 4b4368e..8f4cf37 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -975,14 +975,18 @@ static int RunMfGpuDriverSmoke(string[] args) var evidence = new MimirFensalirFieldLowering().BuildCameraObservationFrame([buffer]); var validation = AquariumFieldEvidenceValidator.Validate(evidence); Console.WriteLine( - $"mf-gpu-driver-smoke source={sample.SourceId} {frame?.Width}x{frame?.Height} format={frame?.PixelFormat} liveBytes={sample.ByteLength} handle=0x{frame?.NativeHandle:x} kind={frame?.NativeHandleKind} resources={evidence.Resources.Count} errors={validation.HasErrors}"); + $"mf-gpu-driver-smoke source={sample.SourceId} {frame?.Width}x{frame?.Height} format={frame?.PixelFormat} liveBytes={sample.ByteLength} handle=0x{frame?.NativeHandle:x} kind={frame?.NativeHandleKind} fence=0x{frame?.ProducerFenceHandle:x} fenceValue={frame?.ProducerFenceValue} resources={evidence.Resources.Count} errors={validation.HasErrors}"); return frame != null && frame.Width == width && frame.Height == height && frame.NativeHandle != 0 && frame.NativeHandleKind == "shared-d3d11-texture" && + frame.ProducerFenceHandle != 0 && + frame.ProducerFenceValue > 0 && sample.ByteLength == 0 && evidence.Resources.Count == 1 && + evidence.Resources[0].ProducerFenceHandle != IntPtr.Zero && + evidence.Resources[0].ProducerFenceValue == frame.ProducerFenceValue && evidence.Resources[0].Residency == AquariumFieldResourceResidency.SharedGpu && !validation.HasErrors ? 0 diff --git a/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs b/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs index e194980..758830d 100644 --- a/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs +++ b/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs @@ -49,6 +49,7 @@ public readonly record struct MimirBridgePayloadView( string NativeHandleKind, int ByteLength, string ResourceKey = "", + ulong ProducerFenceHandle = 0, ulong ProducerFenceValue = 0) { public bool HasResource => NativeHandle != 0 || !string.IsNullOrWhiteSpace(ResourceKey); @@ -363,12 +364,14 @@ public static class MimirFensalirBridgeMapper var nativeHandle = sample.VideoFrame?.NativeHandle ?? sample.AudioBlock?.NativeHandle ?? sample.PayloadHandle; var nativeHandleKind = sample.VideoFrame?.NativeHandleKind ?? sample.AudioBlock?.NativeHandleKind ?? ""; var resourceKey = sample.VideoFrame?.ResourceKey ?? ""; + var producerFenceHandle = sample.VideoFrame?.ProducerFenceHandle ?? 0; var producerFenceValue = sample.VideoFrame?.ProducerFenceValue ?? 0; return new MimirBridgePayloadView( nativeHandle, nativeHandleKind, sample.ByteLength, string.IsNullOrWhiteSpace(resourceKey) ? ResourceKeyForPayload(nativeHandle, nativeHandleKind) : resourceKey, + producerFenceHandle, producerFenceValue); } diff --git a/src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs b/src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs index a914a0e..ce85097 100644 --- a/src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs +++ b/src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs @@ -193,7 +193,9 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt ValidUntilNs: window.EdgeNs, Version: window.SequenceId, NativeHandle: new IntPtr(unchecked((long)window.Payload.NativeHandle)), - NativeHandleKind: window.Payload.NativeHandleKind); + NativeHandleKind: window.Payload.NativeHandleKind, + ProducerFenceHandle: new IntPtr(unchecked((long)window.Payload.ProducerFenceHandle)), + ProducerFenceValue: window.Payload.ProducerFenceValue); } private static AquariumFieldResourceDeclaration ResourceForObservation(MimirObservation observation) @@ -223,7 +225,9 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt ValidUntilNs: observation.CanonicalTimeEstimateNs + Math.Max(0, observation.UncertaintyNs), Version: observation.Provenance.SequenceId, NativeHandle: new IntPtr(unchecked((long)observation.Payload.NativeHandle)), - NativeHandleKind: observation.Payload.NativeHandleKind); + NativeHandleKind: observation.Payload.NativeHandleKind, + ProducerFenceHandle: new IntPtr(unchecked((long)observation.Payload.ProducerFenceHandle)), + ProducerFenceValue: observation.Payload.ProducerFenceValue); } private static AquariumFieldDomain DomainForObservation(MimirObservation observation) => diff --git a/src/Mimir.Runtime/Synchronization/MimirMediaFoundationGpuVideoCaptureDriver.cs b/src/Mimir.Runtime/Synchronization/MimirMediaFoundationGpuVideoCaptureDriver.cs index ac28d64..f08a638 100644 --- a/src/Mimir.Runtime/Synchronization/MimirMediaFoundationGpuVideoCaptureDriver.cs +++ b/src/Mimir.Runtime/Synchronization/MimirMediaFoundationGpuVideoCaptureDriver.cs @@ -58,7 +58,8 @@ public sealed unsafe class MimirMediaFoundationGpuVideoCaptureDriver : IMimirVid out var height, formatBytes, out var timestampNs, - out var sequence) || + out var sequence, + out var producerFenceHandle) || sharedHandle == 0) { frame = default!; @@ -76,6 +77,7 @@ public sealed unsafe class MimirMediaFoundationGpuVideoCaptureDriver : IMimirVid NativeHandle: unchecked((ulong)sharedHandle), NativeHandleKind: "shared-d3d11-texture", ResourceKey: MimirFensalirTextureLeaseClient.ResourceKeyForSource(sourceId), + ProducerFenceHandle: unchecked((ulong)producerFenceHandle), ProducerFenceValue: sequence); data = default; return true; @@ -126,14 +128,14 @@ public sealed unsafe class MimirMediaFoundationGpuVideoCaptureDriver : IMimirVid { private readonly nint library; private readonly delegate* unmanaged[Stdcall] create; - private readonly delegate* unmanaged[Stdcall] read; + private readonly delegate* unmanaged[Stdcall] read; private readonly delegate* unmanaged[Stdcall] destroy; private Native(nint library) { this.library = library; create = (delegate* unmanaged[Stdcall])NativeLibrary.GetExport(library, "mimir_mf_gpu_create"); - read = (delegate* unmanaged[Stdcall])NativeLibrary.GetExport(library, "mimir_mf_gpu_read"); + read = (delegate* unmanaged[Stdcall])NativeLibrary.GetExport(library, "mimir_mf_gpu_read"); destroy = (delegate* unmanaged[Stdcall])NativeLibrary.GetExport(library, "mimir_mf_gpu_destroy"); } @@ -159,11 +161,12 @@ public sealed unsafe class MimirMediaFoundationGpuVideoCaptureDriver : IMimirVid out int height, Span format, out long timestampNs, - out ulong sequence) + out ulong sequence, + out nint producerFenceHandle) { fixed (byte* formatPointer = format) { - return read(capture, out sharedHandle, out width, out height, formatPointer, format.Length, out timestampNs, out sequence) != 0; + return read(capture, out sharedHandle, out width, out height, formatPointer, format.Length, out timestampNs, out sequence, out producerFenceHandle) != 0; } } diff --git a/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs b/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs index 24d13e6..18a9223 100644 --- a/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs +++ b/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs @@ -24,5 +24,6 @@ public sealed record MimirVideoFrameDescriptor( ulong NativeHandle = 0, string NativeHandleKind = "", string ResourceKey = "", + ulong ProducerFenceHandle = 0, ulong ProducerFenceValue = 0, int UnavoidableCopyCount = 0); diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 07a99df..fee22a1 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -181,3 +181,4 @@ {"ts": "2026-05-27T14:25:26+01:00", "type": "verification", "status": "ok", "note": "The new in-process KS/UVC driver pulled a real LeapUVC 640x240 YUY2 frame through native/camera_capture, MimirKsVideoCaptureDriver, and MimirVideoCaptureDriverSource, then uploaded 307200 bytes into the Fensalir texture lease for mimir:resource:camera:leap-stereo-ir:texture2d with UnavoidableCopyCount=1. Managed live payload bytes were cleared after upload."} {"ts": "2026-05-27T14:31:40+01:00", "type": "verification", "status": "ok", "note": "Direct camera driver smokes pulled and uploaded real camera pixels into Fensalir texture leases: LeapUVC 640x240 YUY2 uploaded 307200 bytes, PS3 Eye 0 and 1 each uploaded 320x240 Bayer8 / 76800 bytes, Kiyo Pro uploaded 1920x1080 YUY2 / 4147200 bytes, and regular Kiyo uploaded 640x480 YUY2 / 614400 bytes. All successful paths cleared live managed payload bytes after upload and reported UnavoidableCopyCount=1. Regular Kiyo 1280x720 YUY2 did not open; compressed MJPG/H264 still require a device/GPU decode producer."} {"ts": "2026-05-27T16:35:32+01:00", "type": "verification", "status": "ok", "note": "Media Foundation GPU decode is now a Mimir-owned producer path. native/camera_capture/mimir_mf_gpu_capture.dll runs SourceReader with a D3D11 device manager, selects Kiyo Pro compressed modes, decodes MJPG/H264 on the GPU, copies the decoded GPU surface into a shared BGRA D3D11 texture, and MimirMediaFoundationGpuVideoCaptureDriver publishes a shared-d3d11-texture handle with empty live CPU payload bytes. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed and lowered to valid Fensalir camera resources. Direct shared NV12 texture creation failed with E_INVALIDARG, so planar NV12 remains future plane-aware interop or GPU bridge-copy work."} +{"ts": "2026-05-27T17:19:41+01:00", "type": "verification", "status": "ok", "note": "The MF GPU direct texture path now carries D3D11/D3D12 synchronization instead of relying on flush timing. native/camera_capture/mimir_mf_gpu_capture.dll creates a shared D3D11 fence, signals it after copying the decoded GPU surface into the shared BGRA texture, and MimirMediaFoundationGpuVideoCaptureDriver carries the fence handle/value through MimirVideoFrameDescriptor into Fensalir field resource declarations. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed with zero live CPU payload bytes, nonzero shared texture handles, nonzero producer fence handles, and valid Fensalir camera resources."} diff --git a/state/map.yaml b/state/map.yaml index dc1b754..52ffe26 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -341,8 +341,13 @@ accepted_design: `native/camera_capture/mimir_mf_gpu_capture.dll` to run Media Foundation SourceReader with a D3D11 device manager, decode MJPG/H264 on the GPU, copy the decoded GPU surface into a shared BGRA D3D11 texture, and publish a - `shared-d3d11-texture` handle with zero live CPU payload bytes. Kiyo Pro - 1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed. Direct shared NV12 + `shared-d3d11-texture` handle with zero live CPU payload bytes. The MF GPU + producer now also creates a shared D3D11 fence, signals it after the + texture copy, and carries the fence handle/value through + `MimirVideoFrameDescriptor`, `MimirBridgePayloadView`, and the Fensalir + field resource declaration so D3D12 can wait before sampling. Kiyo Pro + 1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed with nonzero shared + texture and producer fence handles. Direct shared NV12 texture creation failed in D3D11, so NV12 remains future plane-aware interop or GPU bridge-copy work. `Mimir.BufferSmoke --fensalir-camera-observation-smoke`