Signal MF GPU camera producer fences

native/camera_capture/mimir_mf_gpu_capture.cpp

@@ -27,7 +27,10 @@ struct MimirMfGpuCapture
     ComPtr<ID3D11DeviceContext> context;
     ComPtr<IMFDXGIDeviceManager> deviceManager;
     ComPtr<ID3D11Texture2D> sharedTexture;
+    ComPtr<ID3D11Fence> producerFence;
+    ComPtr<ID3D11DeviceContext4> fenceContext;
     HANDLE sharedHandle = nullptr;
+    HANDLE producerFenceHandle = nullptr;
     int width = 0;
     int height = 0;
     DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
@@ -339,6 +342,38 @@ bool createSharedTexture(MimirMfGpuCapture& capture, const GUID& outputSubtype)
     return true;
 }
 
+bool createProducerFence(MimirMfGpuCapture& capture)
+{
+    ComPtr<ID3D11Device5> device5;
+    if (FAILED(capture.device.As(&device5)) ||
+        FAILED(capture.context.As(&capture.fenceContext)))
+    {
+        debugLog("D3D11 fence interfaces unavailable; shared texture will be unsynchronized");
+        return true;
+    }
+
+    const HRESULT fenceHr = device5->CreateFence(
+        0,
+        D3D11_FENCE_FLAG_SHARED,
+        IID_PPV_ARGS(&capture.producerFence));
+    if (FAILED(fenceHr))
+    {
+        debugLog("CreateFence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(fenceHr));
+        capture.fenceContext.Reset();
+        return true;
+    }
+
+    const HRESULT handleHr = capture.producerFence->CreateSharedHandle(nullptr, GENERIC_ALL, nullptr, &capture.producerFenceHandle);
+    if (FAILED(handleHr))
+    {
+        debugLog("CreateSharedHandle fence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(handleHr));
+        capture.producerFence.Reset();
+        capture.fenceContext.Reset();
+    }
+
+    return true;
+}
+
 } // namespace
 
 extern "C"
@@ -430,7 +465,8 @@ __declspec(dllexport) MimirMfGpuCapture* mimir_mf_gpu_create(
         outputSubtype == GUID_NULL ||
         !setInputType(capture->reader.Get(), width, height, inputSubtype, minFps) ||
         !setOutputType(capture->reader.Get(), outputSubtype) ||
-        !createSharedTexture(*capture, outputSubtype))
+        !createSharedTexture(*capture, outputSubtype) ||
+        !createProducerFence(*capture))
     {
         debugLog("create failed input=%s output=%s", inputSubtypeText ? inputSubtypeText : "", outputSubtypeText ? outputSubtypeText : "");
         delete capture;
@@ -448,7 +484,8 @@ __declspec(dllexport) int mimir_mf_gpu_read(
     char* format,
     int formatCapacity,
     std::int64_t* timestampNs,
-    std::uint64_t* sequence)
+    std::uint64_t* sequence,
+    HANDLE* producerFenceHandle)
 {
     if (!capture || !capture->reader || !capture->sharedTexture || !sharedHandle)
     {
@@ -483,7 +520,13 @@ __declspec(dllexport) int mimir_mf_gpu_read(
         return 0;
     }
 
+    const auto nextSequence = capture->sequence + 1;
     capture->context->CopySubresourceRegion(capture->sharedTexture.Get(), 0, 0, 0, 0, decodedTexture.Get(), subresource, nullptr);
+    if (capture->producerFence && capture->fenceContext)
+    {
+        capture->fenceContext->Signal(capture->producerFence.Get(), nextSequence);
+    }
+
     capture->context->Flush();
     *sharedHandle = capture->sharedHandle;
     if (width) *width = capture->width;
@@ -498,7 +541,9 @@ __declspec(dllexport) int mimir_mf_gpu_read(
         std::snprintf(format, static_cast<std::size_t>(formatCapacity), "%s", text);
     }
     if (timestampNs) *timestampNs = timestamp * 100;
-    if (sequence) *sequence = ++capture->sequence;
+    capture->sequence = nextSequence;
+    if (sequence) *sequence = capture->sequence;
+    if (producerFenceHandle) *producerFenceHandle = capture->producerFenceHandle;
     return 1;
 }
 
@@ -513,6 +558,10 @@ __declspec(dllexport) void mimir_mf_gpu_destroy(MimirMfGpuCapture* capture)
     {
         CloseHandle(capture->sharedHandle);
     }
+    if (capture->producerFenceHandle)
+    {
+        CloseHandle(capture->producerFenceHandle);
+    }
     if (capture->source)
     {
         capture->source->Shutdown();

src/Mimir.BufferSmoke/Program.cs

@@ -975,14 +975,18 @@ static int RunMfGpuDriverSmoke(string[] args)
         var evidence = new MimirFensalirFieldLowering().BuildCameraObservationFrame([buffer]);
         var validation = AquariumFieldEvidenceValidator.Validate(evidence);
         Console.WriteLine(
-            $"mf-gpu-driver-smoke source={sample.SourceId} {frame?.Width}x{frame?.Height} format={frame?.PixelFormat} liveBytes={sample.ByteLength} handle=0x{frame?.NativeHandle:x} kind={frame?.NativeHandleKind} resources={evidence.Resources.Count} errors={validation.HasErrors}");
+            $"mf-gpu-driver-smoke source={sample.SourceId} {frame?.Width}x{frame?.Height} format={frame?.PixelFormat} liveBytes={sample.ByteLength} handle=0x{frame?.NativeHandle:x} kind={frame?.NativeHandleKind} fence=0x{frame?.ProducerFenceHandle:x} fenceValue={frame?.ProducerFenceValue} resources={evidence.Resources.Count} errors={validation.HasErrors}");
         return frame != null &&
             frame.Width == width &&
             frame.Height == height &&
             frame.NativeHandle != 0 &&
             frame.NativeHandleKind == "shared-d3d11-texture" &&
+            frame.ProducerFenceHandle != 0 &&
+            frame.ProducerFenceValue > 0 &&
             sample.ByteLength == 0 &&
             evidence.Resources.Count == 1 &&
+            evidence.Resources[0].ProducerFenceHandle != IntPtr.Zero &&
+            evidence.Resources[0].ProducerFenceValue == frame.ProducerFenceValue &&
             evidence.Resources[0].Residency == AquariumFieldResourceResidency.SharedGpu &&
             !validation.HasErrors
             ? 0

src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs

@@ -49,6 +49,7 @@ public readonly record struct MimirBridgePayloadView(
     string NativeHandleKind,
     int ByteLength,
     string ResourceKey = "",
+    ulong ProducerFenceHandle = 0,
     ulong ProducerFenceValue = 0)
 {
     public bool HasResource => NativeHandle != 0 || !string.IsNullOrWhiteSpace(ResourceKey);
@@ -363,12 +364,14 @@ public static class MimirFensalirBridgeMapper
         var nativeHandle = sample.VideoFrame?.NativeHandle ?? sample.AudioBlock?.NativeHandle ?? sample.PayloadHandle;
         var nativeHandleKind = sample.VideoFrame?.NativeHandleKind ?? sample.AudioBlock?.NativeHandleKind ?? "";
         var resourceKey = sample.VideoFrame?.ResourceKey ?? "";
+        var producerFenceHandle = sample.VideoFrame?.ProducerFenceHandle ?? 0;
         var producerFenceValue = sample.VideoFrame?.ProducerFenceValue ?? 0;
         return new MimirBridgePayloadView(
             nativeHandle,
             nativeHandleKind,
             sample.ByteLength,
             string.IsNullOrWhiteSpace(resourceKey) ? ResourceKeyForPayload(nativeHandle, nativeHandleKind) : resourceKey,
+            producerFenceHandle,
             producerFenceValue);
     }
 

src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs

@@ -193,7 +193,9 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
             ValidUntilNs: window.EdgeNs,
             Version: window.SequenceId,
             NativeHandle: new IntPtr(unchecked((long)window.Payload.NativeHandle)),
-            NativeHandleKind: window.Payload.NativeHandleKind);
+            NativeHandleKind: window.Payload.NativeHandleKind,
+            ProducerFenceHandle: new IntPtr(unchecked((long)window.Payload.ProducerFenceHandle)),
+            ProducerFenceValue: window.Payload.ProducerFenceValue);
     }
 
     private static AquariumFieldResourceDeclaration ResourceForObservation(MimirObservation observation)
@@ -223,7 +225,9 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
             ValidUntilNs: observation.CanonicalTimeEstimateNs + Math.Max(0, observation.UncertaintyNs),
             Version: observation.Provenance.SequenceId,
             NativeHandle: new IntPtr(unchecked((long)observation.Payload.NativeHandle)),
-            NativeHandleKind: observation.Payload.NativeHandleKind);
+            NativeHandleKind: observation.Payload.NativeHandleKind,
+            ProducerFenceHandle: new IntPtr(unchecked((long)observation.Payload.ProducerFenceHandle)),
+            ProducerFenceValue: observation.Payload.ProducerFenceValue);
     }
 
     private static AquariumFieldDomain DomainForObservation(MimirObservation observation) =>

src/Mimir.Runtime/Synchronization/MimirMediaFoundationGpuVideoCaptureDriver.cs

@@ -58,7 +58,8 @@ public sealed unsafe class MimirMediaFoundationGpuVideoCaptureDriver : IMimirVid
                 out var height,
                 formatBytes,
                 out var timestampNs,
-                out var sequence) ||
+                out var sequence,
+                out var producerFenceHandle) ||
             sharedHandle == 0)
         {
             frame = default!;
@@ -76,6 +77,7 @@ public sealed unsafe class MimirMediaFoundationGpuVideoCaptureDriver : IMimirVid
             NativeHandle: unchecked((ulong)sharedHandle),
             NativeHandleKind: "shared-d3d11-texture",
             ResourceKey: MimirFensalirTextureLeaseClient.ResourceKeyForSource(sourceId),
+            ProducerFenceHandle: unchecked((ulong)producerFenceHandle),
             ProducerFenceValue: sequence);
         data = default;
         return true;
@@ -126,14 +128,14 @@ public sealed unsafe class MimirMediaFoundationGpuVideoCaptureDriver : IMimirVid
     {
         private readonly nint library;
         private readonly delegate* unmanaged[Stdcall]<byte*, int, int, byte*, byte*, double, nint> create;
-        private readonly delegate* unmanaged[Stdcall]<nint, out nint, out int, out int, byte*, int, out long, out ulong, int> read;
+        private readonly delegate* unmanaged[Stdcall]<nint, out nint, out int, out int, byte*, int, out long, out ulong, out nint, int> read;
         private readonly delegate* unmanaged[Stdcall]<nint, void> destroy;
 
         private Native(nint library)
         {
             this.library = library;
             create = (delegate* unmanaged[Stdcall]<byte*, int, int, byte*, byte*, double, nint>)NativeLibrary.GetExport(library, "mimir_mf_gpu_create");
-            read = (delegate* unmanaged[Stdcall]<nint, out nint, out int, out int, byte*, int, out long, out ulong, int>)NativeLibrary.GetExport(library, "mimir_mf_gpu_read");
+            read = (delegate* unmanaged[Stdcall]<nint, out nint, out int, out int, byte*, int, out long, out ulong, out nint, int>)NativeLibrary.GetExport(library, "mimir_mf_gpu_read");
             destroy = (delegate* unmanaged[Stdcall]<nint, void>)NativeLibrary.GetExport(library, "mimir_mf_gpu_destroy");
         }
 
@@ -159,11 +161,12 @@ public sealed unsafe class MimirMediaFoundationGpuVideoCaptureDriver : IMimirVid
             out int height,
             Span<byte> format,
             out long timestampNs,
-            out ulong sequence)
+            out ulong sequence,
+            out nint producerFenceHandle)
         {
             fixed (byte* formatPointer = format)
             {
-                return read(capture, out sharedHandle, out width, out height, formatPointer, format.Length, out timestampNs, out sequence) != 0;
+                return read(capture, out sharedHandle, out width, out height, formatPointer, format.Length, out timestampNs, out sequence, out producerFenceHandle) != 0;
             }
         }
 

src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs

@@ -24,5 +24,6 @@ public sealed record MimirVideoFrameDescriptor(
     ulong NativeHandle = 0,
     string NativeHandleKind = "",
     string ResourceKey = "",
+    ulong ProducerFenceHandle = 0,
     ulong ProducerFenceValue = 0,
     int UnavoidableCopyCount = 0);

state/evidence.jsonl

@@ -181,3 +181,4 @@
 {"ts": "2026-05-27T14:25:26+01:00", "type": "verification", "status": "ok", "note": "The new in-process KS/UVC driver pulled a real LeapUVC 640x240 YUY2 frame through native/camera_capture, MimirKsVideoCaptureDriver, and MimirVideoCaptureDriverSource, then uploaded 307200 bytes into the Fensalir texture lease for mimir:resource:camera:leap-stereo-ir:texture2d with UnavoidableCopyCount=1. Managed live payload bytes were cleared after upload."}
 {"ts": "2026-05-27T14:31:40+01:00", "type": "verification", "status": "ok", "note": "Direct camera driver smokes pulled and uploaded real camera pixels into Fensalir texture leases: LeapUVC 640x240 YUY2 uploaded 307200 bytes, PS3 Eye 0 and 1 each uploaded 320x240 Bayer8 / 76800 bytes, Kiyo Pro uploaded 1920x1080 YUY2 / 4147200 bytes, and regular Kiyo uploaded 640x480 YUY2 / 614400 bytes. All successful paths cleared live managed payload bytes after upload and reported UnavoidableCopyCount=1. Regular Kiyo 1280x720 YUY2 did not open; compressed MJPG/H264 still require a device/GPU decode producer."}
 {"ts": "2026-05-27T16:35:32+01:00", "type": "verification", "status": "ok", "note": "Media Foundation GPU decode is now a Mimir-owned producer path. native/camera_capture/mimir_mf_gpu_capture.dll runs SourceReader with a D3D11 device manager, selects Kiyo Pro compressed modes, decodes MJPG/H264 on the GPU, copies the decoded GPU surface into a shared BGRA D3D11 texture, and MimirMediaFoundationGpuVideoCaptureDriver publishes a shared-d3d11-texture handle with empty live CPU payload bytes. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed and lowered to valid Fensalir camera resources. Direct shared NV12 texture creation failed with E_INVALIDARG, so planar NV12 remains future plane-aware interop or GPU bridge-copy work."}
+{"ts": "2026-05-27T17:19:41+01:00", "type": "verification", "status": "ok", "note": "The MF GPU direct texture path now carries D3D11/D3D12 synchronization instead of relying on flush timing. native/camera_capture/mimir_mf_gpu_capture.dll creates a shared D3D11 fence, signals it after copying the decoded GPU surface into the shared BGRA texture, and MimirMediaFoundationGpuVideoCaptureDriver carries the fence handle/value through MimirVideoFrameDescriptor into Fensalir field resource declarations. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed with zero live CPU payload bytes, nonzero shared texture handles, nonzero producer fence handles, and valid Fensalir camera resources."}

state/map.yaml

@@ -341,8 +341,13 @@ accepted_design:
     `native/camera_capture/mimir_mf_gpu_capture.dll` to run Media Foundation
     SourceReader with a D3D11 device manager, decode MJPG/H264 on the GPU, copy
     the decoded GPU surface into a shared BGRA D3D11 texture, and publish a
-    `shared-d3d11-texture` handle with zero live CPU payload bytes. Kiyo Pro
-    1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed. Direct shared NV12
+    `shared-d3d11-texture` handle with zero live CPU payload bytes. The MF GPU
+    producer now also creates a shared D3D11 fence, signals it after the
+    texture copy, and carries the fence handle/value through
+    `MimirVideoFrameDescriptor`, `MimirBridgePayloadView`, and the Fensalir
+    field resource declaration so D3D12 can wait before sampling. Kiyo Pro
+    1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed with nonzero shared
+    texture and producer fence handles. Direct shared NV12
     texture creation failed in D3D11, so NV12 remains future plane-aware interop
     or GPU bridge-copy work.
     `Mimir.BufferSmoke --fensalir-camera-observation-smoke`