From 7f3df7979b0ca41e5c5ec2159ecba744b402a3e0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 27 May 2026 14:11:48 +0100 Subject: [PATCH] Upload CPU camera frames into Fensalir leases --- docs/implementation-plan.md | 6 +++ notes/current-system-map.md | 5 ++ notes/fresh-workspace-handoff.md | 4 ++ src/Mimir.BufferSmoke/Program.cs | 51 ++++++++++++++++--- .../MimirFensalirTextureLeaseClient.cs | 23 +++++++++ .../MimirVideoCaptureDriverSource.cs | 14 +++++ .../MimirVideoFrameDescriptor.cs | 3 +- state/evidence.jsonl | 1 + state/map.yaml | 6 +++ 9 files changed, 105 insertions(+), 8 deletions(-) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 4fca6ca..ff4d759 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -192,6 +192,12 @@ a named invariant that the native runtime cannot protect yet. `MimirVideoCaptureDriverSource` forwards the lease client to drivers that implement `IMimirFensalirTextureLeaseReceiver`, so native/direct camera drivers can allocate their Fensalir destination texture before capture/decode. + If a raw single-plane device path must return CPU bytes, + `MimirVideoCaptureDriverSource` uploads them directly into the leased texture + through the broker, clears the live managed payload, and increments + `UnavoidableCopyCount`. NV12 CPU upload is rejected until the engine owns a + real planar copy path; GPU/native producers should write the shared texture + themselves and stay at zero copies. Metadata-only cadence frames remain observations with empty payload handles; they do not become fake render requests. The old direct `AquariumGpuSensorFrame` builder has been removed from Mimir's proof path. diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 9bcc091..33418e1 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -339,6 +339,11 @@ request the destination texture before emitting a frame descriptor. Each camera backend must use the closest-to-device path available and expose its unavoidable copy count. Managed/process wrappers are diagnostic bring-up surfaces, not the production camera hot path. +For raw single-plane system-memory frames, `MimirVideoCaptureDriverSource` +performs the one declared CPU-to-GPU upload into the Fensalir lease, drops the +managed payload from the live sample, and increments `UnavoidableCopyCount`. +NV12 CPU upload is rejected until Fensalir owns a real planar upload path; +device/GPU NV12 producers should write the leased texture directly. Metadata-only cadence frames do not create camera surface intents or fake payload requests. The old direct `AquariumGpuSensorFrame` bridge proof has been removed from Mimir's active proof path. Camera image claims currently defer diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index 2bfabdc..2ce385c 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -51,6 +51,10 @@ Get-Content .\state\evidence.jsonl -Tail 8 that implement `IMimirFensalirTextureLeaseReceiver`. Camera backends must use the closest-to-device path available and report unavoidable copies; process or managed convenience layers are diagnostics. + Raw single-plane CPU-origin frames can use the explicit broker upload lane, + which increments `UnavoidableCopyCount` and clears live managed payload bytes + after upload. NV12 CPU upload is rejected until Fensalir owns planar copy; + device/GPU NV12 producers should write the leased texture directly. Foreign shared texture handles are import edges, not the primary hot path. - The current Mimir/Fensalir rendering teardown map is [[docs/fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index cd02588..d123320 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -757,17 +757,19 @@ static int RunFensalirTextureLeaseSmoke() } var receiverDriver = new FakeTextureLeaseVideoDriver(); + var receiverSource = new MimirVideoCaptureDriverSource( + new MimirStreamDescriptor("driver-camera", MimirStreamKind.Video, MimirStreamOrigin.LocalDevice), + receiverDriver, + static () => 0); using (var runtime = new MimirRuntime( new AquariumRuntimeOptions(Headless: true, CultCachePath: null), new MimirSynchronizationSettings(), - [ - new MimirVideoCaptureDriverSource( - new MimirStreamDescriptor("driver-camera", MimirStreamKind.Video, MimirStreamOrigin.LocalDevice), - receiverDriver, - static () => 0) - ])) + [receiverSource])) { runtime.AttachServices(new AquariumRuntimeServices(broker)); + runtime.OnSceneReady(); + runtime.Update(0.016f, new InputState()); + _ = runtime.Frame; } var buffer = new MimirRollingStreamBuffer( @@ -790,10 +792,12 @@ static int RunFensalirTextureLeaseSmoke() var expectedResourceKey = MimirFensalirTextureLeaseClient.ResourceKeyForSource("kiyo-pro-rgb"); Console.WriteLine( - $"fensalir-texture-lease-smoke leased={lease.IsValid} committed={broker.CommittedFenceValue > 0} driverClient={receiverDriver.HasClient} resources={frame.Resources.Count} claims={frame.Claims.Count} errors={validation.HasErrors}"); + $"fensalir-texture-lease-smoke leased={lease.IsValid} committed={broker.CommittedFenceValue > 0} driverClient={receiverDriver.HasClient} uploaded={broker.UploadedResourceKey} copies={receiverSource.LastUploadedCopyCount} resources={frame.Resources.Count} claims={frame.Claims.Count} errors={validation.HasErrors}"); return lease.IsValid && receiverDriver.HasClient && + receiverSource.LastUploadedCopyCount == 1 && + broker.UploadedResourceKey == MimirFensalirTextureLeaseClient.ResourceKeyForSource("driver-camera") && broker.CommittedResourceKey == expectedResourceKey && broker.CommittedVersion == 0 && broker.CommittedFenceValue == 41 && @@ -6049,6 +6053,8 @@ sealed class FakeFieldResourceBroker : IAquariumFieldResourceBroker public ulong CommittedFenceValue { get; private set; } + public string UploadedResourceKey { get; private set; } = ""; + public AquariumFieldResourceLease LeaseTexture2D(AquariumTexture2DLeaseRequest request) { var declaration = new AquariumFieldResourceDeclaration( @@ -6082,6 +6088,12 @@ sealed class FakeFieldResourceBroker : IAquariumFieldResourceBroker CommittedFenceValue = producerFenceValue; return true; } + + public bool UploadTexture2D(AquariumTexture2DUpload upload) + { + UploadedResourceKey = upload.ResourceKey; + return upload.IsValid; + } } sealed class FakeTextureLeaseVideoDriver : IMimirVideoCaptureDriver, IMimirFensalirTextureLeaseReceiver @@ -6090,13 +6102,38 @@ sealed class FakeTextureLeaseVideoDriver : IMimirVideoCaptureDriver, IMimirFensa public bool HasClient { get; private set; } + private MimirFensalirTextureLeaseClient? client; + private bool emitted; + public void AttachTextureLeaseClient(MimirFensalirTextureLeaseClient? client) { + this.client = client; HasClient = client != null; } public bool TryCapture(out MimirVideoFrameDescriptor frame, out ReadOnlyMemory data) { + if (!emitted && + client != null && + client.TryLeaseTexture2D(new MimirFensalirTextureLeaseRequest( + "driver-camera", + 2, + 2, + MimirVideoPixelFormat.Bgra8, + 8, + DeviceTimestampNs: 2_000_000_000L, + Version: 1), out var lease)) + { + emitted = true; + frame = lease.Frame; + data = new byte[] + { + 0, 0, 0, 255, 255, 255, 255, 255, + 255, 0, 0, 255, 0, 255, 0, 255, + }; + return true; + } + frame = default!; data = default; return false; diff --git a/src/Mimir.Runtime/Synchronization/MimirFensalirTextureLeaseClient.cs b/src/Mimir.Runtime/Synchronization/MimirFensalirTextureLeaseClient.cs index 1105511..e30ba68 100644 --- a/src/Mimir.Runtime/Synchronization/MimirFensalirTextureLeaseClient.cs +++ b/src/Mimir.Runtime/Synchronization/MimirFensalirTextureLeaseClient.cs @@ -68,6 +68,27 @@ public sealed class MimirFensalirTextureLeaseClient(IAquariumFieldResourceBroker public bool Commit(string resourceKey, ulong version, ulong producerFenceValue) => broker.CommitLeaseVersion(resourceKey, version, producerFenceValue); + public bool UploadCpuFrame(MimirVideoFrameDescriptor frame, ReadOnlyMemory data) + { + if (string.IsNullOrWhiteSpace(frame.ResourceKey) || + frame.Width <= 0 || + frame.Height <= 0 || + frame.StrideBytes <= 0 || + data.IsEmpty) + { + return false; + } + + return broker.UploadTexture2D(new AquariumTexture2DUpload( + frame.ResourceKey, + frame.Width, + frame.Height, + FormatForPixelFormat(frame.PixelFormat), + frame.StrideBytes, + data, + frame.ProducerFenceValue)); + } + public static string ResourceKeyForSource(string sourceId) { var normalized = sourceId.Trim().ToLowerInvariant().Replace('\\', '/'); @@ -78,7 +99,9 @@ public sealed class MimirFensalirTextureLeaseClient(IAquariumFieldResourceBroker pixelFormat switch { MimirVideoPixelFormat.Gray8 or MimirVideoPixelFormat.R8 => "R8Unorm", + MimirVideoPixelFormat.Bayer8 => "R8Unorm", MimirVideoPixelFormat.Rg8 or MimirVideoPixelFormat.LeapStereoIr => "R8G8_UNorm", + MimirVideoPixelFormat.Yuy2 => "YUY2", MimirVideoPixelFormat.Bgra8 => "Bgra8", MimirVideoPixelFormat.Nv12 => "Nv12", _ => "Rgba8Unorm", diff --git a/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs b/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs index 19ebe19..cd70b03 100644 --- a/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs +++ b/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs @@ -4,6 +4,7 @@ public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFe { private readonly IMimirVideoCaptureDriver driver; private readonly MimirNativeIngestStreamSource nativeSource; + private MimirFensalirTextureLeaseClient? textureLeaseClient; public MimirVideoCaptureDriverSource( MimirStreamDescriptor descriptor, @@ -25,8 +26,11 @@ public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFe public Func ReadArrivalNs { get; } + public int LastUploadedCopyCount { get; private set; } + public void AttachTextureLeaseClient(MimirFensalirTextureLeaseClient? client) { + textureLeaseClient = client; if (driver is IMimirFensalirTextureLeaseReceiver receiver) { receiver.AttachTextureLeaseClient(client); @@ -45,6 +49,16 @@ public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFe return false; } + if (!data.IsEmpty && !string.IsNullOrWhiteSpace(frame.ResourceKey)) + { + if (textureLeaseClient?.UploadCpuFrame(frame, data) == true) + { + frame = frame with { UnavoidableCopyCount = frame.UnavoidableCopyCount + 1 }; + LastUploadedCopyCount = frame.UnavoidableCopyCount; + data = default; + } + } + nativeSource.PushVideoFrame(frame, ReadArrivalNs(), data); return nativeSource.TryRead(out sample); } diff --git a/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs b/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs index 331496e..24d13e6 100644 --- a/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs +++ b/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs @@ -24,4 +24,5 @@ public sealed record MimirVideoFrameDescriptor( ulong NativeHandle = 0, string NativeHandleKind = "", string ResourceKey = "", - ulong ProducerFenceValue = 0); + ulong ProducerFenceValue = 0, + int UnavoidableCopyCount = 0); diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 847c71e..bb80122 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -176,3 +176,4 @@ {"ts": "2026-05-27T13:26:42+01:00", "type": "decision", "status": "ok", "note": "Fensalir-owned Texture2D leases are the preferred live camera payload authority: Mimir receives AquariumRuntimeServices, requests keyed D3D12 texture/fence leases, writes decoded frames into the engine-owned shared texture, commits producer fence values, and lowers the resource key through FieldEvidence. Imported shared texture handles remain fallback/import edges."} {"ts": "2026-05-27T13:32:24+01:00", "type": "decision", "status": "ok", "note": "Texture lease services now propagate from MimirRuntime into direct video capture sources. MimirVideoCaptureDriverSource forwards a MimirFensalirTextureLeaseClient to drivers that implement IMimirFensalirTextureLeaseReceiver, so a direct camera driver can request its Fensalir-owned destination texture before it emits the frame descriptor and producer fence value."} {"ts": "2026-05-27T13:37:42+01:00", "type": "decision", "status": "ok", "note": "Camera ingest policy is now closest-to-device first. Fensalir-owned Texture2D leases remain the render-input authority, but each native camera backend must choose the most direct device path available and report unavoidable copy count. Managed wrappers, process bridges, and system-memory rings are diagnostics or fallbacks, not production hot-path owners."} +{"ts": "2026-05-27T14:07:32+01:00", "type": "decision", "status": "ok", "note": "Raw single-plane CPU-origin camera frames now have an explicit Fensalir texture upload fallback. MimirVideoCaptureDriverSource uploads returned frame bytes into the leased Fensalir Texture2D through the broker, clears the live managed payload after successful upload, and increments MimirVideoFrameDescriptor.UnavoidableCopyCount. CPU NV12 upload is rejected until Fensalir has a real planar copy path; GPU/direct producers should bypass this lane and remain at zero copies."} diff --git a/state/map.yaml b/state/map.yaml index 40a709c..7591593 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -317,6 +317,12 @@ accepted_design: Camera backends must use the closest-to-device path available and report unavoidable copies; process or managed convenience layers are diagnostics, not production camera hot-path owners. + Raw single-plane system-memory frames use the explicit fallback upload lane: + `MimirVideoCaptureDriverSource` uploads the bytes into the Fensalir lease, + drops the managed payload from the live sample, and increments + `UnavoidableCopyCount`. NV12 CPU upload is rejected until Fensalir has a + real planar upload path; device/GPU NV12 producers should write the leased + texture directly. `Mimir.BufferSmoke --fensalir-camera-observation-smoke` verifies the split, and `--fensalir-texture-lease-smoke` verifies the engine-owned lease path. Camera image claims currently defer until Fensalir