From 493b9ac3c7c65209414b47dea16c6b8bf3a5e53c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 21 May 2026 18:31:36 +0100 Subject: [PATCH] Add native video frame ingest descriptor --- config/mimir-runtime.example.json | 7 ++++++ docs/implementation-plan.md | 4 +++ docs/viable-stream-app.md | 7 ++++++ notes/current-system-map.md | 4 ++- notes/fresh-workspace-handoff.md | 3 +++ .../MimirNativeIngestStreamSource.cs | 25 +++++++++++++++++-- .../MimirProcessStreamSource.cs | 3 ++- .../MimirRuntimeConfiguration.cs | 5 ++++ .../Synchronization/MimirStreamSample.cs | 3 ++- .../MimirVideoFrameDescriptor.cs | 21 ++++++++++++++++ state/evidence.jsonl | 1 + state/map.yaml | 4 +++ state/scratch.md | 2 ++ tests/test_mimir_app_contract.py | 5 ++++ 14 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs diff --git a/config/mimir-runtime.example.json b/config/mimir-runtime.example.json index 6bcb3d3..c709466 100644 --- a/config/mimir-runtime.example.json +++ b/config/mimir-runtime.example.json @@ -8,6 +8,13 @@ "enabled": true, "adapter": "native" }, + { + "sourceId": "leap-stereo-ir", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "native" + }, { "sourceId": "host-focusrite", "kind": "Audio", diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 2be4dd7..fedfc71 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -57,6 +57,10 @@ latency, and confidence. Local devices should feed through native push adapters; process-backed sources are compatibility edges for network/diagnostic command ingest, not the six-camera capture foundation. + - `MimirVideoFrameDescriptor` is the first video-frame hot-path descriptor: + dimensions, pixel format, stride, device timestamp, and optional native/GPU + handle. Leap stereo IR should enter Mimir through this native descriptor, + with OpenCV kept out of the hot loop. - `config/perfect-machine.example.json` declares the contract shape for six cameras, six microphones, reservoir typed views, native authorities, outputs, and the demotion of bridge scripts to tooling. - `native/reservoir` is the first native Rust crate. It implements one shared-edge five-second rolling buffer with typed views and proves that diff --git a/docs/viable-stream-app.md b/docs/viable-stream-app.md index 1acdcf5..9097e72 100644 --- a/docs/viable-stream-app.md +++ b/docs/viable-stream-app.md @@ -96,6 +96,9 @@ The first cut should be deliberately boring: - `IMimirStreamSource` is the adapter seam for local devices and network feeds. - `MimirNativeIngestStreamSource` is the low-overhead push seam for native camera/audio adapters that already own buffers or handles. +- `MimirVideoFrameDescriptor` carries video shape, pixel format, device + timestamp, stride, and optional native/GPU handle metadata. Leap stereo IR + should enter here, not through OpenCV. - `MimirProcessStreamSource` is a compatibility adapter for network or diagnostic command sources. It is not the six-camera local capture foundation. @@ -118,6 +121,10 @@ through `MimirNativeIngestStreamSource`. Process-backed FFmpeg ingest is allowed for remote SRT/diagnostic edges, but local six-camera ingest should stay close to the driver, GPU, and capture APIs. +Leap should be treated as the timing-camera candidate only when the native +adapter preserves device timestamps or captures a monotonic timestamp +immediately on frame receipt. OpenCV arrival timing is too soft for that job. + ## Cut Line Do not add a new registry, bridge daemon, JSON live store, OBS plugin, or diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 2a8f03f..19b05a4 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -70,7 +70,9 @@ rolling buffer per configured audio/video stream, source polling through `IMimirStreamSource`, and runtime UI readouts for stream/buffer health. Local six-camera ingest should push native handles through `MimirNativeIngestStreamSource`; process-backed ingest remains a compatibility -edge for network SRT or diagnostics. +edge for network SRT or diagnostics. `MimirVideoFrameDescriptor` carries +dimensions, pixel format, stride, device timestamp, and optional native/GPU +handle metadata so Leap stereo IR can enter Mimir without OpenCV owning timing. ```mermaid flowchart TD diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index c93fc39..c02e3d4 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -52,6 +52,9 @@ Get-Content .\state\evidence.jsonl -Tail 8 adapters feed buffers through `IMimirStreamSource`. Local six-camera ingest should use native push adapters such as `MimirNativeIngestStreamSource`, not FFmpeg stdout/process capture except as a network or diagnostic edge. + `MimirVideoFrameDescriptor` now carries video shape, pixel format, stride, + device timestamp, and optional native/GPU handle metadata for close-to-metal + frame ingest; Leap stereo IR is the timing-camera candidate for this path. - The reservoir is now one native time-ordered rolling buffer with typed indexes/views. The previous typed-ring crate is gone. - Native reservoir samples carry provenance flags; diagnostic/fallback samples diff --git a/src/Mimir.Runtime/Synchronization/MimirNativeIngestStreamSource.cs b/src/Mimir.Runtime/Synchronization/MimirNativeIngestStreamSource.cs index 8982eac..04ff34e 100644 --- a/src/Mimir.Runtime/Synchronization/MimirNativeIngestStreamSource.cs +++ b/src/Mimir.Runtime/Synchronization/MimirNativeIngestStreamSource.cs @@ -19,7 +19,8 @@ public sealed class MimirNativeIngestStreamSource : IMimirStreamSource long arrivalNs, ulong payloadHandle, int byteLength = 0, - ReadOnlyMemory data = default) + ReadOnlyMemory data = default, + MimirVideoFrameDescriptor? videoFrame = null) { samples.Enqueue(new MimirStreamSample( Descriptor.SourceId, @@ -30,7 +31,27 @@ public sealed class MimirNativeIngestStreamSource : IMimirStreamSource sequence++, payloadHandle, byteLength, - data)); + data, + videoFrame)); + } + + public void PushVideoFrame( + MimirVideoFrameDescriptor frame, + long arrivalNs, + ReadOnlyMemory data = default) + { + if (Descriptor.Kind != MimirStreamKind.Video) + { + throw new InvalidOperationException("Video frames can only be pushed into a video stream source."); + } + + Push( + frame.DeviceTimestampNs, + arrivalNs, + frame.NativeHandle, + data.Length, + data, + frame); } public bool TryRead(out MimirStreamSample sample) diff --git a/src/Mimir.Runtime/Synchronization/MimirProcessStreamSource.cs b/src/Mimir.Runtime/Synchronization/MimirProcessStreamSource.cs index c5144e6..fbe8c52 100644 --- a/src/Mimir.Runtime/Synchronization/MimirProcessStreamSource.cs +++ b/src/Mimir.Runtime/Synchronization/MimirProcessStreamSource.cs @@ -85,7 +85,8 @@ public sealed class MimirProcessStreamSource : IMimirStreamSource sequence++, 0, buffer.Length, - buffer)); + buffer, + null)); } } diff --git a/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs b/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs index 58019be..3e7b0f6 100644 --- a/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs +++ b/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs @@ -93,6 +93,11 @@ public sealed class MimirRuntimeConfiguration private static IMimirStreamSource? TryCreateSource(MimirStreamConfig stream) { + if (string.Equals(stream.Adapter, "native", StringComparison.OrdinalIgnoreCase)) + { + return new MimirNativeIngestStreamSource(ToDescriptor(stream)); + } + if (!string.Equals(stream.Adapter, "process", StringComparison.OrdinalIgnoreCase)) { return null; diff --git a/src/Mimir.Runtime/Synchronization/MimirStreamSample.cs b/src/Mimir.Runtime/Synchronization/MimirStreamSample.cs index a59f745..00b4c4d 100644 --- a/src/Mimir.Runtime/Synchronization/MimirStreamSample.cs +++ b/src/Mimir.Runtime/Synchronization/MimirStreamSample.cs @@ -9,4 +9,5 @@ public readonly record struct MimirStreamSample( ulong Sequence, ulong PayloadHandle, int ByteLength = 0, - ReadOnlyMemory Data = default); + ReadOnlyMemory Data = default, + MimirVideoFrameDescriptor? VideoFrame = null); diff --git a/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs b/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs new file mode 100644 index 0000000..735dbc2 --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs @@ -0,0 +1,21 @@ +namespace Mimir.Runtime.Synchronization; + +public enum MimirVideoPixelFormat +{ + Unknown, + Gray8, + R8, + Rg8, + Bgra8, + Nv12, + LeapStereoIr, +} + +public sealed record MimirVideoFrameDescriptor( + int Width, + int Height, + MimirVideoPixelFormat PixelFormat, + int StrideBytes, + long DeviceTimestampNs, + ulong NativeHandle = 0, + string NativeHandleKind = ""); diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 2a817ff..9e63a04 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -99,3 +99,4 @@ {"ts": "2026-05-21T12:14:26+00:00", "type": "implementation", "status": "ok", "note": "Mimir now owns a C# app solution: Mimir.slnx contains src/Mimir.App, which launches AquariumHost from Aquarium Engine, and src/Mimir.Runtime, which provides the Aquarium runtime factory over Aquarium.LocalCast. Aquarium Engine remains the windowing/rendering/D3D12 library reference."} {"ts": "2026-05-21T12:33:01+00:00", "type": "implementation", "status": "ok", "note": "Mimir.Runtime now owns the first synchronization spine: MimirRuntime wraps Aquarium LocalCast rendering, initializes per-stream rolling buffers through MimirSynchronizationHub, polls registered IMimirStreamSource adapters each update, and defaults buffer duration to five seconds with env overrides for local/network audio/video stream ids."} {"ts": "2026-05-21T17:27:12+00:00", "type": "decision", "status": "ok", "note": "For six local cameras, Mimir ingest should stay close to driver/GPU capture surfaces. MimirNativeIngestStreamSource is the preferred local adapter seam carrying payload handles and optional memory into rolling buffers; MimirProcessStreamSource remains a network/diagnostic compatibility edge, not the local camera foundation."} +{"ts": "2026-05-21T17:31:13+00:00", "type": "implementation", "status": "ok", "note": "OpenCV is cut out of the Mimir video hot-path contract: MimirVideoFrameDescriptor carries dimensions, pixel format, stride, device timestamp, and optional native/GPU handle; MimirNativeIngestStreamSource can push LeapStereoIr or other camera frames directly into rolling buffers while process-backed ingest remains a network/diagnostic edge."} diff --git a/state/map.yaml b/state/map.yaml index 68882b6..dba83df 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -81,6 +81,10 @@ accepted_design: optional memory spans into Mimir buffers. `MimirProcessStreamSource` is a compatibility path for network SRT or diagnostics, not the local camera foundation. + - `MimirVideoFrameDescriptor` is the first native video-frame descriptor for + the C# runtime: width, height, pixel format, stride, device timestamp, and + optional native/GPU handle. Leap stereo IR should use this path as the + timing-camera candidate instead of OpenCV arrival timing. - `native/reservoir` is the first native Rust crate. It now uses one time-ordered rolling buffer with typed views, so retention has exactly one owner while Aquarium/Faust still get kind-scoped queries. diff --git a/state/scratch.md b/state/scratch.md index 5b5805a..b2fc181 100644 --- a/state/scratch.md +++ b/state/scratch.md @@ -17,5 +17,7 @@ No active scratch subgoal. - Local six-camera ingest should stay close to the metal: native push sources carry payload handles into buffers; process sources are network/diagnostic compatibility, not the main local camera path. +- `MimirVideoFrameDescriptor` is the hot-path video descriptor for native camera + workers. Leap stereo IR should push device timestamps/handles through it. - Keep scratch short and disposable. - Promote only durable lessons into `state/evidence.jsonl` or the map. diff --git a/tests/test_mimir_app_contract.py b/tests/test_mimir_app_contract.py index cf861fb..85fdc69 100644 --- a/tests/test_mimir_app_contract.py +++ b/tests/test_mimir_app_contract.py @@ -49,9 +49,14 @@ class MimirAppContractTests(unittest.TestCase): self.assertIn("MIMIR_LOCAL_AUDIO_STREAMS", settings) native_source = (ROOT / "src" / "Mimir.Runtime" / "Synchronization" / "MimirNativeIngestStreamSource.cs").read_text(encoding="utf-8") process_source = (ROOT / "src" / "Mimir.Runtime" / "Synchronization" / "MimirProcessStreamSource.cs").read_text(encoding="utf-8") + frame_descriptor = (ROOT / "src" / "Mimir.Runtime" / "Synchronization" / "MimirVideoFrameDescriptor.cs").read_text(encoding="utf-8") self.assertIn("payloadHandle", native_source) + self.assertIn("PushVideoFrame", native_source) self.assertIn("ReadOnlyMemory", native_source) + self.assertIn("LeapStereoIr", frame_descriptor) + self.assertIn("DeviceTimestampNs", frame_descriptor) + self.assertIn("NativeHandle", frame_descriptor) self.assertIn("RedirectStandardOutput", process_source)