From d1eca0776e10dffee334eccf827279e6295d2537 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 21 May 2026 18:41:02 +0100 Subject: [PATCH] Add direct video capture driver seam --- config/sensor-fusion.example.json | 5 +- docs/calibration.md | 6 +-- docs/implementation-plan.md | 3 ++ docs/sensor-fusion-architecture.md | 18 ++++--- docs/viable-stream-app.md | 7 ++- notes/current-system-map.md | 3 ++ notes/fresh-workspace-handoff.md | 4 ++ .../IMimirVideoCaptureDriver.cs | 8 +++ .../MimirVideoCaptureDriverSource.cs | 49 +++++++++++++++++++ state/evidence.jsonl | 1 + state/map.yaml | 4 ++ state/scratch.md | 2 + tests/test_mimir_app_contract.py | 4 ++ 13 files changed, 100 insertions(+), 14 deletions(-) create mode 100644 src/Mimir.Runtime/Synchronization/IMimirVideoCaptureDriver.cs create mode 100644 src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs diff --git a/config/sensor-fusion.example.json b/config/sensor-fusion.example.json index 4228387..968ba55 100644 --- a/config/sensor-fusion.example.json +++ b/config/sensor-fusion.example.json @@ -74,8 +74,9 @@ "id": "kiyo_rgb", "role": "rgb_projection", "capture": { - "adapter": "opencv-dshow", - "index": 1, + "adapter": "native-driver", + "driver": "media-foundation-or-directshow", + "device_name": "Razer Kiyo", "width": 640, "height": 480, "fps": 30 diff --git a/docs/calibration.md b/docs/calibration.md index d5248d4..520c164 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -85,7 +85,7 @@ Observed video access: Mode probe notes: -- `msmf:0` LeapUVC accepts a `320x240x120` request and reports about 115 fps, with rough OpenCV measured capture around 52 fps in the first short probe. Treat that as "tracking mode is plausible," not calibrated throughput. +- Historical diagnostic only: `msmf:0` LeapUVC accepted a `320x240x120` request and reported about 115 fps, while the old OpenCV probe measured around 52 fps. Treat that as evidence that the sensor mode is plausible and that the diagnostic path is too soft for live timing. Mimir live ingest should talk directly to the driver path. - Kiyo RGB devices produce snapshots, but initial isolated mode probes against Kiyo indices timed out. Do not trust FPS configuration for Kiyos until a backend-specific capture path is chosen. - PS3 Eye default DirectShow capture works at `640x480` near 30 fps. High-FPS tracking modes are not proven yet; mode probes against the DirectShow filter can hang or collide with another open handle, so test them sequentially. @@ -192,9 +192,9 @@ After registration, this machine exposes: Known caveats: -- OpenCV `mode-probe` calls that set width/height/fps may time out against the PS3EyeDirectShow filter. +- Historical diagnostic only: OpenCV `mode-probe` calls that set width/height/fps may time out against the PS3EyeDirectShow filter. - Parallel probes can race the same libusb/backend state and produce `Access is denied`; test PS3 Eye mode changes one handle at a time. -- OpenCV by index does not negotiate high-rate PS3 Eye modes correctly. It can report a requested FPS while delivering the wrong shape or frame rate. +- Historical diagnostic only: OpenCV by index did not negotiate high-rate PS3 Eye modes correctly. It can report a requested FPS while delivering the wrong shape or frame rate. - FFmpeg by DirectShow device name does negotiate the filter pins correctly. Verified modes: - single `PS3 Eye Universal` at `640x480@30/40/50/60`; `640x480@60` delivered about 56 fps over three seconds - single `PS3 Eye Universal` at `320x240@30/40/50/60`; `320x240@60` delivered 180 frames over three seconds diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index fedfc71..f8e825d 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -61,6 +61,9 @@ latency, and confidence. dimensions, pixel format, stride, device timestamp, and optional native/GPU handle. Leap stereo IR should enter Mimir through this native descriptor, with OpenCV kept out of the hot loop. + - `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` define the + live driver-facing seam. LeapUVC/libusb, LeapC images, Media Foundation, + DirectShow, or GPU shared texture capture should plug in here. - `config/perfect-machine.example.json` declares the contract shape for six cameras, six microphones, reservoir typed views, native authorities, outputs, and the demotion of bridge scripts to tooling. - `native/reservoir` is the first native Rust crate. It implements one shared-edge five-second rolling buffer with typed views and proves that diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index 12ad966..328da0d 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -9,8 +9,10 @@ Produce an aligned, timestamped point cloud from the local visual rig while allo The rig currently has usable capture surfaces: - PS3 Eyes through FFmpeg DirectShow device names `PS3 Eye Universal` and `PS3 Eye Universal2`, verified at dual `320x240@60`. -- Kiyo-class RGB cameras through OpenCV/DirectShow/MSMF for color evidence. -- LeapUVC raw IR through OpenCV for near-field evidence. +- Kiyo-class RGB cameras through native Windows camera APIs such as Media + Foundation or DirectShow for color evidence. +- LeapUVC raw IR through native LeapUVC/libusb or LeapC image APIs for + near-field timing evidence. - ChArUco tooling for intrinsics capture, but no shared fusion state yet. That is capture. Capture is not a world model. The previous live producer also @@ -67,7 +69,7 @@ flowchart TD - `localcast.sensor_fusion.calibration_space` owns fixed-board common-space calibration: ChArUco corner observations plus intrinsics become `CameraModel.world_from_sensor` transforms in one room frame. - `localcast.sensor_fusion.surface_features` owns cross-view feature observations, descriptor matching, and triangulated surface tracks. It consumes calibrated cameras; it does not invent camera geometry. - `localcast.sensor_fusion.stochastic_mapping` owns transient feature hypothesis sampling, image-pixel to scene-ray lowering, learned image-ray bias cells, and multi-LOD scene cache packets for later compute-shader reconciliation. -- `localcast.sensor_fusion.camera_control` owns measurement-quality policy: luminance, clipping, contrast, and sharpness in; normalized exposure/gain/focus commands out. Driver adapters own translating those commands to OpenCV, DirectShow, vendor tools, or no-op mocks. +- `localcast.sensor_fusion.camera_control` owns measurement-quality policy: luminance, clipping, contrast, and sharpness in; normalized exposure/gain/focus commands out. Driver adapters own translating those commands to native camera APIs, vendor tools, or no-op mocks. - `localcast.sensor_fusion.dense_stereo` owns a debug CPU reference for dense calibrated RGB surface claims. It is useful for tests and shader parity, but the live million-splat path belongs to Aquarium/GPU compute. - `localcast.sensor_fusion.active_illumination` owns deliberate light pulses as calibration telemetry. A Tuya bulb is an optional local-network actuator, not a rendering authority; its pulse timestamps are evidence for camera exposure/depth response. - `localcast.sensor_fusion.clap_calibration` owns deliberate clap detection: chirplet-synced audio transients nominate oracle timestamps, camera frame windows around that oracle look for motion derivatives, calibrated camera peaks triangulate the impact point, and only audio-plus-visual agreement becomes a clap calibration event. @@ -189,7 +191,7 @@ That lets the final stream compositor buffer point-cloud visuals until they alig Current live deadline rule: `source_time_max_ns - source_time_min_ns` is five seconds. Late-arriving samples can fill the reservoir while they are inside that window; anything older is evidence for offline calibration, not the live program surface. -The Spout/audio overlay applies the same rule at the consumer boundary. If the visual producer or a camera driver stalls, OBS receives a packet clamped to the current audio edge with stale points removed. The current live fallback mode uses cached/synthetic RGB and Leap frames because direct OpenCV reads measured slower than the reservoir; real cameras need nonblocking ingress feeding the same reservoir instead of blocking the producer loop. +The Spout/audio overlay applies the same rule at the consumer boundary. If the visual producer or a camera driver stalls, OBS receives a packet clamped to the current audio edge with stale points removed. The current live fallback mode uses cached/synthetic RGB and Leap frames because the old diagnostic capture path measured slower than the reservoir; real cameras need nonblocking native-driver ingress feeding the same reservoir instead of blocking the producer loop. ## Next Cut @@ -198,8 +200,8 @@ The Spout/audio overlay applies the same rule at the consumer boundary. If the v 3. Use `scripts/triangulate_surface_features.py` on calibrated image pairs to prove real cross-view surface tracks before promoting the matcher into the live producer. 4. Replace the synthetic stochastic transient observations in `localcast.diagnostics.visual_producer` with real native/Aquarium ORB/flow - observations from PS3 Eye/Kiyo frames, then delete the Python diagnostic - producer. + observations from driver-ingested PS3 Eye/Kiyo/Leap frames, then delete the + Python diagnostic producer. 5. Move dense matching and LOD reconciliation from CPU block search/JSON to GPU-resident stereo/flow and compute-shader cache reduction so the million-sample target is not murdered by Python loops. 6. If a Tuya light is available on the local network, run `scripts/pulse_tuya_light.py` with its local key and align pulse timestamps against camera brightness response. 7. Add detector adapters that turn PS3 Eye frames into `Observation2D` marker detections. @@ -210,7 +212,9 @@ The Spout/audio overlay applies the same rule at the consumer boundary. If the v ## References -- OpenCV multi-camera calibration and triangulation are the geometry spine. See `research/visual-spatial-map/summary.md`. +- Multi-camera calibration and triangulation are the geometry spine. OpenCV may + remain an offline calibration library; it is not live frame ingest. See + `research/visual-spatial-map/summary.md`. - Jacob and Haeb-Umbach 2015 motivates audio-visual coordinate alignment. See `research/visual-spatial-map/mirrors/krekovic-2015-audio-visual-geometry-calibration.pdf`. - Kerbl et al. 2023 and RTG-SLAM motivate splats as render packets, not state ownership. See `research/visual-spatial-map/summary.md` and `research/gpu-fusion-splatting/summary.md`. - Spout2 and the OBS Spout2 plugin are the Windows/OBS texture-sharing bridge. See `research/spout-obs-bridge/summary.md`. diff --git a/docs/viable-stream-app.md b/docs/viable-stream-app.md index 9097e72..8276c69 100644 --- a/docs/viable-stream-app.md +++ b/docs/viable-stream-app.md @@ -98,7 +98,10 @@ The first cut should be deliberately boring: camera/audio adapters that already own buffers or handles. - `MimirVideoFrameDescriptor` carries video shape, pixel format, device timestamp, stride, and optional native/GPU handle metadata. Leap stereo IR - should enter here, not through OpenCV. + should enter here through a native driver path. +- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the + driver-facing seam for polling/pushing frames from LeapUVC/libusb, LeapC image + APIs, Media Foundation, DirectShow, or a future GPU shared-texture capture. - `MimirProcessStreamSource` is a compatibility adapter for network or diagnostic command sources. It is not the six-camera local capture foundation. @@ -123,7 +126,7 @@ to the driver, GPU, and capture APIs. Leap should be treated as the timing-camera candidate only when the native adapter preserves device timestamps or captures a monotonic timestamp -immediately on frame receipt. OpenCV arrival timing is too soft for that job. +immediately on frame receipt. Do not put OpenCV in the live ingest path. ## Cut Line diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 19b05a4..7cdcde9 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -73,6 +73,9 @@ Local six-camera ingest should push native handles through edge for network SRT or diagnostics. `MimirVideoFrameDescriptor` carries dimensions, pixel format, stride, device timestamp, and optional native/GPU handle metadata so Leap stereo IR can enter Mimir without OpenCV owning timing. +`IMimirVideoCaptureDriver` is the driver-facing seam for LeapUVC/libusb, LeapC +images, Media Foundation, DirectShow, or GPU shared textures. OpenCV is only an +offline calibration or diagnostic library now. ```mermaid flowchart TD diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index c02e3d4..cf625a7 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -55,6 +55,10 @@ Get-Content .\state\evidence.jsonl -Tail 8 `MimirVideoFrameDescriptor` now carries video shape, pixel format, stride, device timestamp, and optional native/GPU handle metadata for close-to-metal frame ingest; Leap stereo IR is the timing-camera candidate for this path. + `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the live + driver seam; future LeapUVC/libusb, LeapC image, Media Foundation, DirectShow, + or GPU shared-texture adapters should plug in there. OpenCV is no longer a + live ingest target. - The reservoir is now one native time-ordered rolling buffer with typed indexes/views. The previous typed-ring crate is gone. - Native reservoir samples carry provenance flags; diagnostic/fallback samples diff --git a/src/Mimir.Runtime/Synchronization/IMimirVideoCaptureDriver.cs b/src/Mimir.Runtime/Synchronization/IMimirVideoCaptureDriver.cs new file mode 100644 index 0000000..6fc97af --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/IMimirVideoCaptureDriver.cs @@ -0,0 +1,8 @@ +namespace Mimir.Runtime.Synchronization; + +public interface IMimirVideoCaptureDriver : IDisposable +{ + string DriverName { get; } + + bool TryCapture(out MimirVideoFrameDescriptor frame, out ReadOnlyMemory data); +} diff --git a/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs b/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs new file mode 100644 index 0000000..a94ed6e --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs @@ -0,0 +1,49 @@ +namespace Mimir.Runtime.Synchronization; + +public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource +{ + private readonly IMimirVideoCaptureDriver driver; + private readonly MimirNativeIngestStreamSource nativeSource; + + public MimirVideoCaptureDriverSource( + MimirStreamDescriptor descriptor, + IMimirVideoCaptureDriver driver, + Func readArrivalNs) + { + if (descriptor.Kind != MimirStreamKind.Video) + { + throw new ArgumentException("Driver video sources require a video stream descriptor.", nameof(descriptor)); + } + + Descriptor = descriptor; + this.driver = driver; + ReadArrivalNs = readArrivalNs; + nativeSource = new MimirNativeIngestStreamSource(descriptor); + } + + public MimirStreamDescriptor Descriptor { get; } + + public Func ReadArrivalNs { get; } + + public bool TryRead(out MimirStreamSample sample) + { + if (nativeSource.TryRead(out sample)) + { + return true; + } + + if (!driver.TryCapture(out var frame, out var data)) + { + return false; + } + + nativeSource.PushVideoFrame(frame, ReadArrivalNs(), data); + return nativeSource.TryRead(out sample); + } + + public void Dispose() + { + nativeSource.Dispose(); + driver.Dispose(); + } +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 9e63a04..51a7839 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -100,3 +100,4 @@ {"ts": "2026-05-21T12:33:01+00:00", "type": "implementation", "status": "ok", "note": "Mimir.Runtime now owns the first synchronization spine: MimirRuntime wraps Aquarium LocalCast rendering, initializes per-stream rolling buffers through MimirSynchronizationHub, polls registered IMimirStreamSource adapters each update, and defaults buffer duration to five seconds with env overrides for local/network audio/video stream ids."} {"ts": "2026-05-21T17:27:12+00:00", "type": "decision", "status": "ok", "note": "For six local cameras, Mimir ingest should stay close to driver/GPU capture surfaces. MimirNativeIngestStreamSource is the preferred local adapter seam carrying payload handles and optional memory into rolling buffers; MimirProcessStreamSource remains a network/diagnostic compatibility edge, not the local camera foundation."} {"ts": "2026-05-21T17:31:13+00:00", "type": "implementation", "status": "ok", "note": "OpenCV is cut out of the Mimir video hot-path contract: MimirVideoFrameDescriptor carries dimensions, pixel format, stride, device timestamp, and optional native/GPU handle; MimirNativeIngestStreamSource can push LeapStereoIr or other camera frames directly into rolling buffers while process-backed ingest remains a network/diagnostic edge."} +{"ts": "2026-05-21T17:40:33+00:00", "type": "decision", "status": "ok", "note": "Mimir live camera ingest should talk directly to drivers: LeapUVC/libusb, LeapC images, Media Foundation, DirectShow, or GPU shared textures through IMimirVideoCaptureDriver. OpenCV remains offline calibration or historical diagnostics only, not a live ingest target."} diff --git a/state/map.yaml b/state/map.yaml index dba83df..9afa697 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -85,6 +85,10 @@ accepted_design: the C# runtime: width, height, pixel format, stride, device timestamp, and optional native/GPU handle. Leap stereo IR should use this path as the timing-camera candidate instead of OpenCV arrival timing. + - `IMimirVideoCaptureDriver` plus `MimirVideoCaptureDriverSource` is the live + driver-facing seam. LeapUVC/libusb, LeapC images, Media Foundation, + DirectShow, or GPU shared textures should plug in there; OpenCV remains + offline calibration/diagnostic tooling, not live ingest. - `native/reservoir` is the first native Rust crate. It now uses one time-ordered rolling buffer with typed views, so retention has exactly one owner while Aquarium/Faust still get kind-scoped queries. diff --git a/state/scratch.md b/state/scratch.md index b2fc181..510afec 100644 --- a/state/scratch.md +++ b/state/scratch.md @@ -19,5 +19,7 @@ No active scratch subgoal. compatibility, not the main local camera path. - `MimirVideoFrameDescriptor` is the hot-path video descriptor for native camera workers. Leap stereo IR should push device timestamps/handles through it. +- `IMimirVideoCaptureDriver` is the live driver seam. Future Leap/Windows + camera adapters should target that, not OpenCV. - Keep scratch short and disposable. - Promote only durable lessons into `state/evidence.jsonl` or the map. diff --git a/tests/test_mimir_app_contract.py b/tests/test_mimir_app_contract.py index 85fdc69..e7ded45 100644 --- a/tests/test_mimir_app_contract.py +++ b/tests/test_mimir_app_contract.py @@ -50,6 +50,8 @@ class MimirAppContractTests(unittest.TestCase): native_source = (ROOT / "src" / "Mimir.Runtime" / "Synchronization" / "MimirNativeIngestStreamSource.cs").read_text(encoding="utf-8") process_source = (ROOT / "src" / "Mimir.Runtime" / "Synchronization" / "MimirProcessStreamSource.cs").read_text(encoding="utf-8") frame_descriptor = (ROOT / "src" / "Mimir.Runtime" / "Synchronization" / "MimirVideoFrameDescriptor.cs").read_text(encoding="utf-8") + driver = (ROOT / "src" / "Mimir.Runtime" / "Synchronization" / "IMimirVideoCaptureDriver.cs").read_text(encoding="utf-8") + driver_source = (ROOT / "src" / "Mimir.Runtime" / "Synchronization" / "MimirVideoCaptureDriverSource.cs").read_text(encoding="utf-8") self.assertIn("payloadHandle", native_source) self.assertIn("PushVideoFrame", native_source) @@ -57,6 +59,8 @@ class MimirAppContractTests(unittest.TestCase): self.assertIn("LeapStereoIr", frame_descriptor) self.assertIn("DeviceTimestampNs", frame_descriptor) self.assertIn("NativeHandle", frame_descriptor) + self.assertIn("TryCapture", driver) + self.assertIn("MimirNativeIngestStreamSource", driver_source) self.assertIn("RedirectStandardOutput", process_source)