From 17aa027c790473be2186295f93fb2c4d27ba48e5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 22 May 2026 00:37:24 +0100 Subject: [PATCH] Route probe frame events into runtime buffers --- .gitignore | 1 + config/mimir-runtime.example.json | 54 +++- config/mimir-runtime.probes.example.json | 41 +++ docs/implementation-plan.md | 9 +- docs/native-capture-cadence.md | 11 +- .../ks_camera_cadence/ks_camera_cadence.cpp | 66 ++++- native/probes/ps3eye_cadence/CMakeLists.txt | 43 +++ .../ps3eye_cadence/ps3eye_dual_cadence.cpp | 185 ++++++++++++ notes/current-system-map.md | 6 + src/Mimir.Runtime/MimirRuntime.cs | 23 +- .../MimirFrameEventProcessStreamSource.cs | 274 ++++++++++++++++++ .../MimirRuntimeConfiguration.cs | 38 ++- .../MimirVideoFrameDescriptor.cs | 4 + state/evidence.jsonl | 1 + state/map.yaml | 12 +- 15 files changed, 742 insertions(+), 26 deletions(-) create mode 100644 config/mimir-runtime.probes.example.json create mode 100644 native/probes/ps3eye_cadence/CMakeLists.txt create mode 100644 native/probes/ps3eye_cadence/ps3eye_dual_cadence.cpp create mode 100644 src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs diff --git a/.gitignore b/.gitignore index 178c444..5e5675e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ tools/obs-spout2-plugin/ tools/src/ target/ native/**/target/ +native/**/build/ bin/ obj/ **/bin/ diff --git a/config/mimir-runtime.example.json b/config/mimir-runtime.example.json index c709466..4bd381d 100644 --- a/config/mimir-runtime.example.json +++ b/config/mimir-runtime.example.json @@ -2,7 +2,35 @@ "bufferSeconds": 5.0, "streams": [ { - "sourceId": "kiyo-primary", + "sourceId": "leap-stereo-ir", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "native" + }, + { + "sourceId": "kiyo-pro-rgb", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "native" + }, + { + "sourceId": "kiyo-rgb", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "native" + }, + { + "sourceId": "ps3-eye-0", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "native" + }, + { + "sourceId": "ps3-eye-1", "kind": "Video", "origin": "LocalDevice", "enabled": true, @@ -12,8 +40,28 @@ "sourceId": "leap-stereo-ir", "kind": "Video", "origin": "LocalDevice", - "enabled": true, - "adapter": "native" + "enabled": false, + "adapter": "frame-events", + "command": "native/probes/ks_camera_cadence/build/Release/ks_camera_cadence.exe", + "arguments": [ + "--multi-ks-baseline", + "--emit-json-frames" + ] + }, + { + "sourceId": "ps3-eye-0", + "kind": "Video", + "origin": "LocalDevice", + "enabled": false, + "adapter": "frame-events", + "command": "native/probes/ps3eye_cadence/build/Release/ps3eye_dual_cadence.exe", + "arguments": [ + "320", + "240", + "187", + "30", + "--emit-json-frames" + ] }, { "sourceId": "host-focusrite", diff --git a/config/mimir-runtime.probes.example.json b/config/mimir-runtime.probes.example.json new file mode 100644 index 0000000..3cffa2a --- /dev/null +++ b/config/mimir-runtime.probes.example.json @@ -0,0 +1,41 @@ +{ + "bufferSeconds": 5.0, + "streams": [ + { + "sourceId": "local-ks-probe", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "frame-events", + "command": "../native/probes/ks_camera_cadence/build/Release/ks_camera_cadence.exe", + "arguments": [ + "--multi-ks-baseline", + "--emit-json-frames" + ], + "acceptSourceIds": [ + "leap-stereo-ir", + "kiyo-pro-rgb", + "kiyo-rgb" + ] + }, + { + "sourceId": "local-ps3eye-probe", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "frame-events", + "command": "../native/probes/ps3eye_cadence/build/Release/ps3eye_dual_cadence.exe", + "arguments": [ + "320", + "240", + "187", + "30", + "--emit-json-frames" + ], + "acceptSourceIds": [ + "ps3-eye-0", + "ps3-eye-1" + ] + } + ] +} diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 1b272f1..b2af13c 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -29,6 +29,10 @@ a named invariant that the native runtime cannot protect yet. and video streams. - `MimirNativeIngestStreamSource` for direct push ingest into runtime buffers. - `MimirProcessStreamSource` for bridge/network command edges. +- `MimirFrameEventProcessStreamSource` for temporary JSON-line frame metadata + from native probes into the same rolling buffers Aquarium inspects. One probe + process can accept multiple emitted `sourceId` values so it does not reopen + the same camera set once per stream. - `MimirVideoFrameDescriptor` for dimensions, pixel format, stride, device timestamp, and native/GPU handle metadata. - `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` as the live @@ -48,12 +52,15 @@ a named invariant that the native runtime cannot protect yet. testing so OBS can preserve independent controls. - Process-backed stream sources are only acceptable for network bridge feeds or diagnostics. Six-camera local ingest belongs behind direct capture drivers. +- Frame-event process sources are diagnostic only. They prove source cadence and + runtime plumbing without dragging stdout bytes into the pixel hot loop. - Calibration artifacts may remain on disk as evidence, but live state must be in memory inside Mimir/Aquarium/native runtime surfaces. ## Next -1. Implement concrete direct capture drivers for Leap stereo IR first, then the +1. Replace the frame-event diagnostic bridge with concrete direct capture + drivers for Leap stereo IR first, then the other cameras. 2. Feed those drivers into `MimirVideoCaptureDriverSource` and prove sustained frame cadence in the rolling buffers. diff --git a/docs/native-capture-cadence.md b/docs/native-capture-cadence.md index a24da1c..34183e4 100644 --- a/docs/native-capture-cadence.md +++ b/docs/native-capture-cadence.md @@ -5,7 +5,9 @@ promise. ## 2026-05-21: PS3 Eye Raw USB Probe -Probe location: temp workspace only, outside this repo. +Original probe location: temp workspace only, outside this repo. The dual-camera +probe now has a repo-owned copy at `native/probes/ps3eye_cadence` and can emit +JSON frame events for Mimir's diagnostic rolling-buffer adapter. Probe path: @@ -46,6 +48,11 @@ Conclusion: the two PS3 Eyes can be treated as roughly 183 fps timing/marker witnesses at 320x240 on the current USB topology when using the raw USB path. At 640x480 they are roughly 57-59 fps. +Runtime note: `ps3eye_dual_cadence --emit-json-frames` emits one JSON line per +captured frame with `sourceId` values `ps3-eye-0` and `ps3-eye-1`; Mimir.Runtime +uses this only as a diagnostic witness until the native capture driver owns the +buffer handles directly. + The probe reported one unrelated invalid USB descriptor from `USB\VID_0000&PID_0002`; it did not prevent PS3 Eye capture. @@ -68,6 +75,8 @@ Probe path: Probe source: - `native/probes/ks_camera_cadence` +- `--multi-ks-baseline --emit-json-frames` emits JSON frame metadata for + `leap-stereo-ir`, `kiyo-pro-rgb`, and `kiyo-rgb`. Advertised modes and measured queued KS pull rates: diff --git a/native/probes/ks_camera_cadence/ks_camera_cadence.cpp b/native/probes/ks_camera_cadence/ks_camera_cadence.cpp index e6f6408..f438a3c 100644 --- a/native/probes/ks_camera_cadence/ks_camera_cadence.cpp +++ b/native/probes/ks_camera_cadence/ks_camera_cadence.cpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace { @@ -99,6 +100,7 @@ struct PinCandidate struct MultiKsProfile { const char* pathNeedle = ""; + const char* sourceId = ""; const char* label = ""; LONG width = 0; LONG height = 0; @@ -142,6 +144,14 @@ std::string lastErrorText(); std::string wideToUtf8(const wchar_t* value); std::string guidText(const GUID& guid); +std::mutex stdoutMutex; + +long long nowNs() +{ + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + constexpr GUID UsbHubInterfaceGuid = {0xf18a0e88, 0xc30c, 0x11d0, {0x88, 0x15, 0x00, 0xa0, 0xc9, 0x06, 0xbe, 0xd8}}; @@ -1223,7 +1233,13 @@ std::vector inspectPins(HANDLE filter) return candidates; } -bool measureCandidate(HANDLE filter, const PinCandidate& candidate, int queueDepth, std::atomic* startGate = nullptr) +bool measureCandidate( + HANDLE filter, + const PinCandidate& candidate, + int queueDepth, + std::atomic* startGate = nullptr, + const char* eventSourceId = nullptr, + bool emitJsonFrames = false) { const ULONG connectBytes = sizeof(KSPIN_CONNECT) + static_cast(candidate.format.size()); std::vector connectStorage(connectBytes); @@ -1366,6 +1382,26 @@ bool measureCandidate(HANDLE filter, const PinCandidate& candidate, int queueDep std::uint64_t frames = 0; std::uint64_t warmupFrames = 0; bool usingPendingReads = false; + const std::string pixelFormat = fourccOrGuid(candidate.subtype); + + auto emitFrameEvent = [&](std::uint64_t sequence, ULONG byteLength) { + if (!emitJsonFrames || eventSourceId == nullptr || eventSourceId[0] == '\0') + { + return; + } + + std::lock_guard lock(stdoutMutex); + std::printf( + "{\"type\":\"video-frame\",\"sourceId\":\"%s\",\"timestampNs\":%lld,\"sequence\":%llu,\"width\":%ld,\"height\":%ld,\"pixelFormat\":\"%s\",\"strideBytes\":%ld,\"byteLength\":%lu}\n", + eventSourceId, + nowNs(), + static_cast(sequence), + candidate.width, + candidate.height, + pixelFormat.c_str(), + candidate.imageBytes > 0 && candidate.height > 0 ? candidate.imageBytes / candidate.height : 0, + static_cast(byteLength)); + }; for (auto& slot : slots) { @@ -1383,6 +1419,7 @@ bool measureCandidate(HANDLE filter, const PinCandidate& candidate, int queueDep if (std::chrono::steady_clock::now() >= measureStart) { ++frames; + emitFrameEvent(frames, slot.header.DataUsed); } else { @@ -1407,6 +1444,7 @@ bool measureCandidate(HANDLE filter, const PinCandidate& candidate, int queueDep if (std::chrono::steady_clock::now() >= measureStart) { ++frames; + emitFrameEvent(frames, slots.front().header.DataUsed); } else { @@ -1444,6 +1482,7 @@ bool measureCandidate(HANDLE filter, const PinCandidate& candidate, int queueDep if (std::chrono::steady_clock::now() >= measureStart) { ++frames; + emitFrameEvent(frames, slot.header.DataUsed); } else { @@ -1537,12 +1576,12 @@ const PinCandidate* findCandidate( return nullptr; } -int runMultiKsBaseline() +int runMultiKsBaseline(bool emitJsonFrames = false) { const MultiKsProfile profiles[] = { - {"vid_f182&pid_0003&mi_00", "LeapUVC full stereo IR", 640, 240, "YUY2", 100.0, false}, - {"vid_1532&pid_0e05&mi_00", "Kiyo Pro RGB", 1920, 1080, "MJPG", 25.0, false}, - {"vid_1532&pid_0e03&mi_00", "Kiyo RGB", 1920, 1080, "MJPG", 25.0, true}, + {"vid_f182&pid_0003&mi_00", "leap-stereo-ir", "LeapUVC full stereo IR", 640, 240, "YUY2", 100.0, false}, + {"vid_1532&pid_0e05&mi_00", "kiyo-pro-rgb", "Kiyo Pro RGB", 1920, 1080, "MJPG", 25.0, false}, + {"vid_1532&pid_0e03&mi_00", "kiyo-rgb", "Kiyo RGB", 1920, 1080, "MJPG", 25.0, true}, }; struct Worker @@ -1582,7 +1621,7 @@ int runMultiKsBaseline() std::atomic startGate(false); for (auto& worker : workers) { - worker.thread = std::thread([&startGate, &worker]() { + worker.thread = std::thread([&startGate, &worker, emitJsonFrames]() { std::printf("multi-ks preparing: %s path=%s\n", worker.profile.label, worker.path.c_str()); Handle filter(CreateFileA( worker.path.c_str(), @@ -1631,7 +1670,13 @@ int runMultiKsBaseline() candidate->height, fourccOrGuid(candidate->subtype).c_str(), candidate->interval100ns > 0 ? 10000000.0 / static_cast(candidate->interval100ns) : 0.0); - worker.measured = measureCandidate(filter.value, *candidate, 8, &startGate); + worker.measured = measureCandidate( + filter.value, + *candidate, + 8, + &startGate, + worker.profile.sourceId, + emitJsonFrames); }); } @@ -1677,7 +1722,12 @@ int main(int argc, char** argv) } if (targetVid == "--multi-ks-baseline") { - return runMultiKsBaseline(); + bool emitJsonFrames = false; + for (int arg = 2; arg < argc; ++arg) + { + emitJsonFrames = emitJsonFrames || std::string(argv[arg]) == "--emit-json-frames"; + } + return runMultiKsBaseline(emitJsonFrames); } bool controlsOnly = false; diff --git a/native/probes/ps3eye_cadence/CMakeLists.txt b/native/probes/ps3eye_cadence/CMakeLists.txt new file mode 100644 index 0000000..e4a2c3d --- /dev/null +++ b/native/probes/ps3eye_cadence/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.20) + +project(ps3eye_cadence LANGUAGES C CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(PSMOVEAPI_ROOT "$ENV{TEMP}/mimir-driver-probes/psmoveapi" CACHE PATH "Root of a psmoveapi checkout with external/libusb-1.0 and external/PS3EYEDriver.") +set(LIBUSB_ROOT "${PSMOVEAPI_ROOT}/external/libusb-1.0" CACHE PATH "libusb source root.") +set(PS3EYE_ROOT "${PSMOVEAPI_ROOT}/external/PS3EYEDriver" CACHE PATH "PS3EYEDriver source root.") + +add_library(libusb_static STATIC + ${LIBUSB_ROOT}/libusb/core.c + ${LIBUSB_ROOT}/libusb/descriptor.c + ${LIBUSB_ROOT}/libusb/hotplug.c + ${LIBUSB_ROOT}/libusb/io.c + ${LIBUSB_ROOT}/libusb/strerror.c + ${LIBUSB_ROOT}/libusb/sync.c + ${LIBUSB_ROOT}/libusb/os/events_windows.c + ${LIBUSB_ROOT}/libusb/os/threads_windows.c + ${LIBUSB_ROOT}/libusb/os/windows_common.c + ${LIBUSB_ROOT}/libusb/os/windows_usbdk.c + ${LIBUSB_ROOT}/libusb/os/windows_winusb.c) +target_include_directories(libusb_static PUBLIC + ${LIBUSB_ROOT}/libusb + ${LIBUSB_ROOT}/libusb/os + ${LIBUSB_ROOT}/msvc) +target_compile_definitions(libusb_static PRIVATE + _CRT_SECURE_NO_WARNINGS + _WIN32_WINNT=0x0601) +target_link_libraries(libusb_static PRIVATE setupapi ole32 advapi32) + +add_library(ps3eye STATIC + ${PS3EYE_ROOT}/src/ps3eye.cpp + ${PS3EYE_ROOT}/src/ps3eye_capi.cpp) +target_include_directories(ps3eye PUBLIC + ${PS3EYE_ROOT}/src + ${LIBUSB_ROOT}/libusb) +target_link_libraries(ps3eye PUBLIC libusb_static) + +add_executable(ps3eye_dual_cadence ps3eye_dual_cadence.cpp) +target_include_directories(ps3eye_dual_cadence PRIVATE ${PS3EYE_ROOT}/src) +target_link_libraries(ps3eye_dual_cadence PRIVATE ps3eye) diff --git a/native/probes/ps3eye_cadence/ps3eye_dual_cadence.cpp b/native/probes/ps3eye_cadence/ps3eye_dual_cadence.cpp new file mode 100644 index 0000000..19b947d --- /dev/null +++ b/native/probes/ps3eye_cadence/ps3eye_dual_cadence.cpp @@ -0,0 +1,185 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ps3eye_capi.h" + +namespace +{ +long long nowNs() +{ + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +bool hasArg(int argc, char** argv, const char* value) +{ + for (int index = 1; index < argc; ++index) + { + if (std::strcmp(argv[index], value) == 0) + { + return true; + } + } + + return false; +} + +struct Stats +{ + int camera = 0; + int frames = 0; + long long minDelta = INT64_MAX; + long long maxDelta = 0; + long long sumDelta = 0; + int deltas = 0; + bool opened = false; +}; +} + +int main(int argc, char** argv) +{ + std::setvbuf(stdout, nullptr, _IONBF, 0); + + const int width = argc > 1 ? std::atoi(argv[1]) : 320; + const int height = argc > 2 ? std::atoi(argv[2]) : 240; + const int fps = argc > 3 ? std::atoi(argv[3]) : 187; + const double seconds = argc > 4 ? std::atof(argv[4]) : 5.0; + const bool emitJsonFrames = hasArg(argc, argv, "--emit-json-frames"); + + ps3eye_init(); + const int count = ps3eye_count_connected(); + std::printf("ps3eye connected=%d\n", count); + if (count < 1) + { + ps3eye_uninit(); + return 2; + } + + const int cameras = count < 2 ? count : 2; + std::vector eyes(cameras, nullptr); + for (int index = 0; index < cameras; ++index) + { + eyes[index] = ps3eye_open(index, width, height, fps, PS3EYE_FORMAT_BAYER); + if (!eyes[index]) + { + std::fprintf(stderr, "ps3eye_open failed camera=%d mode=%dx%d@%d\n", index, width, height, fps); + } + } + + std::atomic start{false}; + std::vector stats(cameras); + std::vector threads; + std::mutex stdoutMutex; + long long deadline = 0; + + for (int index = 0; index < cameras; ++index) + { + stats[index].camera = index; + if (!eyes[index]) + { + continue; + } + + stats[index].opened = true; + threads.emplace_back([&, index]() { + std::vector frame(static_cast(width) * static_cast(height)); + while (!start.load(std::memory_order_acquire)) + { + std::this_thread::yield(); + } + + long long last = 0; + while (nowNs() < deadline) + { + ps3eye_grab_frame(eyes[index], frame.data()); + const long long timestamp = nowNs(); + if (last != 0) + { + const long long delta = timestamp - last; + if (delta < stats[index].minDelta) + { + stats[index].minDelta = delta; + } + if (delta > stats[index].maxDelta) + { + stats[index].maxDelta = delta; + } + stats[index].sumDelta += delta; + ++stats[index].deltas; + } + + last = timestamp; + ++stats[index].frames; + + if (emitJsonFrames) + { + std::lock_guard lock(stdoutMutex); + std::printf( + "{\"type\":\"video-frame\",\"sourceId\":\"ps3-eye-%d\",\"timestampNs\":%lld,\"sequence\":%d,\"width\":%d,\"height\":%d,\"pixelFormat\":\"Bayer8\",\"strideBytes\":%d,\"byteLength\":%zu}\n", + index, + timestamp, + stats[index].frames, + width, + height, + width, + frame.size()); + } + } + }); + } + + const long long begin = nowNs(); + deadline = begin + static_cast(seconds * 1000000000.0); + start.store(true, std::memory_order_release); + for (auto& thread : threads) + { + thread.join(); + } + + const long long end = nowNs(); + const double elapsed = (end - begin) / 1000000000.0; + + for (int index = 0; index < cameras; ++index) + { + if (!stats[index].opened) + { + std::printf("camera=%d open=false\n", index); + continue; + } + + const double delivered = stats[index].frames / elapsed; + const double avgDeltaMs = stats[index].deltas + ? (stats[index].sumDelta / static_cast(stats[index].deltas)) / 1000000.0 + : 0.0; + std::printf( + "camera=%d requested=%dx%d@%d elapsed=%.3f frames=%d delivered_fps=%.2f avg_delta_ms=%.3f min_delta_ms=%.3f max_delta_ms=%.3f\n", + index, + width, + height, + fps, + elapsed, + stats[index].frames, + delivered, + avgDeltaMs, + stats[index].minDelta == INT64_MAX ? 0.0 : stats[index].minDelta / 1000000.0, + stats[index].maxDelta / 1000000.0); + } + + for (auto* eye : eyes) + { + if (eye) + { + ps3eye_close(eye); + } + } + + ps3eye_uninit(); + return 0; +} diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 76750ca..591145a 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -54,12 +54,18 @@ separately controllable audio stems. - `IMimirStreamSource`; - `MimirNativeIngestStreamSource`; - `MimirProcessStreamSource`; +- `MimirFrameEventProcessStreamSource`; - `MimirVideoFrameDescriptor`; - `IMimirVideoCaptureDriver`; - `MimirVideoCaptureDriverSource`. Process-backed sources are bridge/network edges. Local cameras should feed native descriptors with device timestamps and optional native/GPU handles. +The `frame-events`/`json-lines` adapter is a diagnostic witness only: native +probes can emit per-frame JSON metadata so Aquarium sees real sensor cadence in +the rolling buffers while the direct ABI driver is being cut. It does not carry +pixels and does not own the final six-camera hot path. Multi-camera probes are +one process with declared accepted source ids, not one process per camera. ## OBS Bridge Utility diff --git a/src/Mimir.Runtime/MimirRuntime.cs b/src/Mimir.Runtime/MimirRuntime.cs index b5cb4c2..9ebae2f 100644 --- a/src/Mimir.Runtime/MimirRuntime.cs +++ b/src/Mimir.Runtime/MimirRuntime.cs @@ -108,12 +108,23 @@ public sealed class MimirRuntime : IAquariumRuntime panel.Readout("Sources", () => $"{synchronization.SourceCount}"); panel.Readout("Last poll", () => $"{lastPollCount} samples"); panel.Readout("Ingested", () => $"{synchronization.IngestedSamples}"); - foreach (var buffer in synchronization.Buffers.Buffers) - { - panel.Readout( - buffer.Descriptor.SourceId, - () => $"{buffer.Descriptor.Kind}/{buffer.Descriptor.Origin} {buffer.Count} samples edge {buffer.EdgeNs}"); - } + panel.Readout("Buffer details", DescribeBuffers); }); } + + private string DescribeBuffers() + { + return string.Join(" | ", synchronization.Buffers.Buffers.Select(DescribeBuffer)); + } + + private static string DescribeBuffer(MimirRollingStreamBuffer buffer) + { + var latest = buffer.Latest; + if (latest?.VideoFrame is { } frame) + { + return $"{buffer.Descriptor.SourceId}: {buffer.Count} {frame.Width}x{frame.Height} {frame.PixelFormat} bytes {latest.Value.ByteLength} edge {buffer.EdgeNs}"; + } + + return $"{buffer.Descriptor.SourceId}: {buffer.Count} edge {buffer.EdgeNs}"; + } } diff --git a/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs b/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs new file mode 100644 index 0000000..7e74a2f --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs @@ -0,0 +1,274 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Mimir.Runtime.Synchronization; + +public sealed record MimirFrameEventProcessStreamSourceOptions( + string Command, + IReadOnlyList Arguments, + IReadOnlySet? AcceptedSourceIds = null); + +public sealed class MimirFrameEventProcessStreamSource : IMimirStreamSource +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter() }, + }; + + private readonly ConcurrentQueue samples = new(); + private readonly CancellationTokenSource cancellation = new(); + private readonly Process process; + private readonly Task stdoutTask; + private readonly Task stderrTask; + private ulong fallbackSequence; + private bool disposed; + + public MimirFrameEventProcessStreamSource( + MimirStreamDescriptor descriptor, + MimirFrameEventProcessStreamSourceOptions options) + { + Descriptor = descriptor; + Options = options; + process = StartProcess(options); + stdoutTask = Task.Run(() => ReadStdout(cancellation.Token)); + stderrTask = Task.Run(() => DrainStderr(cancellation.Token)); + } + + public MimirStreamDescriptor Descriptor { get; } + + public MimirFrameEventProcessStreamSourceOptions Options { get; } + + public bool TryRead(out MimirStreamSample sample) + { + return samples.TryDequeue(out sample); + } + + private static Process StartProcess(MimirFrameEventProcessStreamSourceOptions options) + { + var startInfo = new ProcessStartInfo(options.Command) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + + foreach (var argument in options.Arguments) + { + startInfo.ArgumentList.Add(argument); + } + + return Process.Start(startInfo) + ?? throw new InvalidOperationException($"Could not start frame-event source process: {options.Command}"); + } + + private async Task ReadStdout(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + var line = await process.StandardOutput.ReadLineAsync(token).ConfigureAwait(false); + if (line == null) + { + break; + } + + if (!line.TrimStart().StartsWith('{')) + { + continue; + } + + if (TryParseFrameEvent(line, out var sample)) + { + samples.Enqueue(sample); + } + } + } + + private bool TryParseFrameEvent(string line, out MimirStreamSample sample) + { + sample = default; + + MimirFrameEvent? frameEvent; + try + { + frameEvent = JsonSerializer.Deserialize(line, JsonOptions); + } + catch (JsonException) + { + return false; + } + + if (frameEvent == null || !frameEvent.IsVideoFrame) + { + return false; + } + + var sourceId = string.IsNullOrWhiteSpace(frameEvent.SourceId) + ? Descriptor.SourceId + : frameEvent.SourceId; + if (!AcceptsSourceId(sourceId)) + { + return false; + } + + var timestampNs = frameEvent.TimestampNs > 0 + ? frameEvent.TimestampNs + : StopwatchTicksToNs(Stopwatch.GetTimestamp()); + var arrivalNs = StopwatchTicksToNs(Stopwatch.GetTimestamp()); + var byteLength = Math.Max(0, frameEvent.ByteLength); + var strideBytes = frameEvent.StrideBytes > 0 + ? frameEvent.StrideBytes + : InferStride(frameEvent.Width, frameEvent.PixelFormat, byteLength, frameEvent.Height); + var sequence = frameEvent.Sequence ?? fallbackSequence++; + var payloadHandle = frameEvent.NativeHandle; + var videoFrame = new MimirVideoFrameDescriptor( + frameEvent.Width, + frameEvent.Height, + ParsePixelFormat(frameEvent.PixelFormat), + strideBytes, + timestampNs, + payloadHandle, + frameEvent.NativeHandleKind ?? ""); + + sample = new MimirStreamSample( + sourceId, + Descriptor.Kind, + Descriptor.Origin, + timestampNs, + arrivalNs, + sequence, + payloadHandle, + byteLength, + default, + videoFrame); + return true; + } + + private bool AcceptsSourceId(string sourceId) + { + if (Options.AcceptedSourceIds is { Count: > 0 } acceptedSourceIds) + { + return acceptedSourceIds.Contains(sourceId); + } + + return string.Equals(sourceId, Descriptor.SourceId, StringComparison.Ordinal); + } + + private async Task DrainStderr(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + var line = await process.StandardError.ReadLineAsync(token).ConfigureAwait(false); + if (line == null) + { + break; + } + } + } + + private static MimirVideoPixelFormat ParsePixelFormat(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return MimirVideoPixelFormat.Unknown; + } + + return value.Trim().ToUpperInvariant() switch + { + "BAYER" or "BAYER8" => MimirVideoPixelFormat.Bayer8, + "GRAY8" or "Y8" => MimirVideoPixelFormat.Gray8, + "R8" => MimirVideoPixelFormat.R8, + "RG8" => MimirVideoPixelFormat.Rg8, + "YUY2" => MimirVideoPixelFormat.Yuy2, + "MJPG" or "MJPEG" => MimirVideoPixelFormat.Mjpg, + "H264" => MimirVideoPixelFormat.H264, + "NV12" => MimirVideoPixelFormat.Nv12, + "BGRA8" => MimirVideoPixelFormat.Bgra8, + "LEAP_STEREO_IR" or "LEAPSTEREOIR" => MimirVideoPixelFormat.LeapStereoIr, + _ => MimirVideoPixelFormat.Unknown, + }; + } + + private static int InferStride(int width, string? pixelFormat, int byteLength, int height) + { + if (width <= 0) + { + return 0; + } + + if (height > 0 && byteLength > 0 && byteLength % height == 0) + { + return byteLength / height; + } + + return (pixelFormat ?? "").Trim().ToUpperInvariant() switch + { + "BAYER" or "BAYER8" or "GRAY8" or "Y8" or "R8" => width, + "RG8" or "YUY2" or "LEAP_STEREO_IR" or "LEAPSTEREOIR" => checked(width * 2), + "BGRA8" => checked(width * 4), + _ => 0, + }; + } + + private static long StopwatchTicksToNs(long ticks) + { + return checked((long)(ticks * (1_000_000_000.0 / Stopwatch.Frequency))); + } + + public void Dispose() + { + if (disposed) + { + return; + } + + disposed = true; + cancellation.Cancel(); + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch + { + } + + process.Dispose(); + cancellation.Dispose(); + } + + private sealed class MimirFrameEvent + { + public string Type { get; set; } = ""; + + public string SourceId { get; set; } = ""; + + public long TimestampNs { get; set; } + + public ulong? Sequence { get; set; } + + public int Width { get; set; } + + public int Height { get; set; } + + public string PixelFormat { get; set; } = ""; + + public int StrideBytes { get; set; } + + public int ByteLength { get; set; } + + public ulong NativeHandle { get; set; } + + public string? NativeHandleKind { get; set; } + + public bool IsVideoFrame => + string.Equals(Type, "video-frame", StringComparison.OrdinalIgnoreCase) && + Width > 0 && + Height > 0; + } +} diff --git a/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs b/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs index 3e7b0f6..cb24cb2 100644 --- a/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs +++ b/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs @@ -37,7 +37,7 @@ public sealed class MimirRuntimeConfiguration .Select(ToDescriptor) .ToArray(); var sources = sourceModels - .Select(TryCreateSource) + .Select(stream => TryCreateSource(stream, Path.GetDirectoryName(path))) .Where(source => source != null) .Cast() .ToArray(); @@ -91,13 +91,31 @@ public sealed class MimirRuntimeConfiguration stream.Enabled); } - private static IMimirStreamSource? TryCreateSource(MimirStreamConfig stream) + private static IMimirStreamSource? TryCreateSource(MimirStreamConfig stream, string? configDirectory) { if (string.Equals(stream.Adapter, "native", StringComparison.OrdinalIgnoreCase)) { return new MimirNativeIngestStreamSource(ToDescriptor(stream)); } + if (string.Equals(stream.Adapter, "frame-events", StringComparison.OrdinalIgnoreCase) + || string.Equals(stream.Adapter, "json-lines", StringComparison.OrdinalIgnoreCase)) + { + if (string.IsNullOrWhiteSpace(stream.Command)) + { + return null; + } + + return new MimirFrameEventProcessStreamSource( + ToDescriptor(stream), + new MimirFrameEventProcessStreamSourceOptions( + ResolveCommand(stream.Command, configDirectory), + stream.Arguments, + stream.AcceptSourceIds.Length > 0 + ? new HashSet(stream.AcceptSourceIds, StringComparer.Ordinal) + : null)); + } + if (!string.Equals(stream.Adapter, "process", StringComparison.OrdinalIgnoreCase)) { return null; @@ -111,11 +129,23 @@ public sealed class MimirRuntimeConfiguration return new MimirProcessStreamSource( ToDescriptor(stream), new MimirProcessStreamSourceOptions( - stream.Command, + ResolveCommand(stream.Command, configDirectory), stream.Arguments, Math.Max(1024, stream.ChunkBytes))); } + private static string ResolveCommand(string command, string? configDirectory) + { + if (string.IsNullOrWhiteSpace(command) + || Path.IsPathRooted(command) + || string.IsNullOrWhiteSpace(configDirectory)) + { + return command; + } + + return Path.GetFullPath(Path.Combine(configDirectory, command)); + } + private static MimirStreamKind ParseKind(string value) { return Enum.TryParse(value, ignoreCase: true, out var kind) @@ -154,5 +184,7 @@ public sealed class MimirStreamConfig public string[] Arguments { get; set; } = []; + public string[] AcceptSourceIds { get; set; } = []; + public int ChunkBytes { get; set; } = 65_536; } diff --git a/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs b/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs index 735dbc2..0726dfb 100644 --- a/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs +++ b/src/Mimir.Runtime/Synchronization/MimirVideoFrameDescriptor.cs @@ -6,6 +6,10 @@ public enum MimirVideoPixelFormat Gray8, R8, Rg8, + Bayer8, + Yuy2, + Mjpg, + H264, Bgra8, Nv12, LeapStereoIr, diff --git a/state/evidence.jsonl b/state/evidence.jsonl index e4393dd..3db6d2b 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -23,3 +23,4 @@ {"ts":"2026-05-22T02:55:00+00:00","type":"verification","status":"ok","note":"Moved Kiyo Pro back to the Mimir host. Local enumeration showed LeapUVC, Kiyo Pro, and two PS3 Eyes; regular Kiyo was not local. Kiyo Pro remained on root_hub30#7&18eea21b port 8 at UsbHighSpeed(2), bcdUSB 0x0320. With both PS3 Eyes at 320x240@187: Leap 110.80 fps, Kiyo Pro 24.97 fps, PS3 Eye 0 187.46 fps, PS3 Eye 1 186.89 fps. With both PS3 Eyes at 640x480@60: Leap 110.72 fps, Kiyo Pro 24.87 fps, PS3 Eye 0 59.91 fps, PS3 Eye 1 59.19 fps. Kiyo Pro does not poison Leap cadence in the current layout; it remains a ~25 fps high-speed USB RGB source."} {"ts":"2026-05-22T03:15:00+00:00","type":"verification","status":"ok","note":"Moved regular Kiyo from front panel to rear port and mouse into Leap USB neighborhood. All five camera devices / six optical sensors were visible locally: LeapUVC stereo, Kiyo Pro, regular Kiyo, and two PS3 Eyes. With both PS3 Eyes at 320x240@187: Leap 110.84 fps, Kiyo Pro 25.08 fps, Kiyo 30.18 fps, PS3 Eye 0 187.59 fps, PS3 Eye 1 187.02 fps. With both PS3 Eyes at 640x480@60: Leap 110.76 fps, Kiyo Pro 25.14 fps, Kiyo 30.18 fps, PS3 Eye 0 59.66 fps, PS3 Eye 1 59.95 fps. The full intended local optical set is viable; Kiyo Pro remains cadence-limited by high-speed USB negotiation."} {"ts":"2026-05-22T03:35:00+00:00","type":"verification","status":"ok","note":"Rechecked the final full local topology after the regular Kiyo was moved to a rear port and a mouse moved into the Leap USB neighborhood. All five camera devices / six optical sensors remained visible and cadence stayed healthy. High-rate Eye run: Leap 110.84 fps, Kiyo Pro 25.08 fps, regular Kiyo 30.18 fps, PS3 Eye 0 187.59 fps, PS3 Eye 1 187.02 fps. Full-frame Eye run: Leap 110.76 fps, Kiyo Pro 25.14 fps, regular Kiyo 30.18 fps, PS3 Eye 0 59.66 fps, PS3 Eye 1 59.95 fps. Mouse traffic does not reproduce the Focusrite-induced Leap collapse."} +{"ts":"2026-05-22T04:10:00+00:00","type":"implementation","status":"ok","note":"Added a diagnostic frame-event path from native camera probes into Mimir.Runtime rolling buffers. MimirFrameEventProcessStreamSource reads JSON-line video-frame metadata into MimirVideoFrameDescriptor samples; KS and repo-owned PS3 Eye probes can emit those frame events. This is a playground witness for Aquarium UI/cadence, not the final pixel hot path."} diff --git a/state/map.yaml b/state/map.yaml index a95bd6d..26396ce 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -53,6 +53,9 @@ accepted_design: direct native ingest, process bridge ingest, and video capture driver seams. - `MimirVideoFrameDescriptor` carries width, height, pixel format, stride, device timestamp, and optional native/GPU handle. + - `MimirFrameEventProcessStreamSource` is a temporary diagnostic adapter for + JSON-line frame metadata from native probes into runtime rolling buffers; it + does not carry pixels or own the final local-camera hot path. - `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the live seam for Leap, Media Foundation, DirectShow, libusb, LeapC, or shared texture capture. @@ -149,8 +152,9 @@ current_status: Kiyo 30.18 fps, PS3 Eye 0 59.66 fps, and PS3 Eye 1 59.95 fps. A mouse in the Leap USB neighborhood does not reproduce the Focusrite-induced collapse. next_action: > - Move the raw PS3 Eye probe into the repo or replace it with the first - runtime capture driver using the current viable local USB topology. Keep - the Focusrite away from the Leap USB neighborhood, preserve the full local - sensor baseline, and treat Kiyo Pro as a ~25 fps source until its + Run Mimir with frame-event probe sources enabled so Aquarium can inspect + live local sensor cadence in the rolling buffers, then replace the + diagnostic JSON-line process bridge with direct native capture drivers that + push buffer handles into Mimir.Runtime. Keep the Focusrite away from the + Leap USB neighborhood and treat Kiyo Pro as a ~25 fps source until its SuperSpeed negotiation is fixed.