diff --git a/Mimir.slnx b/Mimir.slnx index 7746f6b..84ccf97 100644 --- a/Mimir.slnx +++ b/Mimir.slnx @@ -2,5 +2,6 @@ + diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index b2af13c..87edbdd 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -33,6 +33,9 @@ a named invariant that the native runtime cannot protect yet. from native probes into the same rolling buffers Aquarium inspects. One probe process can accept multiple emitted `sourceId` values so it does not reopen the same camera set once per stream. +- `src/Mimir.BufferSmoke` loads the runtime config, polls the synchronization + hub, and prints the actual rolling buffers. Use `--require-samples` when an + empty declared sensor buffer should fail the run. - `MimirVideoFrameDescriptor` for dimensions, pixel format, stride, device timestamp, and native/GPU handle metadata. - `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` as the live @@ -59,18 +62,21 @@ a named invariant that the native runtime cannot protect yet. ## Next -1. Replace the frame-event diagnostic bridge with concrete direct capture +1. Recover the Kiyo Pro KS acquire path, which currently enumerates but leaves + its runtime smoke buffer empty with `KSSTATE_ACQUIRE failed: The handle is + invalid`. +2. Replace the frame-event diagnostic bridge with concrete direct capture drivers for Leap stereo IR first, then the other cameras. -2. Feed those drivers into `MimirVideoCaptureDriverSource` and prove sustained +3. Feed those drivers into `MimirVideoCaptureDriverSource` and prove sustained frame cadence in the rolling buffers. -3. Add native audio capture workers for local mic, loopback, and network audio +4. Add native audio capture workers for local mic, loopback, and network audio feeds. -4. Bind Aquarium UI to the synchronization hub so buffer depth, stream cadence, +5. Bind Aquarium UI to the synchronization hub so buffer depth, stream cadence, source timestamps, and output settings are visible and adjustable. -5. Move GPU feature extraction, fusion, material fitting, render budgeting, and +6. Move GPU feature extraction, fusion, material fitting, render budgeting, and Spout2 publication into Aquarium. -6. Move mic alignment, room suppression, voice separation, spatialization, and +7. Move mic alignment, room suppression, voice separation, spatialization, and stem generation into Faust/native DSP. -7. Keep the OBS bridge witness ledger as evidence before expanding receiver +8. Keep the OBS bridge witness ledger as evidence before expanding receiver machinery. diff --git a/src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj b/src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj new file mode 100644 index 0000000..a72fb6e --- /dev/null +++ b/src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj @@ -0,0 +1,16 @@ + + + + Exe + net10.0 + Mimir.BufferSmoke + Mimir.BufferSmoke + enable + enable + + + + + + + diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs new file mode 100644 index 0000000..b187372 --- /dev/null +++ b/src/Mimir.BufferSmoke/Program.cs @@ -0,0 +1,92 @@ +using Mimir.Runtime.Synchronization; + +var duration = TimeSpan.FromSeconds(ParseDoubleOption(args, "--seconds", 10.0)); +var pollDelay = TimeSpan.FromMilliseconds(ParseDoubleOption(args, "--poll-ms", 10.0)); +var requireSamples = args.Any(arg => string.Equals(arg, "--require-samples", StringComparison.OrdinalIgnoreCase)); +using var configuration = new DisposableConfiguration(MimirRuntimeConfiguration.Load()); +using var hub = new MimirSynchronizationHub(configuration.Value.Settings); + +foreach (var source in configuration.Value.Sources) +{ + configuration.Detach(source); + hub.AddSource(source); +} + +var deadline = DateTimeOffset.UtcNow + duration; +while (DateTimeOffset.UtcNow < deadline) +{ + hub.PollSources(maxSamplesPerSource: 4096); + await Task.Delay(pollDelay).ConfigureAwait(false); +} + +hub.PollSources(maxSamplesPerSource: 16384); + +Console.WriteLine($"sources={hub.SourceCount} ingested={hub.IngestedSamples} buffers={hub.Buffers.Buffers.Count}"); +var emptyBuffers = new List(); +foreach (var buffer in hub.Buffers.Buffers.OrderBy(buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal)) +{ + var latest = buffer.Latest; + if (latest?.VideoFrame is { } frame) + { + Console.WriteLine( + $"{buffer.Descriptor.SourceId}: count={buffer.Count} edgeNs={buffer.EdgeNs} latest={frame.Width}x{frame.Height} {frame.PixelFormat} bytes={latest.Value.ByteLength}"); + } + else + { + Console.WriteLine($"{buffer.Descriptor.SourceId}: count={buffer.Count} edgeNs={buffer.EdgeNs}"); + } + + if (buffer.Count == 0) + { + emptyBuffers.Add(buffer.Descriptor.SourceId); + } +} + +if (requireSamples && emptyBuffers.Count > 0) +{ + Console.Error.WriteLine($"empty buffers: {string.Join(", ", emptyBuffers)}"); + return 1; +} + +return 0; + +static double ParseDoubleOption(IReadOnlyList args, string name, double fallback) +{ + for (var index = 0; index < args.Count - 1; index++) + { + if (string.Equals(args[index], name, StringComparison.OrdinalIgnoreCase) + && double.TryParse(args[index + 1], out var parsed) + && parsed > 0) + { + return parsed; + } + } + + return fallback; +} + +internal sealed class DisposableConfiguration : IDisposable +{ + private readonly List ownedSources; + + public DisposableConfiguration(MimirRuntimeConfiguration value) + { + Value = value; + ownedSources = value.Sources.ToList(); + } + + public MimirRuntimeConfiguration Value { get; } + + public void Detach(IMimirStreamSource source) + { + ownedSources.Remove(source); + } + + public void Dispose() + { + foreach (var source in ownedSources) + { + source.Dispose(); + } + } +} diff --git a/src/Mimir.Runtime/Synchronization/IMimirStreamSource.cs b/src/Mimir.Runtime/Synchronization/IMimirStreamSource.cs index ea581af..c06d392 100644 --- a/src/Mimir.Runtime/Synchronization/IMimirStreamSource.cs +++ b/src/Mimir.Runtime/Synchronization/IMimirStreamSource.cs @@ -4,5 +4,7 @@ public interface IMimirStreamSource : IDisposable { MimirStreamDescriptor Descriptor { get; } + bool ExposesDescriptorBuffer => true; + bool TryRead(out MimirStreamSample sample); } diff --git a/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs b/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs index 7e74a2f..fa43848 100644 --- a/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs +++ b/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs @@ -41,6 +41,8 @@ public sealed class MimirFrameEventProcessStreamSource : IMimirStreamSource public MimirFrameEventProcessStreamSourceOptions Options { get; } + public bool ExposesDescriptorBuffer => Options.AcceptedSourceIds is not { Count: > 0 }; + public bool TryRead(out MimirStreamSample sample) { return samples.TryDequeue(out sample); diff --git a/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs b/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs index cb24cb2..0c12081 100644 --- a/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs +++ b/src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs @@ -34,7 +34,7 @@ public sealed class MimirRuntimeConfiguration .Where(stream => stream.Enabled) .ToArray(); var streams = sourceModels - .Select(ToDescriptor) + .SelectMany(ToDescriptors) .ToArray(); var sources = sourceModels .Select(stream => TryCreateSource(stream, Path.GetDirectoryName(path))) @@ -91,6 +91,24 @@ public sealed class MimirRuntimeConfiguration stream.Enabled); } + private static IEnumerable ToDescriptors(MimirStreamConfig stream) + { + if (stream.AcceptSourceIds.Length == 0) + { + yield return ToDescriptor(stream); + yield break; + } + + foreach (var sourceId in stream.AcceptSourceIds.Where(sourceId => !string.IsNullOrWhiteSpace(sourceId))) + { + yield return new MimirStreamDescriptor( + sourceId, + ParseKind(stream.Kind), + ParseOrigin(stream.Origin), + stream.Enabled); + } + } + private static IMimirStreamSource? TryCreateSource(MimirStreamConfig stream, string? configDirectory) { if (string.Equals(stream.Adapter, "native", StringComparison.OrdinalIgnoreCase)) diff --git a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs index 05fde74..fad9d88 100644 --- a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs +++ b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs @@ -27,7 +27,10 @@ public sealed class MimirSynchronizationHub : IDisposable { ObjectDisposedException.ThrowIf(disposed, this); sources.Add(source); - Buffers.EnsureBuffer(source.Descriptor); + if (source.ExposesDescriptorBuffer) + { + Buffers.EnsureBuffer(source.Descriptor); + } } public int PollSources(int maxSamplesPerSource = 256) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 3db6d2b..5906a66 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -24,3 +24,4 @@ {"ts":"2026-05-22T03:15:00+00:00","type":"verification","status":"ok","note":"Moved regular Kiyo from front panel to rear port and mouse into Leap USB neighborhood. All five camera devices / six optical sensors were visible locally: LeapUVC stereo, Kiyo Pro, regular Kiyo, and two PS3 Eyes. With both PS3 Eyes at 320x240@187: Leap 110.84 fps, Kiyo Pro 25.08 fps, Kiyo 30.18 fps, PS3 Eye 0 187.59 fps, PS3 Eye 1 187.02 fps. With both PS3 Eyes at 640x480@60: Leap 110.76 fps, Kiyo Pro 25.14 fps, Kiyo 30.18 fps, PS3 Eye 0 59.66 fps, PS3 Eye 1 59.95 fps. The full intended local optical set is viable; Kiyo Pro remains cadence-limited by high-speed USB negotiation."} {"ts":"2026-05-22T03:35:00+00:00","type":"verification","status":"ok","note":"Rechecked the final full local topology after the regular Kiyo was moved to a rear port and a mouse moved into the Leap USB neighborhood. All five camera devices / six optical sensors remained visible and cadence stayed healthy. High-rate Eye run: Leap 110.84 fps, Kiyo Pro 25.08 fps, regular Kiyo 30.18 fps, PS3 Eye 0 187.59 fps, PS3 Eye 1 187.02 fps. Full-frame Eye run: Leap 110.76 fps, Kiyo Pro 25.14 fps, regular Kiyo 30.18 fps, PS3 Eye 0 59.66 fps, PS3 Eye 1 59.95 fps. Mouse traffic does not reproduce the Focusrite-induced Leap collapse."} {"ts":"2026-05-22T04:10:00+00:00","type":"implementation","status":"ok","note":"Added a diagnostic frame-event path from native camera probes into Mimir.Runtime rolling buffers. MimirFrameEventProcessStreamSource reads JSON-line video-frame metadata into MimirVideoFrameDescriptor samples; KS and repo-owned PS3 Eye probes can emit those frame events. This is a playground witness for Aquarium UI/cadence, not the final pixel hot path."} +{"ts":"2026-05-22T04:45:00+00:00","type":"verification","status":"partial","note":"Mimir.BufferSmoke loaded the probe runtime config, started the KS and PS3 Eye frame-event probes, polled MimirSynchronizationHub, and confirmed live samples in rolling buffers for leap-stereo-ir, kiyo-rgb, ps3-eye-0, and ps3-eye-1. Kiyo Pro is declared but empty: the KS path enumerates the device and modes, then fails at KSSTATE_ACQUIRE with ERROR_INVALID_HANDLE. PnP camera interface reports OK, composite reports OK, media interface reports Error, and non-elevated pnputil restart is denied."} diff --git a/state/map.yaml b/state/map.yaml index 26396ce..5111b75 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -152,8 +152,11 @@ current_status: Kiyo 30.18 fps, PS3 Eye 0 59.66 fps, and PS3 Eye 1 59.95 fps. A mouse in the Leap USB neighborhood does not reproduce the Focusrite-induced collapse. next_action: > - Run Mimir with frame-event probe sources enabled so Aquarium can inspect - live local sensor cadence in the rolling buffers, then replace the + Recover or replug the Kiyo Pro KS acquire path, then rerun + Mimir.BufferSmoke with `--require-samples` until all declared local optical + buffers are non-empty. Current smoke proves live rolling-buffer samples for + Leap stereo IR, regular Kiyo, and both PS3 Eyes; Kiyo Pro still enumerates + but fails KSSTATE_ACQUIRE with ERROR_INVALID_HANDLE. After that, replace the diagnostic JSON-line process bridge with direct native capture drivers that push buffer handles into Mimir.Runtime. Keep the Focusrite away from the Leap USB neighborhood and treat Kiyo Pro as a ~25 fps source until its