From 269a8de350f2e8010b7acf80abd813b88a0e4fc4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 22 May 2026 01:07:30 +0100 Subject: [PATCH] Feed WASAPI audio blocks into runtime buffers --- config/mimir-runtime.probes.example.json | 103 +++++ docs/audio-field.md | 13 +- docs/implementation-plan.md | 7 +- .../wasapi_audio_cadence/CMakeLists.txt | 15 + .../wasapi_audio_cadence.cpp | 375 ++++++++++++++++++ src/Mimir.BufferSmoke/Program.cs | 5 + src/Mimir.Runtime/MimirRuntime.cs | 5 + .../MimirAudioBlockDescriptor.cs | 19 + .../MimirFrameEventProcessStreamSource.cs | 82 +++- .../Synchronization/MimirStreamSample.cs | 3 +- state/evidence.jsonl | 1 + state/map.yaml | 16 +- 12 files changed, 617 insertions(+), 27 deletions(-) create mode 100644 native/probes/wasapi_audio_cadence/CMakeLists.txt create mode 100644 native/probes/wasapi_audio_cadence/wasapi_audio_cadence.cpp create mode 100644 src/Mimir.Runtime/Synchronization/MimirAudioBlockDescriptor.cs diff --git a/config/mimir-runtime.probes.example.json b/config/mimir-runtime.probes.example.json index 3cffa2a..169bf4d 100644 --- a/config/mimir-runtime.probes.example.json +++ b/config/mimir-runtime.probes.example.json @@ -36,6 +36,109 @@ "ps3-eye-0", "ps3-eye-1" ] + }, + { + "sourceId": "mic-focusrite-local", + "kind": "Audio", + "origin": "LocalDevice", + "enabled": true, + "adapter": "frame-events", + "command": "../native/probes/wasapi_audio_cadence/build/Release/wasapi_audio_cadence.exe", + "arguments": [ + "--source-id", + "mic-focusrite-local", + "--endpoint", + "{0.0.1.00000000}.{16af9455-4539-4de2-8172-563d991d9e34}", + "--seconds", + "30", + "--emit-json-blocks" + ] + }, + { + "sourceId": "mic-kiyo-pro", + "kind": "Audio", + "origin": "LocalDevice", + "enabled": true, + "adapter": "frame-events", + "command": "../native/probes/wasapi_audio_cadence/build/Release/wasapi_audio_cadence.exe", + "arguments": [ + "--source-id", + "mic-kiyo-pro", + "--endpoint", + "{0.0.1.00000000}.{abc7fd11-3b7d-4cfa-82ee-8767dae13fac}", + "--seconds", + "30", + "--emit-json-blocks" + ] + }, + { + "sourceId": "mic-kiyo", + "kind": "Audio", + "origin": "LocalDevice", + "enabled": true, + "adapter": "frame-events", + "command": "../native/probes/wasapi_audio_cadence/build/Release/wasapi_audio_cadence.exe", + "arguments": [ + "--source-id", + "mic-kiyo", + "--endpoint", + "{0.0.1.00000000}.{ef1fdd6f-c721-4d5a-b183-5bd81dc57c1f}", + "--seconds", + "30", + "--emit-json-blocks" + ] + }, + { + "sourceId": "mic-usb-camera-0", + "kind": "Audio", + "origin": "LocalDevice", + "enabled": true, + "adapter": "frame-events", + "command": "../native/probes/wasapi_audio_cadence/build/Release/wasapi_audio_cadence.exe", + "arguments": [ + "--source-id", + "mic-usb-camera-0", + "--endpoint", + "{0.0.1.00000000}.{57befef3-599c-4c73-a0cd-9539ffc2f54f}", + "--seconds", + "30", + "--emit-json-blocks" + ] + }, + { + "sourceId": "mic-usb-camera-1", + "kind": "Audio", + "origin": "LocalDevice", + "enabled": false, + "adapter": "frame-events", + "command": "../native/probes/wasapi_audio_cadence/build/Release/wasapi_audio_cadence.exe", + "arguments": [ + "--source-id", + "mic-usb-camera-1", + "--endpoint", + "{0.0.1.00000000}.{f26c55af-5018-438a-8cc1-6dbead205a3c}", + "--seconds", + "30", + "--emit-json-blocks" + ] + }, + { + "sourceId": "loopback-scarlett-speakers", + "kind": "Audio", + "origin": "LocalDevice", + "enabled": true, + "adapter": "frame-events", + "command": "../native/probes/wasapi_audio_cadence/build/Release/wasapi_audio_cadence.exe", + "arguments": [ + "--loopback", + "--source-id", + "loopback-scarlett-speakers", + "--endpoint", + "{0.0.0.00000000}.{defdb2ab-ea1a-4eba-ab59-57038754f387}", + "--seconds", + "30", + "--emit-json-blocks" + ] } ] } diff --git a/docs/audio-field.md b/docs/audio-field.md index dc76331..2e3c6d0 100644 --- a/docs/audio-field.md +++ b/docs/audio-field.md @@ -35,6 +35,13 @@ flowchart TD ## Next Cut -Add native audio capture workers that append typed blocks into -`Mimir.Runtime`, then expose buffer depth, clock state, delay estimates, and -stem routing in Aquarium UI. +The current diagnostic witness is `native/probes/wasapi_audio_cadence`, which +emits timestamped WASAPI `audio-block` metadata into `Mimir.Runtime` through the +frame-event adapter. It has proven Focusrite mic, Kiyo Pro mic, Kiyo mic, one +USB Camera mic, and Scarlett speaker loopback in rolling buffers when loopback +audio is actively playing. A second USB Camera microphone endpoint enumerates +but currently produces zero WASAPI packets. + +Next, replace the diagnostic bridge with native audio capture workers that +append typed blocks into `Mimir.Runtime`, then expose buffer depth, clock state, +delay estimates, and stem routing in Aquarium UI. diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index fc3ce09..fb13d4e 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -36,6 +36,9 @@ a named invariant that the native runtime cannot protect yet. - `src/Mimir.BufferSmoke` loads the runtime config, polls the synchronization hub, and prints the actual rolling buffers. Use `--require-samples` when an empty declared sensor buffer should fail the run. +- `native/probes/wasapi_audio_cadence` captures WASAPI mic or render-loopback + block metadata and emits `audio-block` JSON events for the diagnostic runtime + adapter. - `MimirVideoFrameDescriptor` for dimensions, pixel format, stride, device timestamp, and native/GPU handle metadata. - `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` as the live @@ -67,8 +70,8 @@ a named invariant that the native runtime cannot protect yet. other cameras. 2. Feed those drivers into `MimirVideoCaptureDriverSource` and prove sustained frame cadence in the rolling buffers. -3. Add native audio capture workers for local mic, loopback, and network audio - feeds. +3. Replace the WASAPI frame-event diagnostic bridge with native audio capture + workers for local mic, loopback, and network audio feeds. 4. Bind Aquarium UI to the synchronization hub so buffer depth, stream cadence, source timestamps, and output settings are visible and adjustable. 5. Move GPU feature extraction, fusion, material fitting, render budgeting, and diff --git a/native/probes/wasapi_audio_cadence/CMakeLists.txt b/native/probes/wasapi_audio_cadence/CMakeLists.txt new file mode 100644 index 0000000..f3a038b --- /dev/null +++ b/native/probes/wasapi_audio_cadence/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.20) + +project(wasapi_audio_cadence LANGUAGES CXX) + +add_executable(wasapi_audio_cadence + wasapi_audio_cadence.cpp) + +target_compile_features(wasapi_audio_cadence PRIVATE cxx_std_20) +target_compile_definitions(wasapi_audio_cadence PRIVATE + WIN32_LEAN_AND_MEAN + NOMINMAX + _WIN32_WINNT=0x0A00) +target_link_libraries(wasapi_audio_cadence PRIVATE + ole32 + avrt) diff --git a/native/probes/wasapi_audio_cadence/wasapi_audio_cadence.cpp b/native/probes/wasapi_audio_cadence/wasapi_audio_cadence.cpp new file mode 100644 index 0000000..d26255e --- /dev/null +++ b/native/probes/wasapi_audio_cadence/wasapi_audio_cadence.cpp @@ -0,0 +1,375 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace +{ +struct ComInit +{ + HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + ~ComInit() + { + if (SUCCEEDED(hr)) + { + CoUninitialize(); + } + } +}; + +template +struct ComPtr +{ + T* value = nullptr; + ~ComPtr() + { + if (value) + { + value->Release(); + } + } + T** operator&() { return &value; } + T* operator->() const { return value; } + explicit operator bool() const { return value != nullptr; } +}; + +struct PropVariant +{ + PROPVARIANT value{}; + PropVariant() { PropVariantInit(&value); } + ~PropVariant() { PropVariantClear(&value); } +}; + +long long nowNs() +{ + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +std::wstring widen(const char* value) +{ + if (!value) + { + return {}; + } + + const int bytes = MultiByteToWideChar(CP_UTF8, 0, value, -1, nullptr, 0); + std::wstring result(static_cast(bytes > 0 ? bytes - 1 : 0), L'\0'); + if (bytes > 1) + { + MultiByteToWideChar(CP_UTF8, 0, value, -1, result.data(), bytes); + } + return result; +} + +std::string narrow(const wchar_t* value) +{ + if (!value) + { + return {}; + } + + const int bytes = WideCharToMultiByte(CP_UTF8, 0, value, -1, nullptr, 0, nullptr, nullptr); + std::string result(static_cast(bytes > 0 ? bytes - 1 : 0), '\0'); + if (bytes > 1) + { + WideCharToMultiByte(CP_UTF8, 0, value, -1, result.data(), bytes, nullptr, nullptr); + } + return result; +} + +std::string sampleFormat(const WAVEFORMATEX* format) +{ + if (format->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) + { + return "Float32"; + } + + if (format->wFormatTag == WAVE_FORMAT_PCM) + { + return format->wBitsPerSample == 16 ? "Int16" : + format->wBitsPerSample == 24 ? "Int24" : + format->wBitsPerSample == 32 ? "Int32" : "Unknown"; + } + + if (format->wFormatTag == WAVE_FORMAT_EXTENSIBLE && + format->cbSize >= (sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX))) + { + const auto* extensible = reinterpret_cast(format); + if (extensible->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT) + { + return "Float32"; + } + if (extensible->SubFormat == KSDATAFORMAT_SUBTYPE_PCM) + { + const WORD bits = extensible->Samples.wValidBitsPerSample != 0 + ? extensible->Samples.wValidBitsPerSample + : format->wBitsPerSample; + return bits == 16 ? "Int16" : bits == 24 ? "Int24" : bits == 32 ? "Int32" : "Unknown"; + } + } + + return "Unknown"; +} + +bool hasArg(int argc, char** argv, const char* name) +{ + for (int index = 1; index < argc; ++index) + { + if (std::strcmp(argv[index], name) == 0) + { + return true; + } + } + + return false; +} + +const char* option(int argc, char** argv, const char* name, const char* fallback) +{ + for (int index = 1; index < argc - 1; ++index) + { + if (std::strcmp(argv[index], name) == 0) + { + return argv[index + 1]; + } + } + + return fallback; +} + +void listEndpoints(IMMDeviceEnumerator* enumerator) +{ + for (const auto flow : { eCapture, eRender }) + { + ComPtr collection; + if (FAILED(enumerator->EnumAudioEndpoints(flow, DEVICE_STATE_ACTIVE, &collection.value)) || !collection) + { + continue; + } + + UINT count = 0; + collection->GetCount(&count); + std::printf("%s endpoints=%u\n", flow == eCapture ? "capture" : "render", count); + for (UINT index = 0; index < count; ++index) + { + ComPtr device; + if (FAILED(collection->Item(index, &device.value)) || !device) + { + continue; + } + + LPWSTR id = nullptr; + device->GetId(&id); + ComPtr store; + PropVariant name; + if (SUCCEEDED(device->OpenPropertyStore(STGM_READ, &store.value)) && store) + { + store->GetValue(PKEY_Device_FriendlyName, &name.value); + } + + std::printf( + " %u: %s id=%s\n", + index, + name.value.vt == VT_LPWSTR ? narrow(name.value.pwszVal).c_str() : "(unnamed)", + id ? narrow(id).c_str() : ""); + CoTaskMemFree(id); + } + } +} + +IMMDevice* openEndpoint(IMMDeviceEnumerator* enumerator, EDataFlow flow, const char* endpoint) +{ + IMMDevice* device = nullptr; + if (std::strcmp(endpoint, "default") == 0) + { + if (SUCCEEDED(enumerator->GetDefaultAudioEndpoint(flow, eConsole, &device))) + { + return device; + } + return nullptr; + } + + const std::wstring id = widen(endpoint); + if (SUCCEEDED(enumerator->GetDevice(id.c_str(), &device))) + { + return device; + } + + return nullptr; +} +} + +int main(int argc, char** argv) +{ + std::setvbuf(stdout, nullptr, _IONBF, 0); + ComInit com; + if (FAILED(com.hr)) + { + std::fprintf(stderr, "CoInitializeEx failed: 0x%08lx\n", static_cast(com.hr)); + return 2; + } + + ComPtr enumerator; + HRESULT hr = CoCreateInstance( + __uuidof(MMDeviceEnumerator), + nullptr, + CLSCTX_ALL, + __uuidof(IMMDeviceEnumerator), + reinterpret_cast(&enumerator.value)); + if (FAILED(hr) || !enumerator) + { + std::fprintf(stderr, "MMDeviceEnumerator failed: 0x%08lx\n", static_cast(hr)); + return 3; + } + + if (hasArg(argc, argv, "--list")) + { + listEndpoints(enumerator.value); + return 0; + } + + const bool loopback = hasArg(argc, argv, "--loopback"); + const char* endpointId = option(argc, argv, "--endpoint", "default"); + const char* sourceId = option(argc, argv, "--source-id", loopback ? "host-loopback" : "host-mic"); + const double seconds = std::atof(option(argc, argv, "--seconds", "0")); + const bool emitJsonBlocks = hasArg(argc, argv, "--emit-json-blocks"); + const EDataFlow flow = loopback ? eRender : eCapture; + + ComPtr device; + device.value = openEndpoint(enumerator.value, flow, endpointId); + if (!device) + { + std::fprintf(stderr, "Could not open %s endpoint: %s\n", loopback ? "render" : "capture", endpointId); + return 4; + } + + ComPtr store; + PropVariant endpointName; + if (SUCCEEDED(device->OpenPropertyStore(STGM_READ, &store.value)) && store) + { + store->GetValue(PKEY_Device_FriendlyName, &endpointName.value); + } + std::printf( + "wasapi endpoint: source=%s mode=%s name=%s\n", + sourceId, + loopback ? "loopback" : "capture", + endpointName.value.vt == VT_LPWSTR ? narrow(endpointName.value.pwszVal).c_str() : "(unnamed)"); + + ComPtr audioClient; + hr = device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, reinterpret_cast(&audioClient.value)); + if (FAILED(hr) || !audioClient) + { + std::fprintf(stderr, "IAudioClient activation failed: 0x%08lx\n", static_cast(hr)); + return 5; + } + + WAVEFORMATEX* rawFormat = nullptr; + hr = audioClient->GetMixFormat(&rawFormat); + if (FAILED(hr) || !rawFormat) + { + std::fprintf(stderr, "GetMixFormat failed: 0x%08lx\n", static_cast(hr)); + return 6; + } + + const REFERENCE_TIME requestedDuration = 1000000; // 100 ms + hr = audioClient->Initialize( + AUDCLNT_SHAREMODE_SHARED, + loopback ? AUDCLNT_STREAMFLAGS_LOOPBACK : 0, + requestedDuration, + 0, + rawFormat, + nullptr); + if (FAILED(hr)) + { + std::fprintf(stderr, "IAudioClient Initialize failed: 0x%08lx\n", static_cast(hr)); + CoTaskMemFree(rawFormat); + return 7; + } + + ComPtr captureClient; + hr = audioClient->GetService(__uuidof(IAudioCaptureClient), reinterpret_cast(&captureClient.value)); + if (FAILED(hr) || !captureClient) + { + std::fprintf(stderr, "IAudioCaptureClient failed: 0x%08lx\n", static_cast(hr)); + CoTaskMemFree(rawFormat); + return 8; + } + + DWORD taskIndex = 0; + HANDLE avrtHandle = AvSetMmThreadCharacteristicsW(L"Pro Audio", &taskIndex); + audioClient->Start(); + + const long long start = nowNs(); + const long long deadline = seconds > 0.0 ? start + static_cast(seconds * 1000000000.0) : 0; + const std::string format = sampleFormat(rawFormat); + unsigned long long sequence = 0; + unsigned long long totalFrames = 0; + + while (deadline == 0 || nowNs() < deadline) + { + Sleep(5); + UINT32 packetFrames = 0; + while (SUCCEEDED(captureClient->GetNextPacketSize(&packetFrames)) && packetFrames > 0) + { + BYTE* data = nullptr; + UINT32 frames = 0; + DWORD flags = 0; + hr = captureClient->GetBuffer(&data, &frames, &flags, nullptr, nullptr); + if (FAILED(hr)) + { + std::fprintf(stderr, "GetBuffer failed: 0x%08lx\n", static_cast(hr)); + audioClient->Stop(); + CoTaskMemFree(rawFormat); + return 9; + } + + const int byteLength = static_cast(frames * rawFormat->nBlockAlign); + totalFrames += frames; + ++sequence; + if (emitJsonBlocks) + { + std::printf( + "{\"type\":\"audio-block\",\"sourceId\":\"%s\",\"timestampNs\":%lld,\"sequence\":%llu,\"sampleRate\":%lu,\"channels\":%u,\"sampleFormat\":\"%s\",\"frameCount\":%u,\"byteLength\":%d}\n", + sourceId, + nowNs(), + sequence, + static_cast(rawFormat->nSamplesPerSec), + static_cast(rawFormat->nChannels), + format.c_str(), + static_cast(frames), + byteLength); + } + + captureClient->ReleaseBuffer(frames); + } + } + + audioClient->Stop(); + if (avrtHandle) + { + AvRevertMmThreadCharacteristics(avrtHandle); + } + + const double elapsed = (nowNs() - start) / 1000000000.0; + std::printf( + "measured: source=%s blocks=%llu frames=%llu elapsed=%.3f sampleRate=%lu channels=%u format=%s\n", + sourceId, + sequence, + totalFrames, + elapsed, + static_cast(rawFormat->nSamplesPerSec), + static_cast(rawFormat->nChannels), + format.c_str()); + CoTaskMemFree(rawFormat); + return 0; +} diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index b187372..5889165 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -31,6 +31,11 @@ foreach (var buffer in hub.Buffers.Buffers.OrderBy(buffer => buffer.Descriptor.S Console.WriteLine( $"{buffer.Descriptor.SourceId}: count={buffer.Count} edgeNs={buffer.EdgeNs} latest={frame.Width}x{frame.Height} {frame.PixelFormat} bytes={latest.Value.ByteLength}"); } + else if (latest?.AudioBlock is { } block) + { + Console.WriteLine( + $"{buffer.Descriptor.SourceId}: count={buffer.Count} edgeNs={buffer.EdgeNs} latest={block.Channels}ch {block.SampleRate}Hz {block.SampleFormat} frames={block.FrameCount} bytes={latest.Value.ByteLength}"); + } else { Console.WriteLine($"{buffer.Descriptor.SourceId}: count={buffer.Count} edgeNs={buffer.EdgeNs}"); diff --git a/src/Mimir.Runtime/MimirRuntime.cs b/src/Mimir.Runtime/MimirRuntime.cs index 9ebae2f..443a970 100644 --- a/src/Mimir.Runtime/MimirRuntime.cs +++ b/src/Mimir.Runtime/MimirRuntime.cs @@ -125,6 +125,11 @@ public sealed class MimirRuntime : IAquariumRuntime return $"{buffer.Descriptor.SourceId}: {buffer.Count} {frame.Width}x{frame.Height} {frame.PixelFormat} bytes {latest.Value.ByteLength} edge {buffer.EdgeNs}"; } + if (latest?.AudioBlock is { } block) + { + return $"{buffer.Descriptor.SourceId}: {buffer.Count} {block.Channels}ch {block.SampleRate}Hz {block.SampleFormat} frames {block.FrameCount} bytes {latest.Value.ByteLength} edge {buffer.EdgeNs}"; + } + return $"{buffer.Descriptor.SourceId}: {buffer.Count} edge {buffer.EdgeNs}"; } } diff --git a/src/Mimir.Runtime/Synchronization/MimirAudioBlockDescriptor.cs b/src/Mimir.Runtime/Synchronization/MimirAudioBlockDescriptor.cs new file mode 100644 index 0000000..8cd3bfe --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirAudioBlockDescriptor.cs @@ -0,0 +1,19 @@ +namespace Mimir.Runtime.Synchronization; + +public enum MimirAudioSampleFormat +{ + Unknown, + Float32, + Int16, + Int24, + Int32, +} + +public sealed record MimirAudioBlockDescriptor( + int SampleRate, + int Channels, + MimirAudioSampleFormat SampleFormat, + int FrameCount, + long DeviceTimestampNs, + ulong NativeHandle = 0, + string NativeHandleKind = ""); diff --git a/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs b/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs index fa43848..3f73e7c 100644 --- a/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs +++ b/src/Mimir.Runtime/Synchronization/MimirFrameEventProcessStreamSource.cs @@ -103,7 +103,7 @@ public sealed class MimirFrameEventProcessStreamSource : IMimirStreamSource return false; } - if (frameEvent == null || !frameEvent.IsVideoFrame) + if (frameEvent == null || !frameEvent.IsKnownEvent) { return false; } @@ -121,23 +121,41 @@ public sealed class MimirFrameEventProcessStreamSource : IMimirStreamSource : StopwatchTicksToNs(Stopwatch.GetTimestamp()); var arrivalNs = StopwatchTicksToNs(Stopwatch.GetTimestamp()); var byteLength = Math.Max(0, frameEvent.ByteLength); - var strideBytes = frameEvent.StrideBytes > 0 - ? frameEvent.StrideBytes - : InferStride(frameEvent.Width, frameEvent.PixelFormat, byteLength, frameEvent.Height); - var sequence = frameEvent.Sequence ?? fallbackSequence++; var payloadHandle = frameEvent.NativeHandle; - var videoFrame = new MimirVideoFrameDescriptor( - frameEvent.Width, - frameEvent.Height, - ParsePixelFormat(frameEvent.PixelFormat), - strideBytes, - timestampNs, - payloadHandle, - frameEvent.NativeHandleKind ?? ""); + var sampleKind = frameEvent.IsAudioBlock ? MimirStreamKind.Audio : Descriptor.Kind; + MimirVideoFrameDescriptor? videoFrame = null; + MimirAudioBlockDescriptor? audioBlock = null; + if (frameEvent.IsVideoFrame) + { + var strideBytes = frameEvent.StrideBytes > 0 + ? frameEvent.StrideBytes + : InferStride(frameEvent.Width, frameEvent.PixelFormat, byteLength, frameEvent.Height); + videoFrame = new MimirVideoFrameDescriptor( + frameEvent.Width, + frameEvent.Height, + ParsePixelFormat(frameEvent.PixelFormat), + strideBytes, + timestampNs, + payloadHandle, + frameEvent.NativeHandleKind ?? ""); + } + + if (frameEvent.IsAudioBlock) + { + audioBlock = new MimirAudioBlockDescriptor( + frameEvent.SampleRate, + frameEvent.Channels, + ParseSampleFormat(frameEvent.SampleFormat), + frameEvent.FrameCount, + timestampNs, + payloadHandle, + frameEvent.NativeHandleKind ?? ""); + } + var sequence = frameEvent.Sequence ?? fallbackSequence++; sample = new MimirStreamSample( sourceId, - Descriptor.Kind, + sampleKind, Descriptor.Origin, timestampNs, arrivalNs, @@ -145,7 +163,8 @@ public sealed class MimirFrameEventProcessStreamSource : IMimirStreamSource payloadHandle, byteLength, default, - videoFrame); + videoFrame, + audioBlock); return true; } @@ -215,6 +234,23 @@ public sealed class MimirFrameEventProcessStreamSource : IMimirStreamSource }; } + private static MimirAudioSampleFormat ParseSampleFormat(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return MimirAudioSampleFormat.Unknown; + } + + return value.Trim().ToUpperInvariant() switch + { + "FLOAT32" or "IEEE_FLOAT" => MimirAudioSampleFormat.Float32, + "INT16" or "PCM16" => MimirAudioSampleFormat.Int16, + "INT24" or "PCM24" => MimirAudioSampleFormat.Int24, + "INT32" or "PCM32" => MimirAudioSampleFormat.Int32, + _ => MimirAudioSampleFormat.Unknown, + }; + } + private static long StopwatchTicksToNs(long ticks) { return checked((long)(ticks * (1_000_000_000.0 / Stopwatch.Frequency))); @@ -264,6 +300,14 @@ public sealed class MimirFrameEventProcessStreamSource : IMimirStreamSource public int ByteLength { get; set; } + public int SampleRate { get; set; } + + public int Channels { get; set; } + + public string SampleFormat { get; set; } = ""; + + public int FrameCount { get; set; } + public ulong NativeHandle { get; set; } public string? NativeHandleKind { get; set; } @@ -272,5 +316,13 @@ public sealed class MimirFrameEventProcessStreamSource : IMimirStreamSource string.Equals(Type, "video-frame", StringComparison.OrdinalIgnoreCase) && Width > 0 && Height > 0; + + public bool IsAudioBlock => + string.Equals(Type, "audio-block", StringComparison.OrdinalIgnoreCase) && + SampleRate > 0 && + Channels > 0 && + FrameCount > 0; + + public bool IsKnownEvent => IsVideoFrame || IsAudioBlock; } } diff --git a/src/Mimir.Runtime/Synchronization/MimirStreamSample.cs b/src/Mimir.Runtime/Synchronization/MimirStreamSample.cs index 00b4c4d..00ab704 100644 --- a/src/Mimir.Runtime/Synchronization/MimirStreamSample.cs +++ b/src/Mimir.Runtime/Synchronization/MimirStreamSample.cs @@ -10,4 +10,5 @@ public readonly record struct MimirStreamSample( ulong PayloadHandle, int ByteLength = 0, ReadOnlyMemory Data = default, - MimirVideoFrameDescriptor? VideoFrame = null); + MimirVideoFrameDescriptor? VideoFrame = null, + MimirAudioBlockDescriptor? AudioBlock = null); diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 339ced7..bb19bce 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -26,3 +26,4 @@ {"ts":"2026-05-22T04:10:00+00:00","type":"implementation","status":"ok","note":"Added a diagnostic frame-event path from native camera probes into Mimir.Runtime rolling buffers. MimirFrameEventProcessStreamSource reads JSON-line video-frame metadata into MimirVideoFrameDescriptor samples; KS and repo-owned PS3 Eye probes can emit those frame events. This is a playground witness for Aquarium UI/cadence, not the final pixel hot path."} {"ts":"2026-05-22T04:45:00+00:00","type":"verification","status":"partial","note":"Mimir.BufferSmoke loaded the probe runtime config, started the KS and PS3 Eye frame-event probes, polled MimirSynchronizationHub, and confirmed live samples in rolling buffers for leap-stereo-ir, kiyo-rgb, ps3-eye-0, and ps3-eye-1. Kiyo Pro is declared but empty: the KS path enumerates the device and modes, then fails at KSSTATE_ACQUIRE with ERROR_INVALID_HANDLE. PnP camera interface reports OK, composite reports OK, media interface reports Error, and non-elevated pnputil restart is denied."} {"ts":"2026-05-22T05:05:00+00:00","type":"verification","status":"ok","note":"After replugging the Kiyo Pro, all Pro PnP interfaces reported OK and the KS probe again measured 1280x720 MJPG at 24.97 fps and 1920x1080 MJPG at 25.19 fps. Mimir.BufferSmoke with --require-samples then confirmed all five declared local optical buffers received samples: kiyo-pro-rgb 99, kiyo-rgb 120, leap-stereo-ir 401, ps3-eye-0 937, ps3-eye-1 937 over the 8 second smoke run. Pro remains UsbHighSpeed and ~25 fps, but it is now in the runtime buffers."} +{"ts":"2026-05-22T05:35:00+00:00","type":"verification","status":"ok","note":"Added native/probes/wasapi_audio_cadence and MimirAudioBlockDescriptor so WASAPI audio-block metadata enters Mimir.Runtime rolling buffers through the diagnostic event adapter. Endpoint inventory found five capture endpoints and four render endpoints. Full Mimir.BufferSmoke with --require-samples and active music on Scarlett speakers proved ten non-empty buffers: five camera buffers, Focusrite mic, Kiyo mic, Kiyo Pro mic, one USB Camera mic, and Scarlett speaker loopback. Scarlett loopback is silent/no-packet when no render stream is active; the second USB Camera mic endpoint enumerates OK but produced zero WASAPI packets in standalone and simultaneous tests."} diff --git a/state/map.yaml b/state/map.yaml index ca386e8..e63a11a 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -53,9 +53,12 @@ accepted_design: direct native ingest, process bridge ingest, and video capture driver seams. - `MimirVideoFrameDescriptor` carries width, height, pixel format, stride, device timestamp, and optional native/GPU handle. + - `MimirAudioBlockDescriptor` carries sample rate, channels, sample format, + frame count, timestamp, and optional native handle. - `MimirFrameEventProcessStreamSource` is a temporary diagnostic adapter for - JSON-line frame metadata from native probes into runtime rolling buffers; it - does not carry pixels or own the final local-camera hot path. + JSON-line video frame and audio block metadata from native probes into + runtime rolling buffers; it does not carry pixels/audio payloads or own the + final local-device hot path. - `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the live seam for Leap, Media Foundation, DirectShow, libusb, LeapC, or shared texture capture. @@ -155,7 +158,8 @@ current_status: Replace the diagnostic JSON-line process bridge with direct native capture drivers that push buffer handles into Mimir.Runtime. Current Mimir.BufferSmoke with `--require-samples` proves all five declared local - optical buffers receive samples after replugging the Kiyo Pro: Leap stereo - IR, Kiyo Pro, regular Kiyo, and both PS3 Eyes. Keep the Focusrite away from - the Leap USB neighborhood and treat Kiyo Pro as a ~25 fps source until its - SuperSpeed negotiation is fixed. + optical buffers plus Focusrite mic, Kiyo Pro mic, Kiyo mic, one USB Camera + mic, and active Scarlett speaker loopback receive samples. A second USB + Camera mic endpoint enumerates OK but produces zero WASAPI packets. Keep the + Focusrite away from the Leap USB neighborhood and treat Kiyo Pro as a ~25 + fps source until its SuperSpeed negotiation is fixed.