From 5ea539bc7dff3082ff7ba43a54ea681e6dd390e2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 27 May 2026 18:07:04 +0100 Subject: [PATCH] Queue audio actuator controls into Fensalir --- docs/implementation-plan.md | 3 ++- notes/current-system-map.md | 3 ++- src/Mimir.BufferSmoke/Program.cs | 23 +++++++++++++++++++++-- src/Mimir.Runtime/MimirRuntime.cs | 25 +++++++++++++++++++++++++ state/evidence.jsonl | 1 + state/map.yaml | 7 ++++--- 6 files changed, 55 insertions(+), 7 deletions(-) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 73f3f4e..cf8a90b 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -145,7 +145,8 @@ a named invariant that the native runtime cannot protect yet. service and can emit live sync telemetry with `MIMIR_SYNC_TELEMETRY_SECONDS`. UI and telemetry read cached reports/states; they do not run synchronization analysis. The same cadence now publishes the - latest actuator command frame for telemetry/readout; Faust/native DSP still + latest actuator command frame for telemetry/readout and queues it into + `AquariumAudioDocument` as an audio control frame; Faust/native DSP still owns actual sample movement. - `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame` spectrum dashboard. Live spectrum visualization authority is the diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 1ddaeb8..8b67a19 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -306,7 +306,8 @@ runtime command owner above that Faust surface: it converts smoothed sync states into nonnegative holdback commands, keeps source-to-`sourceN` slots stable, reports overflow beyond the six-source profile, and exposes the reference holdback separately. `MimirRuntime` updates that command frame on the sync -analysis cadence and telemetry can print the current DSP targets. This is +analysis cadence, queues it into `AquariumAudioDocument` as an engine audio +control frame, and telemetry can print the current DSP targets. This is control-plane publication only; Faust/native DSP still has to be wired as the hot sample mover. diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index b09f2d6..00ceaca 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -3,6 +3,7 @@ using System.Numerics; using System.Security.Cryptography; using System.Text.Json; using Aquarium.Engine; +using Aquarium.Engine.Audio; using Aquarium.Engine.Input; using Aquarium.Engine.Render; using GameCult.Caching; @@ -1847,10 +1848,26 @@ static int RunAudioActuatorBankSmoke() var early = frame.Commands.First(command => string.Equals(command.SourceId, "mic-1-early", StringComparison.Ordinal)); var late = frame.Commands.First(command => string.Equals(command.SourceId, "mic-0-late", StringComparison.Ordinal)); var mid = frame.Commands.First(command => string.Equals(command.SourceId, "mic-2-mid", StringComparison.Ordinal)); + var audio = new AquariumAudioDocument(); + audio.EnqueueControlFrame(new AquariumAudioControlFrame( + MimirAlignmentActuatorProfile.SixSourceFaust.Id, + frame.ReferenceSourceId, + frame.ReferenceHoldbackSamples, + frame.Commands + .Select(command => new AquariumAudioControlCommand( + command.SourceId, + command.TargetDelaySamples, + command.ResampleRatio, + command.Confidence, + command.FaustControls)) + .ToArray(), + frame.TruncatedSourceCount, + Sequence: 1)); + var drained = audio.DrainControlFrames(); Console.WriteLine( $"audio-actuator-bank-smoke reference={frame.ReferenceSourceId} referenceHoldback={frame.ReferenceHoldbackSamples:0.000} commands={frame.Commands.Count} truncated={frame.TruncatedSourceCount} " + - $"lateDelay={late.TargetDelaySamples:0.000} midDelay={mid.TargetDelaySamples:0.000} earlyDelay={early.TargetDelaySamples:0.000} ratio={early.ResampleRatio:0.000000000}"); + $"lateDelay={late.TargetDelaySamples:0.000} midDelay={mid.TargetDelaySamples:0.000} earlyDelay={early.TargetDelaySamples:0.000} ratio={early.ResampleRatio:0.000000000} audioFrames={drained.Count}"); return string.Equals(frame.ReferenceSourceId, "loopback-scarlett-speakers", StringComparison.Ordinal) && Math.Abs(frame.ReferenceHoldbackSamples - 120.0) < 0.001 && @@ -1858,7 +1875,9 @@ static int RunAudioActuatorBankSmoke() frame.TruncatedSourceCount == 1 && late.TargetDelaySamples <= mid.TargetDelaySamples && mid.TargetDelaySamples < early.TargetDelaySamples && - early.FaustControls.ContainsKey("source1/delay_samples") + early.FaustControls.ContainsKey("source1/delay_samples") && + drained.Count == 1 && + drained[0].Commands.Count == frame.Commands.Count ? 0 : 1; } diff --git a/src/Mimir.Runtime/MimirRuntime.cs b/src/Mimir.Runtime/MimirRuntime.cs index 2ddc618..e6be397 100644 --- a/src/Mimir.Runtime/MimirRuntime.cs +++ b/src/Mimir.Runtime/MimirRuntime.cs @@ -73,6 +73,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec private IReadOnlyList lastAudioSynchronizationReports = []; private IReadOnlyList lastAudioSpectra = []; private long spectrumHistorySequence; + private long audioActuatorFrameSequence; private readonly Queue spectrumHistory = new(); private AquariumRuntimeServices runtimeServices = AquariumRuntimeServices.Empty; private MimirActuatorFrame lastAudioActuatorFrame = MimirActuatorFrame.Empty; @@ -903,9 +904,33 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec lastAudioActuatorFrame = audioActuatorBank.Update( synchronization.AudioSynchronizationStates, audioSyncUpdateIntervalSeconds); + PublishAudioActuatorFrame(lastAudioActuatorFrame); nextAudioSyncSeconds = runtimeSeconds + audioSyncUpdateIntervalSeconds; } + private void PublishAudioActuatorFrame(MimirActuatorFrame frame) + { + if (frame.Commands.Count == 0) + { + return; + } + + audio.EnqueueControlFrame(new AquariumAudioControlFrame( + MimirAlignmentActuatorProfile.SixSourceFaust.Id, + frame.ReferenceSourceId, + frame.ReferenceHoldbackSamples, + frame.Commands + .Select(command => new AquariumAudioControlCommand( + command.SourceId, + command.TargetDelaySamples, + command.ResampleRatio, + command.Confidence, + command.FaustControls)) + .ToArray(), + frame.TruncatedSourceCount, + ++audioActuatorFrameSequence)); + } + private void UpdateAudioSpectra() { if (runtimeSeconds < nextSpectrumSeconds) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index db83767..00a0055 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -184,3 +184,4 @@ {"ts": "2026-05-27T17:19:41+01:00", "type": "verification", "status": "ok", "note": "The MF GPU direct texture path now carries D3D11/D3D12 synchronization instead of relying on flush timing. native/camera_capture/mimir_mf_gpu_capture.dll creates a shared D3D11 fence, signals it after copying the decoded GPU surface into the shared BGRA texture, and MimirMediaFoundationGpuVideoCaptureDriver carries the fence handle/value through MimirVideoFrameDescriptor into Fensalir field resource declarations. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed with zero live CPU payload bytes, nonzero shared texture handles, nonzero producer fence handles, and valid Fensalir camera resources."} {"ts": "2026-05-27T17:53:38+01:00", "type": "decision", "status": "ok", "note": "Synchronized camera Texture2D resources are not a mandate to invent production visual fusion now. Camera image claims remain debug-view-only until the renderer and audio work are fixed; future camera fusion/feature-extraction lowerings need a separate selected policy decision."} {"ts": "2026-05-27T18:03:11+01:00", "type": "decision", "status": "ok", "note": "Runtime audio sync states now derive explicit actuator command frames through MimirAlignmentActuatorBank: measured delay remains analyzer truth, the bank converts offsets into nonnegative Faust holdback controls with stable source slots, reference holdback, and overflow reporting. This is control-plane publication only; Faust/native DSP still owns live sample movement."} +{"ts": "2026-05-27T18:06:41+01:00", "type": "decision", "status": "ok", "note": "MimirRuntime now queues the latest MimirAlignmentActuatorBank frame into AquariumAudioDocument as an engine audio control frame. This creates the Mimir-to-Fensalir control contract for Faust/native DSP targets, but remains control-plane only until a streaming DSP sample mover consumes it."} diff --git a/state/map.yaml b/state/map.yaml index f27d788..281d899 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -182,9 +182,10 @@ accepted_design: actuator surface for six channels of bounded fractional delay/gain. `MimirAlignmentActuatorBank` now converts runtime sync states into explicit nonnegative Faust holdback commands with stable source slots, a separate - reference holdback, and overflow reporting; this publishes DSP control - targets but does not move live samples until the Faust/native hot path is - wired. The synthetic + reference holdback, and overflow reporting, then queues those commands into + `AquariumAudioDocument` as engine audio control frames; this publishes DSP + control targets but does not move live samples until the Faust/native hot + path is wired. The synthetic codebook proof is `dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`. `Mimir.BufferSmoke --render-chirp-bin-f32` plus