From 86130d43497213463f4496fc05e7febd7d6513d7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 27 May 2026 20:19:26 +0100 Subject: [PATCH] Feed actuator audio blocks to Fensalir --- docs/implementation-plan.md | 7 +- notes/current-system-map.md | 8 +- src/Mimir.Runtime/MimirRuntime.cs | 154 ++++++++++++++++++++++++++++++ state/evidence.jsonl | 1 + state/map.yaml | 7 +- 5 files changed, 170 insertions(+), 7 deletions(-) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 8eec640..bc9d22d 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -149,8 +149,11 @@ a named invariant that the native runtime cannot protect yet. `AquariumAudioDocument` as an audio control frame. Mimir also declares the `faust/mimir_alignment_actuator.dsp` program as an `AquariumStreamingDspProgram`, so Fensalir has both the persistent Faust DSP - graph and the matching controls. Device I/O/stem routing is still the next - cut before live samples are actually aligned. + graph and the matching controls. The runtime now also queues sample-bearing + `AquariumStreamingAudioBlock` lanes from the latest actuator source buffers, + using the bank-assigned Faust `sourceN` slot as channel authority. Durable + stem routing/publication is still the next cut before aligned audio becomes a + separately mixable program output. - `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame` spectrum dashboard. Live spectrum visualization authority is the `AquariumFieldEvidenceFrame`: Mimir declares the rolling resource, emits a diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 0ad87a5..e0dbbbf 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -309,9 +309,11 @@ holdback separately. `MimirRuntime` updates that command frame on the sync analysis cadence, queues it into `AquariumAudioDocument` as an engine audio control frame, declares `faust/mimir_alignment_actuator.dsp` as an `AquariumStreamingDspProgram`, and telemetry can print the current DSP targets. -Fensalir can now compile/host that persistent Faust graph, but device I/O and -stem routing still have to feed live blocks through it before audio is actually -aligned. +Mimir also queues sample-bearing `AquariumStreamingAudioBlock` lanes from the +latest actuator source buffers, using the bank-assigned Faust `sourceN` control +path as channel authority. Fensalir can compile/host/process that persistent +Faust graph now; durable stem routing and OBS-facing publication still have to +turn processed output lanes into separately mixable program audio. ## Visual Fusion diff --git a/src/Mimir.Runtime/MimirRuntime.cs b/src/Mimir.Runtime/MimirRuntime.cs index 9c2706d..1fe9d05 100644 --- a/src/Mimir.Runtime/MimirRuntime.cs +++ b/src/Mimir.Runtime/MimirRuntime.cs @@ -5,6 +5,7 @@ using Aquarium.Engine.Render; using Aquarium.Engine.Ui; using CultMath; using Mimir.Runtime.Synchronization; +using System.Buffers.Binary; using System.Diagnostics; using System.Numerics; using System.Text; @@ -932,6 +933,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec .ToArray(), frame.TruncatedSourceCount, ++audioActuatorFrameSequence)); + QueueStreamingActuatorAudioBlock(frame, audioActuatorFrameSequence); } private void QueueAudioActuatorProgram() @@ -957,6 +959,77 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec audioActuatorProgramQueued = true; } + private void QueueStreamingActuatorAudioBlock(MimirActuatorFrame frame, long sequence) + { + var commands = frame.Commands.OrderBy(command => command.SourceId, StringComparer.Ordinal).ToArray(); + if (commands.Length == 0) + { + return; + } + + var buffers = synchronization.Buffers.Buffers + .Where(buffer => buffer.Descriptor.Kind == MimirStreamKind.Audio && buffer.Latest?.AudioBlock != null) + .ToDictionary(buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal); + var channels = new List(commands.Length); + var frameCount = int.MaxValue; + var sampleRate = 0; + for (var index = 0; index < commands.Length; index++) + { + if (!buffers.TryGetValue(commands[index].SourceId, out var buffer) || + buffer.Latest is not { } latest || + latest.AudioBlock is not { } block || + latest.Data.IsEmpty) + { + continue; + } + + if (sampleRate == 0) + { + sampleRate = block.SampleRate; + } + else if (block.SampleRate != sampleRate) + { + continue; + } + + var samples = ExtractMonoAudioBlock(latest, block); + if (samples.Length == 0) + { + continue; + } + + frameCount = Math.Min(frameCount, samples.Length); + var channelIndex = FaustSourceIndex(commands[index]); + if (channelIndex < 0) + { + continue; + } + + channels.Add(new AquariumStreamingAudioChannel(channelIndex, commands[index].SourceId, samples)); + } + + if (channels.Count == 0 || sampleRate <= 0 || frameCount == int.MaxValue) + { + return; + } + + if (channels.Any(channel => channel.Samples.Length != frameCount)) + { + channels = channels + .Select(channel => channel.Samples.Length == frameCount + ? channel + : channel with { Samples = channel.Samples.AsSpan(0, frameCount).ToArray() }) + .ToList(); + } + + audio.EnqueueStreamingAudioBlock(new AquariumStreamingAudioBlock( + MimirAlignmentActuatorProfile.SixSourceFaust.Id, + channels, + frameCount, + sampleRate, + sequence)); + } + private void UpdateAudioSpectra() { if (runtimeSeconds < nextSpectrumSeconds) @@ -1088,6 +1161,87 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec return null; } + private static float[] ExtractMonoAudioBlock(MimirStreamSample sample, MimirAudioBlockDescriptor block) + { + var bytesPerSample = BytesPerSample(block.SampleFormat); + if (bytesPerSample == 0 || block.Channels <= 0 || block.FrameCount <= 0) + { + return []; + } + + var data = sample.Data.Span; + var stride = block.Channels * bytesPerSample; + if (data.Length < stride) + { + return []; + } + + var frameCount = Math.Min(block.FrameCount, data.Length / stride); + var output = new float[frameCount]; + for (var frame = 0; frame < frameCount; frame++) + { + var sum = 0.0f; + var frameOffset = frame * stride; + for (var channel = 0; channel < block.Channels; channel++) + { + var offset = frameOffset + channel * bytesPerSample; + sum += ReadPcmSample(data.Slice(offset, bytesPerSample), block.SampleFormat); + } + + output[frame] = Math.Clamp(sum / block.Channels, -1.0f, 1.0f); + } + + return output; + } + + private static int FaustSourceIndex(MimirActuatorCommand command) + { + foreach (var key in command.FaustControls.Keys) + { + const string prefix = "source"; + const string suffix = "/delay_samples"; + if (!key.StartsWith(prefix, StringComparison.Ordinal) || + !key.EndsWith(suffix, StringComparison.Ordinal)) + { + continue; + } + + var indexText = key[prefix.Length..^suffix.Length]; + if (int.TryParse(indexText, out var index)) + { + return index; + } + } + + return -1; + } + + private static int BytesPerSample(MimirAudioSampleFormat format) => + format switch + { + MimirAudioSampleFormat.Float32 => 4, + MimirAudioSampleFormat.Int16 => 2, + MimirAudioSampleFormat.Int24 => 3, + MimirAudioSampleFormat.Int32 => 4, + _ => 0 + }; + + private static float ReadPcmSample(ReadOnlySpan data, MimirAudioSampleFormat format) => + format switch + { + MimirAudioSampleFormat.Float32 => BitConverter.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(data)), + MimirAudioSampleFormat.Int16 => BinaryPrimitives.ReadInt16LittleEndian(data) / 32768.0f, + MimirAudioSampleFormat.Int24 => ReadInt24LittleEndian(data) / 8388608.0f, + MimirAudioSampleFormat.Int32 => BinaryPrimitives.ReadInt32LittleEndian(data) / 2147483648.0f, + _ => 0.0f + }; + + private static int ReadInt24LittleEndian(ReadOnlySpan data) + { + var value = data[0] | (data[1] << 8) | (data[2] << 16); + return (value & 0x800000) == 0 ? value : value | unchecked((int)0xff000000); + } + private static float ParseAudioSyncIntervalSeconds() { return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_AUDIO_SYNC_INTERVAL_SECONDS"), out var seconds) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 0e23a4a..11cad10 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -186,3 +186,4 @@ {"ts": "2026-05-27T18:03:11+01:00", "type": "decision", "status": "ok", "note": "Runtime audio sync states now derive explicit actuator command frames through MimirAlignmentActuatorBank: measured delay remains analyzer truth, the bank converts offsets into nonnegative Faust holdback controls with stable source slots, reference holdback, and overflow reporting. This is control-plane publication only; Faust/native DSP still owns live sample movement."} {"ts": "2026-05-27T18:06:41+01:00", "type": "decision", "status": "ok", "note": "MimirRuntime now queues the latest MimirAlignmentActuatorBank frame into AquariumAudioDocument as an engine audio control frame. This creates the Mimir-to-Fensalir control contract for Faust/native DSP targets, but remains control-plane only until a streaming DSP sample mover consumes it."} {"ts": "2026-05-27T18:54:28+01:00", "type": "decision", "status": "ok", "note": "Mimir now declares faust/mimir_alignment_actuator.dsp as an AquariumStreamingDspProgram before queuing actuator controls, giving Fensalir both the persistent Faust DSP graph and the control frames. Live sample alignment still waits on the next device I/O/stem-routing cut."} +{"ts": "2026-05-27T20:18:50+01:00", "type": "verification", "status": "ok", "note": "MimirRuntime now queues sample-bearing AquariumStreamingAudioBlock lanes for actuator source buffers after publishing the control frame. Channel indices are derived from the bank-assigned Faust sourceN control path, so source disappearance does not reshuffle lanes. Mimir build passed; Fensalir tests prove declared blocks enter the streaming Faust DSP host."} diff --git a/state/map.yaml b/state/map.yaml index e2be35c..aec9104 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -186,8 +186,11 @@ accepted_design: `AquariumAudioDocument` as engine audio control frames. Mimir also queues `faust/mimir_alignment_actuator.dsp` as an `AquariumStreamingDspProgram` so Fensalir can compile/host the persistent Faust graph that consumes those - controls; live sample movement still waits on device I/O and stem routing - through that graph. The synthetic + controls. The same runtime cadence queues sample-bearing + `AquariumStreamingAudioBlock` lanes from the latest actuator source buffers, + with channel indices derived from the bank-assigned Faust `sourceN` control + path; live stem publication still waits on routing processed DSP output + lanes into program audio. The synthetic codebook proof is `dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`. `Mimir.BufferSmoke --render-chirp-bin-f32` plus