Feed actuator audio blocks to Fensalir

This commit is contained in:
Meta Vens 2026-05-27 20:19:26 +01:00
commit 86130d4349
5 changed files with 170 additions and 7 deletions

View file

@ -149,8 +149,11 @@ a named invariant that the native runtime cannot protect yet.
`AquariumAudioDocument` as an audio control frame. Mimir also declares the
`faust/mimir_alignment_actuator.dsp` program as an
`AquariumStreamingDspProgram`, so Fensalir has both the persistent Faust DSP
graph and the matching controls. Device I/O/stem routing is still the next
cut before live samples are actually aligned.
graph and the matching controls. The runtime now also queues sample-bearing
`AquariumStreamingAudioBlock` lanes from the latest actuator source buffers,
using the bank-assigned Faust `sourceN` slot as channel authority. Durable
stem routing/publication is still the next cut before aligned audio becomes a
separately mixable program output.
- `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame`
spectrum dashboard. Live spectrum visualization authority is the
`AquariumFieldEvidenceFrame`: Mimir declares the rolling resource, emits a

View file

@ -309,9 +309,11 @@ holdback separately. `MimirRuntime` updates that command frame on the sync
analysis cadence, queues it into `AquariumAudioDocument` as an engine audio
control frame, declares `faust/mimir_alignment_actuator.dsp` as an
`AquariumStreamingDspProgram`, and telemetry can print the current DSP targets.
Fensalir can now compile/host that persistent Faust graph, but device I/O and
stem routing still have to feed live blocks through it before audio is actually
aligned.
Mimir also queues sample-bearing `AquariumStreamingAudioBlock` lanes from the
latest actuator source buffers, using the bank-assigned Faust `sourceN` control
path as channel authority. Fensalir can compile/host/process that persistent
Faust graph now; durable stem routing and OBS-facing publication still have to
turn processed output lanes into separately mixable program audio.
## Visual Fusion

View file

@ -5,6 +5,7 @@ using Aquarium.Engine.Render;
using Aquarium.Engine.Ui;
using CultMath;
using Mimir.Runtime.Synchronization;
using System.Buffers.Binary;
using System.Diagnostics;
using System.Numerics;
using System.Text;
@ -932,6 +933,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
.ToArray(),
frame.TruncatedSourceCount,
++audioActuatorFrameSequence));
QueueStreamingActuatorAudioBlock(frame, audioActuatorFrameSequence);
}
private void QueueAudioActuatorProgram()
@ -957,6 +959,77 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
audioActuatorProgramQueued = true;
}
private void QueueStreamingActuatorAudioBlock(MimirActuatorFrame frame, long sequence)
{
var commands = frame.Commands.OrderBy(command => command.SourceId, StringComparer.Ordinal).ToArray();
if (commands.Length == 0)
{
return;
}
var buffers = synchronization.Buffers.Buffers
.Where(buffer => buffer.Descriptor.Kind == MimirStreamKind.Audio && buffer.Latest?.AudioBlock != null)
.ToDictionary(buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal);
var channels = new List<AquariumStreamingAudioChannel>(commands.Length);
var frameCount = int.MaxValue;
var sampleRate = 0;
for (var index = 0; index < commands.Length; index++)
{
if (!buffers.TryGetValue(commands[index].SourceId, out var buffer) ||
buffer.Latest is not { } latest ||
latest.AudioBlock is not { } block ||
latest.Data.IsEmpty)
{
continue;
}
if (sampleRate == 0)
{
sampleRate = block.SampleRate;
}
else if (block.SampleRate != sampleRate)
{
continue;
}
var samples = ExtractMonoAudioBlock(latest, block);
if (samples.Length == 0)
{
continue;
}
frameCount = Math.Min(frameCount, samples.Length);
var channelIndex = FaustSourceIndex(commands[index]);
if (channelIndex < 0)
{
continue;
}
channels.Add(new AquariumStreamingAudioChannel(channelIndex, commands[index].SourceId, samples));
}
if (channels.Count == 0 || sampleRate <= 0 || frameCount == int.MaxValue)
{
return;
}
if (channels.Any(channel => channel.Samples.Length != frameCount))
{
channels = channels
.Select(channel => channel.Samples.Length == frameCount
? channel
: channel with { Samples = channel.Samples.AsSpan(0, frameCount).ToArray() })
.ToList();
}
audio.EnqueueStreamingAudioBlock(new AquariumStreamingAudioBlock(
MimirAlignmentActuatorProfile.SixSourceFaust.Id,
channels,
frameCount,
sampleRate,
sequence));
}
private void UpdateAudioSpectra()
{
if (runtimeSeconds < nextSpectrumSeconds)
@ -1088,6 +1161,87 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
return null;
}
private static float[] ExtractMonoAudioBlock(MimirStreamSample sample, MimirAudioBlockDescriptor block)
{
var bytesPerSample = BytesPerSample(block.SampleFormat);
if (bytesPerSample == 0 || block.Channels <= 0 || block.FrameCount <= 0)
{
return [];
}
var data = sample.Data.Span;
var stride = block.Channels * bytesPerSample;
if (data.Length < stride)
{
return [];
}
var frameCount = Math.Min(block.FrameCount, data.Length / stride);
var output = new float[frameCount];
for (var frame = 0; frame < frameCount; frame++)
{
var sum = 0.0f;
var frameOffset = frame * stride;
for (var channel = 0; channel < block.Channels; channel++)
{
var offset = frameOffset + channel * bytesPerSample;
sum += ReadPcmSample(data.Slice(offset, bytesPerSample), block.SampleFormat);
}
output[frame] = Math.Clamp(sum / block.Channels, -1.0f, 1.0f);
}
return output;
}
private static int FaustSourceIndex(MimirActuatorCommand command)
{
foreach (var key in command.FaustControls.Keys)
{
const string prefix = "source";
const string suffix = "/delay_samples";
if (!key.StartsWith(prefix, StringComparison.Ordinal) ||
!key.EndsWith(suffix, StringComparison.Ordinal))
{
continue;
}
var indexText = key[prefix.Length..^suffix.Length];
if (int.TryParse(indexText, out var index))
{
return index;
}
}
return -1;
}
private static int BytesPerSample(MimirAudioSampleFormat format) =>
format switch
{
MimirAudioSampleFormat.Float32 => 4,
MimirAudioSampleFormat.Int16 => 2,
MimirAudioSampleFormat.Int24 => 3,
MimirAudioSampleFormat.Int32 => 4,
_ => 0
};
private static float ReadPcmSample(ReadOnlySpan<byte> data, MimirAudioSampleFormat format) =>
format switch
{
MimirAudioSampleFormat.Float32 => BitConverter.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(data)),
MimirAudioSampleFormat.Int16 => BinaryPrimitives.ReadInt16LittleEndian(data) / 32768.0f,
MimirAudioSampleFormat.Int24 => ReadInt24LittleEndian(data) / 8388608.0f,
MimirAudioSampleFormat.Int32 => BinaryPrimitives.ReadInt32LittleEndian(data) / 2147483648.0f,
_ => 0.0f
};
private static int ReadInt24LittleEndian(ReadOnlySpan<byte> data)
{
var value = data[0] | (data[1] << 8) | (data[2] << 16);
return (value & 0x800000) == 0 ? value : value | unchecked((int)0xff000000);
}
private static float ParseAudioSyncIntervalSeconds()
{
return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_AUDIO_SYNC_INTERVAL_SECONDS"), out var seconds)

View file

@ -186,3 +186,4 @@
{"ts": "2026-05-27T18:03:11+01:00", "type": "decision", "status": "ok", "note": "Runtime audio sync states now derive explicit actuator command frames through MimirAlignmentActuatorBank: measured delay remains analyzer truth, the bank converts offsets into nonnegative Faust holdback controls with stable source slots, reference holdback, and overflow reporting. This is control-plane publication only; Faust/native DSP still owns live sample movement."}
{"ts": "2026-05-27T18:06:41+01:00", "type": "decision", "status": "ok", "note": "MimirRuntime now queues the latest MimirAlignmentActuatorBank frame into AquariumAudioDocument as an engine audio control frame. This creates the Mimir-to-Fensalir control contract for Faust/native DSP targets, but remains control-plane only until a streaming DSP sample mover consumes it."}
{"ts": "2026-05-27T18:54:28+01:00", "type": "decision", "status": "ok", "note": "Mimir now declares faust/mimir_alignment_actuator.dsp as an AquariumStreamingDspProgram before queuing actuator controls, giving Fensalir both the persistent Faust DSP graph and the control frames. Live sample alignment still waits on the next device I/O/stem-routing cut."}
{"ts": "2026-05-27T20:18:50+01:00", "type": "verification", "status": "ok", "note": "MimirRuntime now queues sample-bearing AquariumStreamingAudioBlock lanes for actuator source buffers after publishing the control frame. Channel indices are derived from the bank-assigned Faust sourceN control path, so source disappearance does not reshuffle lanes. Mimir build passed; Fensalir tests prove declared blocks enter the streaming Faust DSP host."}

View file

@ -186,8 +186,11 @@ accepted_design:
`AquariumAudioDocument` as engine audio control frames. Mimir also queues
`faust/mimir_alignment_actuator.dsp` as an `AquariumStreamingDspProgram`
so Fensalir can compile/host the persistent Faust graph that consumes those
controls; live sample movement still waits on device I/O and stem routing
through that graph. The synthetic
controls. The same runtime cadence queues sample-bearing
`AquariumStreamingAudioBlock` lanes from the latest actuator source buffers,
with channel indices derived from the bank-assigned Faust `sourceN` control
path; live stem publication still waits on routing processed DSP output
lanes into program audio. The synthetic
codebook proof is `dotnet run --project
.\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`.
`Mimir.BufferSmoke --render-chirp-bin-f32` plus