Feed actuator audio blocks to Fensalir
This commit is contained in:
parent
b4308f0c48
commit
86130d4349
5 changed files with 170 additions and 7 deletions
|
|
@ -149,8 +149,11 @@ a named invariant that the native runtime cannot protect yet.
|
|||
`AquariumAudioDocument` as an audio control frame. Mimir also declares the
|
||||
`faust/mimir_alignment_actuator.dsp` program as an
|
||||
`AquariumStreamingDspProgram`, so Fensalir has both the persistent Faust DSP
|
||||
graph and the matching controls. Device I/O/stem routing is still the next
|
||||
cut before live samples are actually aligned.
|
||||
graph and the matching controls. The runtime now also queues sample-bearing
|
||||
`AquariumStreamingAudioBlock` lanes from the latest actuator source buffers,
|
||||
using the bank-assigned Faust `sourceN` slot as channel authority. Durable
|
||||
stem routing/publication is still the next cut before aligned audio becomes a
|
||||
separately mixable program output.
|
||||
- `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame`
|
||||
spectrum dashboard. Live spectrum visualization authority is the
|
||||
`AquariumFieldEvidenceFrame`: Mimir declares the rolling resource, emits a
|
||||
|
|
|
|||
|
|
@ -309,9 +309,11 @@ holdback separately. `MimirRuntime` updates that command frame on the sync
|
|||
analysis cadence, queues it into `AquariumAudioDocument` as an engine audio
|
||||
control frame, declares `faust/mimir_alignment_actuator.dsp` as an
|
||||
`AquariumStreamingDspProgram`, and telemetry can print the current DSP targets.
|
||||
Fensalir can now compile/host that persistent Faust graph, but device I/O and
|
||||
stem routing still have to feed live blocks through it before audio is actually
|
||||
aligned.
|
||||
Mimir also queues sample-bearing `AquariumStreamingAudioBlock` lanes from the
|
||||
latest actuator source buffers, using the bank-assigned Faust `sourceN` control
|
||||
path as channel authority. Fensalir can compile/host/process that persistent
|
||||
Faust graph now; durable stem routing and OBS-facing publication still have to
|
||||
turn processed output lanes into separately mixable program audio.
|
||||
|
||||
## Visual Fusion
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ using Aquarium.Engine.Render;
|
|||
using Aquarium.Engine.Ui;
|
||||
using CultMath;
|
||||
using Mimir.Runtime.Synchronization;
|
||||
using System.Buffers.Binary;
|
||||
using System.Diagnostics;
|
||||
using System.Numerics;
|
||||
using System.Text;
|
||||
|
|
@ -932,6 +933,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
|
|||
.ToArray(),
|
||||
frame.TruncatedSourceCount,
|
||||
++audioActuatorFrameSequence));
|
||||
QueueStreamingActuatorAudioBlock(frame, audioActuatorFrameSequence);
|
||||
}
|
||||
|
||||
private void QueueAudioActuatorProgram()
|
||||
|
|
@ -957,6 +959,77 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
|
|||
audioActuatorProgramQueued = true;
|
||||
}
|
||||
|
||||
private void QueueStreamingActuatorAudioBlock(MimirActuatorFrame frame, long sequence)
|
||||
{
|
||||
var commands = frame.Commands.OrderBy(command => command.SourceId, StringComparer.Ordinal).ToArray();
|
||||
if (commands.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var buffers = synchronization.Buffers.Buffers
|
||||
.Where(buffer => buffer.Descriptor.Kind == MimirStreamKind.Audio && buffer.Latest?.AudioBlock != null)
|
||||
.ToDictionary(buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal);
|
||||
var channels = new List<AquariumStreamingAudioChannel>(commands.Length);
|
||||
var frameCount = int.MaxValue;
|
||||
var sampleRate = 0;
|
||||
for (var index = 0; index < commands.Length; index++)
|
||||
{
|
||||
if (!buffers.TryGetValue(commands[index].SourceId, out var buffer) ||
|
||||
buffer.Latest is not { } latest ||
|
||||
latest.AudioBlock is not { } block ||
|
||||
latest.Data.IsEmpty)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (sampleRate == 0)
|
||||
{
|
||||
sampleRate = block.SampleRate;
|
||||
}
|
||||
else if (block.SampleRate != sampleRate)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var samples = ExtractMonoAudioBlock(latest, block);
|
||||
if (samples.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
frameCount = Math.Min(frameCount, samples.Length);
|
||||
var channelIndex = FaustSourceIndex(commands[index]);
|
||||
if (channelIndex < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
channels.Add(new AquariumStreamingAudioChannel(channelIndex, commands[index].SourceId, samples));
|
||||
}
|
||||
|
||||
if (channels.Count == 0 || sampleRate <= 0 || frameCount == int.MaxValue)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (channels.Any(channel => channel.Samples.Length != frameCount))
|
||||
{
|
||||
channels = channels
|
||||
.Select(channel => channel.Samples.Length == frameCount
|
||||
? channel
|
||||
: channel with { Samples = channel.Samples.AsSpan(0, frameCount).ToArray() })
|
||||
.ToList();
|
||||
}
|
||||
|
||||
audio.EnqueueStreamingAudioBlock(new AquariumStreamingAudioBlock(
|
||||
MimirAlignmentActuatorProfile.SixSourceFaust.Id,
|
||||
channels,
|
||||
frameCount,
|
||||
sampleRate,
|
||||
sequence));
|
||||
}
|
||||
|
||||
private void UpdateAudioSpectra()
|
||||
{
|
||||
if (runtimeSeconds < nextSpectrumSeconds)
|
||||
|
|
@ -1088,6 +1161,87 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
|
|||
return null;
|
||||
}
|
||||
|
||||
private static float[] ExtractMonoAudioBlock(MimirStreamSample sample, MimirAudioBlockDescriptor block)
|
||||
{
|
||||
var bytesPerSample = BytesPerSample(block.SampleFormat);
|
||||
if (bytesPerSample == 0 || block.Channels <= 0 || block.FrameCount <= 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var data = sample.Data.Span;
|
||||
var stride = block.Channels * bytesPerSample;
|
||||
if (data.Length < stride)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var frameCount = Math.Min(block.FrameCount, data.Length / stride);
|
||||
var output = new float[frameCount];
|
||||
for (var frame = 0; frame < frameCount; frame++)
|
||||
{
|
||||
var sum = 0.0f;
|
||||
var frameOffset = frame * stride;
|
||||
for (var channel = 0; channel < block.Channels; channel++)
|
||||
{
|
||||
var offset = frameOffset + channel * bytesPerSample;
|
||||
sum += ReadPcmSample(data.Slice(offset, bytesPerSample), block.SampleFormat);
|
||||
}
|
||||
|
||||
output[frame] = Math.Clamp(sum / block.Channels, -1.0f, 1.0f);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
private static int FaustSourceIndex(MimirActuatorCommand command)
|
||||
{
|
||||
foreach (var key in command.FaustControls.Keys)
|
||||
{
|
||||
const string prefix = "source";
|
||||
const string suffix = "/delay_samples";
|
||||
if (!key.StartsWith(prefix, StringComparison.Ordinal) ||
|
||||
!key.EndsWith(suffix, StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var indexText = key[prefix.Length..^suffix.Length];
|
||||
if (int.TryParse(indexText, out var index))
|
||||
{
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static int BytesPerSample(MimirAudioSampleFormat format) =>
|
||||
format switch
|
||||
{
|
||||
MimirAudioSampleFormat.Float32 => 4,
|
||||
MimirAudioSampleFormat.Int16 => 2,
|
||||
MimirAudioSampleFormat.Int24 => 3,
|
||||
MimirAudioSampleFormat.Int32 => 4,
|
||||
_ => 0
|
||||
};
|
||||
|
||||
private static float ReadPcmSample(ReadOnlySpan<byte> data, MimirAudioSampleFormat format) =>
|
||||
format switch
|
||||
{
|
||||
MimirAudioSampleFormat.Float32 => BitConverter.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(data)),
|
||||
MimirAudioSampleFormat.Int16 => BinaryPrimitives.ReadInt16LittleEndian(data) / 32768.0f,
|
||||
MimirAudioSampleFormat.Int24 => ReadInt24LittleEndian(data) / 8388608.0f,
|
||||
MimirAudioSampleFormat.Int32 => BinaryPrimitives.ReadInt32LittleEndian(data) / 2147483648.0f,
|
||||
_ => 0.0f
|
||||
};
|
||||
|
||||
private static int ReadInt24LittleEndian(ReadOnlySpan<byte> data)
|
||||
{
|
||||
var value = data[0] | (data[1] << 8) | (data[2] << 16);
|
||||
return (value & 0x800000) == 0 ? value : value | unchecked((int)0xff000000);
|
||||
}
|
||||
|
||||
private static float ParseAudioSyncIntervalSeconds()
|
||||
{
|
||||
return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_AUDIO_SYNC_INTERVAL_SECONDS"), out var seconds)
|
||||
|
|
|
|||
|
|
@ -186,3 +186,4 @@
|
|||
{"ts": "2026-05-27T18:03:11+01:00", "type": "decision", "status": "ok", "note": "Runtime audio sync states now derive explicit actuator command frames through MimirAlignmentActuatorBank: measured delay remains analyzer truth, the bank converts offsets into nonnegative Faust holdback controls with stable source slots, reference holdback, and overflow reporting. This is control-plane publication only; Faust/native DSP still owns live sample movement."}
|
||||
{"ts": "2026-05-27T18:06:41+01:00", "type": "decision", "status": "ok", "note": "MimirRuntime now queues the latest MimirAlignmentActuatorBank frame into AquariumAudioDocument as an engine audio control frame. This creates the Mimir-to-Fensalir control contract for Faust/native DSP targets, but remains control-plane only until a streaming DSP sample mover consumes it."}
|
||||
{"ts": "2026-05-27T18:54:28+01:00", "type": "decision", "status": "ok", "note": "Mimir now declares faust/mimir_alignment_actuator.dsp as an AquariumStreamingDspProgram before queuing actuator controls, giving Fensalir both the persistent Faust DSP graph and the control frames. Live sample alignment still waits on the next device I/O/stem-routing cut."}
|
||||
{"ts": "2026-05-27T20:18:50+01:00", "type": "verification", "status": "ok", "note": "MimirRuntime now queues sample-bearing AquariumStreamingAudioBlock lanes for actuator source buffers after publishing the control frame. Channel indices are derived from the bank-assigned Faust sourceN control path, so source disappearance does not reshuffle lanes. Mimir build passed; Fensalir tests prove declared blocks enter the streaming Faust DSP host."}
|
||||
|
|
|
|||
|
|
@ -186,8 +186,11 @@ accepted_design:
|
|||
`AquariumAudioDocument` as engine audio control frames. Mimir also queues
|
||||
`faust/mimir_alignment_actuator.dsp` as an `AquariumStreamingDspProgram`
|
||||
so Fensalir can compile/host the persistent Faust graph that consumes those
|
||||
controls; live sample movement still waits on device I/O and stem routing
|
||||
through that graph. The synthetic
|
||||
controls. The same runtime cadence queues sample-bearing
|
||||
`AquariumStreamingAudioBlock` lanes from the latest actuator source buffers,
|
||||
with channel indices derived from the bank-assigned Faust `sourceN` control
|
||||
path; live stem publication still waits on routing processed DSP output
|
||||
lanes into program audio. The synthetic
|
||||
codebook proof is `dotnet run --project
|
||||
.\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`.
|
||||
`Mimir.BufferSmoke --render-chirp-bin-f32` plus
|
||||
|
|
|
|||
Loading…
Reference in a new issue