Render live audio spectra after scene ready

This commit is contained in:
Meta Vens 2026-05-25 07:06:25 +01:00
commit 6e370ff2ac
3 changed files with 450 additions and 3 deletions

View file

@ -6,6 +6,7 @@ using Aquarium.Engine.Ui;
using Mimir.Runtime.Synchronization;
using System.Diagnostics;
using System.Numerics;
using System.Text;
namespace Mimir.Runtime;
@ -14,26 +15,33 @@ public sealed class MimirRuntime : IAquariumRuntime
private const float DefaultAudioSyncUpdateIntervalSeconds = 0.1f;
private const double HybridPassiveConfidenceThreshold = 0.12;
private const double CalibrationStartSeconds = 0.5;
private const int CalibrationBatchSegments = 120;
private const int CalibrationBatchSegments = 4;
private const int HybridWatermarkIntervalSegments = 4;
private const float DefaultSpectrumUpdateIntervalSeconds = 0.2f;
private readonly MimirSynchronizationHub synchronization;
private readonly MimirAudioSpectrumAnalyzer spectrumAnalyzer;
private readonly AquariumUiDocument ui;
private readonly AquariumAudioDocument audio = new();
private readonly MimirAudioSynchronizationSettings audioSyncSettings;
private readonly float telemetryIntervalSeconds;
private readonly float audioSyncUpdateIntervalSeconds;
private readonly float spectrumUpdateIntervalSeconds;
private readonly float calibrationGain;
private readonly float watermarkGain;
private readonly MimirBioacousticContestantRenderer? complexContourWitness;
private int lastPollCount;
private float runtimeSeconds;
private float nextAudioSyncSeconds;
private float nextSpectrumSeconds;
private float nextTelemetrySeconds;
private bool sceneReady;
private ulong calibrationSegmentIndex;
private long lastHybridWatermarkSegment = -1;
private double lastAudioSyncAnalysisMilliseconds;
private double lastSpectrumAnalysisMilliseconds;
private double lastPassiveSynchronizationConfidence;
private IReadOnlyList<MimirAudioSynchronizationReport> lastAudioSynchronizationReports = [];
private IReadOnlyList<MimirAudioSpectrumSnapshot> lastAudioSpectra = [];
public MimirRuntime(AquariumRuntimeOptions options)
: this(options, MimirRuntimeConfiguration.Load())
@ -57,9 +65,11 @@ public sealed class MimirRuntime : IAquariumRuntime
{
Options = options;
synchronization = new MimirSynchronizationHub(settings);
spectrumAnalyzer = new MimirAudioSpectrumAnalyzer(ParseSpectrumFftSize());
audioSyncSettings = settings.Audio;
telemetryIntervalSeconds = ParseTelemetryIntervalSeconds();
audioSyncUpdateIntervalSeconds = ParseAudioSyncIntervalSeconds();
spectrumUpdateIntervalSeconds = ParseSpectrumUpdateIntervalSeconds();
calibrationGain = settings.Audio.CalibrationGain;
watermarkGain = settings.Audio.WatermarkGain;
complexContourWitness = settings.Audio.EnableComplexContourRuntime
@ -115,12 +125,30 @@ public sealed class MimirRuntime : IAquariumRuntime
public void Update(float deltaSeconds, InputState input)
{
runtimeSeconds += Math.Max(deltaSeconds, 0.0f);
QueueCalibrationTimeline();
lastPollCount = synchronization.PollSources();
UpdateAudioSynchronization();
if (sceneReady)
{
QueueCalibrationTimeline();
UpdateAudioSpectra();
UpdateAudioSynchronization();
}
EmitTelemetry();
}
public void OnSceneReady()
{
if (sceneReady)
{
return;
}
sceneReady = true;
nextAudioSyncSeconds = runtimeSeconds + audioSyncUpdateIntervalSeconds;
nextSpectrumSeconds = runtimeSeconds;
Console.WriteLine($"Mimir Fensalir scene ready at {runtimeSeconds:0.000}s; runtime audio tests and spectra enabled.");
}
public AquariumFrame ComposeFrame(AquariumFrame frame, AquariumFrameInput input)
{
return frame;
@ -151,6 +179,18 @@ public sealed class MimirRuntime : IAquariumRuntime
panel.Readout("Audio sync state", DescribeAudioSyncState);
panel.Readout("Aligned audio", DescribeAlignedAudio);
panel.Readout("Chirplet reference", DescribeChirpletReference);
panel.Section("Live FFT");
panel.Readout("Spectrum cadence", () => sceneReady
? $"{lastAudioSpectra.Count} spectra fft={lastAudioSpectra.FirstOrDefault()?.FftSize ?? 0} analyze={lastSpectrumAnalysisMilliseconds:0.00}ms"
: "waiting for Fensalir scene ready");
panel.TextBox(
"Spectra",
DescribeSpectra,
_ => { },
lines: 12,
acceptsReturn: false,
monospace: true,
tooltip: "Cached FFT spectra from the runtime rolling audio buffers.");
});
}
@ -311,6 +351,20 @@ public sealed class MimirRuntime : IAquariumRuntime
nextAudioSyncSeconds = runtimeSeconds + audioSyncUpdateIntervalSeconds;
}
private void UpdateAudioSpectra()
{
if (runtimeSeconds < nextSpectrumSeconds)
{
return;
}
var stopwatch = Stopwatch.StartNew();
lastAudioSpectra = spectrumAnalyzer.Analyze(synchronization.Buffers.Buffers, audioSyncSettings.ReferenceSourceId);
stopwatch.Stop();
lastSpectrumAnalysisMilliseconds = stopwatch.Elapsed.TotalMilliseconds;
nextSpectrumSeconds = runtimeSeconds + spectrumUpdateIntervalSeconds;
}
private void EmitTelemetry()
{
if (telemetryIntervalSeconds <= 0.0f || runtimeSeconds < nextTelemetrySeconds)
@ -324,6 +378,12 @@ public sealed class MimirRuntime : IAquariumRuntime
Console.WriteLine(
$"mimir-sync-telemetry t={runtimeSeconds:0.00}s audioSync={audioSyncSettings.Mode} loopbackCount={loopback?.Count ?? 0} loopbackEdgeNs={loopback?.EdgeNs ?? 0} reports={lastAudioSynchronizationReports.Count} states={states.Count} analyzeMs={lastAudioSyncAnalysisMilliseconds:0.0} aligned={DescribeAlignedAudio()}");
Console.WriteLine($"mimir-sync-buffers {DescribeAudioBuffers()}");
foreach (var spectrum in lastAudioSpectra.OrderBy(snapshot => snapshot.SourceId, StringComparer.Ordinal))
{
Console.WriteLine(
$"mimir-spectrum {spectrum.SourceId} rate={spectrum.SampleRate} fft={spectrum.FftSize} rms={spectrum.Rms:0.000000} peak={spectrum.Peak:0.000000} floorDb={spectrum.NoiseFloorDb:0.0} peaks={DescribeSpectrumPeaks(spectrum)}");
}
foreach (var report in lastAudioSynchronizationReports.OrderBy(report => report.SourceId, StringComparer.Ordinal))
{
Console.WriteLine(
@ -365,6 +425,20 @@ public sealed class MimirRuntime : IAquariumRuntime
: DefaultAudioSyncUpdateIntervalSeconds;
}
private static float ParseSpectrumUpdateIntervalSeconds()
{
return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_SPECTRUM_INTERVAL_SECONDS"), out var seconds)
? Math.Clamp(seconds, 0.05f, 10.0f)
: DefaultSpectrumUpdateIntervalSeconds;
}
private static int ParseSpectrumFftSize()
{
return int.TryParse(Environment.GetEnvironmentVariable("MIMIR_SPECTRUM_FFT_SIZE"), out var value)
? Math.Clamp(value, 1024, 32768)
: 8192;
}
private bool ShouldEmitCalibrationTimeline()
{
if (!HasReferenceAudioBuffer())
@ -407,6 +481,65 @@ public sealed class MimirRuntime : IAquariumRuntime
.Select(buffer => $"{buffer.Descriptor.SourceId}:{buffer.Count}@{buffer.EdgeNs}"));
}
private string DescribeSpectra()
{
if (!sceneReady)
{
return "Fensalir scene not ready.";
}
if (lastAudioSpectra.Count == 0)
{
return "No sample-bearing audio buffers yet.";
}
var builder = new StringBuilder();
foreach (var spectrum in lastAudioSpectra)
{
builder
.Append(spectrum.SourceId)
.Append(" rms=")
.Append(spectrum.Rms.ToString("0.000000"))
.Append(" peak=")
.Append(spectrum.Peak.ToString("0.000000"))
.Append(" ")
.Append(SpectrumBars(spectrum))
.AppendLine();
builder
.Append(" ")
.Append(DescribeSpectrumPeaks(spectrum))
.AppendLine();
}
return builder.ToString().TrimEnd();
}
private static string SpectrumBars(MimirAudioSpectrumSnapshot spectrum)
{
const string ramp = " .:-=+*#%@";
if (spectrum.BandDecibels.Count == 0)
{
return "";
}
var floor = Math.Min(spectrum.NoiseFloorDb, spectrum.BandDecibels.Min());
var ceiling = Math.Max(-24.0, spectrum.BandDecibels.Max());
var span = Math.Max(12.0, ceiling - floor);
var chars = spectrum.BandDecibels.Select(value =>
{
var normalized = Math.Clamp((value - floor) / span, 0.0, 1.0);
return ramp[(int)Math.Round(normalized * (ramp.Length - 1))];
});
return new string(chars.ToArray());
}
private static string DescribeSpectrumPeaks(MimirAudioSpectrumSnapshot spectrum)
{
return spectrum.Peaks.Count == 0
? "no FFT peaks"
: string.Join(", ", spectrum.Peaks.Select(peak => $"{peak.FrequencyHz / 1000.0:0.00}kHz {peak.Decibels:0.0}dB"));
}
private void UpdatePassiveSynchronizationConfidence(IReadOnlyList<MimirAudioSynchronizationReport> reports)
{
var passiveConfidence = reports

View file

@ -0,0 +1,313 @@
using System.Buffers.Binary;
using System.Numerics;
namespace Mimir.Runtime.Synchronization;
public sealed record MimirAudioSpectrumBin(double FrequencyHz, double Decibels, double Magnitude);
public sealed record MimirAudioSpectrumSnapshot(
string SourceId,
int SampleRate,
int FftSize,
int WindowSamples,
double Rms,
double Peak,
double NoiseFloorDb,
IReadOnlyList<MimirAudioSpectrumBin> Peaks,
IReadOnlyList<double> BandDecibels,
long EdgeNs);
public sealed class MimirAudioSpectrumAnalyzer
{
private const int MinimumFftSize = 1024;
private const int MaximumFftSize = 32768;
private readonly int fftSize;
private readonly int displayBandCount;
private readonly Complex[] spectrum;
public MimirAudioSpectrumAnalyzer(int fftSize = 8192, int displayBandCount = 48)
{
this.fftSize = Math.Clamp(NextPowerOfTwo(Math.Max(MinimumFftSize, fftSize)), MinimumFftSize, MaximumFftSize);
this.displayBandCount = Math.Clamp(displayBandCount, 16, 96);
spectrum = new Complex[this.fftSize];
}
public IReadOnlyList<MimirAudioSpectrumSnapshot> Analyze(
IEnumerable<MimirRollingStreamBuffer> buffers,
string referenceSourceId,
int maxNonReferenceSources = 3)
{
var audioBuffers = buffers
.Where(buffer => buffer.Descriptor.Kind == MimirStreamKind.Audio && buffer.Latest?.AudioBlock != null)
.OrderBy(buffer => string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal) ? 0 : 1)
.ThenBy(buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal)
.ToArray();
if (audioBuffers.Length == 0)
{
return [];
}
var snapshots = new List<MimirAudioSpectrumSnapshot>();
var nonReferenceCount = 0;
foreach (var buffer in audioBuffers)
{
var isReference = string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal);
if (!isReference && nonReferenceCount++ >= maxNonReferenceSources)
{
continue;
}
var snapshot = AnalyzeBuffer(buffer);
if (snapshot != null)
{
snapshots.Add(snapshot);
}
}
return snapshots;
}
private MimirAudioSpectrumSnapshot? AnalyzeBuffer(MimirRollingStreamBuffer buffer)
{
var latestBlock = buffer.Latest?.AudioBlock;
if (latestBlock == null || latestBlock.Channels <= 0 || latestBlock.SampleRate <= 0)
{
return null;
}
Array.Clear(spectrum);
var written = FillLatestMonoWindow(buffer, latestBlock, spectrum);
if (written < MinimumFftSize / 2)
{
return null;
}
var rms = 0.0;
var peak = 0.0;
for (var index = 0; index < written; index++)
{
var sample = spectrum[index].Real;
rms += sample * sample;
peak = Math.Max(peak, Math.Abs(sample));
}
rms = Math.Sqrt(rms / Math.Max(1, written));
ApplyHannWindow(spectrum, written);
FastFourierTransform(spectrum, inverse: false);
var magnitudes = new double[fftSize / 2];
for (var index = 1; index < magnitudes.Length; index++)
{
magnitudes[index] = spectrum[index].Magnitude / Math.Max(1, written);
}
var bands = BuildLogBands(magnitudes, latestBlock.SampleRate);
var peaks = FindPeaks(magnitudes, latestBlock.SampleRate, count: 6);
var floor = bands.Count == 0 ? -120.0 : bands.OrderBy(value => value).ElementAt(Math.Clamp(bands.Count / 4, 0, bands.Count - 1));
return new MimirAudioSpectrumSnapshot(
buffer.Descriptor.SourceId,
latestBlock.SampleRate,
fftSize,
written,
rms,
peak,
floor,
peaks,
bands,
buffer.EdgeNs);
}
private int FillLatestMonoWindow(
MimirRollingStreamBuffer buffer,
MimirAudioBlockDescriptor latestBlock,
Complex[] destination)
{
var write = destination.Length;
foreach (var sample in buffer.Snapshot()
.Where(sample => sample.AudioBlock != null && !sample.Data.IsEmpty)
.Reverse())
{
var block = sample.AudioBlock!;
if (!IsSupportedPcmFormat(block.SampleFormat) || block.Channels <= 0 || block.SampleRate != latestBlock.SampleRate)
{
continue;
}
var bytesPerSample = BytesPerSample(block.SampleFormat);
var frameCount = sample.Data.Length / (bytesPerSample * block.Channels);
var data = sample.Data.Span;
for (var frame = frameCount - 1; frame >= 0 && write > 0; frame--)
{
var offset = frame * block.Channels * bytesPerSample;
destination[--write] = new Complex(ReadPcmSample(data.Slice(offset, bytesPerSample), block.SampleFormat), 0.0);
}
if (write == 0)
{
break;
}
}
var written = destination.Length - write;
if (write > 0 && written > 0)
{
Array.Copy(destination, write, destination, 0, written);
Array.Clear(destination, written, destination.Length - written);
}
return written;
}
private IReadOnlyList<double> BuildLogBands(IReadOnlyList<double> magnitudes, int sampleRate)
{
var bands = new double[displayBandCount];
var nyquist = sampleRate * 0.5;
var minHz = 40.0;
var maxHz = Math.Max(minHz * 2.0, nyquist);
for (var band = 0; band < bands.Length; band++)
{
var start = band / (double)bands.Length;
var end = (band + 1) / (double)bands.Length;
var lowHz = minHz * Math.Pow(maxHz / minHz, start);
var highHz = minHz * Math.Pow(maxHz / minHz, end);
var firstBin = Math.Clamp((int)Math.Floor(lowHz * fftSize / sampleRate), 1, magnitudes.Count - 1);
var lastBin = Math.Clamp((int)Math.Ceiling(highHz * fftSize / sampleRate), firstBin, magnitudes.Count - 1);
var sum = 0.0;
var count = 0;
for (var bin = firstBin; bin <= lastBin; bin++)
{
sum += magnitudes[bin] * magnitudes[bin];
count++;
}
bands[band] = ToDecibels(Math.Sqrt(sum / Math.Max(1, count)));
}
return bands;
}
private IReadOnlyList<MimirAudioSpectrumBin> FindPeaks(IReadOnlyList<double> magnitudes, int sampleRate, int count)
{
var peaks = new List<MimirAudioSpectrumBin>();
var minBin = Math.Max(1, (int)Math.Round(20.0 * fftSize / sampleRate));
var maxBin = Math.Min(magnitudes.Count - 2, (int)Math.Round(Math.Min(sampleRate * 0.48, 24000.0) * fftSize / sampleRate));
for (var bin = minBin; bin <= maxBin; bin++)
{
var magnitude = magnitudes[bin];
if (magnitude <= magnitudes[bin - 1] || magnitude < magnitudes[bin + 1])
{
continue;
}
peaks.Add(new MimirAudioSpectrumBin(bin * sampleRate / (double)fftSize, ToDecibels(magnitude), magnitude));
}
return peaks
.OrderByDescending(peak => peak.Magnitude)
.Take(count)
.OrderBy(peak => peak.FrequencyHz)
.ToArray();
}
private static void ApplyHannWindow(Complex[] values, int count)
{
for (var index = 0; index < count; index++)
{
var window = 0.5 - 0.5 * Math.Cos(2.0 * Math.PI * index / Math.Max(1, count - 1));
values[index] *= window;
}
}
private static bool IsSupportedPcmFormat(MimirAudioSampleFormat sampleFormat) =>
sampleFormat is MimirAudioSampleFormat.Float32 or
MimirAudioSampleFormat.Int16 or
MimirAudioSampleFormat.Int24 or
MimirAudioSampleFormat.Int32;
private static int BytesPerSample(MimirAudioSampleFormat sampleFormat) =>
sampleFormat switch
{
MimirAudioSampleFormat.Float32 => sizeof(float),
MimirAudioSampleFormat.Int16 => sizeof(short),
MimirAudioSampleFormat.Int24 => 3,
MimirAudioSampleFormat.Int32 => sizeof(int),
_ => 0,
};
private static float ReadPcmSample(ReadOnlySpan<byte> data, MimirAudioSampleFormat sampleFormat) =>
sampleFormat switch
{
MimirAudioSampleFormat.Float32 => BitConverter.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(data)),
MimirAudioSampleFormat.Int16 => BinaryPrimitives.ReadInt16LittleEndian(data) / 32768.0f,
MimirAudioSampleFormat.Int24 => ReadInt24LittleEndian(data) / 8388608.0f,
MimirAudioSampleFormat.Int32 => BinaryPrimitives.ReadInt32LittleEndian(data) / 2147483648.0f,
_ => 0.0f,
};
private static int ReadInt24LittleEndian(ReadOnlySpan<byte> data)
{
var value = data[0] | (data[1] << 8) | (data[2] << 16);
return (value & 0x800000) != 0 ? value | unchecked((int)0xff000000) : value;
}
private static double ToDecibels(double value) => 20.0 * Math.Log10(Math.Max(value, 1.0e-12));
private static int NextPowerOfTwo(int value)
{
var power = 1;
while (power < value)
{
power <<= 1;
}
return power;
}
private static void FastFourierTransform(Complex[] values, bool inverse)
{
var n = values.Length;
for (int index = 1, swap = 0; index < n; index++)
{
var bit = n >> 1;
for (; (swap & bit) != 0; bit >>= 1)
{
swap ^= bit;
}
swap ^= bit;
if (index < swap)
{
(values[index], values[swap]) = (values[swap], values[index]);
}
}
for (var length = 2; length <= n; length <<= 1)
{
var angle = 2.0 * Math.PI / length * (inverse ? 1.0 : -1.0);
var step = new Complex(Math.Cos(angle), Math.Sin(angle));
for (var start = 0; start < n; start += length)
{
var rotation = Complex.One;
var half = length >> 1;
for (var offset = 0; offset < half; offset++)
{
var even = values[start + offset];
var odd = values[start + offset + half] * rotation;
values[start + offset] = even + odd;
values[start + offset + half] = even - odd;
rotation *= step;
}
}
}
if (!inverse)
{
return;
}
for (var index = 0; index < n; index++)
{
values[index] /= n;
}
}
}

View file

@ -109,3 +109,4 @@
{"ts": "2026-05-25T06:37:00+01:00", "type": "verification", "status": "partial", "note": "Ran a fresh live Scarlett complex-contour pass through the physical output/input path. Rendered `artifacts/asio/canary-packet-live-20260525-063229-f32.raw`, played it through Focusrite ASIO at 192 kHz, and captured `artifacts/asio/scarlett-canary-packet-live-20260525-063229-f32.raw` from all four inputs. Loopback 1 -> Loopback 2 measured -0.014 us with 216 direct hits and 0.008 sample MAE. With `calibration/bioacoustic/complex-contour-channel-model.json` loaded, shotgun measured -3.984 us from prior seed and cardioid measured +1.547 us from prior seed. This proves fresh meatspace viability, but physical mic confidence remains low; next work is stronger direct-path confidence and independent distance/path truth. Receipt: `calibration/bioacoustic/complex-contour-live-20260525-063229.md`."}
{"ts": "2026-05-25T06:50:00+01:00", "type": "implementation", "status": "ok", "note": "Corrected `MimirDirectPathTracker` confidence scoring. The old score normalized direct-cluster confidence against all alternate matched-filter lobes, which made a nearly perfect loopback lock report only 0.126 confidence. Confidence now primarily belongs to the selected direct cluster: direct-hit count, residual tightness, phase coherence, and prediction agreement; nearby competing lobes apply only an ambiguity penalty. Reanalysis of `scarlett-canary-packet-live-20260525-063229-f32.raw` leaves delays unchanged but raises Loopback 1 -> Loopback 2 confidence to 0.971, cardioid confidence to 0.488, and the weaker/disconnected-looking channel to 0.335."}
{"ts": "2026-05-25T06:59:00+01:00", "type": "correction", "status": "invalidated", "note": "Invalidated the apparent cardioid timing proof in `calibration/bioacoustic/complex-contour-live-20260525-063229.md`. The log-mel spectrogram and channel statistics show `asio-ch0` is not a real acoustic signal in that capture: RMS is almost flat (`0.00208350` to `0.00209815` by 0.1s block), peak is only `0.00334287`, and zero-lag correlation with loopback is `0.000032`. The near-seed +1.547 us contour estimate was a seeded tracker fit against noise, not physical mic evidence. The receipt remains valid as a loopback/interface proof (`asio-ch2` -> `asio-ch3` at -0.014 us, confidence 0.971) plus a weak/non-authoritative input witness."}
{"ts": "2026-05-25T07:05:34+01:00", "type": "implementation", "status": "ok", "note": "Added runtime FFT spectra and Fensalir readiness gating. Fensalir now calls IAquariumRuntime.OnSceneReady after the first presented ready frame; Mimir waits for that callback before emitting bioacoustic test audio, running sync analysis, or updating the cached spectrum diagnostics. MimirAudioSpectrumAnalyzer computes bounded Hann-windowed radix-2 FFT snapshots from rolling audio buffers, exposes top peaks/log bands/RMS/peak, and Mimir renders cached spectra in the DirectWrite UI plus telemetry. Verification run with ASIO config showed scene-ready before bioacoustic chunks and live 8192-point spectra for asio-ch0 through asio-ch3; asio-ch0 remains flat low-level noise while loopback/mic-bearing channels show song-band peaks."}