From 6e370ff2ac72be35649482d33ff6495d0087a8dc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 25 May 2026 07:06:25 +0100 Subject: [PATCH] Render live audio spectra after scene ready --- src/Mimir.Runtime/MimirRuntime.cs | 139 +++++++- .../MimirAudioSpectrumAnalyzer.cs | 313 ++++++++++++++++++ state/evidence.jsonl | 1 + 3 files changed, 450 insertions(+), 3 deletions(-) create mode 100644 src/Mimir.Runtime/Synchronization/MimirAudioSpectrumAnalyzer.cs diff --git a/src/Mimir.Runtime/MimirRuntime.cs b/src/Mimir.Runtime/MimirRuntime.cs index ce4858b..18c6914 100644 --- a/src/Mimir.Runtime/MimirRuntime.cs +++ b/src/Mimir.Runtime/MimirRuntime.cs @@ -6,6 +6,7 @@ using Aquarium.Engine.Ui; using Mimir.Runtime.Synchronization; using System.Diagnostics; using System.Numerics; +using System.Text; namespace Mimir.Runtime; @@ -14,26 +15,33 @@ public sealed class MimirRuntime : IAquariumRuntime private const float DefaultAudioSyncUpdateIntervalSeconds = 0.1f; private const double HybridPassiveConfidenceThreshold = 0.12; private const double CalibrationStartSeconds = 0.5; - private const int CalibrationBatchSegments = 120; + private const int CalibrationBatchSegments = 4; private const int HybridWatermarkIntervalSegments = 4; + private const float DefaultSpectrumUpdateIntervalSeconds = 0.2f; private readonly MimirSynchronizationHub synchronization; + private readonly MimirAudioSpectrumAnalyzer spectrumAnalyzer; private readonly AquariumUiDocument ui; private readonly AquariumAudioDocument audio = new(); private readonly MimirAudioSynchronizationSettings audioSyncSettings; private readonly float telemetryIntervalSeconds; private readonly float audioSyncUpdateIntervalSeconds; + private readonly float spectrumUpdateIntervalSeconds; private readonly float calibrationGain; private readonly float watermarkGain; private readonly MimirBioacousticContestantRenderer? complexContourWitness; private int lastPollCount; private float runtimeSeconds; private float nextAudioSyncSeconds; + private float nextSpectrumSeconds; private float nextTelemetrySeconds; + private bool sceneReady; private ulong calibrationSegmentIndex; private long lastHybridWatermarkSegment = -1; private double lastAudioSyncAnalysisMilliseconds; + private double lastSpectrumAnalysisMilliseconds; private double lastPassiveSynchronizationConfidence; private IReadOnlyList lastAudioSynchronizationReports = []; + private IReadOnlyList lastAudioSpectra = []; public MimirRuntime(AquariumRuntimeOptions options) : this(options, MimirRuntimeConfiguration.Load()) @@ -57,9 +65,11 @@ public sealed class MimirRuntime : IAquariumRuntime { Options = options; synchronization = new MimirSynchronizationHub(settings); + spectrumAnalyzer = new MimirAudioSpectrumAnalyzer(ParseSpectrumFftSize()); audioSyncSettings = settings.Audio; telemetryIntervalSeconds = ParseTelemetryIntervalSeconds(); audioSyncUpdateIntervalSeconds = ParseAudioSyncIntervalSeconds(); + spectrumUpdateIntervalSeconds = ParseSpectrumUpdateIntervalSeconds(); calibrationGain = settings.Audio.CalibrationGain; watermarkGain = settings.Audio.WatermarkGain; complexContourWitness = settings.Audio.EnableComplexContourRuntime @@ -115,12 +125,30 @@ public sealed class MimirRuntime : IAquariumRuntime public void Update(float deltaSeconds, InputState input) { runtimeSeconds += Math.Max(deltaSeconds, 0.0f); - QueueCalibrationTimeline(); lastPollCount = synchronization.PollSources(); - UpdateAudioSynchronization(); + if (sceneReady) + { + QueueCalibrationTimeline(); + UpdateAudioSpectra(); + UpdateAudioSynchronization(); + } + EmitTelemetry(); } + public void OnSceneReady() + { + if (sceneReady) + { + return; + } + + sceneReady = true; + nextAudioSyncSeconds = runtimeSeconds + audioSyncUpdateIntervalSeconds; + nextSpectrumSeconds = runtimeSeconds; + Console.WriteLine($"Mimir Fensalir scene ready at {runtimeSeconds:0.000}s; runtime audio tests and spectra enabled."); + } + public AquariumFrame ComposeFrame(AquariumFrame frame, AquariumFrameInput input) { return frame; @@ -151,6 +179,18 @@ public sealed class MimirRuntime : IAquariumRuntime panel.Readout("Audio sync state", DescribeAudioSyncState); panel.Readout("Aligned audio", DescribeAlignedAudio); panel.Readout("Chirplet reference", DescribeChirpletReference); + panel.Section("Live FFT"); + panel.Readout("Spectrum cadence", () => sceneReady + ? $"{lastAudioSpectra.Count} spectra fft={lastAudioSpectra.FirstOrDefault()?.FftSize ?? 0} analyze={lastSpectrumAnalysisMilliseconds:0.00}ms" + : "waiting for Fensalir scene ready"); + panel.TextBox( + "Spectra", + DescribeSpectra, + _ => { }, + lines: 12, + acceptsReturn: false, + monospace: true, + tooltip: "Cached FFT spectra from the runtime rolling audio buffers."); }); } @@ -311,6 +351,20 @@ public sealed class MimirRuntime : IAquariumRuntime nextAudioSyncSeconds = runtimeSeconds + audioSyncUpdateIntervalSeconds; } + private void UpdateAudioSpectra() + { + if (runtimeSeconds < nextSpectrumSeconds) + { + return; + } + + var stopwatch = Stopwatch.StartNew(); + lastAudioSpectra = spectrumAnalyzer.Analyze(synchronization.Buffers.Buffers, audioSyncSettings.ReferenceSourceId); + stopwatch.Stop(); + lastSpectrumAnalysisMilliseconds = stopwatch.Elapsed.TotalMilliseconds; + nextSpectrumSeconds = runtimeSeconds + spectrumUpdateIntervalSeconds; + } + private void EmitTelemetry() { if (telemetryIntervalSeconds <= 0.0f || runtimeSeconds < nextTelemetrySeconds) @@ -324,6 +378,12 @@ public sealed class MimirRuntime : IAquariumRuntime Console.WriteLine( $"mimir-sync-telemetry t={runtimeSeconds:0.00}s audioSync={audioSyncSettings.Mode} loopbackCount={loopback?.Count ?? 0} loopbackEdgeNs={loopback?.EdgeNs ?? 0} reports={lastAudioSynchronizationReports.Count} states={states.Count} analyzeMs={lastAudioSyncAnalysisMilliseconds:0.0} aligned={DescribeAlignedAudio()}"); Console.WriteLine($"mimir-sync-buffers {DescribeAudioBuffers()}"); + foreach (var spectrum in lastAudioSpectra.OrderBy(snapshot => snapshot.SourceId, StringComparer.Ordinal)) + { + Console.WriteLine( + $"mimir-spectrum {spectrum.SourceId} rate={spectrum.SampleRate} fft={spectrum.FftSize} rms={spectrum.Rms:0.000000} peak={spectrum.Peak:0.000000} floorDb={spectrum.NoiseFloorDb:0.0} peaks={DescribeSpectrumPeaks(spectrum)}"); + } + foreach (var report in lastAudioSynchronizationReports.OrderBy(report => report.SourceId, StringComparer.Ordinal)) { Console.WriteLine( @@ -365,6 +425,20 @@ public sealed class MimirRuntime : IAquariumRuntime : DefaultAudioSyncUpdateIntervalSeconds; } + private static float ParseSpectrumUpdateIntervalSeconds() + { + return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_SPECTRUM_INTERVAL_SECONDS"), out var seconds) + ? Math.Clamp(seconds, 0.05f, 10.0f) + : DefaultSpectrumUpdateIntervalSeconds; + } + + private static int ParseSpectrumFftSize() + { + return int.TryParse(Environment.GetEnvironmentVariable("MIMIR_SPECTRUM_FFT_SIZE"), out var value) + ? Math.Clamp(value, 1024, 32768) + : 8192; + } + private bool ShouldEmitCalibrationTimeline() { if (!HasReferenceAudioBuffer()) @@ -407,6 +481,65 @@ public sealed class MimirRuntime : IAquariumRuntime .Select(buffer => $"{buffer.Descriptor.SourceId}:{buffer.Count}@{buffer.EdgeNs}")); } + private string DescribeSpectra() + { + if (!sceneReady) + { + return "Fensalir scene not ready."; + } + + if (lastAudioSpectra.Count == 0) + { + return "No sample-bearing audio buffers yet."; + } + + var builder = new StringBuilder(); + foreach (var spectrum in lastAudioSpectra) + { + builder + .Append(spectrum.SourceId) + .Append(" rms=") + .Append(spectrum.Rms.ToString("0.000000")) + .Append(" peak=") + .Append(spectrum.Peak.ToString("0.000000")) + .Append(" ") + .Append(SpectrumBars(spectrum)) + .AppendLine(); + builder + .Append(" ") + .Append(DescribeSpectrumPeaks(spectrum)) + .AppendLine(); + } + + return builder.ToString().TrimEnd(); + } + + private static string SpectrumBars(MimirAudioSpectrumSnapshot spectrum) + { + const string ramp = " .:-=+*#%@"; + if (spectrum.BandDecibels.Count == 0) + { + return ""; + } + + var floor = Math.Min(spectrum.NoiseFloorDb, spectrum.BandDecibels.Min()); + var ceiling = Math.Max(-24.0, spectrum.BandDecibels.Max()); + var span = Math.Max(12.0, ceiling - floor); + var chars = spectrum.BandDecibels.Select(value => + { + var normalized = Math.Clamp((value - floor) / span, 0.0, 1.0); + return ramp[(int)Math.Round(normalized * (ramp.Length - 1))]; + }); + return new string(chars.ToArray()); + } + + private static string DescribeSpectrumPeaks(MimirAudioSpectrumSnapshot spectrum) + { + return spectrum.Peaks.Count == 0 + ? "no FFT peaks" + : string.Join(", ", spectrum.Peaks.Select(peak => $"{peak.FrequencyHz / 1000.0:0.00}kHz {peak.Decibels:0.0}dB")); + } + private void UpdatePassiveSynchronizationConfidence(IReadOnlyList reports) { var passiveConfidence = reports diff --git a/src/Mimir.Runtime/Synchronization/MimirAudioSpectrumAnalyzer.cs b/src/Mimir.Runtime/Synchronization/MimirAudioSpectrumAnalyzer.cs new file mode 100644 index 0000000..407d933 --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirAudioSpectrumAnalyzer.cs @@ -0,0 +1,313 @@ +using System.Buffers.Binary; +using System.Numerics; + +namespace Mimir.Runtime.Synchronization; + +public sealed record MimirAudioSpectrumBin(double FrequencyHz, double Decibels, double Magnitude); + +public sealed record MimirAudioSpectrumSnapshot( + string SourceId, + int SampleRate, + int FftSize, + int WindowSamples, + double Rms, + double Peak, + double NoiseFloorDb, + IReadOnlyList Peaks, + IReadOnlyList BandDecibels, + long EdgeNs); + +public sealed class MimirAudioSpectrumAnalyzer +{ + private const int MinimumFftSize = 1024; + private const int MaximumFftSize = 32768; + private readonly int fftSize; + private readonly int displayBandCount; + private readonly Complex[] spectrum; + + public MimirAudioSpectrumAnalyzer(int fftSize = 8192, int displayBandCount = 48) + { + this.fftSize = Math.Clamp(NextPowerOfTwo(Math.Max(MinimumFftSize, fftSize)), MinimumFftSize, MaximumFftSize); + this.displayBandCount = Math.Clamp(displayBandCount, 16, 96); + spectrum = new Complex[this.fftSize]; + } + + public IReadOnlyList Analyze( + IEnumerable buffers, + string referenceSourceId, + int maxNonReferenceSources = 3) + { + var audioBuffers = buffers + .Where(buffer => buffer.Descriptor.Kind == MimirStreamKind.Audio && buffer.Latest?.AudioBlock != null) + .OrderBy(buffer => string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal) ? 0 : 1) + .ThenBy(buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal) + .ToArray(); + if (audioBuffers.Length == 0) + { + return []; + } + + var snapshots = new List(); + var nonReferenceCount = 0; + foreach (var buffer in audioBuffers) + { + var isReference = string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal); + if (!isReference && nonReferenceCount++ >= maxNonReferenceSources) + { + continue; + } + + var snapshot = AnalyzeBuffer(buffer); + if (snapshot != null) + { + snapshots.Add(snapshot); + } + } + + return snapshots; + } + + private MimirAudioSpectrumSnapshot? AnalyzeBuffer(MimirRollingStreamBuffer buffer) + { + var latestBlock = buffer.Latest?.AudioBlock; + if (latestBlock == null || latestBlock.Channels <= 0 || latestBlock.SampleRate <= 0) + { + return null; + } + + Array.Clear(spectrum); + var written = FillLatestMonoWindow(buffer, latestBlock, spectrum); + if (written < MinimumFftSize / 2) + { + return null; + } + + var rms = 0.0; + var peak = 0.0; + for (var index = 0; index < written; index++) + { + var sample = spectrum[index].Real; + rms += sample * sample; + peak = Math.Max(peak, Math.Abs(sample)); + } + + rms = Math.Sqrt(rms / Math.Max(1, written)); + ApplyHannWindow(spectrum, written); + FastFourierTransform(spectrum, inverse: false); + + var magnitudes = new double[fftSize / 2]; + for (var index = 1; index < magnitudes.Length; index++) + { + magnitudes[index] = spectrum[index].Magnitude / Math.Max(1, written); + } + + var bands = BuildLogBands(magnitudes, latestBlock.SampleRate); + var peaks = FindPeaks(magnitudes, latestBlock.SampleRate, count: 6); + var floor = bands.Count == 0 ? -120.0 : bands.OrderBy(value => value).ElementAt(Math.Clamp(bands.Count / 4, 0, bands.Count - 1)); + return new MimirAudioSpectrumSnapshot( + buffer.Descriptor.SourceId, + latestBlock.SampleRate, + fftSize, + written, + rms, + peak, + floor, + peaks, + bands, + buffer.EdgeNs); + } + + private int FillLatestMonoWindow( + MimirRollingStreamBuffer buffer, + MimirAudioBlockDescriptor latestBlock, + Complex[] destination) + { + var write = destination.Length; + foreach (var sample in buffer.Snapshot() + .Where(sample => sample.AudioBlock != null && !sample.Data.IsEmpty) + .Reverse()) + { + var block = sample.AudioBlock!; + if (!IsSupportedPcmFormat(block.SampleFormat) || block.Channels <= 0 || block.SampleRate != latestBlock.SampleRate) + { + continue; + } + + var bytesPerSample = BytesPerSample(block.SampleFormat); + var frameCount = sample.Data.Length / (bytesPerSample * block.Channels); + var data = sample.Data.Span; + for (var frame = frameCount - 1; frame >= 0 && write > 0; frame--) + { + var offset = frame * block.Channels * bytesPerSample; + destination[--write] = new Complex(ReadPcmSample(data.Slice(offset, bytesPerSample), block.SampleFormat), 0.0); + } + + if (write == 0) + { + break; + } + } + + var written = destination.Length - write; + if (write > 0 && written > 0) + { + Array.Copy(destination, write, destination, 0, written); + Array.Clear(destination, written, destination.Length - written); + } + + return written; + } + + private IReadOnlyList BuildLogBands(IReadOnlyList magnitudes, int sampleRate) + { + var bands = new double[displayBandCount]; + var nyquist = sampleRate * 0.5; + var minHz = 40.0; + var maxHz = Math.Max(minHz * 2.0, nyquist); + for (var band = 0; band < bands.Length; band++) + { + var start = band / (double)bands.Length; + var end = (band + 1) / (double)bands.Length; + var lowHz = minHz * Math.Pow(maxHz / minHz, start); + var highHz = minHz * Math.Pow(maxHz / minHz, end); + var firstBin = Math.Clamp((int)Math.Floor(lowHz * fftSize / sampleRate), 1, magnitudes.Count - 1); + var lastBin = Math.Clamp((int)Math.Ceiling(highHz * fftSize / sampleRate), firstBin, magnitudes.Count - 1); + var sum = 0.0; + var count = 0; + for (var bin = firstBin; bin <= lastBin; bin++) + { + sum += magnitudes[bin] * magnitudes[bin]; + count++; + } + + bands[band] = ToDecibels(Math.Sqrt(sum / Math.Max(1, count))); + } + + return bands; + } + + private IReadOnlyList FindPeaks(IReadOnlyList magnitudes, int sampleRate, int count) + { + var peaks = new List(); + var minBin = Math.Max(1, (int)Math.Round(20.0 * fftSize / sampleRate)); + var maxBin = Math.Min(magnitudes.Count - 2, (int)Math.Round(Math.Min(sampleRate * 0.48, 24000.0) * fftSize / sampleRate)); + for (var bin = minBin; bin <= maxBin; bin++) + { + var magnitude = magnitudes[bin]; + if (magnitude <= magnitudes[bin - 1] || magnitude < magnitudes[bin + 1]) + { + continue; + } + + peaks.Add(new MimirAudioSpectrumBin(bin * sampleRate / (double)fftSize, ToDecibels(magnitude), magnitude)); + } + + return peaks + .OrderByDescending(peak => peak.Magnitude) + .Take(count) + .OrderBy(peak => peak.FrequencyHz) + .ToArray(); + } + + private static void ApplyHannWindow(Complex[] values, int count) + { + for (var index = 0; index < count; index++) + { + var window = 0.5 - 0.5 * Math.Cos(2.0 * Math.PI * index / Math.Max(1, count - 1)); + values[index] *= window; + } + } + + private static bool IsSupportedPcmFormat(MimirAudioSampleFormat sampleFormat) => + sampleFormat is MimirAudioSampleFormat.Float32 or + MimirAudioSampleFormat.Int16 or + MimirAudioSampleFormat.Int24 or + MimirAudioSampleFormat.Int32; + + private static int BytesPerSample(MimirAudioSampleFormat sampleFormat) => + sampleFormat switch + { + MimirAudioSampleFormat.Float32 => sizeof(float), + MimirAudioSampleFormat.Int16 => sizeof(short), + MimirAudioSampleFormat.Int24 => 3, + MimirAudioSampleFormat.Int32 => sizeof(int), + _ => 0, + }; + + private static float ReadPcmSample(ReadOnlySpan data, MimirAudioSampleFormat sampleFormat) => + sampleFormat switch + { + MimirAudioSampleFormat.Float32 => BitConverter.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(data)), + MimirAudioSampleFormat.Int16 => BinaryPrimitives.ReadInt16LittleEndian(data) / 32768.0f, + MimirAudioSampleFormat.Int24 => ReadInt24LittleEndian(data) / 8388608.0f, + MimirAudioSampleFormat.Int32 => BinaryPrimitives.ReadInt32LittleEndian(data) / 2147483648.0f, + _ => 0.0f, + }; + + private static int ReadInt24LittleEndian(ReadOnlySpan data) + { + var value = data[0] | (data[1] << 8) | (data[2] << 16); + return (value & 0x800000) != 0 ? value | unchecked((int)0xff000000) : value; + } + + private static double ToDecibels(double value) => 20.0 * Math.Log10(Math.Max(value, 1.0e-12)); + + private static int NextPowerOfTwo(int value) + { + var power = 1; + while (power < value) + { + power <<= 1; + } + + return power; + } + + private static void FastFourierTransform(Complex[] values, bool inverse) + { + var n = values.Length; + for (int index = 1, swap = 0; index < n; index++) + { + var bit = n >> 1; + for (; (swap & bit) != 0; bit >>= 1) + { + swap ^= bit; + } + + swap ^= bit; + if (index < swap) + { + (values[index], values[swap]) = (values[swap], values[index]); + } + } + + for (var length = 2; length <= n; length <<= 1) + { + var angle = 2.0 * Math.PI / length * (inverse ? 1.0 : -1.0); + var step = new Complex(Math.Cos(angle), Math.Sin(angle)); + for (var start = 0; start < n; start += length) + { + var rotation = Complex.One; + var half = length >> 1; + for (var offset = 0; offset < half; offset++) + { + var even = values[start + offset]; + var odd = values[start + offset + half] * rotation; + values[start + offset] = even + odd; + values[start + offset + half] = even - odd; + rotation *= step; + } + } + } + + if (!inverse) + { + return; + } + + for (var index = 0; index < n; index++) + { + values[index] /= n; + } + } +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 630a35c..89b9787 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -109,3 +109,4 @@ {"ts": "2026-05-25T06:37:00+01:00", "type": "verification", "status": "partial", "note": "Ran a fresh live Scarlett complex-contour pass through the physical output/input path. Rendered `artifacts/asio/canary-packet-live-20260525-063229-f32.raw`, played it through Focusrite ASIO at 192 kHz, and captured `artifacts/asio/scarlett-canary-packet-live-20260525-063229-f32.raw` from all four inputs. Loopback 1 -> Loopback 2 measured -0.014 us with 216 direct hits and 0.008 sample MAE. With `calibration/bioacoustic/complex-contour-channel-model.json` loaded, shotgun measured -3.984 us from prior seed and cardioid measured +1.547 us from prior seed. This proves fresh meatspace viability, but physical mic confidence remains low; next work is stronger direct-path confidence and independent distance/path truth. Receipt: `calibration/bioacoustic/complex-contour-live-20260525-063229.md`."} {"ts": "2026-05-25T06:50:00+01:00", "type": "implementation", "status": "ok", "note": "Corrected `MimirDirectPathTracker` confidence scoring. The old score normalized direct-cluster confidence against all alternate matched-filter lobes, which made a nearly perfect loopback lock report only 0.126 confidence. Confidence now primarily belongs to the selected direct cluster: direct-hit count, residual tightness, phase coherence, and prediction agreement; nearby competing lobes apply only an ambiguity penalty. Reanalysis of `scarlett-canary-packet-live-20260525-063229-f32.raw` leaves delays unchanged but raises Loopback 1 -> Loopback 2 confidence to 0.971, cardioid confidence to 0.488, and the weaker/disconnected-looking channel to 0.335."} {"ts": "2026-05-25T06:59:00+01:00", "type": "correction", "status": "invalidated", "note": "Invalidated the apparent cardioid timing proof in `calibration/bioacoustic/complex-contour-live-20260525-063229.md`. The log-mel spectrogram and channel statistics show `asio-ch0` is not a real acoustic signal in that capture: RMS is almost flat (`0.00208350` to `0.00209815` by 0.1s block), peak is only `0.00334287`, and zero-lag correlation with loopback is `0.000032`. The near-seed +1.547 us contour estimate was a seeded tracker fit against noise, not physical mic evidence. The receipt remains valid as a loopback/interface proof (`asio-ch2` -> `asio-ch3` at -0.014 us, confidence 0.971) plus a weak/non-authoritative input witness."} +{"ts": "2026-05-25T07:05:34+01:00", "type": "implementation", "status": "ok", "note": "Added runtime FFT spectra and Fensalir readiness gating. Fensalir now calls IAquariumRuntime.OnSceneReady after the first presented ready frame; Mimir waits for that callback before emitting bioacoustic test audio, running sync analysis, or updating the cached spectrum diagnostics. MimirAudioSpectrumAnalyzer computes bounded Hann-windowed radix-2 FFT snapshots from rolling audio buffers, exposes top peaks/log bands/RMS/peak, and Mimir renders cached spectra in the DirectWrite UI plus telemetry. Verification run with ASIO config showed scene-ready before bioacoustic chunks and live 8192-point spectra for asio-ch0 through asio-ch3; asio-ch0 remains flat low-level noise while loopback/mic-bearing channels show song-band peaks."}