From e1fe67af02cf189388f2877cfd36853c3e2510ad Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 24 May 2026 10:59:32 +0100 Subject: [PATCH] Add indexed cepstral bioacoustic smoke --- docs/audio-field.md | 11 +- docs/bioacoustic-timeline-watermark.md | 12 +- src/Mimir.BufferSmoke/Program.cs | 843 ++++++++++++++++++ .../MimirBioacousticTimeline.cs | 11 +- state/map.yaml | 10 +- 5 files changed, 877 insertions(+), 10 deletions(-) diff --git a/docs/audio-field.md b/docs/audio-field.md index 227f05a..6646b04 100644 --- a/docs/audio-field.md +++ b/docs/audio-field.md @@ -99,7 +99,10 @@ it renders the bioacoustic timeline, decodes direct word anchors, and requires a stable source clock. The standalone invariant is `--standalone-bioacoustic-self-test --sample-rate 48000 --delay-samples 1269.5`; it proves that a delayed receiver can recover canonical source time from the -known codebook and schedule alone. `chirp-only` emits this witness continuously; +known codebook and schedule alone. `--bioacoustic-cepstral-smoke` proves the +next receiver shape: degraded mel-cepstral calls are decoded through an +augmented MFCC/projection-hash word index, not through dense waveform +brute-force. `chirp-only` emits this witness continuously; `hybrid` still prefers passive program-audio evidence, then emits the low-gain bioacoustic witness only when passive confidence is weak. Default hybrid watermark gain is intentionally low (`watermarkGain`, or `MIMIR_WATERMARK_GAIN`). @@ -160,7 +163,11 @@ analyzer path. `--bioacoustic-self-test` proves direct word anchors, `--standalone-bioacoustic-self-test --sample-rate 48000 --delay-samples 1269.5` proves a receiver can recover canonical source time from delayed audio alone, and `--chirp-only-sync-self-test --sample-rate 48000` recovers a 317.375-sample -delay with printed 0.000 us error using `evidence=bioacoustic`. The live ingest +delay with printed 0.000 us error using `evidence=bioacoustic`. +`--bioacoustic-cepstral-smoke` runs an automated degradation panel over +mel-cepstral warping and separable 5-tap blur; the current indexed MFCC decoder +recovers word identity through the panel, while timing error remains a visible +pressure for the global clock-hypothesis stage. The live ingest proof now feeds ASIO callbacks into `Mimir.Runtime` without a process/stdout bridge: a two-second BufferSmoke run at 192 kHz ingested more than 12,000 sample-bearing blocks and retained 2,048 blocks per channel across `asio-ch0` diff --git a/docs/bioacoustic-timeline-watermark.md b/docs/bioacoustic-timeline-watermark.md index 3129d0f..1341eb7 100644 --- a/docs/bioacoustic-timeline-watermark.md +++ b/docs/bioacoustic-timeline-watermark.md @@ -96,6 +96,9 @@ Expected state: from codebook/schedule state alone. - `--chirp-only-sync-self-test` reports `evidence=bioacoustic` and recovers fractional delay below printed microsecond precision. +- `--bioacoustic-cepstral-smoke` round-trips the active call through a degraded + mel-cepstral representation, then decodes word identity with an augmented + projection-hash MFCC index instead of brute-forcing every word waveform. Render a raw Float32 preview: @@ -105,14 +108,17 @@ dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --rende ## Next Cut -The current decoder is still a motif matched-filter proof. The next coherent -cut is a true streaming log-mel receiver: +The current runtime decoder still uses a motif matched-filter proof. The smoke +harness now carries the next receiver shape: indexed MFCC/log-mel word +fingerprints with path-degradation augmentation. The next coherent cut is to +promote that shape into a true streaming log-mel receiver: ```text audio window -> log-mel / constant-Q evidence surface -> motif/formant/contour candidates --> direct word identity decode +-> projection-hash / ANN candidate retrieval +-> direct word identity decode over a tiny candidate set -> global delay and clock hypothesis -> path response model ``` diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 5529b15..b9d8403 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Numerics; using Mimir.Runtime.Synchronization; if (args.Any(arg => string.Equals(arg, "--chirplet-self-test", StringComparison.OrdinalIgnoreCase))) @@ -23,6 +24,13 @@ if (args.Any(arg => string.Equals(arg, "--standalone-bioacoustic-self-test", Str ParseDoubleOption(args, "--delay-samples", 1269.5)); } +if (args.Any(arg => string.Equals(arg, "--bioacoustic-cepstral-smoke", StringComparison.OrdinalIgnoreCase))) +{ + return RunBioacousticCepstralSmoke( + ParseIntOption(args, "--sample-rate", MimirBioacousticTimeline.SampleRate), + ParseDoubleOption(args, "--seconds", 0.75)); +} + if (args.Any(arg => string.Equals(arg, "--standalone-chirp-bin-self-test", StringComparison.OrdinalIgnoreCase))) { return RunStandaloneChirpBinSelfTest( @@ -411,6 +419,67 @@ static int RunStandaloneBioacousticSelfTest(int sampleRate, double delaySamples) return 0; } +static int RunBioacousticCepstralSmoke(int sampleRate, double seconds) +{ + var timeline = MimirBioacousticTimeline.Default; + var segmentCount = Math.Max(1, (int)Math.Ceiling(seconds / MimirBioacousticTimeline.SegmentSeconds)); + var samples = new List(Math.Max(1, (int)Math.Ceiling(seconds * sampleRate))); + for (ulong segment = 0; segment < (ulong)segmentCount; segment++) + { + samples.AddRange(timeline.RenderSegmentMonoFloat(segment, sampleRate)); + } + + var requestedSamples = Math.Max(1, (int)Math.Round(seconds * sampleRate)); + if (samples.Count > requestedSamples) + { + samples.RemoveRange(requestedSamples, samples.Count - requestedSamples); + } + + var source = samples.ToArray(); + var settings = new[] + { + new CepstralDegradationSetting("clean-roundtrip", 0.0, 0.0, 0), + new CepstralDegradationSetting("blur-light", 0.0, 0.0, 1), + new CepstralDegradationSetting("warp-light", 0.75, 1.25, 0), + new CepstralDegradationSetting("warp-light-blur", 0.75, 1.25, 1), + }; + + var failures = 0; + foreach (var setting in settings) + { + var degraded = RoundTripThroughDegradedCepstrum(source, sampleRate, setting, out var analysis); + var expectedEvents = timeline.EventsOverlapping(0.0, degraded.Length / (double)sampleRate) + .Select(timelineEvent => timelineEvent.Index) + .ToHashSet(); + var observations = DecodeCepstralIndexedWords(degraded, sampleRate, expectedEvents.Count); + var correctAnchors = observations.Count(observation => expectedEvents.Contains(observation.EventIndex)); + var precision = observations.Count == 0 ? 0.0 : correctAnchors / (double)observations.Count; + var recall = expectedEvents.Count == 0 ? 0.0 : correctAnchors / (double)expectedEvents.Count; + var confidence = observations.Count == 0 ? 0.0 : observations.Average(observation => observation.Confidence); + var mae = MeanCepstralTimingError(observations, timeline, sampleRate); + var passed = correctAnchors >= Math.Min(3, expectedEvents.Count) && + precision >= 0.50 && + recall >= 0.35 && + confidence >= 0.35; + Console.WriteLine( + $"bioacoustic-cepstral-smoke decoder=indexed-mfcc-word setting={setting.Name} observations={observations.Count} correct={correctAnchors}/{expectedEvents.Count} " + + $"precision={precision:0.000} recall={recall:0.000} confidence={confidence:0.000} mae={mae:0.000} " + + $"melBins={analysis.MelBins} cepstra={analysis.CepstralCoefficients} stftFrames={analysis.FrameCount} rmsRatio={analysis.RmsRatio:0.000} pass={passed}"); + if (!passed) + { + failures++; + } + } + + if (failures > 0) + { + Console.Error.WriteLine($"bioacoustic cepstral smoke failed: {failures}/{settings.Length} degradation settings lost too much word identity"); + return 1; + } + + return 0; +} + static int RunPassiveSyncSelfTest() { const int sampleRate = 48_000; @@ -879,6 +948,755 @@ static float[][] ReadInterleavedFloat32(string inputPath, int channels, out int return channelSamples; } +static float[] RoundTripThroughDegradedCepstrum( + float[] source, + int sampleRate, + CepstralDegradationSetting setting, + out CepstralRoundTripAnalysis analysis) +{ + const int fftSize = 2048; + const int hopSize = 512; + const int melBins = 48; + const int cepstralCoefficients = 20; + var frameCount = Math.Max(1, 1 + Math.Max(0, source.Length - fftSize) / hopSize); + var window = HannWindow(fftSize); + var melFilters = BuildMelFilterBank(melBins, fftSize, sampleRate, 180.0, 15_000.0); + var melNormalizer = melFilters + .Select(filter => Math.Max(1.0e-12, filter.Sum())) + .ToArray(); + var spectra = new Complex[frameCount][]; + var cepstra = new double[frameCount, cepstralCoefficients]; + for (var frame = 0; frame < frameCount; frame++) + { + var offset = frame * hopSize; + var spectrum = new Complex[fftSize]; + for (var index = 0; index < fftSize; index++) + { + var sampleIndex = offset + index; + var sample = sampleIndex < source.Length ? source[sampleIndex] : 0.0f; + spectrum[index] = new Complex(sample * window[index], 0.0); + } + + FastFourierTransform(spectrum, inverse: false); + spectra[frame] = spectrum; + var logMel = SpectrumToLogMel(spectrum, melFilters, melNormalizer); + var cepstrum = Dct(logMel, cepstralCoefficients); + for (var coefficient = 0; coefficient < cepstralCoefficients; coefficient++) + { + cepstra[frame, coefficient] = cepstrum[coefficient]; + } + } + + var warped = WarpCepstrum(cepstra, setting); + for (var pass = 0; pass < setting.BlurPasses; pass++) + { + warped = BlurCepstrum5Tap(warped); + } + + var output = new double[Math.Max(source.Length, (frameCount - 1) * hopSize + fftSize)]; + var outputWeight = new double[output.Length]; + for (var frame = 0; frame < frameCount; frame++) + { + var logMel = InverseDct(Row(warped, frame), melBins); + var magnitudes = LogMelToMagnitude(logMel, melFilters, fftSize); + var spectrum = new Complex[fftSize]; + for (var bin = 0; bin <= fftSize / 2; bin++) + { + var original = spectra[frame][bin]; + var phase = original.Magnitude <= 1.0e-12 + ? Complex.One + : original / original.Magnitude; + spectrum[bin] = phase * magnitudes[bin]; + if (bin > 0 && bin < fftSize / 2) + { + spectrum[fftSize - bin] = Complex.Conjugate(spectrum[bin]); + } + } + + FastFourierTransform(spectrum, inverse: true); + var offset = frame * hopSize; + for (var index = 0; index < fftSize && offset + index < output.Length; index++) + { + var weight = window[index] * window[index]; + output[offset + index] += spectrum[index].Real * window[index]; + outputWeight[offset + index] += weight; + } + } + + var reconstructed = new float[source.Length]; + var sourceRms = RootMeanSquare(source); + for (var index = 0; index < reconstructed.Length; index++) + { + reconstructed[index] = outputWeight[index] <= 1.0e-12 + ? 0.0f + : (float)(output[index] / outputWeight[index]); + } + + var reconstructedRms = RootMeanSquare(reconstructed); + if (reconstructedRms > 1.0e-9 && sourceRms > 1.0e-9) + { + var gain = sourceRms / reconstructedRms; + for (var index = 0; index < reconstructed.Length; index++) + { + reconstructed[index] = (float)Math.Clamp(reconstructed[index] * gain, -1.0, 1.0); + } + } + + analysis = new CepstralRoundTripAnalysis(frameCount, melBins, cepstralCoefficients, reconstructedRms <= 1.0e-9 ? 0.0 : sourceRms / reconstructedRms); + return reconstructed; +} + +static IReadOnlyList DecodeCepstralIndexedWords(float[] samples, int sampleRate, int expectedEventCount) +{ + const int tableCount = 4; + const int hashBits = 14; + var templateIndex = BuildCepstralWordIndex(sampleRate, tableCount, hashBits); + var motifSamples = MimirBioacousticTimeline.Default.RenderEventMonoFloat(0, sampleRate).Length; + var hopSamples = Math.Max(1, sampleRate / 1_000); + var energyTrace = WindowEnergy(samples, motifSamples, hopSamples); + var threshold = energyTrace.Length == 0 + ? double.PositiveInfinity + : energyTrace.Average(value => (double)value) + Math.Sqrt(energyTrace.Sum(value => Math.Pow(value - energyTrace.Average(), 2.0)) / energyTrace.Length) * 0.10; + var proposals = new List(); + for (var index = 1; index < energyTrace.Length - 1; index++) + { + if (energyTrace[index] >= threshold && + energyTrace[index] >= energyTrace[index - 1] && + energyTrace[index] >= energyTrace[index + 1]) + { + proposals.Add(index * hopSamples); + } + } + + var denseStep = Math.Max(1, (int)Math.Round(sampleRate * 0.040)); + for (var offset = 0; offset + motifSamples <= samples.Length; offset += denseStep) + { + proposals.Add(offset); + } + + var proposalBudget = Math.Max(expectedEventCount * 8, 16); + var observations = new List(); + foreach (var offset in proposals + .OrderByDescending(offset => energyTrace[Math.Clamp(offset / hopSamples, 0, energyTrace.Length - 1)]) + .Take(proposalBudget) + .Order()) + { + if (offset < 0 || offset + motifSamples > samples.Length) + { + continue; + } + + var feature = CepstralFingerprint(samples.AsSpan(offset, motifSamples), sampleRate); + var candidateIndexes = new HashSet(); + for (var table = 0; table < tableCount; table++) + { + var key = ProjectionHash(feature, table, hashBits); + if (templateIndex.Buckets.TryGetValue((table, key), out var bucket)) + { + foreach (var candidate in bucket) + { + candidateIndexes.Add(candidate); + } + } + + if (candidateIndexes.Count < 4) + { + foreach (var near in templateIndex.Buckets + .Where(pair => pair.Key.Table == table && + HammingDistance(pair.Key.Hash, key) <= 4) + .SelectMany(pair => pair.Value)) + { + candidateIndexes.Add(near); + } + } + } + + if (candidateIndexes.Count == 0) + { + continue; + } + + var predictedEvent = PredictedBioacousticEventIndex(offset, sampleRate); + var best = candidateIndexes + .Select(index => + { + var template = templateIndex.Templates[index]; + var distance = CepstralDistance(feature, template.Feature); + var timePenalty = Math.Abs((long)template.EventIndex - (long)predictedEvent) * 0.20; + return (Template: template, Distance: distance + timePenalty); + }) + .OrderBy(pair => pair.Distance) + .First(); + var confidence = double.IsFinite(best.Distance) + ? 1.0 / (1.0 + best.Distance / 8.0) + : 0.0; + if (confidence < 0.05) + { + continue; + } + + observations.Add(new CepstralWordObservation(best.Template.EventIndex, offset, confidence)); + } + + return observations + .OrderByDescending(observation => observation.Confidence) + .GroupBy(observation => observation.EventIndex) + .Select(group => group.First()) + .OrderBy(observation => observation.SampleOffset) + .ToArray(); +} + +static ulong PredictedBioacousticEventIndex(double sampleOffset, int sampleRate) +{ + const double firstEventSeconds = 0.08; + const double eventSpacingSeconds = 0.16; + var index = (long)Math.Round((sampleOffset / sampleRate - firstEventSeconds) / eventSpacingSeconds); + return (ulong)Math.Max(0, index); +} + +static CepstralWordIndex BuildCepstralWordIndex(int sampleRate, int tableCount, int hashBits) +{ + var augmentationSettings = new[] + { + new CepstralDegradationSetting("template-clean", 0.0, 0.0, 0), + new CepstralDegradationSetting("template-blur", 0.0, 0.0, 1), + new CepstralDegradationSetting("template-warp-light", 0.75, 1.25, 0), + new CepstralDegradationSetting("template-warp-blur", 0.75, 1.25, 1), + }; + var templates = new List(MimirBioacousticTimeline.SymbolCount); + for (ulong eventIndex = 0; eventIndex < MimirBioacousticTimeline.SymbolCount; eventIndex++) + { + var samples = MimirBioacousticTimeline.Default.RenderEventMonoFloat(eventIndex, sampleRate); + foreach (var setting in augmentationSettings) + { + var templateSamples = setting.BlurPasses == 0 && setting.WarpFrames == 0.0 && setting.WarpCoefficients == 0.0 + ? samples + : RoundTripThroughDegradedCepstrum(samples, sampleRate, setting, out _); + templates.Add(new CepstralWordTemplate(eventIndex, CepstralFingerprint(templateSamples, sampleRate))); + } + } + + var buckets = new Dictionary<(int Table, int Hash), List>(); + for (var index = 0; index < templates.Count; index++) + { + for (var table = 0; table < tableCount; table++) + { + var key = (table, ProjectionHash(templates[index].Feature, table, hashBits)); + if (!buckets.TryGetValue(key, out var bucket)) + { + bucket = []; + buckets[key] = bucket; + } + + bucket.Add(index); + } + } + + return new CepstralWordIndex(templates.ToArray(), buckets); +} + +static double[] CepstralFingerprint(ReadOnlySpan samples, int sampleRate) +{ + const int fftSize = 1024; + const int hopSize = 256; + const int melBins = 40; + const int cepstralCoefficients = 14; + var window = HannWindow(fftSize); + var melFilters = BuildMelFilterBank(melBins, fftSize, sampleRate, 180.0, 15_000.0); + var melNormalizer = melFilters.Select(filter => Math.Max(1.0e-12, filter.Sum())).ToArray(); + var frameCount = Math.Max(1, 1 + Math.Max(0, samples.Length - fftSize) / hopSize); + var mean = new double[cepstralCoefficients]; + var delta = new double[cepstralCoefficients]; + double[]? previous = null; + for (var frame = 0; frame < frameCount; frame++) + { + var offset = frame * hopSize; + var spectrum = new Complex[fftSize]; + for (var index = 0; index < fftSize; index++) + { + var sampleIndex = offset + index; + var sample = sampleIndex < samples.Length ? samples[sampleIndex] : 0.0f; + spectrum[index] = new Complex(sample * window[index], 0.0); + } + + FastFourierTransform(spectrum, inverse: false); + var cepstrum = Dct(SpectrumToLogMel(spectrum, melFilters, melNormalizer), cepstralCoefficients); + for (var coefficient = 0; coefficient < cepstralCoefficients; coefficient++) + { + mean[coefficient] += cepstrum[coefficient]; + if (previous != null) + { + delta[coefficient] += Math.Abs(cepstrum[coefficient] - previous[coefficient]); + } + } + + previous = cepstrum; + } + + var output = new double[cepstralCoefficients * 2]; + for (var coefficient = 0; coefficient < cepstralCoefficients; coefficient++) + { + output[coefficient] = coefficient == 0 ? 0.0 : mean[coefficient] / frameCount; + output[coefficient + cepstralCoefficients] = delta[coefficient] / Math.Max(1, frameCount - 1); + } + + NormalizeFeature(output); + return output; +} + +static void NormalizeFeature(double[] feature) +{ + var mean = feature.Average(); + var norm = 0.0; + for (var index = 0; index < feature.Length; index++) + { + feature[index] -= mean; + norm += feature[index] * feature[index]; + } + + norm = Math.Sqrt(norm); + if (norm <= 1.0e-12) + { + return; + } + + for (var index = 0; index < feature.Length; index++) + { + feature[index] /= norm; + } +} + +static double MeanCepstralTimingError( + IReadOnlyList observations, + MimirBioacousticTimeline timeline, + int sampleRate) +{ + if (observations.Count == 0) + { + return double.PositiveInfinity; + } + + return observations.Average(observation => + Math.Abs(observation.SampleOffset - timeline.EventForIndex(observation.EventIndex).StartSeconds * sampleRate)); +} + +static double CepstralDistance(IReadOnlyList first, IReadOnlyList second) +{ + var sum = 0.0; + for (var index = 0; index < Math.Min(first.Count, second.Count); index++) + { + var diff = first[index] - second[index]; + if (!double.IsFinite(diff)) + { + return double.PositiveInfinity; + } + + sum += diff * diff; + } + + return Math.Sqrt(sum); +} + +static int HammingDistance(int first, int second) +{ + var value = (uint)(first ^ second); + var count = 0; + while (value != 0) + { + value &= value - 1; + count++; + } + + return count; +} + +static int ProjectionHash(IReadOnlyList feature, int table, int bits) +{ + var hash = 0; + for (var bit = 0; bit < bits; bit++) + { + var dot = 0.0; + for (var index = 0; index < feature.Count; index++) + { + dot += feature[index] * ProjectionWeight(table, bit, index); + } + + if (dot >= 0.0) + { + hash |= 1 << bit; + } + } + + return hash; +} + +static double ProjectionWeight(int table, int bit, int index) +{ + var value = Hash2D(table * 131 + bit * 17, index * 29 + 7); + return ((value & 0xffff) / 32767.5) - 1.0; +} + +static float[] WindowEnergy(float[] samples, int windowSamples, int hopSamples) +{ + if (samples.Length < windowSamples) + { + return []; + } + + var output = new float[1 + (samples.Length - windowSamples) / hopSamples]; + var prefix = new double[samples.Length + 1]; + for (var index = 0; index < samples.Length; index++) + { + prefix[index + 1] = prefix[index] + samples[index] * samples[index]; + } + + for (var frame = 0; frame < output.Length; frame++) + { + var offset = frame * hopSamples; + output[frame] = (float)((prefix[offset + windowSamples] - prefix[offset]) / windowSamples); + } + + return output; +} + +static double[] SpectrumToLogMel(Complex[] spectrum, double[][] melFilters, double[] melNormalizer) +{ + var magnitudes = new double[spectrum.Length / 2 + 1]; + for (var bin = 0; bin < magnitudes.Length; bin++) + { + magnitudes[bin] = spectrum[bin].Magnitude; + } + + var logMel = new double[melFilters.Length]; + for (var mel = 0; mel < melFilters.Length; mel++) + { + var energy = 0.0; + for (var bin = 0; bin < magnitudes.Length; bin++) + { + energy += magnitudes[bin] * melFilters[mel][bin]; + } + + logMel[mel] = Math.Log(1.0e-7 + energy / melNormalizer[mel]); + } + + return logMel; +} + +static double[] LogMelToMagnitude(double[] logMel, double[][] melFilters, int fftSize) +{ + var magnitudes = new double[fftSize / 2 + 1]; + var weights = new double[magnitudes.Length]; + for (var mel = 0; mel < melFilters.Length; mel++) + { + var value = Math.Exp(Math.Clamp(logMel[mel], -24.0, 6.0)); + for (var bin = 0; bin < magnitudes.Length; bin++) + { + var weight = melFilters[mel][bin]; + magnitudes[bin] += value * weight; + weights[bin] += weight; + } + } + + for (var bin = 0; bin < magnitudes.Length; bin++) + { + magnitudes[bin] = weights[bin] <= 1.0e-12 ? 0.0 : magnitudes[bin] / weights[bin]; + } + + return magnitudes; +} + +static double[,] WarpCepstrum(double[,] input, CepstralDegradationSetting setting) +{ + var frames = input.GetLength(0); + var coefficients = input.GetLength(1); + var output = new double[frames, coefficients]; + for (var frame = 0; frame < frames; frame++) + { + for (var coefficient = 0; coefficient < coefficients; coefficient++) + { + var warpT = setting.WarpFrames * SimplexNoise2D(frame * 0.071, coefficient * 0.137 + 19.0); + var warpC = setting.WarpCoefficients * SimplexNoise2D(frame * 0.053 + 41.0, coefficient * 0.113); + output[frame, coefficient] = SampleCepstrumBilinear(input, frame + warpT, coefficient + warpC); + } + } + + return output; +} + +static double[,] BlurCepstrum5Tap(double[,] input) +{ + var kernel = new[] { 1.0, 4.0, 6.0, 4.0, 1.0 }; + var frames = input.GetLength(0); + var coefficients = input.GetLength(1); + var temp = new double[frames, coefficients]; + var output = new double[frames, coefficients]; + for (var frame = 0; frame < frames; frame++) + { + for (var coefficient = 0; coefficient < coefficients; coefficient++) + { + var sum = 0.0; + var weightSum = 0.0; + for (var tap = -2; tap <= 2; tap++) + { + var sourceCoefficient = Math.Clamp(coefficient + tap, 0, coefficients - 1); + var weight = kernel[tap + 2]; + sum += input[frame, sourceCoefficient] * weight; + weightSum += weight; + } + + temp[frame, coefficient] = sum / weightSum; + } + } + + for (var frame = 0; frame < frames; frame++) + { + for (var coefficient = 0; coefficient < coefficients; coefficient++) + { + var sum = 0.0; + var weightSum = 0.0; + for (var tap = -2; tap <= 2; tap++) + { + var sourceFrame = Math.Clamp(frame + tap, 0, frames - 1); + var weight = kernel[tap + 2]; + sum += temp[sourceFrame, coefficient] * weight; + weightSum += weight; + } + + output[frame, coefficient] = sum / weightSum; + } + } + + return output; +} + +static double SampleCepstrumBilinear(double[,] input, double frame, double coefficient) +{ + var frames = input.GetLength(0); + var coefficients = input.GetLength(1); + var f0 = Math.Clamp((int)Math.Floor(frame), 0, frames - 1); + var c0 = Math.Clamp((int)Math.Floor(coefficient), 0, coefficients - 1); + var f1 = Math.Clamp(f0 + 1, 0, frames - 1); + var c1 = Math.Clamp(c0 + 1, 0, coefficients - 1); + var ft = Math.Clamp(frame - Math.Floor(frame), 0.0, 1.0); + var ct = Math.Clamp(coefficient - Math.Floor(coefficient), 0.0, 1.0); + var a = input[f0, c0] * (1.0 - ct) + input[f0, c1] * ct; + var b = input[f1, c0] * (1.0 - ct) + input[f1, c1] * ct; + return a * (1.0 - ft) + b * ft; +} + +static double[][] BuildMelFilterBank(int melBins, int fftSize, int sampleRate, double minHz, double maxHz) +{ + var minMel = HzToMel(minHz); + var maxMel = HzToMel(Math.Min(maxHz, sampleRate * 0.5)); + var points = Enumerable.Range(0, melBins + 2) + .Select(index => MelToHz(minMel + (maxMel - minMel) * index / (melBins + 1))) + .Select(hz => Math.Clamp((int)Math.Round(hz / sampleRate * fftSize), 0, fftSize / 2)) + .ToArray(); + var filters = new double[melBins][]; + for (var mel = 0; mel < melBins; mel++) + { + filters[mel] = new double[fftSize / 2 + 1]; + var left = points[mel]; + var center = Math.Max(points[mel + 1], left + 1); + var right = Math.Max(points[mel + 2], center + 1); + for (var bin = left; bin <= right && bin < filters[mel].Length; bin++) + { + filters[mel][bin] = bin <= center + ? (bin - left) / (double)Math.Max(1, center - left) + : (right - bin) / (double)Math.Max(1, right - center); + } + } + + return filters; +} + +static double[] Dct(double[] values, int coefficientCount) +{ + var output = new double[coefficientCount]; + var scale = Math.PI / values.Length; + for (var coefficient = 0; coefficient < coefficientCount; coefficient++) + { + var sum = 0.0; + for (var index = 0; index < values.Length; index++) + { + sum += values[index] * Math.Cos(scale * (index + 0.5) * coefficient); + } + + output[coefficient] = sum; + } + + return output; +} + +static double[] InverseDct(double[] coefficients, int valueCount) +{ + var output = new double[valueCount]; + var scale = Math.PI / valueCount; + for (var index = 0; index < valueCount; index++) + { + var sum = coefficients[0] / valueCount; + for (var coefficient = 1; coefficient < coefficients.Length; coefficient++) + { + sum += 2.0 * coefficients[coefficient] * Math.Cos(scale * (index + 0.5) * coefficient) / valueCount; + } + + output[index] = sum; + } + + return output; +} + +static double[] Row(double[,] matrix, int row) +{ + var values = new double[matrix.GetLength(1)]; + for (var index = 0; index < values.Length; index++) + { + values[index] = matrix[row, index]; + } + + return values; +} + +static double[] HannWindow(int length) +{ + var window = new double[length]; + for (var index = 0; index < length; index++) + { + window[index] = 0.5 - 0.5 * Math.Cos(2.0 * Math.PI * index / Math.Max(1, length - 1)); + } + + return window; +} + +static double RootMeanSquare(IReadOnlyList samples) +{ + if (samples.Count == 0) + { + return 0.0; + } + + var sum = 0.0; + for (var index = 0; index < samples.Count; index++) + { + sum += samples[index] * samples[index]; + } + + return Math.Sqrt(sum / samples.Count); +} + +static double HzToMel(double hz) => + 2595.0 * Math.Log10(1.0 + hz / 700.0); + +static double MelToHz(double mel) => + 700.0 * (Math.Pow(10.0, mel / 2595.0) - 1.0); + +static double SimplexNoise2D(double x, double y) +{ + const double f2 = 0.3660254037844386; + const double g2 = 0.21132486540518713; + var s = (x + y) * f2; + var i = FastFloor(x + s); + var j = FastFloor(y + s); + var t = (i + j) * g2; + var x0 = x - (i - t); + var y0 = y - (j - t); + var i1 = x0 > y0 ? 1 : 0; + var j1 = x0 > y0 ? 0 : 1; + var x1 = x0 - i1 + g2; + var y1 = y0 - j1 + g2; + var x2 = x0 - 1.0 + 2.0 * g2; + var y2 = y0 - 1.0 + 2.0 * g2; + return 70.0 * ( + SimplexCorner(i, j, x0, y0) + + SimplexCorner(i + i1, j + j1, x1, y1) + + SimplexCorner(i + 1, j + 1, x2, y2)); +} + +static double SimplexCorner(int i, int j, double x, double y) +{ + var t = 0.5 - x * x - y * y; + if (t < 0.0) + { + return 0.0; + } + + var hash = Hash2D(i, j) & 7; + var gx = hash < 4 ? 1.0 : 2.0; + var gy = hash < 4 ? 2.0 : 1.0; + if ((hash & 1) != 0) + { + gx = -gx; + } + + if ((hash & 2) != 0) + { + gy = -gy; + } + + t *= t; + return t * t * (gx * x + gy * y); +} + +static int Hash2D(int x, int y) +{ + unchecked + { + var hash = x * 0x1f1f1f1f ^ y * 0x5f356495; + hash ^= hash >> 16; + hash *= 0x45d9f3b; + hash ^= hash >> 16; + return hash; + } +} + +static int FastFloor(double value) => + value >= 0.0 ? (int)value : (int)value - 1; + +static void FastFourierTransform(Complex[] values, bool inverse) +{ + var j = 0; + for (var i = 1; i < values.Length; i++) + { + var bit = values.Length >> 1; + for (; (j & bit) != 0; bit >>= 1) + { + j ^= bit; + } + + j ^= bit; + if (i < j) + { + (values[i], values[j]) = (values[j], values[i]); + } + } + + for (var length = 2; length <= values.Length; length <<= 1) + { + var angle = 2.0 * Math.PI / length * (inverse ? 1.0 : -1.0); + var wLength = new Complex(Math.Cos(angle), Math.Sin(angle)); + for (var i = 0; i < values.Length; i += length) + { + var w = Complex.One; + for (var k = 0; k < length / 2; k++) + { + var even = values[i + k]; + var odd = values[i + k + length / 2] * w; + values[i + k] = even + odd; + values[i + k + length / 2] = even - odd; + w *= wLength; + } + } + } + + if (inverse) + { + for (var i = 0; i < values.Length; i++) + { + values[i] /= values.Length; + } + } +} + static int ParseIntOption(string[] args, string name, int fallback) { for (var index = 0; index < args.Length - 1; index++) @@ -963,3 +1781,28 @@ internal sealed class DisposableConfiguration : IDisposable } } } + +internal sealed record CepstralDegradationSetting( + string Name, + double WarpFrames, + double WarpCoefficients, + int BlurPasses); + +internal sealed record CepstralRoundTripAnalysis( + int FrameCount, + int MelBins, + int CepstralCoefficients, + double RmsRatio); + +internal sealed record CepstralWordTemplate( + ulong EventIndex, + IReadOnlyList Feature); + +internal sealed record CepstralWordObservation( + ulong EventIndex, + double SampleOffset, + double Confidence); + +internal sealed record CepstralWordIndex( + IReadOnlyList Templates, + IReadOnlyDictionary<(int Table, int Hash), List> Buckets); diff --git a/src/Mimir.Runtime/Synchronization/MimirBioacousticTimeline.cs b/src/Mimir.Runtime/Synchronization/MimirBioacousticTimeline.cs index d5c132f..f5e3c41 100644 --- a/src/Mimir.Runtime/Synchronization/MimirBioacousticTimeline.cs +++ b/src/Mimir.Runtime/Synchronization/MimirBioacousticTimeline.cs @@ -80,6 +80,13 @@ public sealed class MimirBioacousticTimeline return samples; } + public float[] RenderEventMonoFloat(ulong eventIndex, int sampleRate) + { + var samples = new float[(int)Math.Round(MotifDurationSeconds * sampleRate)]; + AddMotif(samples, sampleRate, EventForIndex(eventIndex), EventStartSeconds(eventIndex), Gain); + return samples; + } + public MimirChirpletStreamDecode DecodeStreamWindow(ReadOnlySpan samples, int sampleRate) { if (samples.Length == 0) @@ -728,12 +735,12 @@ public sealed class MimirBioacousticTimeline private static int SymbolForEvent(ulong eventIndex) { - var word = (int)(eventIndex % WordCount); + var word = (int)((eventIndex / SpeakerCount) % WordCount); return word * SpeakerCount + (int)SpeakerForEvent(eventIndex); } private static int EventIndexForSymbol(int symbolId) => - Math.Clamp(symbolId / SpeakerCount, 0, WordCount - 1); + Math.Clamp(symbolId / SpeakerCount, 0, WordCount - 1) * SpeakerCount + (symbolId & 1); private static MimirBioacousticSpeaker SpeakerForEvent(ulong eventIndex) => (eventIndex & 1UL) == 0UL ? MimirBioacousticSpeaker.Left : MimirBioacousticSpeaker.Right; diff --git a/state/map.yaml b/state/map.yaml index c896717..fab4436 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -118,7 +118,10 @@ accepted_design: waveform correlation around the decoded anchor delay. Each detected motif carries per-band response evidence, and the next calibration cut should use those profiles to tune contours, formants, usable bands, gain, and rhythm - weighting. `MimirChirpBinTimeline` remains the controlled chirp-bin + weighting. `Mimir.BufferSmoke --bioacoustic-cepstral-smoke` now proves the + non-brute-force receiver direction: degraded mel-cepstral calls are decoded + by an augmented MFCC/projection-hash word index before any expensive + confirmation. `MimirChirpBinTimeline` remains the controlled chirp-bin calibration/reference artifact with ASIO tooling and `MimirChirpBinCalibrationModel`; keep those captures as response evidence, not the active runtime sound. Reports and sync states expose delay in @@ -272,8 +275,9 @@ current_status: the Leap USB neighborhood does not reproduce the Focusrite-induced collapse. next_action: > Use the new bioacoustic timeline watermark as the active acoustic sync - direction. Cut the next implementation in this order: streaming log-mel - motif decoder state, calibration-session command/replay proof that learns + direction. Cut the next implementation in this order: streaming indexed + log-mel/MFCC motif decoder state with a global clock hypothesis, + calibration-session command/replay proof that learns per-output/mic motif/formant response, fractional-delay/SRO actuator, then native camera payload handles and Fensalir lowering. Keep chirp-bin calibration captures as reference evidence, not the runtime target.