Add indexed cepstral bioacoustic smoke
This commit is contained in:
parent
3cae265d22
commit
e1fe67af02
5 changed files with 877 additions and 10 deletions
|
|
@ -99,7 +99,10 @@ it renders the bioacoustic timeline, decodes direct word anchors, and requires a
|
|||
stable source clock. The standalone invariant is
|
||||
`--standalone-bioacoustic-self-test --sample-rate 48000 --delay-samples 1269.5`;
|
||||
it proves that a delayed receiver can recover canonical source time from the
|
||||
known codebook and schedule alone. `chirp-only` emits this witness continuously;
|
||||
known codebook and schedule alone. `--bioacoustic-cepstral-smoke` proves the
|
||||
next receiver shape: degraded mel-cepstral calls are decoded through an
|
||||
augmented MFCC/projection-hash word index, not through dense waveform
|
||||
brute-force. `chirp-only` emits this witness continuously;
|
||||
`hybrid` still prefers passive program-audio evidence, then emits the low-gain
|
||||
bioacoustic witness only when passive confidence is weak. Default hybrid
|
||||
watermark gain is intentionally low (`watermarkGain`, or `MIMIR_WATERMARK_GAIN`).
|
||||
|
|
@ -160,7 +163,11 @@ analyzer path. `--bioacoustic-self-test` proves direct word anchors,
|
|||
`--standalone-bioacoustic-self-test --sample-rate 48000 --delay-samples 1269.5`
|
||||
proves a receiver can recover canonical source time from delayed audio alone,
|
||||
and `--chirp-only-sync-self-test --sample-rate 48000` recovers a 317.375-sample
|
||||
delay with printed 0.000 us error using `evidence=bioacoustic`. The live ingest
|
||||
delay with printed 0.000 us error using `evidence=bioacoustic`.
|
||||
`--bioacoustic-cepstral-smoke` runs an automated degradation panel over
|
||||
mel-cepstral warping and separable 5-tap blur; the current indexed MFCC decoder
|
||||
recovers word identity through the panel, while timing error remains a visible
|
||||
pressure for the global clock-hypothesis stage. The live ingest
|
||||
proof now feeds ASIO callbacks into `Mimir.Runtime` without a process/stdout
|
||||
bridge: a two-second BufferSmoke run at 192 kHz ingested more than 12,000
|
||||
sample-bearing blocks and retained 2,048 blocks per channel across `asio-ch0`
|
||||
|
|
|
|||
|
|
@ -96,6 +96,9 @@ Expected state:
|
|||
from codebook/schedule state alone.
|
||||
- `--chirp-only-sync-self-test` reports `evidence=bioacoustic` and recovers
|
||||
fractional delay below printed microsecond precision.
|
||||
- `--bioacoustic-cepstral-smoke` round-trips the active call through a degraded
|
||||
mel-cepstral representation, then decodes word identity with an augmented
|
||||
projection-hash MFCC index instead of brute-forcing every word waveform.
|
||||
|
||||
Render a raw Float32 preview:
|
||||
|
||||
|
|
@ -105,14 +108,17 @@ dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --rende
|
|||
|
||||
## Next Cut
|
||||
|
||||
The current decoder is still a motif matched-filter proof. The next coherent
|
||||
cut is a true streaming log-mel receiver:
|
||||
The current runtime decoder still uses a motif matched-filter proof. The smoke
|
||||
harness now carries the next receiver shape: indexed MFCC/log-mel word
|
||||
fingerprints with path-degradation augmentation. The next coherent cut is to
|
||||
promote that shape into a true streaming log-mel receiver:
|
||||
|
||||
```text
|
||||
audio window
|
||||
-> log-mel / constant-Q evidence surface
|
||||
-> motif/formant/contour candidates
|
||||
-> direct word identity decode
|
||||
-> projection-hash / ANN candidate retrieval
|
||||
-> direct word identity decode over a tiny candidate set
|
||||
-> global delay and clock hypothesis
|
||||
-> path response model
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using System.Diagnostics;
|
||||
using System.Numerics;
|
||||
using Mimir.Runtime.Synchronization;
|
||||
|
||||
if (args.Any(arg => string.Equals(arg, "--chirplet-self-test", StringComparison.OrdinalIgnoreCase)))
|
||||
|
|
@ -23,6 +24,13 @@ if (args.Any(arg => string.Equals(arg, "--standalone-bioacoustic-self-test", Str
|
|||
ParseDoubleOption(args, "--delay-samples", 1269.5));
|
||||
}
|
||||
|
||||
if (args.Any(arg => string.Equals(arg, "--bioacoustic-cepstral-smoke", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return RunBioacousticCepstralSmoke(
|
||||
ParseIntOption(args, "--sample-rate", MimirBioacousticTimeline.SampleRate),
|
||||
ParseDoubleOption(args, "--seconds", 0.75));
|
||||
}
|
||||
|
||||
if (args.Any(arg => string.Equals(arg, "--standalone-chirp-bin-self-test", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return RunStandaloneChirpBinSelfTest(
|
||||
|
|
@ -411,6 +419,67 @@ static int RunStandaloneBioacousticSelfTest(int sampleRate, double delaySamples)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int RunBioacousticCepstralSmoke(int sampleRate, double seconds)
|
||||
{
|
||||
var timeline = MimirBioacousticTimeline.Default;
|
||||
var segmentCount = Math.Max(1, (int)Math.Ceiling(seconds / MimirBioacousticTimeline.SegmentSeconds));
|
||||
var samples = new List<float>(Math.Max(1, (int)Math.Ceiling(seconds * sampleRate)));
|
||||
for (ulong segment = 0; segment < (ulong)segmentCount; segment++)
|
||||
{
|
||||
samples.AddRange(timeline.RenderSegmentMonoFloat(segment, sampleRate));
|
||||
}
|
||||
|
||||
var requestedSamples = Math.Max(1, (int)Math.Round(seconds * sampleRate));
|
||||
if (samples.Count > requestedSamples)
|
||||
{
|
||||
samples.RemoveRange(requestedSamples, samples.Count - requestedSamples);
|
||||
}
|
||||
|
||||
var source = samples.ToArray();
|
||||
var settings = new[]
|
||||
{
|
||||
new CepstralDegradationSetting("clean-roundtrip", 0.0, 0.0, 0),
|
||||
new CepstralDegradationSetting("blur-light", 0.0, 0.0, 1),
|
||||
new CepstralDegradationSetting("warp-light", 0.75, 1.25, 0),
|
||||
new CepstralDegradationSetting("warp-light-blur", 0.75, 1.25, 1),
|
||||
};
|
||||
|
||||
var failures = 0;
|
||||
foreach (var setting in settings)
|
||||
{
|
||||
var degraded = RoundTripThroughDegradedCepstrum(source, sampleRate, setting, out var analysis);
|
||||
var expectedEvents = timeline.EventsOverlapping(0.0, degraded.Length / (double)sampleRate)
|
||||
.Select(timelineEvent => timelineEvent.Index)
|
||||
.ToHashSet();
|
||||
var observations = DecodeCepstralIndexedWords(degraded, sampleRate, expectedEvents.Count);
|
||||
var correctAnchors = observations.Count(observation => expectedEvents.Contains(observation.EventIndex));
|
||||
var precision = observations.Count == 0 ? 0.0 : correctAnchors / (double)observations.Count;
|
||||
var recall = expectedEvents.Count == 0 ? 0.0 : correctAnchors / (double)expectedEvents.Count;
|
||||
var confidence = observations.Count == 0 ? 0.0 : observations.Average(observation => observation.Confidence);
|
||||
var mae = MeanCepstralTimingError(observations, timeline, sampleRate);
|
||||
var passed = correctAnchors >= Math.Min(3, expectedEvents.Count) &&
|
||||
precision >= 0.50 &&
|
||||
recall >= 0.35 &&
|
||||
confidence >= 0.35;
|
||||
Console.WriteLine(
|
||||
$"bioacoustic-cepstral-smoke decoder=indexed-mfcc-word setting={setting.Name} observations={observations.Count} correct={correctAnchors}/{expectedEvents.Count} " +
|
||||
$"precision={precision:0.000} recall={recall:0.000} confidence={confidence:0.000} mae={mae:0.000} " +
|
||||
$"melBins={analysis.MelBins} cepstra={analysis.CepstralCoefficients} stftFrames={analysis.FrameCount} rmsRatio={analysis.RmsRatio:0.000} pass={passed}");
|
||||
if (!passed)
|
||||
{
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
|
||||
if (failures > 0)
|
||||
{
|
||||
Console.Error.WriteLine($"bioacoustic cepstral smoke failed: {failures}/{settings.Length} degradation settings lost too much word identity");
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int RunPassiveSyncSelfTest()
|
||||
{
|
||||
const int sampleRate = 48_000;
|
||||
|
|
@ -879,6 +948,755 @@ static float[][] ReadInterleavedFloat32(string inputPath, int channels, out int
|
|||
return channelSamples;
|
||||
}
|
||||
|
||||
static float[] RoundTripThroughDegradedCepstrum(
|
||||
float[] source,
|
||||
int sampleRate,
|
||||
CepstralDegradationSetting setting,
|
||||
out CepstralRoundTripAnalysis analysis)
|
||||
{
|
||||
const int fftSize = 2048;
|
||||
const int hopSize = 512;
|
||||
const int melBins = 48;
|
||||
const int cepstralCoefficients = 20;
|
||||
var frameCount = Math.Max(1, 1 + Math.Max(0, source.Length - fftSize) / hopSize);
|
||||
var window = HannWindow(fftSize);
|
||||
var melFilters = BuildMelFilterBank(melBins, fftSize, sampleRate, 180.0, 15_000.0);
|
||||
var melNormalizer = melFilters
|
||||
.Select(filter => Math.Max(1.0e-12, filter.Sum()))
|
||||
.ToArray();
|
||||
var spectra = new Complex[frameCount][];
|
||||
var cepstra = new double[frameCount, cepstralCoefficients];
|
||||
for (var frame = 0; frame < frameCount; frame++)
|
||||
{
|
||||
var offset = frame * hopSize;
|
||||
var spectrum = new Complex[fftSize];
|
||||
for (var index = 0; index < fftSize; index++)
|
||||
{
|
||||
var sampleIndex = offset + index;
|
||||
var sample = sampleIndex < source.Length ? source[sampleIndex] : 0.0f;
|
||||
spectrum[index] = new Complex(sample * window[index], 0.0);
|
||||
}
|
||||
|
||||
FastFourierTransform(spectrum, inverse: false);
|
||||
spectra[frame] = spectrum;
|
||||
var logMel = SpectrumToLogMel(spectrum, melFilters, melNormalizer);
|
||||
var cepstrum = Dct(logMel, cepstralCoefficients);
|
||||
for (var coefficient = 0; coefficient < cepstralCoefficients; coefficient++)
|
||||
{
|
||||
cepstra[frame, coefficient] = cepstrum[coefficient];
|
||||
}
|
||||
}
|
||||
|
||||
var warped = WarpCepstrum(cepstra, setting);
|
||||
for (var pass = 0; pass < setting.BlurPasses; pass++)
|
||||
{
|
||||
warped = BlurCepstrum5Tap(warped);
|
||||
}
|
||||
|
||||
var output = new double[Math.Max(source.Length, (frameCount - 1) * hopSize + fftSize)];
|
||||
var outputWeight = new double[output.Length];
|
||||
for (var frame = 0; frame < frameCount; frame++)
|
||||
{
|
||||
var logMel = InverseDct(Row(warped, frame), melBins);
|
||||
var magnitudes = LogMelToMagnitude(logMel, melFilters, fftSize);
|
||||
var spectrum = new Complex[fftSize];
|
||||
for (var bin = 0; bin <= fftSize / 2; bin++)
|
||||
{
|
||||
var original = spectra[frame][bin];
|
||||
var phase = original.Magnitude <= 1.0e-12
|
||||
? Complex.One
|
||||
: original / original.Magnitude;
|
||||
spectrum[bin] = phase * magnitudes[bin];
|
||||
if (bin > 0 && bin < fftSize / 2)
|
||||
{
|
||||
spectrum[fftSize - bin] = Complex.Conjugate(spectrum[bin]);
|
||||
}
|
||||
}
|
||||
|
||||
FastFourierTransform(spectrum, inverse: true);
|
||||
var offset = frame * hopSize;
|
||||
for (var index = 0; index < fftSize && offset + index < output.Length; index++)
|
||||
{
|
||||
var weight = window[index] * window[index];
|
||||
output[offset + index] += spectrum[index].Real * window[index];
|
||||
outputWeight[offset + index] += weight;
|
||||
}
|
||||
}
|
||||
|
||||
var reconstructed = new float[source.Length];
|
||||
var sourceRms = RootMeanSquare(source);
|
||||
for (var index = 0; index < reconstructed.Length; index++)
|
||||
{
|
||||
reconstructed[index] = outputWeight[index] <= 1.0e-12
|
||||
? 0.0f
|
||||
: (float)(output[index] / outputWeight[index]);
|
||||
}
|
||||
|
||||
var reconstructedRms = RootMeanSquare(reconstructed);
|
||||
if (reconstructedRms > 1.0e-9 && sourceRms > 1.0e-9)
|
||||
{
|
||||
var gain = sourceRms / reconstructedRms;
|
||||
for (var index = 0; index < reconstructed.Length; index++)
|
||||
{
|
||||
reconstructed[index] = (float)Math.Clamp(reconstructed[index] * gain, -1.0, 1.0);
|
||||
}
|
||||
}
|
||||
|
||||
analysis = new CepstralRoundTripAnalysis(frameCount, melBins, cepstralCoefficients, reconstructedRms <= 1.0e-9 ? 0.0 : sourceRms / reconstructedRms);
|
||||
return reconstructed;
|
||||
}
|
||||
|
||||
static IReadOnlyList<CepstralWordObservation> DecodeCepstralIndexedWords(float[] samples, int sampleRate, int expectedEventCount)
|
||||
{
|
||||
const int tableCount = 4;
|
||||
const int hashBits = 14;
|
||||
var templateIndex = BuildCepstralWordIndex(sampleRate, tableCount, hashBits);
|
||||
var motifSamples = MimirBioacousticTimeline.Default.RenderEventMonoFloat(0, sampleRate).Length;
|
||||
var hopSamples = Math.Max(1, sampleRate / 1_000);
|
||||
var energyTrace = WindowEnergy(samples, motifSamples, hopSamples);
|
||||
var threshold = energyTrace.Length == 0
|
||||
? double.PositiveInfinity
|
||||
: energyTrace.Average(value => (double)value) + Math.Sqrt(energyTrace.Sum(value => Math.Pow(value - energyTrace.Average(), 2.0)) / energyTrace.Length) * 0.10;
|
||||
var proposals = new List<int>();
|
||||
for (var index = 1; index < energyTrace.Length - 1; index++)
|
||||
{
|
||||
if (energyTrace[index] >= threshold &&
|
||||
energyTrace[index] >= energyTrace[index - 1] &&
|
||||
energyTrace[index] >= energyTrace[index + 1])
|
||||
{
|
||||
proposals.Add(index * hopSamples);
|
||||
}
|
||||
}
|
||||
|
||||
var denseStep = Math.Max(1, (int)Math.Round(sampleRate * 0.040));
|
||||
for (var offset = 0; offset + motifSamples <= samples.Length; offset += denseStep)
|
||||
{
|
||||
proposals.Add(offset);
|
||||
}
|
||||
|
||||
var proposalBudget = Math.Max(expectedEventCount * 8, 16);
|
||||
var observations = new List<CepstralWordObservation>();
|
||||
foreach (var offset in proposals
|
||||
.OrderByDescending(offset => energyTrace[Math.Clamp(offset / hopSamples, 0, energyTrace.Length - 1)])
|
||||
.Take(proposalBudget)
|
||||
.Order())
|
||||
{
|
||||
if (offset < 0 || offset + motifSamples > samples.Length)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var feature = CepstralFingerprint(samples.AsSpan(offset, motifSamples), sampleRate);
|
||||
var candidateIndexes = new HashSet<int>();
|
||||
for (var table = 0; table < tableCount; table++)
|
||||
{
|
||||
var key = ProjectionHash(feature, table, hashBits);
|
||||
if (templateIndex.Buckets.TryGetValue((table, key), out var bucket))
|
||||
{
|
||||
foreach (var candidate in bucket)
|
||||
{
|
||||
candidateIndexes.Add(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
if (candidateIndexes.Count < 4)
|
||||
{
|
||||
foreach (var near in templateIndex.Buckets
|
||||
.Where(pair => pair.Key.Table == table &&
|
||||
HammingDistance(pair.Key.Hash, key) <= 4)
|
||||
.SelectMany(pair => pair.Value))
|
||||
{
|
||||
candidateIndexes.Add(near);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (candidateIndexes.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var predictedEvent = PredictedBioacousticEventIndex(offset, sampleRate);
|
||||
var best = candidateIndexes
|
||||
.Select(index =>
|
||||
{
|
||||
var template = templateIndex.Templates[index];
|
||||
var distance = CepstralDistance(feature, template.Feature);
|
||||
var timePenalty = Math.Abs((long)template.EventIndex - (long)predictedEvent) * 0.20;
|
||||
return (Template: template, Distance: distance + timePenalty);
|
||||
})
|
||||
.OrderBy(pair => pair.Distance)
|
||||
.First();
|
||||
var confidence = double.IsFinite(best.Distance)
|
||||
? 1.0 / (1.0 + best.Distance / 8.0)
|
||||
: 0.0;
|
||||
if (confidence < 0.05)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
observations.Add(new CepstralWordObservation(best.Template.EventIndex, offset, confidence));
|
||||
}
|
||||
|
||||
return observations
|
||||
.OrderByDescending(observation => observation.Confidence)
|
||||
.GroupBy(observation => observation.EventIndex)
|
||||
.Select(group => group.First())
|
||||
.OrderBy(observation => observation.SampleOffset)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
static ulong PredictedBioacousticEventIndex(double sampleOffset, int sampleRate)
|
||||
{
|
||||
const double firstEventSeconds = 0.08;
|
||||
const double eventSpacingSeconds = 0.16;
|
||||
var index = (long)Math.Round((sampleOffset / sampleRate - firstEventSeconds) / eventSpacingSeconds);
|
||||
return (ulong)Math.Max(0, index);
|
||||
}
|
||||
|
||||
static CepstralWordIndex BuildCepstralWordIndex(int sampleRate, int tableCount, int hashBits)
|
||||
{
|
||||
var augmentationSettings = new[]
|
||||
{
|
||||
new CepstralDegradationSetting("template-clean", 0.0, 0.0, 0),
|
||||
new CepstralDegradationSetting("template-blur", 0.0, 0.0, 1),
|
||||
new CepstralDegradationSetting("template-warp-light", 0.75, 1.25, 0),
|
||||
new CepstralDegradationSetting("template-warp-blur", 0.75, 1.25, 1),
|
||||
};
|
||||
var templates = new List<CepstralWordTemplate>(MimirBioacousticTimeline.SymbolCount);
|
||||
for (ulong eventIndex = 0; eventIndex < MimirBioacousticTimeline.SymbolCount; eventIndex++)
|
||||
{
|
||||
var samples = MimirBioacousticTimeline.Default.RenderEventMonoFloat(eventIndex, sampleRate);
|
||||
foreach (var setting in augmentationSettings)
|
||||
{
|
||||
var templateSamples = setting.BlurPasses == 0 && setting.WarpFrames == 0.0 && setting.WarpCoefficients == 0.0
|
||||
? samples
|
||||
: RoundTripThroughDegradedCepstrum(samples, sampleRate, setting, out _);
|
||||
templates.Add(new CepstralWordTemplate(eventIndex, CepstralFingerprint(templateSamples, sampleRate)));
|
||||
}
|
||||
}
|
||||
|
||||
var buckets = new Dictionary<(int Table, int Hash), List<int>>();
|
||||
for (var index = 0; index < templates.Count; index++)
|
||||
{
|
||||
for (var table = 0; table < tableCount; table++)
|
||||
{
|
||||
var key = (table, ProjectionHash(templates[index].Feature, table, hashBits));
|
||||
if (!buckets.TryGetValue(key, out var bucket))
|
||||
{
|
||||
bucket = [];
|
||||
buckets[key] = bucket;
|
||||
}
|
||||
|
||||
bucket.Add(index);
|
||||
}
|
||||
}
|
||||
|
||||
return new CepstralWordIndex(templates.ToArray(), buckets);
|
||||
}
|
||||
|
||||
static double[] CepstralFingerprint(ReadOnlySpan<float> samples, int sampleRate)
|
||||
{
|
||||
const int fftSize = 1024;
|
||||
const int hopSize = 256;
|
||||
const int melBins = 40;
|
||||
const int cepstralCoefficients = 14;
|
||||
var window = HannWindow(fftSize);
|
||||
var melFilters = BuildMelFilterBank(melBins, fftSize, sampleRate, 180.0, 15_000.0);
|
||||
var melNormalizer = melFilters.Select(filter => Math.Max(1.0e-12, filter.Sum())).ToArray();
|
||||
var frameCount = Math.Max(1, 1 + Math.Max(0, samples.Length - fftSize) / hopSize);
|
||||
var mean = new double[cepstralCoefficients];
|
||||
var delta = new double[cepstralCoefficients];
|
||||
double[]? previous = null;
|
||||
for (var frame = 0; frame < frameCount; frame++)
|
||||
{
|
||||
var offset = frame * hopSize;
|
||||
var spectrum = new Complex[fftSize];
|
||||
for (var index = 0; index < fftSize; index++)
|
||||
{
|
||||
var sampleIndex = offset + index;
|
||||
var sample = sampleIndex < samples.Length ? samples[sampleIndex] : 0.0f;
|
||||
spectrum[index] = new Complex(sample * window[index], 0.0);
|
||||
}
|
||||
|
||||
FastFourierTransform(spectrum, inverse: false);
|
||||
var cepstrum = Dct(SpectrumToLogMel(spectrum, melFilters, melNormalizer), cepstralCoefficients);
|
||||
for (var coefficient = 0; coefficient < cepstralCoefficients; coefficient++)
|
||||
{
|
||||
mean[coefficient] += cepstrum[coefficient];
|
||||
if (previous != null)
|
||||
{
|
||||
delta[coefficient] += Math.Abs(cepstrum[coefficient] - previous[coefficient]);
|
||||
}
|
||||
}
|
||||
|
||||
previous = cepstrum;
|
||||
}
|
||||
|
||||
var output = new double[cepstralCoefficients * 2];
|
||||
for (var coefficient = 0; coefficient < cepstralCoefficients; coefficient++)
|
||||
{
|
||||
output[coefficient] = coefficient == 0 ? 0.0 : mean[coefficient] / frameCount;
|
||||
output[coefficient + cepstralCoefficients] = delta[coefficient] / Math.Max(1, frameCount - 1);
|
||||
}
|
||||
|
||||
NormalizeFeature(output);
|
||||
return output;
|
||||
}
|
||||
|
||||
static void NormalizeFeature(double[] feature)
|
||||
{
|
||||
var mean = feature.Average();
|
||||
var norm = 0.0;
|
||||
for (var index = 0; index < feature.Length; index++)
|
||||
{
|
||||
feature[index] -= mean;
|
||||
norm += feature[index] * feature[index];
|
||||
}
|
||||
|
||||
norm = Math.Sqrt(norm);
|
||||
if (norm <= 1.0e-12)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
for (var index = 0; index < feature.Length; index++)
|
||||
{
|
||||
feature[index] /= norm;
|
||||
}
|
||||
}
|
||||
|
||||
static double MeanCepstralTimingError(
|
||||
IReadOnlyList<CepstralWordObservation> observations,
|
||||
MimirBioacousticTimeline timeline,
|
||||
int sampleRate)
|
||||
{
|
||||
if (observations.Count == 0)
|
||||
{
|
||||
return double.PositiveInfinity;
|
||||
}
|
||||
|
||||
return observations.Average(observation =>
|
||||
Math.Abs(observation.SampleOffset - timeline.EventForIndex(observation.EventIndex).StartSeconds * sampleRate));
|
||||
}
|
||||
|
||||
static double CepstralDistance(IReadOnlyList<double> first, IReadOnlyList<double> second)
|
||||
{
|
||||
var sum = 0.0;
|
||||
for (var index = 0; index < Math.Min(first.Count, second.Count); index++)
|
||||
{
|
||||
var diff = first[index] - second[index];
|
||||
if (!double.IsFinite(diff))
|
||||
{
|
||||
return double.PositiveInfinity;
|
||||
}
|
||||
|
||||
sum += diff * diff;
|
||||
}
|
||||
|
||||
return Math.Sqrt(sum);
|
||||
}
|
||||
|
||||
static int HammingDistance(int first, int second)
|
||||
{
|
||||
var value = (uint)(first ^ second);
|
||||
var count = 0;
|
||||
while (value != 0)
|
||||
{
|
||||
value &= value - 1;
|
||||
count++;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
static int ProjectionHash(IReadOnlyList<double> feature, int table, int bits)
|
||||
{
|
||||
var hash = 0;
|
||||
for (var bit = 0; bit < bits; bit++)
|
||||
{
|
||||
var dot = 0.0;
|
||||
for (var index = 0; index < feature.Count; index++)
|
||||
{
|
||||
dot += feature[index] * ProjectionWeight(table, bit, index);
|
||||
}
|
||||
|
||||
if (dot >= 0.0)
|
||||
{
|
||||
hash |= 1 << bit;
|
||||
}
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
static double ProjectionWeight(int table, int bit, int index)
|
||||
{
|
||||
var value = Hash2D(table * 131 + bit * 17, index * 29 + 7);
|
||||
return ((value & 0xffff) / 32767.5) - 1.0;
|
||||
}
|
||||
|
||||
static float[] WindowEnergy(float[] samples, int windowSamples, int hopSamples)
|
||||
{
|
||||
if (samples.Length < windowSamples)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var output = new float[1 + (samples.Length - windowSamples) / hopSamples];
|
||||
var prefix = new double[samples.Length + 1];
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
prefix[index + 1] = prefix[index] + samples[index] * samples[index];
|
||||
}
|
||||
|
||||
for (var frame = 0; frame < output.Length; frame++)
|
||||
{
|
||||
var offset = frame * hopSamples;
|
||||
output[frame] = (float)((prefix[offset + windowSamples] - prefix[offset]) / windowSamples);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static double[] SpectrumToLogMel(Complex[] spectrum, double[][] melFilters, double[] melNormalizer)
|
||||
{
|
||||
var magnitudes = new double[spectrum.Length / 2 + 1];
|
||||
for (var bin = 0; bin < magnitudes.Length; bin++)
|
||||
{
|
||||
magnitudes[bin] = spectrum[bin].Magnitude;
|
||||
}
|
||||
|
||||
var logMel = new double[melFilters.Length];
|
||||
for (var mel = 0; mel < melFilters.Length; mel++)
|
||||
{
|
||||
var energy = 0.0;
|
||||
for (var bin = 0; bin < magnitudes.Length; bin++)
|
||||
{
|
||||
energy += magnitudes[bin] * melFilters[mel][bin];
|
||||
}
|
||||
|
||||
logMel[mel] = Math.Log(1.0e-7 + energy / melNormalizer[mel]);
|
||||
}
|
||||
|
||||
return logMel;
|
||||
}
|
||||
|
||||
static double[] LogMelToMagnitude(double[] logMel, double[][] melFilters, int fftSize)
|
||||
{
|
||||
var magnitudes = new double[fftSize / 2 + 1];
|
||||
var weights = new double[magnitudes.Length];
|
||||
for (var mel = 0; mel < melFilters.Length; mel++)
|
||||
{
|
||||
var value = Math.Exp(Math.Clamp(logMel[mel], -24.0, 6.0));
|
||||
for (var bin = 0; bin < magnitudes.Length; bin++)
|
||||
{
|
||||
var weight = melFilters[mel][bin];
|
||||
magnitudes[bin] += value * weight;
|
||||
weights[bin] += weight;
|
||||
}
|
||||
}
|
||||
|
||||
for (var bin = 0; bin < magnitudes.Length; bin++)
|
||||
{
|
||||
magnitudes[bin] = weights[bin] <= 1.0e-12 ? 0.0 : magnitudes[bin] / weights[bin];
|
||||
}
|
||||
|
||||
return magnitudes;
|
||||
}
|
||||
|
||||
static double[,] WarpCepstrum(double[,] input, CepstralDegradationSetting setting)
|
||||
{
|
||||
var frames = input.GetLength(0);
|
||||
var coefficients = input.GetLength(1);
|
||||
var output = new double[frames, coefficients];
|
||||
for (var frame = 0; frame < frames; frame++)
|
||||
{
|
||||
for (var coefficient = 0; coefficient < coefficients; coefficient++)
|
||||
{
|
||||
var warpT = setting.WarpFrames * SimplexNoise2D(frame * 0.071, coefficient * 0.137 + 19.0);
|
||||
var warpC = setting.WarpCoefficients * SimplexNoise2D(frame * 0.053 + 41.0, coefficient * 0.113);
|
||||
output[frame, coefficient] = SampleCepstrumBilinear(input, frame + warpT, coefficient + warpC);
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static double[,] BlurCepstrum5Tap(double[,] input)
|
||||
{
|
||||
var kernel = new[] { 1.0, 4.0, 6.0, 4.0, 1.0 };
|
||||
var frames = input.GetLength(0);
|
||||
var coefficients = input.GetLength(1);
|
||||
var temp = new double[frames, coefficients];
|
||||
var output = new double[frames, coefficients];
|
||||
for (var frame = 0; frame < frames; frame++)
|
||||
{
|
||||
for (var coefficient = 0; coefficient < coefficients; coefficient++)
|
||||
{
|
||||
var sum = 0.0;
|
||||
var weightSum = 0.0;
|
||||
for (var tap = -2; tap <= 2; tap++)
|
||||
{
|
||||
var sourceCoefficient = Math.Clamp(coefficient + tap, 0, coefficients - 1);
|
||||
var weight = kernel[tap + 2];
|
||||
sum += input[frame, sourceCoefficient] * weight;
|
||||
weightSum += weight;
|
||||
}
|
||||
|
||||
temp[frame, coefficient] = sum / weightSum;
|
||||
}
|
||||
}
|
||||
|
||||
for (var frame = 0; frame < frames; frame++)
|
||||
{
|
||||
for (var coefficient = 0; coefficient < coefficients; coefficient++)
|
||||
{
|
||||
var sum = 0.0;
|
||||
var weightSum = 0.0;
|
||||
for (var tap = -2; tap <= 2; tap++)
|
||||
{
|
||||
var sourceFrame = Math.Clamp(frame + tap, 0, frames - 1);
|
||||
var weight = kernel[tap + 2];
|
||||
sum += temp[sourceFrame, coefficient] * weight;
|
||||
weightSum += weight;
|
||||
}
|
||||
|
||||
output[frame, coefficient] = sum / weightSum;
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static double SampleCepstrumBilinear(double[,] input, double frame, double coefficient)
|
||||
{
|
||||
var frames = input.GetLength(0);
|
||||
var coefficients = input.GetLength(1);
|
||||
var f0 = Math.Clamp((int)Math.Floor(frame), 0, frames - 1);
|
||||
var c0 = Math.Clamp((int)Math.Floor(coefficient), 0, coefficients - 1);
|
||||
var f1 = Math.Clamp(f0 + 1, 0, frames - 1);
|
||||
var c1 = Math.Clamp(c0 + 1, 0, coefficients - 1);
|
||||
var ft = Math.Clamp(frame - Math.Floor(frame), 0.0, 1.0);
|
||||
var ct = Math.Clamp(coefficient - Math.Floor(coefficient), 0.0, 1.0);
|
||||
var a = input[f0, c0] * (1.0 - ct) + input[f0, c1] * ct;
|
||||
var b = input[f1, c0] * (1.0 - ct) + input[f1, c1] * ct;
|
||||
return a * (1.0 - ft) + b * ft;
|
||||
}
|
||||
|
||||
static double[][] BuildMelFilterBank(int melBins, int fftSize, int sampleRate, double minHz, double maxHz)
|
||||
{
|
||||
var minMel = HzToMel(minHz);
|
||||
var maxMel = HzToMel(Math.Min(maxHz, sampleRate * 0.5));
|
||||
var points = Enumerable.Range(0, melBins + 2)
|
||||
.Select(index => MelToHz(minMel + (maxMel - minMel) * index / (melBins + 1)))
|
||||
.Select(hz => Math.Clamp((int)Math.Round(hz / sampleRate * fftSize), 0, fftSize / 2))
|
||||
.ToArray();
|
||||
var filters = new double[melBins][];
|
||||
for (var mel = 0; mel < melBins; mel++)
|
||||
{
|
||||
filters[mel] = new double[fftSize / 2 + 1];
|
||||
var left = points[mel];
|
||||
var center = Math.Max(points[mel + 1], left + 1);
|
||||
var right = Math.Max(points[mel + 2], center + 1);
|
||||
for (var bin = left; bin <= right && bin < filters[mel].Length; bin++)
|
||||
{
|
||||
filters[mel][bin] = bin <= center
|
||||
? (bin - left) / (double)Math.Max(1, center - left)
|
||||
: (right - bin) / (double)Math.Max(1, right - center);
|
||||
}
|
||||
}
|
||||
|
||||
return filters;
|
||||
}
|
||||
|
||||
static double[] Dct(double[] values, int coefficientCount)
|
||||
{
|
||||
var output = new double[coefficientCount];
|
||||
var scale = Math.PI / values.Length;
|
||||
for (var coefficient = 0; coefficient < coefficientCount; coefficient++)
|
||||
{
|
||||
var sum = 0.0;
|
||||
for (var index = 0; index < values.Length; index++)
|
||||
{
|
||||
sum += values[index] * Math.Cos(scale * (index + 0.5) * coefficient);
|
||||
}
|
||||
|
||||
output[coefficient] = sum;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static double[] InverseDct(double[] coefficients, int valueCount)
|
||||
{
|
||||
var output = new double[valueCount];
|
||||
var scale = Math.PI / valueCount;
|
||||
for (var index = 0; index < valueCount; index++)
|
||||
{
|
||||
var sum = coefficients[0] / valueCount;
|
||||
for (var coefficient = 1; coefficient < coefficients.Length; coefficient++)
|
||||
{
|
||||
sum += 2.0 * coefficients[coefficient] * Math.Cos(scale * (index + 0.5) * coefficient) / valueCount;
|
||||
}
|
||||
|
||||
output[index] = sum;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static double[] Row(double[,] matrix, int row)
|
||||
{
|
||||
var values = new double[matrix.GetLength(1)];
|
||||
for (var index = 0; index < values.Length; index++)
|
||||
{
|
||||
values[index] = matrix[row, index];
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
static double[] HannWindow(int length)
|
||||
{
|
||||
var window = new double[length];
|
||||
for (var index = 0; index < length; index++)
|
||||
{
|
||||
window[index] = 0.5 - 0.5 * Math.Cos(2.0 * Math.PI * index / Math.Max(1, length - 1));
|
||||
}
|
||||
|
||||
return window;
|
||||
}
|
||||
|
||||
static double RootMeanSquare(IReadOnlyList<float> samples)
|
||||
{
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
var sum = 0.0;
|
||||
for (var index = 0; index < samples.Count; index++)
|
||||
{
|
||||
sum += samples[index] * samples[index];
|
||||
}
|
||||
|
||||
return Math.Sqrt(sum / samples.Count);
|
||||
}
|
||||
|
||||
static double HzToMel(double hz) =>
|
||||
2595.0 * Math.Log10(1.0 + hz / 700.0);
|
||||
|
||||
static double MelToHz(double mel) =>
|
||||
700.0 * (Math.Pow(10.0, mel / 2595.0) - 1.0);
|
||||
|
||||
static double SimplexNoise2D(double x, double y)
|
||||
{
|
||||
const double f2 = 0.3660254037844386;
|
||||
const double g2 = 0.21132486540518713;
|
||||
var s = (x + y) * f2;
|
||||
var i = FastFloor(x + s);
|
||||
var j = FastFloor(y + s);
|
||||
var t = (i + j) * g2;
|
||||
var x0 = x - (i - t);
|
||||
var y0 = y - (j - t);
|
||||
var i1 = x0 > y0 ? 1 : 0;
|
||||
var j1 = x0 > y0 ? 0 : 1;
|
||||
var x1 = x0 - i1 + g2;
|
||||
var y1 = y0 - j1 + g2;
|
||||
var x2 = x0 - 1.0 + 2.0 * g2;
|
||||
var y2 = y0 - 1.0 + 2.0 * g2;
|
||||
return 70.0 * (
|
||||
SimplexCorner(i, j, x0, y0) +
|
||||
SimplexCorner(i + i1, j + j1, x1, y1) +
|
||||
SimplexCorner(i + 1, j + 1, x2, y2));
|
||||
}
|
||||
|
||||
static double SimplexCorner(int i, int j, double x, double y)
|
||||
{
|
||||
var t = 0.5 - x * x - y * y;
|
||||
if (t < 0.0)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
var hash = Hash2D(i, j) & 7;
|
||||
var gx = hash < 4 ? 1.0 : 2.0;
|
||||
var gy = hash < 4 ? 2.0 : 1.0;
|
||||
if ((hash & 1) != 0)
|
||||
{
|
||||
gx = -gx;
|
||||
}
|
||||
|
||||
if ((hash & 2) != 0)
|
||||
{
|
||||
gy = -gy;
|
||||
}
|
||||
|
||||
t *= t;
|
||||
return t * t * (gx * x + gy * y);
|
||||
}
|
||||
|
||||
static int Hash2D(int x, int y)
|
||||
{
|
||||
unchecked
|
||||
{
|
||||
var hash = x * 0x1f1f1f1f ^ y * 0x5f356495;
|
||||
hash ^= hash >> 16;
|
||||
hash *= 0x45d9f3b;
|
||||
hash ^= hash >> 16;
|
||||
return hash;
|
||||
}
|
||||
}
|
||||
|
||||
static int FastFloor(double value) =>
|
||||
value >= 0.0 ? (int)value : (int)value - 1;
|
||||
|
||||
static void FastFourierTransform(Complex[] values, bool inverse)
|
||||
{
|
||||
var j = 0;
|
||||
for (var i = 1; i < values.Length; i++)
|
||||
{
|
||||
var bit = values.Length >> 1;
|
||||
for (; (j & bit) != 0; bit >>= 1)
|
||||
{
|
||||
j ^= bit;
|
||||
}
|
||||
|
||||
j ^= bit;
|
||||
if (i < j)
|
||||
{
|
||||
(values[i], values[j]) = (values[j], values[i]);
|
||||
}
|
||||
}
|
||||
|
||||
for (var length = 2; length <= values.Length; length <<= 1)
|
||||
{
|
||||
var angle = 2.0 * Math.PI / length * (inverse ? 1.0 : -1.0);
|
||||
var wLength = new Complex(Math.Cos(angle), Math.Sin(angle));
|
||||
for (var i = 0; i < values.Length; i += length)
|
||||
{
|
||||
var w = Complex.One;
|
||||
for (var k = 0; k < length / 2; k++)
|
||||
{
|
||||
var even = values[i + k];
|
||||
var odd = values[i + k + length / 2] * w;
|
||||
values[i + k] = even + odd;
|
||||
values[i + k + length / 2] = even - odd;
|
||||
w *= wLength;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (inverse)
|
||||
{
|
||||
for (var i = 0; i < values.Length; i++)
|
||||
{
|
||||
values[i] /= values.Length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int ParseIntOption(string[] args, string name, int fallback)
|
||||
{
|
||||
for (var index = 0; index < args.Length - 1; index++)
|
||||
|
|
@ -963,3 +1781,28 @@ internal sealed class DisposableConfiguration : IDisposable
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record CepstralDegradationSetting(
|
||||
string Name,
|
||||
double WarpFrames,
|
||||
double WarpCoefficients,
|
||||
int BlurPasses);
|
||||
|
||||
internal sealed record CepstralRoundTripAnalysis(
|
||||
int FrameCount,
|
||||
int MelBins,
|
||||
int CepstralCoefficients,
|
||||
double RmsRatio);
|
||||
|
||||
internal sealed record CepstralWordTemplate(
|
||||
ulong EventIndex,
|
||||
IReadOnlyList<double> Feature);
|
||||
|
||||
internal sealed record CepstralWordObservation(
|
||||
ulong EventIndex,
|
||||
double SampleOffset,
|
||||
double Confidence);
|
||||
|
||||
internal sealed record CepstralWordIndex(
|
||||
IReadOnlyList<CepstralWordTemplate> Templates,
|
||||
IReadOnlyDictionary<(int Table, int Hash), List<int>> Buckets);
|
||||
|
|
|
|||
|
|
@ -80,6 +80,13 @@ public sealed class MimirBioacousticTimeline
|
|||
return samples;
|
||||
}
|
||||
|
||||
public float[] RenderEventMonoFloat(ulong eventIndex, int sampleRate)
|
||||
{
|
||||
var samples = new float[(int)Math.Round(MotifDurationSeconds * sampleRate)];
|
||||
AddMotif(samples, sampleRate, EventForIndex(eventIndex), EventStartSeconds(eventIndex), Gain);
|
||||
return samples;
|
||||
}
|
||||
|
||||
public MimirChirpletStreamDecode DecodeStreamWindow(ReadOnlySpan<float> samples, int sampleRate)
|
||||
{
|
||||
if (samples.Length == 0)
|
||||
|
|
@ -728,12 +735,12 @@ public sealed class MimirBioacousticTimeline
|
|||
|
||||
private static int SymbolForEvent(ulong eventIndex)
|
||||
{
|
||||
var word = (int)(eventIndex % WordCount);
|
||||
var word = (int)((eventIndex / SpeakerCount) % WordCount);
|
||||
return word * SpeakerCount + (int)SpeakerForEvent(eventIndex);
|
||||
}
|
||||
|
||||
private static int EventIndexForSymbol(int symbolId) =>
|
||||
Math.Clamp(symbolId / SpeakerCount, 0, WordCount - 1);
|
||||
Math.Clamp(symbolId / SpeakerCount, 0, WordCount - 1) * SpeakerCount + (symbolId & 1);
|
||||
|
||||
private static MimirBioacousticSpeaker SpeakerForEvent(ulong eventIndex) =>
|
||||
(eventIndex & 1UL) == 0UL ? MimirBioacousticSpeaker.Left : MimirBioacousticSpeaker.Right;
|
||||
|
|
|
|||
|
|
@ -118,7 +118,10 @@ accepted_design:
|
|||
waveform correlation around the decoded anchor delay. Each detected motif
|
||||
carries per-band response evidence, and the next calibration cut should use
|
||||
those profiles to tune contours, formants, usable bands, gain, and rhythm
|
||||
weighting. `MimirChirpBinTimeline` remains the controlled chirp-bin
|
||||
weighting. `Mimir.BufferSmoke --bioacoustic-cepstral-smoke` now proves the
|
||||
non-brute-force receiver direction: degraded mel-cepstral calls are decoded
|
||||
by an augmented MFCC/projection-hash word index before any expensive
|
||||
confirmation. `MimirChirpBinTimeline` remains the controlled chirp-bin
|
||||
calibration/reference artifact with ASIO tooling and
|
||||
`MimirChirpBinCalibrationModel`; keep those captures as response evidence,
|
||||
not the active runtime sound. Reports and sync states expose delay in
|
||||
|
|
@ -272,8 +275,9 @@ current_status:
|
|||
the Leap USB neighborhood does not reproduce the Focusrite-induced collapse.
|
||||
next_action: >
|
||||
Use the new bioacoustic timeline watermark as the active acoustic sync
|
||||
direction. Cut the next implementation in this order: streaming log-mel
|
||||
motif decoder state, calibration-session command/replay proof that learns
|
||||
direction. Cut the next implementation in this order: streaming indexed
|
||||
log-mel/MFCC motif decoder state with a global clock hypothesis,
|
||||
calibration-session command/replay proof that learns
|
||||
per-output/mic motif/formant response, fractional-delay/SRO actuator, then
|
||||
native camera payload handles and Fensalir lowering. Keep chirp-bin
|
||||
calibration captures as reference evidence, not the runtime target.
|
||||
|
|
|
|||
Loading…
Reference in a new issue