Add structured chirplet calibration phrase

This commit is contained in:
Meta Vens 2026-05-22 12:40:30 +01:00
commit bbb052a8ba
12 changed files with 392 additions and 136 deletions

View file

@ -3,6 +3,7 @@ namespace Mimir.Runtime.Synchronization;
public sealed record MimirAlignedAudioChannel(
string SourceId,
int DelaySamples,
double FractionalDelaySamples,
double Confidence);
public sealed record MimirAlignedAudioFrame(

View file

@ -6,10 +6,8 @@ public sealed class MimirAudioSynchronizationAnalyzer
{
private const int MaxWindowSamples = 48_000 * 5;
private const int MaxLagSamples = 2_400;
private const int ChirpletHopSamples = 64;
private const double ChirpletDurationSeconds = 0.08;
private const double ChirpletStartHz = 9_000.0;
private const double ChirpletEndHz = 16_000.0;
private const int ChirpletHopSamples = 16;
private readonly MimirChirpletCalibrationPhrase chirpletPhrase = MimirChirpletCalibrationPhrase.Default;
public IReadOnlyList<MimirAudioSynchronizationReport> Analyze(
IEnumerable<MimirRollingStreamBuffer> buffers,
@ -66,23 +64,29 @@ public sealed class MimirAudioSynchronizationAnalyzer
continue;
}
var referenceSync = BuildChirpletEnergyTrace(referenceSamples.AsSpan(^compared..), referenceBlock.SampleRate);
var candidateSync = BuildChirpletEnergyTrace(candidateSamples.AsSpan(^compared..), referenceBlock.SampleRate);
var referenceWindow = referenceSamples.AsSpan(^compared..);
var candidateWindow = candidateSamples.AsSpan(^compared..);
var referenceSync = chirpletPhrase.BuildEnergyTrace(referenceWindow, referenceBlock.SampleRate, ChirpletHopSamples);
var candidateSync = chirpletPhrase.BuildEnergyTrace(candidateWindow, referenceBlock.SampleRate, ChirpletHopSamples);
if (referenceSync.Length < 16 || candidateSync.Length < 16)
{
continue;
}
var maxLag = Math.Min(MaxLagSamples / ChirpletHopSamples, Math.Min(referenceSync.Length, candidateSync.Length) / 2);
var (delaySamples, confidence) = EstimateDelay(referenceSync, candidateSync, maxLag);
delaySamples *= ChirpletHopSamples;
var (delayHops, confidence) = EstimateDelay(referenceSync, candidateSync, maxLag);
var fractionalDelaySamples = delayHops * ChirpletHopSamples;
var delaySamples = (int)Math.Round(fractionalDelaySamples);
var bandResponses = chirpletPhrase.EstimateBandResponse(candidateWindow, referenceBlock.SampleRate);
reports.Add(new MimirAudioSynchronizationReport(
reference.Descriptor.SourceId,
buffer.Descriptor.SourceId,
referenceBlock.SampleRate,
delaySamples,
delaySamples * 1000.0 / referenceBlock.SampleRate,
fractionalDelaySamples,
fractionalDelaySamples * 1000.0 / referenceBlock.SampleRate,
confidence,
bandResponses,
compared,
reference.Latest?.Sequence ?? 0,
buffer.Latest?.Sequence ?? 0));
@ -142,7 +146,7 @@ public sealed class MimirAudioSynchronizationAnalyzer
var maxPositiveDelay = Math.Max(0, alignedReports.Select(report => report.DelaySamples).DefaultIfEmpty(0).Max());
var channels = new List<MimirAlignedAudioChannel>
{
new(referenceSourceId, 0, 1.0),
new(referenceSourceId, 0, 0.0, 1.0),
};
var samples = new List<float[]>
{
@ -171,7 +175,11 @@ public sealed class MimirAudioSynchronizationAnalyzer
}
samples.Add(TailEndingBefore(candidateSamples, frameCount, trimSamples));
channels.Add(new MimirAlignedAudioChannel(report.SourceId, report.DelaySamples, report.Confidence));
channels.Add(new MimirAlignedAudioChannel(
report.SourceId,
report.DelaySamples,
report.FractionalDelaySamples,
report.Confidence));
}
return new MimirAlignedAudioFrame(
@ -276,81 +284,12 @@ public sealed class MimirAudioSynchronizationAnalyzer
return output;
}
private static float[] BuildChirpletEnergyTrace(ReadOnlySpan<float> samples, int sampleRate)
{
var kernelLength = Math.Max(8, (int)Math.Round(sampleRate * ChirpletDurationSeconds));
if (samples.Length < kernelLength)
{
return [];
}
var output = new float[1 + (samples.Length - kernelLength) / ChirpletHopSamples];
for (var frame = 0; frame < output.Length; frame++)
{
var offset = frame * ChirpletHopSamples;
var real = 0.0;
var imag = 0.0;
var energy = 0.0;
for (var index = 0; index < kernelLength; index++)
{
var normalized = kernelLength <= 1 ? 1.0 : index / (double)(kernelLength - 1);
var time = index / (double)sampleRate;
var envelope = 0.5 - 0.5 * Math.Cos(2.0 * Math.PI * normalized);
var phase = 2.0 * Math.PI * (ChirpletStartHz * time + 0.5 * (ChirpletEndHz - ChirpletStartHz) * time * normalized);
var sample = samples[offset + index] * envelope;
real += sample * Math.Cos(phase);
imag -= sample * Math.Sin(phase);
energy += sample * sample;
}
var magnitude = Math.Sqrt(real * real + imag * imag);
output[frame] = energy > 1.0e-12 ? (float)(magnitude / Math.Sqrt(energy)) : 0.0f;
}
return ContrastNormalize(output);
}
private static float[] ContrastNormalize(float[] samples)
{
if (samples.Length == 0)
{
return samples;
}
var mean = 0.0;
for (var index = 0; index < samples.Length; index++)
{
mean += samples[index];
}
mean /= samples.Length;
var variance = 0.0;
for (var index = 0; index < samples.Length; index++)
{
var centered = samples[index] - mean;
variance += centered * centered;
}
var deviation = Math.Sqrt(variance / samples.Length);
if (deviation <= 1.0e-12)
{
return samples;
}
var output = new float[samples.Length];
for (var index = 0; index < samples.Length; index++)
{
var z = (samples[index] - mean) / deviation;
output[index] = z > 0.0 ? (float)(z * z) : 0.0f;
}
return output;
}
private static (int DelaySamples, double Confidence) EstimateDelay(ReadOnlySpan<float> reference, ReadOnlySpan<float> candidate, int maxLag)
private static (double DelaySamples, double Confidence) EstimateDelay(ReadOnlySpan<float> reference, ReadOnlySpan<float> candidate, int maxLag)
{
var bestLag = 0;
var best = double.NegativeInfinity;
var previous = double.NegativeInfinity;
var next = double.NegativeInfinity;
for (var lag = -maxLag; lag <= maxLag; lag++)
{
var score = NormalizedCorrelation(reference, candidate, lag);
@ -361,7 +300,34 @@ public sealed class MimirAudioSynchronizationAnalyzer
}
}
return (bestLag, Math.Max(0.0, Math.Min(1.0, best)));
if (bestLag > -maxLag)
{
previous = NormalizedCorrelation(reference, candidate, bestLag - 1);
}
if (bestLag < maxLag)
{
next = NormalizedCorrelation(reference, candidate, bestLag + 1);
}
var offset = ParabolicPeakOffset(previous, best, next);
return (bestLag + offset, Math.Max(0.0, Math.Min(1.0, best)));
}
private static double ParabolicPeakOffset(double left, double center, double right)
{
if (!double.IsFinite(left) || !double.IsFinite(center) || !double.IsFinite(right))
{
return 0.0;
}
var denominator = left - 2.0 * center + right;
if (Math.Abs(denominator) <= 1.0e-12)
{
return 0.0;
}
return Math.Clamp(0.5 * (left - right) / denominator, -0.5, 0.5);
}
private static double NormalizedCorrelation(ReadOnlySpan<float> reference, ReadOnlySpan<float> candidate, int lag)

View file

@ -5,8 +5,10 @@ public sealed record MimirAudioSynchronizationReport(
string SourceId,
int SampleRate,
int DelaySamples,
double FractionalDelaySamples,
double DelayMilliseconds,
double Confidence,
IReadOnlyList<MimirChirpletBandResponse> BandResponses,
int ComparedSamples,
ulong ReferenceSequence,
ulong SourceSequence);

View file

@ -0,0 +1,246 @@
namespace Mimir.Runtime.Synchronization;
public sealed record MimirChirpletTone(
double StartSeconds,
double DurationSeconds,
double StartHz,
double EndHz,
double Gain = 1.0)
{
public double CenterHz => (StartHz + EndHz) * 0.5;
}
public sealed record MimirChirpletBandResponse(
double CenterHz,
double Energy);
public sealed class MimirChirpletCalibrationPhrase
{
private readonly float[] kernel;
private readonly IReadOnlyList<float[]> toneKernels;
public MimirChirpletCalibrationPhrase(
int sampleRate,
double intervalSeconds,
double firstFireSeconds,
double gain,
IReadOnlyList<MimirChirpletTone> tones)
{
SampleRate = sampleRate;
IntervalSeconds = intervalSeconds;
FirstFireSeconds = firstFireSeconds;
Gain = gain;
Tones = tones;
DurationSeconds = tones.Count == 0
? 0.0
: tones.Max(tone => tone.StartSeconds + tone.DurationSeconds);
kernel = RenderMonoFloat(sampleRate);
toneKernels = tones.Select(tone => RenderToneKernel(tone, sampleRate, DurationSeconds)).ToArray();
}
public static MimirChirpletCalibrationPhrase Default { get; } = new(
sampleRate: 48_000,
intervalSeconds: 1.5,
firstFireSeconds: 0.5,
gain: 0.125,
tones:
[
new(0.000, 0.055, 7_680.0, 8_320.0),
new(0.085, 0.055, 9_600.0, 10_400.0),
new(0.170, 0.055, 11_520.0, 12_480.0),
new(0.255, 0.055, 15_360.0, 16_000.0),
]);
public int SampleRate { get; }
public double IntervalSeconds { get; }
public double FirstFireSeconds { get; }
public double Gain { get; }
public double DurationSeconds { get; }
public IReadOnlyList<MimirChirpletTone> Tones { get; }
public string RenderPcm16Base64()
{
var samples = RenderMonoFloat(SampleRate);
var bytes = new byte[samples.Length * sizeof(short)];
for (var index = 0; index < samples.Length; index++)
{
var sample = (short)Math.Round(Math.Clamp(samples[index], -1.0f, 1.0f) * short.MaxValue);
bytes[index * sizeof(short)] = (byte)(sample & 0xff);
bytes[index * sizeof(short) + 1] = (byte)((sample >> 8) & 0xff);
}
return Convert.ToBase64String(bytes);
}
public float[] BuildEnergyTrace(ReadOnlySpan<float> samples, int sampleRate, int hopSamples)
{
var analysisKernel = sampleRate == SampleRate ? kernel : RenderMonoFloat(sampleRate);
return BuildMatchedEnergyTrace(samples, analysisKernel, hopSamples);
}
public IReadOnlyList<MimirChirpletBandResponse> EstimateBandResponse(ReadOnlySpan<float> samples, int sampleRate)
{
if (samples.Length == 0 || Tones.Count == 0)
{
return [];
}
var responses = new List<MimirChirpletBandResponse>(Tones.Count);
for (var index = 0; index < Tones.Count; index++)
{
var tone = Tones[index];
var toneKernel = sampleRate == SampleRate
? toneKernels[index]
: RenderToneKernel(tone, sampleRate, DurationSeconds);
var energy = MaxMatchedEnergy(samples, toneKernel, Math.Max(1, sampleRate / 1_000));
responses.Add(new MimirChirpletBandResponse(tone.CenterHz, energy));
}
return responses;
}
private float[] RenderMonoFloat(int sampleRate)
{
var frameCount = Math.Max(1, (int)Math.Ceiling(DurationSeconds * sampleRate));
var samples = new float[frameCount];
foreach (var tone in Tones)
{
AddTone(samples, sampleRate, tone);
}
var peak = samples.Select(Math.Abs).DefaultIfEmpty(0.0f).Max();
if (peak <= 1.0e-9f)
{
return samples;
}
var scale = (float)(Gain / peak);
for (var index = 0; index < samples.Length; index++)
{
samples[index] *= scale;
}
return samples;
}
private static float[] RenderToneKernel(MimirChirpletTone tone, int sampleRate, double phraseDurationSeconds)
{
var frameCount = Math.Max(1, (int)Math.Ceiling(phraseDurationSeconds * sampleRate));
var samples = new float[frameCount];
AddTone(samples, sampleRate, tone);
return samples;
}
private static void AddTone(float[] samples, int sampleRate, MimirChirpletTone tone)
{
var startFrame = Math.Max(0, (int)Math.Round(tone.StartSeconds * sampleRate));
var frameCount = Math.Max(1, (int)Math.Round(tone.DurationSeconds * sampleRate));
for (var frame = 0; frame < frameCount && startFrame + frame < samples.Length; frame++)
{
var normalized = frameCount <= 1 ? 1.0 : frame / (double)(frameCount - 1);
var t = frame / (double)sampleRate;
var envelope = 0.5 - 0.5 * Math.Cos(2.0 * Math.PI * normalized);
var phase = 2.0 * Math.PI * (tone.StartHz * t + 0.5 * (tone.EndHz - tone.StartHz) * t * normalized);
samples[startFrame + frame] += (float)(Math.Sin(phase) * envelope * tone.Gain);
}
}
private static float[] BuildMatchedEnergyTrace(ReadOnlySpan<float> samples, ReadOnlySpan<float> kernel, int hopSamples)
{
if (samples.Length < kernel.Length || kernel.Length == 0)
{
return [];
}
var output = new float[1 + (samples.Length - kernel.Length) / hopSamples];
for (var frame = 0; frame < output.Length; frame++)
{
var offset = frame * hopSamples;
output[frame] = (float)MatchedEnergy(samples.Slice(offset, kernel.Length), kernel);
}
return ContrastNormalize(output);
}
private static double MatchedEnergy(ReadOnlySpan<float> samples, ReadOnlySpan<float> kernel)
{
var count = Math.Min(samples.Length, kernel.Length);
if (count == 0)
{
return 0.0;
}
var dot = 0.0;
var sampleEnergy = 0.0;
var kernelEnergy = 0.0;
for (var index = 0; index < count; index++)
{
var sample = samples[index];
var basis = kernel[index];
dot += sample * basis;
sampleEnergy += sample * sample;
kernelEnergy += basis * basis;
}
var denominator = Math.Sqrt(sampleEnergy * kernelEnergy);
return denominator > 1.0e-12 ? Math.Abs(dot) / denominator : 0.0;
}
private static double MaxMatchedEnergy(ReadOnlySpan<float> samples, ReadOnlySpan<float> kernel, int hopSamples)
{
if (samples.Length < kernel.Length || kernel.Length == 0)
{
return 0.0;
}
var best = 0.0;
for (var offset = 0; offset <= samples.Length - kernel.Length; offset += hopSamples)
{
best = Math.Max(best, MatchedEnergy(samples.Slice(offset, kernel.Length), kernel));
}
return best;
}
private static float[] ContrastNormalize(float[] samples)
{
if (samples.Length == 0)
{
return samples;
}
var mean = 0.0;
for (var index = 0; index < samples.Length; index++)
{
mean += samples[index];
}
mean /= samples.Length;
var variance = 0.0;
for (var index = 0; index < samples.Length; index++)
{
var centered = samples[index] - mean;
variance += centered * centered;
}
var deviation = Math.Sqrt(variance / samples.Length);
if (deviation <= 1.0e-12)
{
return samples;
}
var output = new float[samples.Length];
for (var index = 0; index < samples.Length; index++)
{
var z = (samples[index] - mean) / deviation;
output[index] = z > 0.0 ? (float)(z * z) : 0.0f;
}
return output;
}
}