Add structured chirplet calibration phrase
This commit is contained in:
parent
5ab8e80cf0
commit
bbb052a8ba
12 changed files with 392 additions and 136 deletions
|
|
@ -3,6 +3,7 @@ namespace Mimir.Runtime.Synchronization;
|
|||
public sealed record MimirAlignedAudioChannel(
|
||||
string SourceId,
|
||||
int DelaySamples,
|
||||
double FractionalDelaySamples,
|
||||
double Confidence);
|
||||
|
||||
public sealed record MimirAlignedAudioFrame(
|
||||
|
|
|
|||
|
|
@ -6,10 +6,8 @@ public sealed class MimirAudioSynchronizationAnalyzer
|
|||
{
|
||||
private const int MaxWindowSamples = 48_000 * 5;
|
||||
private const int MaxLagSamples = 2_400;
|
||||
private const int ChirpletHopSamples = 64;
|
||||
private const double ChirpletDurationSeconds = 0.08;
|
||||
private const double ChirpletStartHz = 9_000.0;
|
||||
private const double ChirpletEndHz = 16_000.0;
|
||||
private const int ChirpletHopSamples = 16;
|
||||
private readonly MimirChirpletCalibrationPhrase chirpletPhrase = MimirChirpletCalibrationPhrase.Default;
|
||||
|
||||
public IReadOnlyList<MimirAudioSynchronizationReport> Analyze(
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers,
|
||||
|
|
@ -66,23 +64,29 @@ public sealed class MimirAudioSynchronizationAnalyzer
|
|||
continue;
|
||||
}
|
||||
|
||||
var referenceSync = BuildChirpletEnergyTrace(referenceSamples.AsSpan(^compared..), referenceBlock.SampleRate);
|
||||
var candidateSync = BuildChirpletEnergyTrace(candidateSamples.AsSpan(^compared..), referenceBlock.SampleRate);
|
||||
var referenceWindow = referenceSamples.AsSpan(^compared..);
|
||||
var candidateWindow = candidateSamples.AsSpan(^compared..);
|
||||
var referenceSync = chirpletPhrase.BuildEnergyTrace(referenceWindow, referenceBlock.SampleRate, ChirpletHopSamples);
|
||||
var candidateSync = chirpletPhrase.BuildEnergyTrace(candidateWindow, referenceBlock.SampleRate, ChirpletHopSamples);
|
||||
if (referenceSync.Length < 16 || candidateSync.Length < 16)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var maxLag = Math.Min(MaxLagSamples / ChirpletHopSamples, Math.Min(referenceSync.Length, candidateSync.Length) / 2);
|
||||
var (delaySamples, confidence) = EstimateDelay(referenceSync, candidateSync, maxLag);
|
||||
delaySamples *= ChirpletHopSamples;
|
||||
var (delayHops, confidence) = EstimateDelay(referenceSync, candidateSync, maxLag);
|
||||
var fractionalDelaySamples = delayHops * ChirpletHopSamples;
|
||||
var delaySamples = (int)Math.Round(fractionalDelaySamples);
|
||||
var bandResponses = chirpletPhrase.EstimateBandResponse(candidateWindow, referenceBlock.SampleRate);
|
||||
reports.Add(new MimirAudioSynchronizationReport(
|
||||
reference.Descriptor.SourceId,
|
||||
buffer.Descriptor.SourceId,
|
||||
referenceBlock.SampleRate,
|
||||
delaySamples,
|
||||
delaySamples * 1000.0 / referenceBlock.SampleRate,
|
||||
fractionalDelaySamples,
|
||||
fractionalDelaySamples * 1000.0 / referenceBlock.SampleRate,
|
||||
confidence,
|
||||
bandResponses,
|
||||
compared,
|
||||
reference.Latest?.Sequence ?? 0,
|
||||
buffer.Latest?.Sequence ?? 0));
|
||||
|
|
@ -142,7 +146,7 @@ public sealed class MimirAudioSynchronizationAnalyzer
|
|||
var maxPositiveDelay = Math.Max(0, alignedReports.Select(report => report.DelaySamples).DefaultIfEmpty(0).Max());
|
||||
var channels = new List<MimirAlignedAudioChannel>
|
||||
{
|
||||
new(referenceSourceId, 0, 1.0),
|
||||
new(referenceSourceId, 0, 0.0, 1.0),
|
||||
};
|
||||
var samples = new List<float[]>
|
||||
{
|
||||
|
|
@ -171,7 +175,11 @@ public sealed class MimirAudioSynchronizationAnalyzer
|
|||
}
|
||||
|
||||
samples.Add(TailEndingBefore(candidateSamples, frameCount, trimSamples));
|
||||
channels.Add(new MimirAlignedAudioChannel(report.SourceId, report.DelaySamples, report.Confidence));
|
||||
channels.Add(new MimirAlignedAudioChannel(
|
||||
report.SourceId,
|
||||
report.DelaySamples,
|
||||
report.FractionalDelaySamples,
|
||||
report.Confidence));
|
||||
}
|
||||
|
||||
return new MimirAlignedAudioFrame(
|
||||
|
|
@ -276,81 +284,12 @@ public sealed class MimirAudioSynchronizationAnalyzer
|
|||
return output;
|
||||
}
|
||||
|
||||
private static float[] BuildChirpletEnergyTrace(ReadOnlySpan<float> samples, int sampleRate)
|
||||
{
|
||||
var kernelLength = Math.Max(8, (int)Math.Round(sampleRate * ChirpletDurationSeconds));
|
||||
if (samples.Length < kernelLength)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var output = new float[1 + (samples.Length - kernelLength) / ChirpletHopSamples];
|
||||
for (var frame = 0; frame < output.Length; frame++)
|
||||
{
|
||||
var offset = frame * ChirpletHopSamples;
|
||||
var real = 0.0;
|
||||
var imag = 0.0;
|
||||
var energy = 0.0;
|
||||
for (var index = 0; index < kernelLength; index++)
|
||||
{
|
||||
var normalized = kernelLength <= 1 ? 1.0 : index / (double)(kernelLength - 1);
|
||||
var time = index / (double)sampleRate;
|
||||
var envelope = 0.5 - 0.5 * Math.Cos(2.0 * Math.PI * normalized);
|
||||
var phase = 2.0 * Math.PI * (ChirpletStartHz * time + 0.5 * (ChirpletEndHz - ChirpletStartHz) * time * normalized);
|
||||
var sample = samples[offset + index] * envelope;
|
||||
real += sample * Math.Cos(phase);
|
||||
imag -= sample * Math.Sin(phase);
|
||||
energy += sample * sample;
|
||||
}
|
||||
|
||||
var magnitude = Math.Sqrt(real * real + imag * imag);
|
||||
output[frame] = energy > 1.0e-12 ? (float)(magnitude / Math.Sqrt(energy)) : 0.0f;
|
||||
}
|
||||
|
||||
return ContrastNormalize(output);
|
||||
}
|
||||
|
||||
private static float[] ContrastNormalize(float[] samples)
|
||||
{
|
||||
if (samples.Length == 0)
|
||||
{
|
||||
return samples;
|
||||
}
|
||||
|
||||
var mean = 0.0;
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
mean += samples[index];
|
||||
}
|
||||
|
||||
mean /= samples.Length;
|
||||
var variance = 0.0;
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
var centered = samples[index] - mean;
|
||||
variance += centered * centered;
|
||||
}
|
||||
|
||||
var deviation = Math.Sqrt(variance / samples.Length);
|
||||
if (deviation <= 1.0e-12)
|
||||
{
|
||||
return samples;
|
||||
}
|
||||
|
||||
var output = new float[samples.Length];
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
var z = (samples[index] - mean) / deviation;
|
||||
output[index] = z > 0.0 ? (float)(z * z) : 0.0f;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
private static (int DelaySamples, double Confidence) EstimateDelay(ReadOnlySpan<float> reference, ReadOnlySpan<float> candidate, int maxLag)
|
||||
private static (double DelaySamples, double Confidence) EstimateDelay(ReadOnlySpan<float> reference, ReadOnlySpan<float> candidate, int maxLag)
|
||||
{
|
||||
var bestLag = 0;
|
||||
var best = double.NegativeInfinity;
|
||||
var previous = double.NegativeInfinity;
|
||||
var next = double.NegativeInfinity;
|
||||
for (var lag = -maxLag; lag <= maxLag; lag++)
|
||||
{
|
||||
var score = NormalizedCorrelation(reference, candidate, lag);
|
||||
|
|
@ -361,7 +300,34 @@ public sealed class MimirAudioSynchronizationAnalyzer
|
|||
}
|
||||
}
|
||||
|
||||
return (bestLag, Math.Max(0.0, Math.Min(1.0, best)));
|
||||
if (bestLag > -maxLag)
|
||||
{
|
||||
previous = NormalizedCorrelation(reference, candidate, bestLag - 1);
|
||||
}
|
||||
|
||||
if (bestLag < maxLag)
|
||||
{
|
||||
next = NormalizedCorrelation(reference, candidate, bestLag + 1);
|
||||
}
|
||||
|
||||
var offset = ParabolicPeakOffset(previous, best, next);
|
||||
return (bestLag + offset, Math.Max(0.0, Math.Min(1.0, best)));
|
||||
}
|
||||
|
||||
private static double ParabolicPeakOffset(double left, double center, double right)
|
||||
{
|
||||
if (!double.IsFinite(left) || !double.IsFinite(center) || !double.IsFinite(right))
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
var denominator = left - 2.0 * center + right;
|
||||
if (Math.Abs(denominator) <= 1.0e-12)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
return Math.Clamp(0.5 * (left - right) / denominator, -0.5, 0.5);
|
||||
}
|
||||
|
||||
private static double NormalizedCorrelation(ReadOnlySpan<float> reference, ReadOnlySpan<float> candidate, int lag)
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ public sealed record MimirAudioSynchronizationReport(
|
|||
string SourceId,
|
||||
int SampleRate,
|
||||
int DelaySamples,
|
||||
double FractionalDelaySamples,
|
||||
double DelayMilliseconds,
|
||||
double Confidence,
|
||||
IReadOnlyList<MimirChirpletBandResponse> BandResponses,
|
||||
int ComparedSamples,
|
||||
ulong ReferenceSequence,
|
||||
ulong SourceSequence);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,246 @@
|
|||
namespace Mimir.Runtime.Synchronization;
|
||||
|
||||
public sealed record MimirChirpletTone(
|
||||
double StartSeconds,
|
||||
double DurationSeconds,
|
||||
double StartHz,
|
||||
double EndHz,
|
||||
double Gain = 1.0)
|
||||
{
|
||||
public double CenterHz => (StartHz + EndHz) * 0.5;
|
||||
}
|
||||
|
||||
public sealed record MimirChirpletBandResponse(
|
||||
double CenterHz,
|
||||
double Energy);
|
||||
|
||||
public sealed class MimirChirpletCalibrationPhrase
|
||||
{
|
||||
private readonly float[] kernel;
|
||||
private readonly IReadOnlyList<float[]> toneKernels;
|
||||
|
||||
public MimirChirpletCalibrationPhrase(
|
||||
int sampleRate,
|
||||
double intervalSeconds,
|
||||
double firstFireSeconds,
|
||||
double gain,
|
||||
IReadOnlyList<MimirChirpletTone> tones)
|
||||
{
|
||||
SampleRate = sampleRate;
|
||||
IntervalSeconds = intervalSeconds;
|
||||
FirstFireSeconds = firstFireSeconds;
|
||||
Gain = gain;
|
||||
Tones = tones;
|
||||
DurationSeconds = tones.Count == 0
|
||||
? 0.0
|
||||
: tones.Max(tone => tone.StartSeconds + tone.DurationSeconds);
|
||||
kernel = RenderMonoFloat(sampleRate);
|
||||
toneKernels = tones.Select(tone => RenderToneKernel(tone, sampleRate, DurationSeconds)).ToArray();
|
||||
}
|
||||
|
||||
public static MimirChirpletCalibrationPhrase Default { get; } = new(
|
||||
sampleRate: 48_000,
|
||||
intervalSeconds: 1.5,
|
||||
firstFireSeconds: 0.5,
|
||||
gain: 0.125,
|
||||
tones:
|
||||
[
|
||||
new(0.000, 0.055, 7_680.0, 8_320.0),
|
||||
new(0.085, 0.055, 9_600.0, 10_400.0),
|
||||
new(0.170, 0.055, 11_520.0, 12_480.0),
|
||||
new(0.255, 0.055, 15_360.0, 16_000.0),
|
||||
]);
|
||||
|
||||
public int SampleRate { get; }
|
||||
|
||||
public double IntervalSeconds { get; }
|
||||
|
||||
public double FirstFireSeconds { get; }
|
||||
|
||||
public double Gain { get; }
|
||||
|
||||
public double DurationSeconds { get; }
|
||||
|
||||
public IReadOnlyList<MimirChirpletTone> Tones { get; }
|
||||
|
||||
public string RenderPcm16Base64()
|
||||
{
|
||||
var samples = RenderMonoFloat(SampleRate);
|
||||
var bytes = new byte[samples.Length * sizeof(short)];
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
var sample = (short)Math.Round(Math.Clamp(samples[index], -1.0f, 1.0f) * short.MaxValue);
|
||||
bytes[index * sizeof(short)] = (byte)(sample & 0xff);
|
||||
bytes[index * sizeof(short) + 1] = (byte)((sample >> 8) & 0xff);
|
||||
}
|
||||
|
||||
return Convert.ToBase64String(bytes);
|
||||
}
|
||||
|
||||
public float[] BuildEnergyTrace(ReadOnlySpan<float> samples, int sampleRate, int hopSamples)
|
||||
{
|
||||
var analysisKernel = sampleRate == SampleRate ? kernel : RenderMonoFloat(sampleRate);
|
||||
return BuildMatchedEnergyTrace(samples, analysisKernel, hopSamples);
|
||||
}
|
||||
|
||||
public IReadOnlyList<MimirChirpletBandResponse> EstimateBandResponse(ReadOnlySpan<float> samples, int sampleRate)
|
||||
{
|
||||
if (samples.Length == 0 || Tones.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var responses = new List<MimirChirpletBandResponse>(Tones.Count);
|
||||
for (var index = 0; index < Tones.Count; index++)
|
||||
{
|
||||
var tone = Tones[index];
|
||||
var toneKernel = sampleRate == SampleRate
|
||||
? toneKernels[index]
|
||||
: RenderToneKernel(tone, sampleRate, DurationSeconds);
|
||||
var energy = MaxMatchedEnergy(samples, toneKernel, Math.Max(1, sampleRate / 1_000));
|
||||
responses.Add(new MimirChirpletBandResponse(tone.CenterHz, energy));
|
||||
}
|
||||
|
||||
return responses;
|
||||
}
|
||||
|
||||
private float[] RenderMonoFloat(int sampleRate)
|
||||
{
|
||||
var frameCount = Math.Max(1, (int)Math.Ceiling(DurationSeconds * sampleRate));
|
||||
var samples = new float[frameCount];
|
||||
foreach (var tone in Tones)
|
||||
{
|
||||
AddTone(samples, sampleRate, tone);
|
||||
}
|
||||
|
||||
var peak = samples.Select(Math.Abs).DefaultIfEmpty(0.0f).Max();
|
||||
if (peak <= 1.0e-9f)
|
||||
{
|
||||
return samples;
|
||||
}
|
||||
|
||||
var scale = (float)(Gain / peak);
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
samples[index] *= scale;
|
||||
}
|
||||
|
||||
return samples;
|
||||
}
|
||||
|
||||
private static float[] RenderToneKernel(MimirChirpletTone tone, int sampleRate, double phraseDurationSeconds)
|
||||
{
|
||||
var frameCount = Math.Max(1, (int)Math.Ceiling(phraseDurationSeconds * sampleRate));
|
||||
var samples = new float[frameCount];
|
||||
AddTone(samples, sampleRate, tone);
|
||||
return samples;
|
||||
}
|
||||
|
||||
private static void AddTone(float[] samples, int sampleRate, MimirChirpletTone tone)
|
||||
{
|
||||
var startFrame = Math.Max(0, (int)Math.Round(tone.StartSeconds * sampleRate));
|
||||
var frameCount = Math.Max(1, (int)Math.Round(tone.DurationSeconds * sampleRate));
|
||||
for (var frame = 0; frame < frameCount && startFrame + frame < samples.Length; frame++)
|
||||
{
|
||||
var normalized = frameCount <= 1 ? 1.0 : frame / (double)(frameCount - 1);
|
||||
var t = frame / (double)sampleRate;
|
||||
var envelope = 0.5 - 0.5 * Math.Cos(2.0 * Math.PI * normalized);
|
||||
var phase = 2.0 * Math.PI * (tone.StartHz * t + 0.5 * (tone.EndHz - tone.StartHz) * t * normalized);
|
||||
samples[startFrame + frame] += (float)(Math.Sin(phase) * envelope * tone.Gain);
|
||||
}
|
||||
}
|
||||
|
||||
private static float[] BuildMatchedEnergyTrace(ReadOnlySpan<float> samples, ReadOnlySpan<float> kernel, int hopSamples)
|
||||
{
|
||||
if (samples.Length < kernel.Length || kernel.Length == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var output = new float[1 + (samples.Length - kernel.Length) / hopSamples];
|
||||
for (var frame = 0; frame < output.Length; frame++)
|
||||
{
|
||||
var offset = frame * hopSamples;
|
||||
output[frame] = (float)MatchedEnergy(samples.Slice(offset, kernel.Length), kernel);
|
||||
}
|
||||
|
||||
return ContrastNormalize(output);
|
||||
}
|
||||
|
||||
private static double MatchedEnergy(ReadOnlySpan<float> samples, ReadOnlySpan<float> kernel)
|
||||
{
|
||||
var count = Math.Min(samples.Length, kernel.Length);
|
||||
if (count == 0)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
var dot = 0.0;
|
||||
var sampleEnergy = 0.0;
|
||||
var kernelEnergy = 0.0;
|
||||
for (var index = 0; index < count; index++)
|
||||
{
|
||||
var sample = samples[index];
|
||||
var basis = kernel[index];
|
||||
dot += sample * basis;
|
||||
sampleEnergy += sample * sample;
|
||||
kernelEnergy += basis * basis;
|
||||
}
|
||||
|
||||
var denominator = Math.Sqrt(sampleEnergy * kernelEnergy);
|
||||
return denominator > 1.0e-12 ? Math.Abs(dot) / denominator : 0.0;
|
||||
}
|
||||
|
||||
private static double MaxMatchedEnergy(ReadOnlySpan<float> samples, ReadOnlySpan<float> kernel, int hopSamples)
|
||||
{
|
||||
if (samples.Length < kernel.Length || kernel.Length == 0)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
var best = 0.0;
|
||||
for (var offset = 0; offset <= samples.Length - kernel.Length; offset += hopSamples)
|
||||
{
|
||||
best = Math.Max(best, MatchedEnergy(samples.Slice(offset, kernel.Length), kernel));
|
||||
}
|
||||
|
||||
return best;
|
||||
}
|
||||
|
||||
private static float[] ContrastNormalize(float[] samples)
|
||||
{
|
||||
if (samples.Length == 0)
|
||||
{
|
||||
return samples;
|
||||
}
|
||||
|
||||
var mean = 0.0;
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
mean += samples[index];
|
||||
}
|
||||
|
||||
mean /= samples.Length;
|
||||
var variance = 0.0;
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
var centered = samples[index] - mean;
|
||||
variance += centered * centered;
|
||||
}
|
||||
|
||||
var deviation = Math.Sqrt(variance / samples.Length);
|
||||
if (deviation <= 1.0e-12)
|
||||
{
|
||||
return samples;
|
||||
}
|
||||
|
||||
var output = new float[samples.Length];
|
||||
for (var index = 0; index < samples.Length; index++)
|
||||
{
|
||||
var z = (samples[index] - mean) / deviation;
|
||||
output[index] = z > 0.0 ? (float)(z * z) : 0.0f;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue