Trace live chirplet synchronization

This commit is contained in:
Meta Vens 2026-05-22 17:35:21 +01:00
commit b0dbf4bbff
7 changed files with 156 additions and 17 deletions

View file

@ -5,6 +5,9 @@ namespace Mimir.Runtime.Synchronization;
public sealed class MimirAudioSynchronizationAnalyzer
{
private const int MaxWindowSamples = 48_000 * 2;
private readonly List<MimirAudioSynchronizationDecodeTrace> lastDecodeTraces = [];
public IReadOnlyList<MimirAudioSynchronizationDecodeTrace> LastDecodeTraces => lastDecodeTraces;
public IReadOnlyList<MimirAudioSynchronizationReport> Analyze(
IEnumerable<MimirRollingStreamBuffer> buffers,
@ -36,6 +39,7 @@ public sealed class MimirAudioSynchronizationAnalyzer
}
var reports = new List<MimirAudioSynchronizationReport>();
lastDecodeTraces.Clear();
foreach (var buffer in audioBuffers)
{
if (ReferenceEquals(buffer, reference))
@ -73,6 +77,20 @@ public sealed class MimirAudioSynchronizationAnalyzer
var referenceDecode = MimirChirpletTimeline.Default.DecodeStreamWindow(referenceWindow, referenceBlock.SampleRate);
var candidateDecode = MimirChirpletTimeline.Default.DecodeStreamWindow(candidateWindow, referenceBlock.SampleRate);
var deterministicFit = EstimateDelayFromDecodedTimeline(referenceDecode, candidateDecode);
lastDecodeTraces.Add(new MimirAudioSynchronizationDecodeTrace(
reference.Descriptor.SourceId,
buffer.Descriptor.SourceId,
referenceBlock.SampleRate,
compared,
referenceDecode.Frames.Count,
referenceDecode.Anchors.Count,
referenceDecode.ClockFit?.Confidence ?? 0.0,
candidateDecode.Frames.Count,
candidateDecode.Anchors.Count,
candidateDecode.ClockFit?.Confidence ?? 0.0,
deterministicFit.MatchedEvents,
deterministicFit.Confidence,
deterministicFit.MatchedEvents >= 3 ? "report" : "insufficient-anchors"));
if (deterministicFit.MatchedEvents < 3)
{
continue;

View file

@ -0,0 +1,16 @@
namespace Mimir.Runtime.Synchronization;
public sealed record MimirAudioSynchronizationDecodeTrace(
string ReferenceSourceId,
string SourceId,
int SampleRate,
int ComparedSamples,
int ReferenceFrames,
int ReferenceAnchors,
double ReferenceClockConfidence,
int CandidateFrames,
int CandidateAnchors,
double CandidateClockConfidence,
int MatchedEvents,
double Confidence,
string Status);

View file

@ -155,13 +155,27 @@ public sealed class MimirChirpletTimeline
return [];
}
var envelope = ContrastNormalize(BuildEnvelopeEnergyTrace(samples, sampleRate, hopSamples));
if (envelope.Length <= 2)
var trace = BuildChirpletEnergyTrace(samples, sampleRate, hopSamples);
if (trace.Count <= 2)
{
return [];
}
var frames = new List<MimirChirpletTransformFrame>();
var candidateOffsets = new SortedSet<int>();
for (var frame = 1; frame < trace.Count - 1; frame++)
{
if (trace[frame] < 0.05 ||
trace[frame] < trace[frame - 1] ||
trace[frame] < trace[frame + 1])
{
continue;
}
candidateOffsets.Add(frame * hopSamples);
}
var envelope = ContrastNormalize(BuildEnvelopeEnergyTrace(samples, sampleRate, hopSamples));
for (var frame = 1; frame < envelope.Length - 1; frame++)
{
if (envelope[frame] < 0.055 ||
@ -171,10 +185,15 @@ public sealed class MimirChirpletTimeline
continue;
}
candidateOffsets.Add(frame * hopSamples);
}
foreach (var offset in candidateOffsets)
{
var transformFrame = ClassifySymbolsAt(
samples,
sampleRate,
frame * hopSamples,
offset,
Math.Max(hopSamples * 3, sampleRate / 100));
if (transformFrame != null)
{
@ -228,7 +247,7 @@ public sealed class MimirChirpletTimeline
}
}
if (bestEnergy >= 0.08)
if (bestEnergy >= 0.035)
{
var refinedOffset = RefineMatchedOffset(samples, kernel, bestOffset, start, end);
var refinedEnergy = MatchedEnergy(
@ -247,7 +266,7 @@ public sealed class MimirChirpletTimeline
.OrderByDescending(candidate => candidate.Energy)
.Take(MaxSymbolCandidatesPerFrame)
.ToArray();
if (ordered[0].Energy < 0.25)
if (ordered[0].Energy < 0.08)
{
return null;
}
@ -659,6 +678,40 @@ public sealed class MimirChirpletTimeline
return Math.Min(1.0, Math.Sqrt(sineDot * sineDot + cosineDot * cosineDot) / denominator);
}
private IReadOnlyList<double> BuildChirpletEnergyTrace(ReadOnlySpan<float> samples, int sampleRate, int hopSamples)
{
var maxKernelLength = Codebook.Symbols
.Select(symbol => sampleRate == SampleRate ? symbolKernels[symbol.SymbolId].Length : RenderToneKernel(symbol.Tone, sampleRate).Length)
.DefaultIfEmpty(0)
.Max();
if (samples.Length < maxKernelLength || maxKernelLength == 0)
{
return [];
}
var output = new double[1 + (samples.Length - maxKernelLength) / hopSamples];
for (var frame = 0; frame < output.Length; frame++)
{
var offset = frame * hopSamples;
var best = 0.0;
for (var symbol = 0; symbol < Codebook.Symbols.Count; symbol++)
{
var tone = Codebook[symbol].Tone;
var kernel = sampleRate == SampleRate ? symbolKernels[symbol] : RenderToneKernel(tone, sampleRate);
if (offset + kernel.Length > samples.Length)
{
continue;
}
best = Math.Max(best, MatchedEnergy(samples.Slice(offset, kernel.Length), kernel));
}
output[frame] = best;
}
return output;
}
private static float[] BuildEnvelopeEnergyTrace(ReadOnlySpan<float> samples, int sampleRate, int hopSamples)
{
var windowSamples = Math.Max(1, (int)Math.Round(MaxEventDurationSeconds * sampleRate));

View file

@ -33,6 +33,9 @@ public sealed class MimirSynchronizationHub : IDisposable
public IReadOnlyList<MimirAudioSynchronizationReport> AudioSynchronizationReports =>
audioSynchronizationReports.Values.OrderBy(report => report.SourceId, StringComparer.Ordinal).ToArray();
public IReadOnlyList<MimirAudioSynchronizationDecodeTrace> AudioSynchronizationDecodeTraces =>
audioSynchronization.LastDecodeTraces;
public void AddSource(IMimirStreamSource source)
{
ObjectDisposedException.ThrowIf(disposed, this);