Bound runtime sync analysis

This commit is contained in:
Meta Vens 2026-05-22 14:16:50 +01:00
commit 200186d4d5
8 changed files with 112 additions and 37 deletions

View file

@ -11,7 +11,7 @@ namespace Mimir.Runtime;
public sealed class MimirRuntime : IAquariumRuntime
{
private const string DefaultAudioSyncReference = "loopback-scarlett-speakers";
private const float AudioSyncUpdateIntervalSeconds = 0.5f;
private const float AudioSyncUpdateIntervalSeconds = 0.1f;
private readonly LocalCastRuntime visualRuntime;
private readonly MimirSynchronizationHub synchronization;
private readonly AquariumUiDocument ui;
@ -164,7 +164,8 @@ public sealed class MimirRuntime : IAquariumRuntime
return;
}
lastAudioSynchronizationReports = synchronization.AnalyzeAudioSynchronization(DefaultAudioSyncReference, runtimeSeconds);
synchronization.AnalyzeAudioSynchronizationStep(DefaultAudioSyncReference, runtimeSeconds);
lastAudioSynchronizationReports = synchronization.AudioSynchronizationReports;
nextAudioSyncSeconds += AudioSyncUpdateIntervalSeconds;
}
@ -229,7 +230,7 @@ public sealed class MimirRuntime : IAquariumRuntime
private string DescribeAudioSync()
{
var reports = synchronization.AnalyzeAudioSynchronization(DefaultAudioSyncReference, runtimeSeconds);
var reports = synchronization.AudioSynchronizationReports;
return reports.Count == 0
? "no payload windows"
: string.Join(" | ", reports.Select(report => $"{report.SourceId}: {report.FractionalDelaySamples:0.0} samples {report.DelayMilliseconds:0.00}ms c={report.Confidence:0.00} events={report.TimelineMatchedEvents}"));
@ -237,10 +238,10 @@ public sealed class MimirRuntime : IAquariumRuntime
private string DescribeAlignedAudio()
{
var frame = synchronization.BuildAlignedAudioFrame(DefaultAudioSyncReference);
return frame == null
? "no aligned frame"
: $"{frame.Channels.Count}ch {frame.SampleRate}Hz {frame.FrameCount} frames";
var states = synchronization.AudioSynchronizationStates;
return states.Count == 0
? "no aligned state"
: $"{states.Count + 1}ch state-ready";
}
private string DescribeAudioSyncState()

View file

@ -11,7 +11,8 @@ public sealed class MimirAudioSynchronizationAnalyzer
public IReadOnlyList<MimirAudioSynchronizationReport> Analyze(
IEnumerable<MimirRollingStreamBuffer> buffers,
string referenceSourceId,
double approximateTimelineSeconds)
double approximateTimelineSeconds,
IReadOnlySet<string>? candidateSourceIds = null)
{
var audioBuffers = buffers
.Where(buffer => buffer.Descriptor.Kind == MimirStreamKind.Audio)
@ -47,6 +48,11 @@ public sealed class MimirAudioSynchronizationAnalyzer
continue;
}
if (candidateSourceIds != null && !candidateSourceIds.Contains(buffer.Descriptor.SourceId))
{
continue;
}
var commonEndNs = Math.Min(referenceLatest.Value.TimestampNs, buffer.Latest?.TimestampNs ?? 0);
var candidateSamples = ExtractMonoWindow(buffer, out var candidateBlock);
if (candidateSamples.Length == 0 || candidateBlock == null)
@ -363,7 +369,11 @@ public sealed class MimirAudioSynchronizationAnalyzer
return (0.0, 0.0, 0);
}
var candidateByEvent = candidateObservations.ToDictionary(observation => observation.EventIndex);
var candidateByEvent = candidateObservations
.GroupBy(observation => observation.EventIndex)
.ToDictionary(
group => group.Key,
group => group.OrderByDescending(observation => observation.Energy).First());
var delays = new List<(double Delay, double Weight)>();
foreach (var reference in referenceObservations)
{

View file

@ -5,7 +5,9 @@ public sealed class MimirSynchronizationHub : IDisposable
private readonly List<IMimirStreamSource> sources = [];
private readonly MimirAudioSynchronizationAnalyzer audioSynchronization = new();
private readonly MimirAudioSynchronizationStateTracker audioSynchronizationState = new();
private readonly Dictionary<string, MimirAudioSynchronizationReport> audioSynchronizationReports = new(StringComparer.Ordinal);
private ulong ingestedSamples;
private int nextAudioSynchronizationCandidate;
public MimirSynchronizationHub(MimirSynchronizationSettings settings)
{
@ -28,6 +30,9 @@ public sealed class MimirSynchronizationHub : IDisposable
public IReadOnlyList<MimirAudioSynchronizationState> AudioSynchronizationStates =>
audioSynchronizationState.States;
public IReadOnlyList<MimirAudioSynchronizationReport> AudioSynchronizationReports =>
audioSynchronizationReports.Values.OrderBy(report => report.SourceId, StringComparer.Ordinal).ToArray();
public void AddSource(IMimirStreamSource source)
{
ObjectDisposedException.ThrowIf(disposed, this);
@ -68,7 +73,42 @@ public sealed class MimirSynchronizationHub : IDisposable
{
ObjectDisposedException.ThrowIf(disposed, this);
var reports = audioSynchronization.Analyze(Buffers.Buffers, referenceSourceId, approximateTimelineSeconds);
audioSynchronizationState.Update(reports);
StoreAudioSynchronizationReports(reports);
return reports;
}
public IReadOnlyList<MimirAudioSynchronizationReport> AnalyzeAudioSynchronizationStep(
string referenceSourceId,
double approximateTimelineSeconds,
int maxCandidates = 1)
{
ObjectDisposedException.ThrowIf(disposed, this);
var candidates = Buffers.Buffers
.Where(buffer =>
buffer.Descriptor.Kind == MimirStreamKind.Audio &&
!string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal) &&
buffer.Latest?.AudioBlock != null)
.OrderBy(buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal)
.Select(buffer => buffer.Descriptor.SourceId)
.ToArray();
if (candidates.Length == 0)
{
return [];
}
var selected = new HashSet<string>(StringComparer.Ordinal);
for (var count = 0; count < Math.Min(maxCandidates, candidates.Length); count++)
{
selected.Add(candidates[nextAudioSynchronizationCandidate % candidates.Length]);
nextAudioSynchronizationCandidate++;
}
var reports = audioSynchronization.Analyze(
Buffers.Buffers,
referenceSourceId,
approximateTimelineSeconds,
selected);
StoreAudioSynchronizationReports(reports);
return reports;
}
@ -87,6 +127,16 @@ public sealed class MimirSynchronizationHub : IDisposable
private bool disposed;
private void StoreAudioSynchronizationReports(IReadOnlyList<MimirAudioSynchronizationReport> reports)
{
foreach (var report in reports)
{
audioSynchronizationReports[report.SourceId] = report;
}
audioSynchronizationState.Update(reports);
}
public void Dispose()
{
if (disposed)