GameCult/Mimir
Watch
9
0
Fork
You've already forked Mimir
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Trace live chirplet synchronization

Browse source
This commit is contained in:
Meta Vens 2026-05-22 17:35:21 +01:00
commit b0dbf4bbff
7 changed files with 156 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/Mimir.BufferSmoke/Program.cs
View file

@ -73,6 +73,12 @@ foreach (var state in hub.AudioSynchronizationStates)
$"sync-state {state.ReferenceSourceId}->{state.SourceId}: smoothedDelaySamples={state.SmoothedDelaySamples:0.000} delayMs={state.DelayMilliseconds:0.000} sroPpm={state.SamplingRateOffsetPpm:0.000} confidence={state.Confidence:0.000} bands={DescribeBands(state.BandResponses)}");
}
foreach (var trace in hub.AudioSynchronizationDecodeTraces.OrderBy(trace => trace.SourceId, StringComparer.Ordinal))
{
Console.WriteLine(
$"sync-decode {trace.ReferenceSourceId}->{trace.SourceId}: status={trace.Status} compared={trace.ComparedSamples} rate={trace.SampleRate} refFrames={trace.ReferenceFrames} refAnchors={trace.ReferenceAnchors} refClock={trace.ReferenceClockConfidence:0.000} candFrames={trace.CandidateFrames} candAnchors={trace.CandidateAnchors} candClock={trace.CandidateClockConfidence:0.000} matched={trace.MatchedEvents} confidence={trace.Confidence:0.000}");
}
if (requireSamples && emptyBuffers.Count > 0)
{
Console.Error.WriteLine($"empty buffers: {string.Join(", ", emptyBuffers)}");

66
src/Mimir.Runtime/MimirRuntime.cs
View file

@ -12,10 +12,14 @@ public sealed class MimirRuntime : IAquariumRuntime
{
private const string DefaultAudioSyncReference = "loopback-scarlett-speakers";
private const float AudioSyncUpdateIntervalSeconds = 0.1f;
private const double CalibrationStartSeconds = 0.5;
private const int CalibrationBatchSegments = 120;
private const float DefaultCalibrationGain = 2.0f;
private readonly LocalCastRuntime visualRuntime;
private readonly MimirSynchronizationHub synchronization;
private readonly AquariumUiDocument ui;
private readonly float telemetryIntervalSeconds;
private readonly float calibrationGain;
private int lastPollCount;
private float runtimeSeconds;
private float nextAudioSyncSeconds = AudioSyncUpdateIntervalSeconds;
@ -47,6 +51,7 @@ public sealed class MimirRuntime : IAquariumRuntime
visualRuntime = new LocalCastRuntime(options);
synchronization = new MimirSynchronizationHub(settings);
telemetryIntervalSeconds = ParseTelemetryIntervalSeconds();
calibrationGain = ParseCalibrationGain();
nextTelemetrySeconds = telemetryIntervalSeconds;
foreach (var source in streamSources)
{
@ -132,24 +137,48 @@ public sealed class MimirRuntime : IAquariumRuntime
private void QueueCalibrationTimeline()
{
var queuedUntilSeconds = calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds;
var targetSeconds = runtimeSeconds + (float)MimirChirpletTimeline.QueueLeadSeconds;
while (queuedUntilSeconds < targetSeconds)
var nextSegmentSeconds = CalibrationStartSeconds + calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds;
if (runtimeSeconds < nextSegmentSeconds)
{
visualRuntime.Audio.EnqueuePcm16Base64(
MimirChirpletTimeline.Default.RenderSegmentPcm16Base64(calibrationSegmentIndex),
MimirChirpletTimeline.SampleRate,
channels: 1,
gain: 1.0f);
calibrationSegmentIndex++;
queuedUntilSeconds = calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds;
return;
}
var batch = RenderCalibrationBatchPcm16Base64(calibrationSegmentIndex, CalibrationBatchSegments, out var peak);
visualRuntime.Audio.EnqueuePcm16Base64(
batch,
MimirChirpletTimeline.SampleRate,
channels: 1,
gain: calibrationGain);
Console.WriteLine(
$"mimir-chirplet-batch firstSegment={calibrationSegmentIndex} segments={CalibrationBatchSegments} seconds={CalibrationBatchSegments * MimirChirpletTimeline.SegmentSeconds:0.00} peak={peak:0.000000} gain={calibrationGain:0.###} base64Bytes={batch.Length}");
calibrationSegmentIndex += CalibrationBatchSegments;
}
private string DescribeChirpletReference()
{
var queuedUntilSeconds = calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds;
return $"{DefaultAudioSyncReference} continuous timeline {MimirChirpletTimeline.SegmentSeconds:0.00}s segments queued to {queuedUntilSeconds:0.00}s";
var emittedUntilSeconds = calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds;
return $"{DefaultAudioSyncReference} continuous timeline {MimirChirpletTimeline.SegmentSeconds:0.00}s segments emitted to {emittedUntilSeconds:0.00}s";
}
private static string RenderCalibrationBatchPcm16Base64(ulong firstSegment, int segmentCount, out float peak)
{
var samplesPerSegment = (int)Math.Round(MimirChirpletTimeline.SegmentSeconds * MimirChirpletTimeline.SampleRate);
var bytes = new byte[samplesPerSegment * Math.Max(1, segmentCount) * sizeof(short)];
var byteIndex = 0;
peak = 0.0f;
for (var segment = 0; segment < segmentCount; segment++)
{
var samples = MimirChirpletTimeline.Default.RenderSegmentMonoFloat(firstSegment + (ulong)segment);
for (var index = 0; index < samples.Length; index++)
{
peak = Math.Max(peak, Math.Abs(samples[index]));
var sample = (short)Math.Round(Math.Clamp(samples[index], -1.0f, 1.0f) * short.MaxValue);
bytes[byteIndex++] = (byte)(sample & 0xff);
bytes[byteIndex++] = (byte)((sample >> 8) & 0xff);
}
}
return Convert.ToBase64String(bytes);
}
private string DescribeBuffers()
@ -194,6 +223,12 @@ public sealed class MimirRuntime : IAquariumRuntime
$"mimir-sync-state {state.ReferenceSourceId}->{state.SourceId} delaySamples={state.SmoothedDelaySamples:0.000} delayMs={state.DelayMilliseconds:0.000} sroPpm={state.SamplingRateOffsetPpm:0.000} confidence={state.Confidence:0.000}");
}
foreach (var trace in synchronization.AudioSynchronizationDecodeTraces.OrderBy(trace => trace.SourceId, StringComparer.Ordinal))
{
Console.WriteLine(
$"mimir-sync-decode {trace.ReferenceSourceId}->{trace.SourceId} status={trace.Status} compared={trace.ComparedSamples} rate={trace.SampleRate} refFrames={trace.ReferenceFrames} refAnchors={trace.ReferenceAnchors} refClock={trace.ReferenceClockConfidence:0.000} candFrames={trace.CandidateFrames} candAnchors={trace.CandidateAnchors} candClock={trace.CandidateClockConfidence:0.000} matched={trace.MatchedEvents} confidence={trace.Confidence:0.000}");
}
nextTelemetrySeconds += telemetryIntervalSeconds;
}
@ -204,6 +239,13 @@ public sealed class MimirRuntime : IAquariumRuntime
: 0.0f;
}
private static float ParseCalibrationGain()
{
return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_CHIRPLET_GAIN"), out var gain)
? Math.Clamp(gain, 0.05f, 4.0f)
: DefaultCalibrationGain;
}
private string DescribeAudioBuffers()
{
return string.Join(" | ", synchronization.Buffers.Buffers

18
src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationAnalyzer.cs
View file

@ -5,6 +5,9 @@ namespace Mimir.Runtime.Synchronization;
public sealed class MimirAudioSynchronizationAnalyzer
{
private const int MaxWindowSamples = 48_000 * 2;
private readonly List<MimirAudioSynchronizationDecodeTrace> lastDecodeTraces = [];
public IReadOnlyList<MimirAudioSynchronizationDecodeTrace> LastDecodeTraces => lastDecodeTraces;
public IReadOnlyList<MimirAudioSynchronizationReport> Analyze(
IEnumerable<MimirRollingStreamBuffer> buffers,
@ -36,6 +39,7 @@ public sealed class MimirAudioSynchronizationAnalyzer
}
var reports = new List<MimirAudioSynchronizationReport>();
lastDecodeTraces.Clear();
foreach (var buffer in audioBuffers)
{
if (ReferenceEquals(buffer, reference))
@ -73,6 +77,20 @@ public sealed class MimirAudioSynchronizationAnalyzer
var referenceDecode = MimirChirpletTimeline.Default.DecodeStreamWindow(referenceWindow, referenceBlock.SampleRate);
var candidateDecode = MimirChirpletTimeline.Default.DecodeStreamWindow(candidateWindow, referenceBlock.SampleRate);
var deterministicFit = EstimateDelayFromDecodedTimeline(referenceDecode, candidateDecode);
lastDecodeTraces.Add(new MimirAudioSynchronizationDecodeTrace(
reference.Descriptor.SourceId,
buffer.Descriptor.SourceId,
referenceBlock.SampleRate,
compared,
referenceDecode.Frames.Count,
referenceDecode.Anchors.Count,
referenceDecode.ClockFit?.Confidence ?? 0.0,
candidateDecode.Frames.Count,
candidateDecode.Anchors.Count,
candidateDecode.ClockFit?.Confidence ?? 0.0,
deterministicFit.MatchedEvents,
deterministicFit.Confidence,
deterministicFit.MatchedEvents >= 3 ? "report" : "insufficient-anchors"));
if (deterministicFit.MatchedEvents < 3)
{
continue;

16
src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationDecodeTrace.cs Normal file
View file

@ -0,0 +1,16 @@
namespace Mimir.Runtime.Synchronization;
public sealed record MimirAudioSynchronizationDecodeTrace(
string ReferenceSourceId,
string SourceId,
int SampleRate,
int ComparedSamples,
int ReferenceFrames,
int ReferenceAnchors,
double ReferenceClockConfidence,
int CandidateFrames,
int CandidateAnchors,
double CandidateClockConfidence,
int MatchedEvents,
double Confidence,
string Status);

63
src/Mimir.Runtime/Synchronization/MimirChirpletTimeline.cs
View file

@ -155,13 +155,27 @@ public sealed class MimirChirpletTimeline
return [];
}
var envelope = ContrastNormalize(BuildEnvelopeEnergyTrace(samples, sampleRate, hopSamples));
if (envelope.Length <= 2)
var trace = BuildChirpletEnergyTrace(samples, sampleRate, hopSamples);
if (trace.Count <= 2)
{
return [];
}
var frames = new List<MimirChirpletTransformFrame>();
var candidateOffsets = new SortedSet<int>();
for (var frame = 1; frame < trace.Count - 1; frame++)
{
if (trace[frame] < 0.05 ||
trace[frame] < trace[frame - 1] ||
trace[frame] < trace[frame + 1])
{
continue;
}
candidateOffsets.Add(frame * hopSamples);
}
var envelope = ContrastNormalize(BuildEnvelopeEnergyTrace(samples, sampleRate, hopSamples));
for (var frame = 1; frame < envelope.Length - 1; frame++)
{
if (envelope[frame] < 0.055 ||
@ -171,10 +185,15 @@ public sealed class MimirChirpletTimeline
continue;
}
candidateOffsets.Add(frame * hopSamples);
}
foreach (var offset in candidateOffsets)
{
var transformFrame = ClassifySymbolsAt(
samples,
sampleRate,
frame * hopSamples,
offset,
Math.Max(hopSamples * 3, sampleRate / 100));
if (transformFrame != null)
{
@ -228,7 +247,7 @@ public sealed class MimirChirpletTimeline
}
}
if (bestEnergy >= 0.08)
if (bestEnergy >= 0.035)
{
var refinedOffset = RefineMatchedOffset(samples, kernel, bestOffset, start, end);
var refinedEnergy = MatchedEnergy(
@ -247,7 +266,7 @@ public sealed class MimirChirpletTimeline
.OrderByDescending(candidate => candidate.Energy)
.Take(MaxSymbolCandidatesPerFrame)
.ToArray();
if (ordered[0].Energy < 0.25)
if (ordered[0].Energy < 0.08)
{
return null;
}
@ -659,6 +678,40 @@ public sealed class MimirChirpletTimeline
return Math.Min(1.0, Math.Sqrt(sineDot * sineDot + cosineDot * cosineDot) / denominator);
}
private IReadOnlyList<double> BuildChirpletEnergyTrace(ReadOnlySpan<float> samples, int sampleRate, int hopSamples)
{
var maxKernelLength = Codebook.Symbols
.Select(symbol => sampleRate == SampleRate ? symbolKernels[symbol.SymbolId].Length : RenderToneKernel(symbol.Tone, sampleRate).Length)
.DefaultIfEmpty(0)
.Max();
if (samples.Length < maxKernelLength || maxKernelLength == 0)
{
return [];
}
var output = new double[1 + (samples.Length - maxKernelLength) / hopSamples];
for (var frame = 0; frame < output.Length; frame++)
{
var offset = frame * hopSamples;
var best = 0.0;
for (var symbol = 0; symbol < Codebook.Symbols.Count; symbol++)
{
var tone = Codebook[symbol].Tone;
var kernel = sampleRate == SampleRate ? symbolKernels[symbol] : RenderToneKernel(tone, sampleRate);
if (offset + kernel.Length > samples.Length)
{
continue;
}
best = Math.Max(best, MatchedEnergy(samples.Slice(offset, kernel.Length), kernel));
}
output[frame] = best;
}
return output;
}
private static float[] BuildEnvelopeEnergyTrace(ReadOnlySpan<float> samples, int sampleRate, int hopSamples)
{
var windowSamples = Math.Max(1, (int)Math.Round(MaxEventDurationSeconds * sampleRate));

3
src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs
View file

@ -33,6 +33,9 @@ public sealed class MimirSynchronizationHub : IDisposable
public IReadOnlyList<MimirAudioSynchronizationReport> AudioSynchronizationReports =>
audioSynchronizationReports.Values.OrderBy(report => report.SourceId, StringComparer.Ordinal).ToArray();
public IReadOnlyList<MimirAudioSynchronizationDecodeTrace> AudioSynchronizationDecodeTraces =>
audioSynchronization.LastDecodeTraces;
public void AddSource(IMimirStreamSource source)
{
ObjectDisposedException.ThrowIf(disposed, this);

1
state/evidence.jsonl
View file

@ -49,3 +49,4 @@
{"ts":"2026-05-22T17:35:00+01:00","type":"implementation","status":"ok","note":"Separated chirplet symbol ownership into MimirChirpletSymbolCodebook and added per-candidate refined sample offsets to transform frames. The codebook now guarantees each of the 32 symbols has a unique chirp shape; rhythm is additional code evidence rather than an acoustic alias. ClassifySymbolsAt now does coarse dictionary search, local sample-resolution search, and parabolic sub-sample refinement per symbol candidate. Synthetic two-second emitted timeline check now detects all 15 emitted chirps, decodes 13 possible triplet anchors for events 0-12, and fits clock=48001.819 Hz with confidence=0.952 and MAE=5.171 samples, improved from 11 anchors / 47978.350 Hz / MAE=8.741."}
{"ts":"2026-05-22T17:45:00+01:00","type":"implementation","status":"ok","note":"Replaced single-phase chirplet matching with quadrature sine/cosine kernels and fixed emitted timeline segment gain so canonical chirps are not amplitude-renormalized per half-second segment. Added `Mimir.BufferSmoke --chirplet-self-test` as a first-class synthetic invariant check. The check renders two seconds of canonical timeline audio, detects all 15 emitted chirps, keeps 13 triplet anchors for events 0-12, fits clock=47999.999990 Hz with confidence=1.000, and holds anchor MAE to 0.000014 samples. The next proof is the same deterministic anchor stability through actual loopback and microphone paths."}
{"ts": "2026-05-22T15:52:26+00:00", "type": "implementation", "status": "ok", "note": "Restored the repo-local Python persistence helpers from the Epiphany-style helper family after checking VoidBot and sibling repos. Mimir now has tools/localcast_state.py for status/evidence/branch edits and tools/localcast_prepare_compaction.py for pre-compaction state audits; AGENTS.md points future agents at both helpers."}
{"ts": "2026-05-22T17:34:45+01:00", "type": "verification", "status": "ok", "note": "Aquarium WASAPI resampler used int multiplication for source.Length * targetSampleRate; a 60s 48k chirplet batch overflowed and queued one 44.1k output frame. Switching to double-length arithmetic restored 2,646,000 output frames and audible chirplets."}