From b0dbf4bbff66e105c08fadfb892062c86b9ec837 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 22 May 2026 17:35:21 +0100 Subject: [PATCH] Trace live chirplet synchronization --- src/Mimir.BufferSmoke/Program.cs | 6 ++ src/Mimir.Runtime/MimirRuntime.cs | 66 +++++++++++++++---- .../MimirAudioSynchronizationAnalyzer.cs | 18 +++++ .../MimirAudioSynchronizationDecodeTrace.cs | 16 +++++ .../Synchronization/MimirChirpletTimeline.cs | 63 ++++++++++++++++-- .../MimirSynchronizationHub.cs | 3 + state/evidence.jsonl | 1 + 7 files changed, 156 insertions(+), 17 deletions(-) create mode 100644 src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationDecodeTrace.cs diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index dd9821a..db5f157 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -73,6 +73,12 @@ foreach (var state in hub.AudioSynchronizationStates) $"sync-state {state.ReferenceSourceId}->{state.SourceId}: smoothedDelaySamples={state.SmoothedDelaySamples:0.000} delayMs={state.DelayMilliseconds:0.000} sroPpm={state.SamplingRateOffsetPpm:0.000} confidence={state.Confidence:0.000} bands={DescribeBands(state.BandResponses)}"); } +foreach (var trace in hub.AudioSynchronizationDecodeTraces.OrderBy(trace => trace.SourceId, StringComparer.Ordinal)) +{ + Console.WriteLine( + $"sync-decode {trace.ReferenceSourceId}->{trace.SourceId}: status={trace.Status} compared={trace.ComparedSamples} rate={trace.SampleRate} refFrames={trace.ReferenceFrames} refAnchors={trace.ReferenceAnchors} refClock={trace.ReferenceClockConfidence:0.000} candFrames={trace.CandidateFrames} candAnchors={trace.CandidateAnchors} candClock={trace.CandidateClockConfidence:0.000} matched={trace.MatchedEvents} confidence={trace.Confidence:0.000}"); +} + if (requireSamples && emptyBuffers.Count > 0) { Console.Error.WriteLine($"empty buffers: {string.Join(", ", emptyBuffers)}"); diff --git a/src/Mimir.Runtime/MimirRuntime.cs b/src/Mimir.Runtime/MimirRuntime.cs index a73a225..f318a65 100644 --- a/src/Mimir.Runtime/MimirRuntime.cs +++ b/src/Mimir.Runtime/MimirRuntime.cs @@ -12,10 +12,14 @@ public sealed class MimirRuntime : IAquariumRuntime { private const string DefaultAudioSyncReference = "loopback-scarlett-speakers"; private const float AudioSyncUpdateIntervalSeconds = 0.1f; + private const double CalibrationStartSeconds = 0.5; + private const int CalibrationBatchSegments = 120; + private const float DefaultCalibrationGain = 2.0f; private readonly LocalCastRuntime visualRuntime; private readonly MimirSynchronizationHub synchronization; private readonly AquariumUiDocument ui; private readonly float telemetryIntervalSeconds; + private readonly float calibrationGain; private int lastPollCount; private float runtimeSeconds; private float nextAudioSyncSeconds = AudioSyncUpdateIntervalSeconds; @@ -47,6 +51,7 @@ public sealed class MimirRuntime : IAquariumRuntime visualRuntime = new LocalCastRuntime(options); synchronization = new MimirSynchronizationHub(settings); telemetryIntervalSeconds = ParseTelemetryIntervalSeconds(); + calibrationGain = ParseCalibrationGain(); nextTelemetrySeconds = telemetryIntervalSeconds; foreach (var source in streamSources) { @@ -132,24 +137,48 @@ public sealed class MimirRuntime : IAquariumRuntime private void QueueCalibrationTimeline() { - var queuedUntilSeconds = calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds; - var targetSeconds = runtimeSeconds + (float)MimirChirpletTimeline.QueueLeadSeconds; - while (queuedUntilSeconds < targetSeconds) + var nextSegmentSeconds = CalibrationStartSeconds + calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds; + if (runtimeSeconds < nextSegmentSeconds) { - visualRuntime.Audio.EnqueuePcm16Base64( - MimirChirpletTimeline.Default.RenderSegmentPcm16Base64(calibrationSegmentIndex), - MimirChirpletTimeline.SampleRate, - channels: 1, - gain: 1.0f); - calibrationSegmentIndex++; - queuedUntilSeconds = calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds; + return; } + + var batch = RenderCalibrationBatchPcm16Base64(calibrationSegmentIndex, CalibrationBatchSegments, out var peak); + visualRuntime.Audio.EnqueuePcm16Base64( + batch, + MimirChirpletTimeline.SampleRate, + channels: 1, + gain: calibrationGain); + Console.WriteLine( + $"mimir-chirplet-batch firstSegment={calibrationSegmentIndex} segments={CalibrationBatchSegments} seconds={CalibrationBatchSegments * MimirChirpletTimeline.SegmentSeconds:0.00} peak={peak:0.000000} gain={calibrationGain:0.###} base64Bytes={batch.Length}"); + calibrationSegmentIndex += CalibrationBatchSegments; } private string DescribeChirpletReference() { - var queuedUntilSeconds = calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds; - return $"{DefaultAudioSyncReference} continuous timeline {MimirChirpletTimeline.SegmentSeconds:0.00}s segments queued to {queuedUntilSeconds:0.00}s"; + var emittedUntilSeconds = calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds; + return $"{DefaultAudioSyncReference} continuous timeline {MimirChirpletTimeline.SegmentSeconds:0.00}s segments emitted to {emittedUntilSeconds:0.00}s"; + } + + private static string RenderCalibrationBatchPcm16Base64(ulong firstSegment, int segmentCount, out float peak) + { + var samplesPerSegment = (int)Math.Round(MimirChirpletTimeline.SegmentSeconds * MimirChirpletTimeline.SampleRate); + var bytes = new byte[samplesPerSegment * Math.Max(1, segmentCount) * sizeof(short)]; + var byteIndex = 0; + peak = 0.0f; + for (var segment = 0; segment < segmentCount; segment++) + { + var samples = MimirChirpletTimeline.Default.RenderSegmentMonoFloat(firstSegment + (ulong)segment); + for (var index = 0; index < samples.Length; index++) + { + peak = Math.Max(peak, Math.Abs(samples[index])); + var sample = (short)Math.Round(Math.Clamp(samples[index], -1.0f, 1.0f) * short.MaxValue); + bytes[byteIndex++] = (byte)(sample & 0xff); + bytes[byteIndex++] = (byte)((sample >> 8) & 0xff); + } + } + + return Convert.ToBase64String(bytes); } private string DescribeBuffers() @@ -194,6 +223,12 @@ public sealed class MimirRuntime : IAquariumRuntime $"mimir-sync-state {state.ReferenceSourceId}->{state.SourceId} delaySamples={state.SmoothedDelaySamples:0.000} delayMs={state.DelayMilliseconds:0.000} sroPpm={state.SamplingRateOffsetPpm:0.000} confidence={state.Confidence:0.000}"); } + foreach (var trace in synchronization.AudioSynchronizationDecodeTraces.OrderBy(trace => trace.SourceId, StringComparer.Ordinal)) + { + Console.WriteLine( + $"mimir-sync-decode {trace.ReferenceSourceId}->{trace.SourceId} status={trace.Status} compared={trace.ComparedSamples} rate={trace.SampleRate} refFrames={trace.ReferenceFrames} refAnchors={trace.ReferenceAnchors} refClock={trace.ReferenceClockConfidence:0.000} candFrames={trace.CandidateFrames} candAnchors={trace.CandidateAnchors} candClock={trace.CandidateClockConfidence:0.000} matched={trace.MatchedEvents} confidence={trace.Confidence:0.000}"); + } + nextTelemetrySeconds += telemetryIntervalSeconds; } @@ -204,6 +239,13 @@ public sealed class MimirRuntime : IAquariumRuntime : 0.0f; } + private static float ParseCalibrationGain() + { + return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_CHIRPLET_GAIN"), out var gain) + ? Math.Clamp(gain, 0.05f, 4.0f) + : DefaultCalibrationGain; + } + private string DescribeAudioBuffers() { return string.Join(" | ", synchronization.Buffers.Buffers diff --git a/src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationAnalyzer.cs b/src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationAnalyzer.cs index b8b8d73..eec6107 100644 --- a/src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationAnalyzer.cs +++ b/src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationAnalyzer.cs @@ -5,6 +5,9 @@ namespace Mimir.Runtime.Synchronization; public sealed class MimirAudioSynchronizationAnalyzer { private const int MaxWindowSamples = 48_000 * 2; + private readonly List lastDecodeTraces = []; + + public IReadOnlyList LastDecodeTraces => lastDecodeTraces; public IReadOnlyList Analyze( IEnumerable buffers, @@ -36,6 +39,7 @@ public sealed class MimirAudioSynchronizationAnalyzer } var reports = new List(); + lastDecodeTraces.Clear(); foreach (var buffer in audioBuffers) { if (ReferenceEquals(buffer, reference)) @@ -73,6 +77,20 @@ public sealed class MimirAudioSynchronizationAnalyzer var referenceDecode = MimirChirpletTimeline.Default.DecodeStreamWindow(referenceWindow, referenceBlock.SampleRate); var candidateDecode = MimirChirpletTimeline.Default.DecodeStreamWindow(candidateWindow, referenceBlock.SampleRate); var deterministicFit = EstimateDelayFromDecodedTimeline(referenceDecode, candidateDecode); + lastDecodeTraces.Add(new MimirAudioSynchronizationDecodeTrace( + reference.Descriptor.SourceId, + buffer.Descriptor.SourceId, + referenceBlock.SampleRate, + compared, + referenceDecode.Frames.Count, + referenceDecode.Anchors.Count, + referenceDecode.ClockFit?.Confidence ?? 0.0, + candidateDecode.Frames.Count, + candidateDecode.Anchors.Count, + candidateDecode.ClockFit?.Confidence ?? 0.0, + deterministicFit.MatchedEvents, + deterministicFit.Confidence, + deterministicFit.MatchedEvents >= 3 ? "report" : "insufficient-anchors")); if (deterministicFit.MatchedEvents < 3) { continue; diff --git a/src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationDecodeTrace.cs b/src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationDecodeTrace.cs new file mode 100644 index 0000000..9b3edda --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationDecodeTrace.cs @@ -0,0 +1,16 @@ +namespace Mimir.Runtime.Synchronization; + +public sealed record MimirAudioSynchronizationDecodeTrace( + string ReferenceSourceId, + string SourceId, + int SampleRate, + int ComparedSamples, + int ReferenceFrames, + int ReferenceAnchors, + double ReferenceClockConfidence, + int CandidateFrames, + int CandidateAnchors, + double CandidateClockConfidence, + int MatchedEvents, + double Confidence, + string Status); diff --git a/src/Mimir.Runtime/Synchronization/MimirChirpletTimeline.cs b/src/Mimir.Runtime/Synchronization/MimirChirpletTimeline.cs index 1b4a6b5..c1f8efc 100644 --- a/src/Mimir.Runtime/Synchronization/MimirChirpletTimeline.cs +++ b/src/Mimir.Runtime/Synchronization/MimirChirpletTimeline.cs @@ -155,13 +155,27 @@ public sealed class MimirChirpletTimeline return []; } - var envelope = ContrastNormalize(BuildEnvelopeEnergyTrace(samples, sampleRate, hopSamples)); - if (envelope.Length <= 2) + var trace = BuildChirpletEnergyTrace(samples, sampleRate, hopSamples); + if (trace.Count <= 2) { return []; } var frames = new List(); + var candidateOffsets = new SortedSet(); + for (var frame = 1; frame < trace.Count - 1; frame++) + { + if (trace[frame] < 0.05 || + trace[frame] < trace[frame - 1] || + trace[frame] < trace[frame + 1]) + { + continue; + } + + candidateOffsets.Add(frame * hopSamples); + } + + var envelope = ContrastNormalize(BuildEnvelopeEnergyTrace(samples, sampleRate, hopSamples)); for (var frame = 1; frame < envelope.Length - 1; frame++) { if (envelope[frame] < 0.055 || @@ -171,10 +185,15 @@ public sealed class MimirChirpletTimeline continue; } + candidateOffsets.Add(frame * hopSamples); + } + + foreach (var offset in candidateOffsets) + { var transformFrame = ClassifySymbolsAt( samples, sampleRate, - frame * hopSamples, + offset, Math.Max(hopSamples * 3, sampleRate / 100)); if (transformFrame != null) { @@ -228,7 +247,7 @@ public sealed class MimirChirpletTimeline } } - if (bestEnergy >= 0.08) + if (bestEnergy >= 0.035) { var refinedOffset = RefineMatchedOffset(samples, kernel, bestOffset, start, end); var refinedEnergy = MatchedEnergy( @@ -247,7 +266,7 @@ public sealed class MimirChirpletTimeline .OrderByDescending(candidate => candidate.Energy) .Take(MaxSymbolCandidatesPerFrame) .ToArray(); - if (ordered[0].Energy < 0.25) + if (ordered[0].Energy < 0.08) { return null; } @@ -659,6 +678,40 @@ public sealed class MimirChirpletTimeline return Math.Min(1.0, Math.Sqrt(sineDot * sineDot + cosineDot * cosineDot) / denominator); } + private IReadOnlyList BuildChirpletEnergyTrace(ReadOnlySpan samples, int sampleRate, int hopSamples) + { + var maxKernelLength = Codebook.Symbols + .Select(symbol => sampleRate == SampleRate ? symbolKernels[symbol.SymbolId].Length : RenderToneKernel(symbol.Tone, sampleRate).Length) + .DefaultIfEmpty(0) + .Max(); + if (samples.Length < maxKernelLength || maxKernelLength == 0) + { + return []; + } + + var output = new double[1 + (samples.Length - maxKernelLength) / hopSamples]; + for (var frame = 0; frame < output.Length; frame++) + { + var offset = frame * hopSamples; + var best = 0.0; + for (var symbol = 0; symbol < Codebook.Symbols.Count; symbol++) + { + var tone = Codebook[symbol].Tone; + var kernel = sampleRate == SampleRate ? symbolKernels[symbol] : RenderToneKernel(tone, sampleRate); + if (offset + kernel.Length > samples.Length) + { + continue; + } + + best = Math.Max(best, MatchedEnergy(samples.Slice(offset, kernel.Length), kernel)); + } + + output[frame] = best; + } + + return output; + } + private static float[] BuildEnvelopeEnergyTrace(ReadOnlySpan samples, int sampleRate, int hopSamples) { var windowSamples = Math.Max(1, (int)Math.Round(MaxEventDurationSeconds * sampleRate)); diff --git a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs index b20632f..619674b 100644 --- a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs +++ b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs @@ -33,6 +33,9 @@ public sealed class MimirSynchronizationHub : IDisposable public IReadOnlyList AudioSynchronizationReports => audioSynchronizationReports.Values.OrderBy(report => report.SourceId, StringComparer.Ordinal).ToArray(); + public IReadOnlyList AudioSynchronizationDecodeTraces => + audioSynchronization.LastDecodeTraces; + public void AddSource(IMimirStreamSource source) { ObjectDisposedException.ThrowIf(disposed, this); diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 2a443a1..dc558c0 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -49,3 +49,4 @@ {"ts":"2026-05-22T17:35:00+01:00","type":"implementation","status":"ok","note":"Separated chirplet symbol ownership into MimirChirpletSymbolCodebook and added per-candidate refined sample offsets to transform frames. The codebook now guarantees each of the 32 symbols has a unique chirp shape; rhythm is additional code evidence rather than an acoustic alias. ClassifySymbolsAt now does coarse dictionary search, local sample-resolution search, and parabolic sub-sample refinement per symbol candidate. Synthetic two-second emitted timeline check now detects all 15 emitted chirps, decodes 13 possible triplet anchors for events 0-12, and fits clock=48001.819 Hz with confidence=0.952 and MAE=5.171 samples, improved from 11 anchors / 47978.350 Hz / MAE=8.741."} {"ts":"2026-05-22T17:45:00+01:00","type":"implementation","status":"ok","note":"Replaced single-phase chirplet matching with quadrature sine/cosine kernels and fixed emitted timeline segment gain so canonical chirps are not amplitude-renormalized per half-second segment. Added `Mimir.BufferSmoke --chirplet-self-test` as a first-class synthetic invariant check. The check renders two seconds of canonical timeline audio, detects all 15 emitted chirps, keeps 13 triplet anchors for events 0-12, fits clock=47999.999990 Hz with confidence=1.000, and holds anchor MAE to 0.000014 samples. The next proof is the same deterministic anchor stability through actual loopback and microphone paths."} {"ts": "2026-05-22T15:52:26+00:00", "type": "implementation", "status": "ok", "note": "Restored the repo-local Python persistence helpers from the Epiphany-style helper family after checking VoidBot and sibling repos. Mimir now has tools/localcast_state.py for status/evidence/branch edits and tools/localcast_prepare_compaction.py for pre-compaction state audits; AGENTS.md points future agents at both helpers."} +{"ts": "2026-05-22T17:34:45+01:00", "type": "verification", "status": "ok", "note": "Aquarium WASAPI resampler used int multiplication for source.Length * targetSampleRate; a 60s 48k chirplet batch overflowed and queued one 44.1k output frame. Switching to double-length arithmetic restored 2,646,000 output frames and audible chirplets."}