diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 331a813..a7beb71 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -594,11 +594,21 @@ a named invariant that the native runtime cannot protect yet. Well profile with capture-page/body-paging enabled. This is the first local paged-recording ABI for offline reconstruction, not the final CultCache shard engine. GPU-handle-only samples remain metadata/body-handle evidence until a - Fensalir/native program-output recorder owns rendered composite bodies. The - current live scar is explicit: ASIO device timestamps and - camera driver timestamps are different domains, so `Mimir.Well` publishes - degraded sync status until a camera-clock/global-time correction owner maps - those domains. + Fensalir/native program-output recorder owns rendered composite bodies. + `Mimir.Well` now publishes the first stream-organ pressure surface inside + `mimir.well_snapshot.v1`: `clockDomains` reports per-domain live edges, + actual overlap, and provisional offsets to the ASIO reference, while + `streamPressure` (`mimir.well_stream_pressure.v1`) reports poll and publish + pressure for the transitional WebSocket/JSON path. The planner feeds + provisional clock-domain edge fits through `MimirSourceTimingCorrection`, + then uses actual oldest buffered sample timestamps instead of the configured + retention boundary when choosing a synchronized window. This makes the live + metric honest: a five-second buffer target is no longer allowed to masquerade + as five seconds of resident samples when body serialization/backpressure + starves a source. The next coherent cut is a CultMesh streaming organ with a + compact typed control lane, a CultCache page/shard append body lane, explicit + cursors/backpressure, and a lossy latest-state dashboard lane; do not keep + expanding inline base64 capture pages as if they were the final transport. Eve now follows the same rule for its own device body: `MimirDistributedWitnessConfigurations.EveLocalSensors` and the `eve-local-sensors` Perfect Machine profile make Eve responsible for local diff --git a/src/Mimir.Runtime/Synchronization/MimirRollingStreamBuffer.cs b/src/Mimir.Runtime/Synchronization/MimirRollingStreamBuffer.cs index 1f0a75e..3ee08bd 100644 --- a/src/Mimir.Runtime/Synchronization/MimirRollingStreamBuffer.cs +++ b/src/Mimir.Runtime/Synchronization/MimirRollingStreamBuffer.cs @@ -18,6 +18,8 @@ public sealed class MimirRollingStreamBuffer public long WindowStartNs => Math.Max(0, EdgeNs - DurationNs); + public long OldestSampleTimestampNs => samples.Count == 0 ? 0L : samples.Peek().TimestampNs; + public int Count => samples.Count; public MimirStreamSample? Latest { get; private set; } diff --git a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs index d04bd76..99ef348 100644 --- a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs +++ b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs @@ -67,10 +67,14 @@ public sealed class MimirSynchronizationHub : IDisposable { ObjectDisposedException.ThrowIf(disposed, this); var delay = presentationDelay ?? TimeSpan.FromTicks(Settings.BufferDuration.Ticks / 2); + var timingCorrections = MimirSynchronizedBufferPlanner + .CorrectionsFromClockDomainEdges(Buffers.Buffers, Settings.Audio.ReferenceSourceId) + .Concat(MimirSynchronizedBufferPlanner.CorrectionsFromAudioStates(audioSynchronizationState.States, Buffers.Buffers)) + .ToArray(); return new MimirSynchronizedBufferPlanner().BuildFrame( Buffers.Buffers, delay, - MimirSynchronizedBufferPlanner.CorrectionsFromAudioStates(audioSynchronizationState.States, Buffers.Buffers)); + timingCorrections); } public void AddSource(IMimirStreamSource source) diff --git a/src/Mimir.Runtime/Synchronization/MimirSynchronizedBufferPlanner.cs b/src/Mimir.Runtime/Synchronization/MimirSynchronizedBufferPlanner.cs index 0f5ea43..ff15522 100644 --- a/src/Mimir.Runtime/Synchronization/MimirSynchronizedBufferPlanner.cs +++ b/src/Mimir.Runtime/Synchronization/MimirSynchronizedBufferPlanner.cs @@ -98,6 +98,49 @@ public sealed class MimirSynchronizedBufferPlanner return corrections; } + public static IReadOnlyList CorrectionsFromClockDomainEdges( + IEnumerable buffers, + string referenceSourceId, + double confidence = 0.25) + { + var activeBuffers = buffers + .Where(static buffer => buffer.Latest.HasValue) + .ToArray(); + var referenceBuffer = activeBuffers.FirstOrDefault(buffer => + string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal)); + var referenceEdgeNs = referenceBuffer?.Latest?.TimestampNs ?? 0L; + if (referenceEdgeNs <= 0) + { + return []; + } + + var referenceClockDomainId = referenceBuffer?.Descriptor.EffectiveClockDomainId ?? ""; + var corrections = new List(); + foreach (var group in activeBuffers.GroupBy(static buffer => buffer.Descriptor.EffectiveClockDomainId, StringComparer.Ordinal)) + { + var domainEdgeNs = group + .Select(static buffer => buffer.Latest?.TimestampNs ?? 0L) + .Where(static timestamp => timestamp > 0) + .DefaultIfEmpty(0L) + .Max(); + if (domainEdgeNs <= 0) + { + continue; + } + + var offsetNs = checked(referenceEdgeNs - domainEdgeNs); + var isReference = string.Equals(group.Key, referenceClockDomainId, StringComparison.Ordinal); + corrections.Add(new MimirSourceTimingCorrection( + isReference ? referenceSourceId : "", + group.Key, + isReference ? 0L : offsetNs, + isReference ? 1.0 : Math.Clamp(confidence, 0.0, 1.0), + isReference ? "reference-clock-domain" : "provisional-clock-domain-edge-fit")); + } + + return corrections; + } + public MimirSynchronizedBufferFrame BuildFrame( IEnumerable buffers, TimeSpan presentationDelay, @@ -113,7 +156,10 @@ public sealed class MimirSynchronizedBufferPlanner } var corrections = BuildCorrectionMap(timingCorrections); - var windowStartNs = activeBuffers.Max(buffer => Correct(corrections, buffer.Descriptor, buffer.WindowStartNs)); + var windowStartNs = activeBuffers.Max(buffer => Correct( + corrections, + buffer.Descriptor, + buffer.OldestSampleTimestampNs > 0 ? buffer.OldestSampleTimestampNs : buffer.WindowStartNs)); var windowEndNs = activeBuffers.Min(buffer => { var latest = buffer.Latest!.Value; @@ -210,15 +256,20 @@ public sealed class MimirSynchronizedBufferPlanner } MimirStreamSample? best = null; + var bestIndex = -1; MimirSynchronizedSliceStatus status = MimirSynchronizedSliceStatus.Missing; long bestDistance = long.MaxValue; - foreach (var sample in samples) + for (var index = 0; index < samples.Count; index++) { + var sample = samples[index]; + var previous = index > 0 ? samples[index - 1] : (MimirStreamSample?)null; + var next = index < samples.Count - 1 ? samples[index + 1] : (MimirStreamSample?)null; var startNs = correction.ToCanonicalNs(sample.TimestampNs); - var endNs = SampleCanonicalEndNs(sample, correction); + var endNs = SampleCanonicalEndNs(sample, correction, previous, next); if (presentationTimeNs >= startNs && presentationTimeNs < endNs) { best = sample; + bestIndex = index; status = MimirSynchronizedSliceStatus.Ready; bestDistance = 0; break; @@ -230,6 +281,7 @@ public sealed class MimirSynchronizedBufferPlanner if (distance < bestDistance) { best = sample; + bestIndex = index; bestDistance = distance; status = presentationTimeNs >= endNs ? MimirSynchronizedSliceStatus.HoldingPreviousSample @@ -239,7 +291,11 @@ public sealed class MimirSynchronizedBufferPlanner var selected = best!.Value; var selectedStartNs = correction.ToCanonicalNs(selected.TimestampNs); - var selectedEndNs = SampleCanonicalEndNs(selected, correction); + var selectedPrevious = bestIndex > 0 ? samples[bestIndex - 1] : (MimirStreamSample?)null; + var selectedNext = bestIndex >= 0 && bestIndex < samples.Count - 1 + ? samples[bestIndex + 1] + : (MimirStreamSample?)null; + var selectedEndNs = SampleCanonicalEndNs(selected, correction, selectedPrevious, selectedNext); return new MimirSynchronizedStreamSlice( buffer.Descriptor.SourceId, buffer.Descriptor.Kind, @@ -256,20 +312,42 @@ public sealed class MimirSynchronizedBufferPlanner correction.EvidenceKind); } - private static long SampleCanonicalEndNs(MimirStreamSample sample, MimirSourceTimingCorrection correction) + private static long SampleCanonicalEndNs( + MimirStreamSample sample, + MimirSourceTimingCorrection correction, + MimirStreamSample? previousSample = null, + MimirStreamSample? nextSample = null) { var startNs = correction.ToCanonicalNs(sample.TimestampNs); - var durationNs = SampleDurationNs(sample); + var durationNs = SampleDurationNs(sample, previousSample, nextSample); return checked(startNs + Math.Max(1, durationNs)); } - private static long SampleDurationNs(MimirStreamSample sample) + private static long SampleDurationNs( + MimirStreamSample sample, + MimirStreamSample? previousSample = null, + MimirStreamSample? nextSample = null) { if (sample.AudioBlock is { SampleRate: > 0, FrameCount: > 0 } block) { return checked((long)Math.Ceiling(block.FrameCount * 1_000_000_000.0 / block.SampleRate)); } + if (sample.VideoFrame is not null) + { + if (nextSample.HasValue && nextSample.Value.TimestampNs > sample.TimestampNs) + { + return nextSample.Value.TimestampNs - sample.TimestampNs; + } + + if (previousSample.HasValue && sample.TimestampNs > previousSample.Value.TimestampNs) + { + return sample.TimestampNs - previousSample.Value.TimestampNs; + } + + return 33_333_334L; + } + return 1; } } diff --git a/src/Mimir.Well/Program.cs b/src/Mimir.Well/Program.cs index 6b6bd0a..43f56df 100644 --- a/src/Mimir.Well/Program.cs +++ b/src/Mimir.Well/Program.cs @@ -1,4 +1,5 @@ using System.Net.WebSockets; +using System.Diagnostics; using System.Text; using System.Text.Json; using Mimir.Runtime.Synchronization; @@ -54,6 +55,9 @@ var nextPublish = DateTimeOffset.MinValue; var nextCapture = DateTimeOffset.MinValue; var nextSync = DateTimeOffset.MinValue; var nextVisualCalibration = DateTimeOffset.MinValue; +var frameDegradedCount = 0L; +var nextFrameDegradedLog = DateTimeOffset.MinValue; +var streamTelemetry = new MimirWellStreamTelemetry(); var presentation = new MimirPresentationControlState(); var exposureController = new MimirCameraExposureController(new MimirCameraExposureControlOptions( options.VisualCalibrationEnabled, @@ -66,7 +70,10 @@ Console.Error.WriteLine($"Mimir Well sources={runtimeConfig.SourceFactories.Coun while (!stopping.IsCancellationRequested) { - hub.PollSources(options.MaxSamplesPerSource); + var pollStopwatch = Stopwatch.StartNew(); + var consumedSamples = hub.PollSources(options.MaxSamplesPerSource); + pollStopwatch.Stop(); + streamTelemetry.ObservePoll(consumedSamples, pollStopwatch.Elapsed); var now = DateTimeOffset.UtcNow; if (now >= nextSync) { @@ -95,28 +102,43 @@ while (!stopping.IsCancellationRequested) if (now >= nextPublish) { presentation.SyncFromBuffers(hub.Buffers.Buffers); - var frame = BuildFrameOrEmpty(hub, TimeSpan.FromMilliseconds(options.PresentationDelayMs)); + var frameResult = BuildFrameOrEmpty(hub, TimeSpan.FromMilliseconds(options.PresentationDelayMs)); + if (!string.IsNullOrWhiteSpace(frameResult.DegradedReason)) + { + frameDegradedCount++; + if (now >= nextFrameDegradedLog) + { + Console.Error.WriteLine( + $"mimir-well synchronized-frame degraded count={frameDegradedCount} reason={frameResult.DegradedReason}"); + nextFrameDegradedLog = now + TimeSpan.FromSeconds(5); + } + } + + var frame = frameResult.Frame; var snapshot = MimirWellSnapshot.Build( options, runtimeConfig, hub, presentation, - frame, + frameResult, exposureController.Statuses, sourceErrors, + streamTelemetry.Snapshot(now, subscribersAreExternal: true), ++sequence, startedAt); - await publisher.PublishAsync(snapshot, stopping.Token).ConfigureAwait(false); + streamTelemetry.ObservePublish( + await publisher.PublishAsync(snapshot, "mimir.well_snapshot.v1", stopping.Token).ConfigureAwait(false)); if (options.CapturePagesEnabled && now >= nextCapture) { var capturePage = MimirWellCapturePage.Build( options, presentation, - frame, + frameResult, hub.Buffers.Buffers, ++captureSequence, startedAt); - await publisher.PublishAsync(capturePage, stopping.Token).ConfigureAwait(false); + streamTelemetry.ObservePublish( + await publisher.PublishAsync(capturePage, "mimir.well_capture_page.v1", stopping.Token).ConfigureAwait(false)); nextCapture = now + TimeSpan.FromMilliseconds(options.CaptureIntervalMs); } @@ -143,24 +165,117 @@ Console.Error.WriteLine($"Mimir Well complete sequence={sequence} ingested={hub. static long NowNs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L; -static MimirSynchronizedBufferFrame BuildFrameOrEmpty(MimirSynchronizationHub hub, TimeSpan presentationDelay) +static MimirWellFrameBuildResult BuildFrameOrEmpty(MimirSynchronizationHub hub, TimeSpan presentationDelay) { try { - return hub.BuildSynchronizedBufferFrame(presentationDelay); + return new MimirWellFrameBuildResult( + hub.BuildSynchronizedBufferFrame(presentationDelay), + "", + ""); } catch (ArgumentException ex) { - Console.Error.WriteLine($"mimir-well synchronized-frame degraded: {ex.Message}"); - return new MimirSynchronizedBufferFrame(0, 0, 0, presentationDelay, []); + return new MimirWellFrameBuildResult( + new MimirSynchronizedBufferFrame(0, 0, 0, presentationDelay, []), + ex.GetType().Name, + ex.Message); } catch (OverflowException ex) { - Console.Error.WriteLine($"mimir-well synchronized-frame degraded: {ex.Message}"); - return new MimirSynchronizedBufferFrame(0, 0, 0, presentationDelay, []); + return new MimirWellFrameBuildResult( + new MimirSynchronizedBufferFrame(0, 0, 0, presentationDelay, []), + ex.GetType().Name, + ex.Message); } } +internal sealed record MimirWellFrameBuildResult( + MimirSynchronizedBufferFrame Frame, + string DegradedKind, + string DegradedReason); + +internal sealed record MimirWellPublishReceipt( + string Document, + int ByteLength, + double SerializeMilliseconds, + double SendMilliseconds, + double TotalMilliseconds); + +internal sealed class MimirWellStreamTelemetry +{ + private long pollIterations; + private long consumedSamples; + private long zeroPollIterations; + private double maxPollMilliseconds; + private double totalPollMilliseconds; + private long publishedDocuments; + private long publishedBytes; + private double maxPublishMilliseconds; + private double totalPublishMilliseconds; + private string lastPublishedDocument = ""; + private int lastPublishedBytes; + private double lastPublishMilliseconds; + + public void ObservePoll(int consumed, TimeSpan elapsed) + { + pollIterations++; + consumedSamples += Math.Max(0, consumed); + if (consumed <= 0) + { + zeroPollIterations++; + } + + var milliseconds = elapsed.TotalMilliseconds; + totalPollMilliseconds += milliseconds; + maxPollMilliseconds = Math.Max(maxPollMilliseconds, milliseconds); + } + + public void ObservePublish(MimirWellPublishReceipt receipt) + { + publishedDocuments++; + publishedBytes += receipt.ByteLength; + totalPublishMilliseconds += receipt.TotalMilliseconds; + maxPublishMilliseconds = Math.Max(maxPublishMilliseconds, receipt.TotalMilliseconds); + lastPublishedDocument = receipt.Document; + lastPublishedBytes = receipt.ByteLength; + lastPublishMilliseconds = receipt.TotalMilliseconds; + } + + public object Snapshot(DateTimeOffset now, bool subscribersAreExternal) => new + { + document = "mimir.well_stream_pressure.v1", + status = "transitional-websocket-json", + authority = "Mimir.Well observes producer pressure; CultMesh streaming organ should own durable body lanes.", + wallClockUtc = now.ToString("O"), + poll = new + { + iterations = pollIterations, + consumedSamples, + zeroPollIterations, + averageMilliseconds = pollIterations == 0 ? 0.0 : totalPollMilliseconds / pollIterations, + maxMilliseconds = maxPollMilliseconds, + }, + publish = new + { + documents = publishedDocuments, + bytes = publishedBytes, + averageMilliseconds = publishedDocuments == 0 ? 0.0 : totalPublishMilliseconds / publishedDocuments, + maxMilliseconds = maxPublishMilliseconds, + lastDocument = lastPublishedDocument, + lastBytes = lastPublishedBytes, + lastMilliseconds = lastPublishMilliseconds, + subscribersAreExternal, + }, + nextOrgan = new + { + controlLane = "CultMesh typed state and compact stream cursors", + bodyLane = "CultCache page/shard append stream with refs, hashes, and backpressure", + realtimeLane = "loss-aware latest-state observation stream for dashboards", + }, + }; +} + internal sealed record MimirWellOptions( Uri PublishUrl, string NodeId, @@ -255,11 +370,24 @@ internal sealed class MimirWellPublisher(Uri url) : IAsyncDisposable await socket.ConnectAsync(url, stopping).ConfigureAwait(false); } - public async Task PublishAsync(object document, CancellationToken stopping) + public async Task PublishAsync( + object document, + string documentName, + CancellationToken stopping) { + var stopwatch = Stopwatch.StartNew(); var json = JsonSerializer.Serialize(document); var bytes = Encoding.UTF8.GetBytes(json); + var serializeMilliseconds = stopwatch.Elapsed.TotalMilliseconds; + stopwatch.Restart(); await socket.SendAsync(bytes, WebSocketMessageType.Text, endOfMessage: true, stopping).ConfigureAwait(false); + stopwatch.Stop(); + return new MimirWellPublishReceipt( + documentName, + bytes.Length, + serializeMilliseconds, + stopwatch.Elapsed.TotalMilliseconds, + serializeMilliseconds + stopwatch.Elapsed.TotalMilliseconds); } public async ValueTask DisposeAsync() @@ -285,11 +413,14 @@ internal static class MimirWellCapturePage public static object Build( MimirWellOptions options, MimirPresentationControlState presentation, - MimirSynchronizedBufferFrame frame, + MimirWellFrameBuildResult frameResult, IEnumerable buffers, long captureSequence, - DateTimeOffset startedAt) => new + DateTimeOffset startedAt) { + var frame = frameResult.Frame; + return new + { type = "cultmesh-observation", document = "mimir.well_capture_page.v1", sourceId = "mimir-well", @@ -304,6 +435,8 @@ internal static class MimirWellCapturePage frame.WindowEndNs, PresentationDelayMs = frame.PresentationDelay.TotalMilliseconds, frame.IsComplete, + degradedKind = frameResult.DegradedKind, + degradedReason = frameResult.DegradedReason, }, storage = new { @@ -340,6 +473,7 @@ internal static class MimirWellCapturePage samples = CaptureSamples(options, captureSequence, frame, buffers) .ToArray(), }; + } private static IEnumerable CaptureSamples( MimirWellOptions options, @@ -484,6 +618,108 @@ internal static class MimirWellCapturePage } } +internal static class MimirWellClockDiagnostics +{ + public static object Build( + IEnumerable buffers, + string referenceSourceId, + MimirWellFrameBuildResult frameResult) + { + var activeBuffers = buffers + .Where(static buffer => buffer.Latest.HasValue) + .ToArray(); + var referenceBuffer = activeBuffers.FirstOrDefault(buffer => + string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal)); + var referenceEdgeNs = referenceBuffer?.Latest?.TimestampNs ?? 0L; + var referenceClockDomain = referenceBuffer?.Descriptor.EffectiveClockDomainId ?? ""; + + var domains = activeBuffers + .GroupBy(static buffer => buffer.Descriptor.EffectiveClockDomainId, StringComparer.Ordinal) + .Select(group => BuildDomain(group.Key, group.ToArray(), referenceClockDomain, referenceEdgeNs)) + .OrderBy(domain => domain.ClockDomainId, StringComparer.Ordinal) + .ToArray(); + + return new + { + status = frameResult.Frame.Slices.Count > 0 ? "frame-built" : "degraded", + frameResult.DegradedKind, + frameResult.DegradedReason, + referenceSourceId, + referenceClockDomain, + referenceEdgeNs, + domainCount = domains.Length, + activeSourceCount = activeBuffers.Length, + domains, + }; + } + + private static ClockDomainDiagnostics BuildDomain( + string clockDomainId, + IReadOnlyList buffers, + string referenceClockDomain, + long referenceEdgeNs) + { + var latestEdges = buffers + .Select(buffer => buffer.Latest?.TimestampNs ?? 0L) + .Where(static value => value > 0) + .ToArray(); + var windowStarts = buffers + .Select(static buffer => buffer.OldestSampleTimestampNs > 0 ? buffer.OldestSampleTimestampNs : buffer.WindowStartNs) + .Where(static value => value > 0) + .ToArray(); + var minLatest = latestEdges.Length == 0 ? 0L : latestEdges.Min(); + var maxLatest = latestEdges.Length == 0 ? 0L : latestEdges.Max(); + var maxWindowStart = windowStarts.Length == 0 ? 0L : windowStarts.Max(); + var minWindowEnd = minLatest; + var overlapNs = minWindowEnd > 0 && maxWindowStart > 0 + ? minWindowEnd - maxWindowStart + : 0L; + var domainEdgeNs = maxLatest; + var provisionalOffsetToReferenceNs = referenceEdgeNs > 0 && domainEdgeNs > 0 + ? referenceEdgeNs - domainEdgeNs + : 0L; + + return new ClockDomainDiagnostics( + clockDomainId, + buffers.Count, + buffers + .OrderBy(static buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal) + .Select(buffer => new + { + buffer.Descriptor.SourceId, + Kind = buffer.Descriptor.Kind.ToString(), + Origin = buffer.Descriptor.Origin.ToString(), + buffer.Count, + buffer.WindowStartNs, + buffer.OldestSampleTimestampNs, + EdgeNs = buffer.Latest?.TimestampNs ?? 0L, + ByteLength = buffer.Latest?.ByteLength ?? 0, + }) + .ToArray(), + maxWindowStart, + minLatest, + maxLatest, + overlapNs, + overlapNs >= 0, + string.Equals(clockDomainId, referenceClockDomain, StringComparison.Ordinal), + provisionalOffsetToReferenceNs, + provisionalOffsetToReferenceNs / 1_000_000.0); + } + + private sealed record ClockDomainDiagnostics( + string ClockDomainId, + int SourceCount, + object[] Sources, + long MaxWindowStartNs, + long MinLatestEdgeNs, + long MaxLatestEdgeNs, + long OverlapNs, + bool HasLocalOverlap, + bool IsReferenceDomain, + long ProvisionalOffsetToReferenceNs, + double ProvisionalOffsetToReferenceMs); +} + internal static class MimirWellSnapshot { public static object Build( @@ -491,12 +727,16 @@ internal static class MimirWellSnapshot MimirRuntimeConfiguration runtimeConfig, MimirSynchronizationHub hub, MimirPresentationControlState presentation, - MimirSynchronizedBufferFrame frame, + MimirWellFrameBuildResult frameResult, IReadOnlyList visualCalibration, IReadOnlyList sourceErrors, + object streamPressure, long sequence, - DateTimeOffset startedAt) => new + DateTimeOffset startedAt) { + var frame = frameResult.Frame; + return new + { type = "cultmesh-observation", document = "mimir.well_snapshot.v1", sourceId = "mimir-well", @@ -516,8 +756,15 @@ internal static class MimirWellSnapshot frame.WindowEndNs, PresentationDelayMs = frame.PresentationDelay.TotalMilliseconds, frame.IsComplete, + degradedKind = frameResult.DegradedKind, + degradedReason = frameResult.DegradedReason, slices = frame.Slices.Select(SliceSnapshot).ToArray(), }, + clockDomains = MimirWellClockDiagnostics.Build( + hub.Buffers.Buffers, + runtimeConfig.Settings.Audio.ReferenceSourceId, + frameResult), + streamPressure, audioSync = new { referenceSourceId = runtimeConfig.Settings.Audio.ReferenceSourceId, @@ -622,6 +869,7 @@ internal static class MimirWellSnapshot }).ToArray(), }, }; + } private static object BufferSnapshot(MimirRollingStreamBuffer buffer) => new { @@ -632,6 +880,7 @@ internal static class MimirWellSnapshot buffer.Descriptor.EffectiveClockDomainId, buffer.Count, buffer.EdgeNs, + buffer.OldestSampleTimestampNs, buffer.WindowStartNs, hasLatest = buffer.Latest.HasValue, latest = buffer.Latest.HasValue ? SampleSnapshot(buffer.Latest.Value) : null, diff --git a/state/evidence.jsonl b/state/evidence.jsonl index c31a8b0..c28cc0c 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -347,3 +347,4 @@ {"ts": "2026-06-02T00:06:12+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Added the first Well-to-Recorder archival payload path. Mimir.Well now publishes bounded mimir.well_capture_page.v1 documents beside the dashboard snapshot; capture pages carry synchronized frame-slice timing, configured composite state, and inline base64 sample bodies when CPU payload bytes exist below the configured limit. Mimir.VerseRecorder strips those inline bodies into paged binary files under bodies/page-*.bin, replaces them with mimir.recorder_body_ref.v1 refs in observations.jsonl, and appends mimir.recorder_body_index.v1 records to bodies/index.jsonl. Verified with dotnet build for Mimir.Well and Mimir.VerseRecorder, supervisor dry-runs, and a relay/recorder synthetic capture-page smoke that wrote a 4-byte page body with the expected SHA-256 ref."} {"ts": "2026-06-02T00:06:13+01:00", "type": "verification", "status": "degraded", "note": "2026-06-02: A short local online-verse daemon run proved recorder startup and session/body directory creation, but live Mimir.Well synchronized-frame planning degraded repeatedly with mixed timestamp-domain range errors such as camera/ASIO timestamps being compared before a shared canonical clock owner exists. Treat this as the known camera-clock/global-time correction scar, not a recorder storage failure."} {"ts": "2026-06-02T00:25:02+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02 shakedown: Fixed two data-loss/disk-pressure scars while leaving the long-running Well stack active. The supervisor now passes --seconds 0 to Mimir.Well for indefinite runs instead of a one-year CancelAfter delay that .NET rejects, and waits for relay /health before starting subscribers/producers so Nightwing does not race dotnet cold start. Mimir.Well capture pages now fall back to latest per-buffer samples with TimingEvidenceKind=unsynchronized-fallback when synchronized-frame planning degrades, so Recorder pages raw CPU bodies even before the canonical clock owner exists. Mimir.EveSensorReceiver still broadcasts full capture pages to subscribers but redacts capture-page bodies from diagnostic stdout to avoid duplicating the paged body store in verse-relay.out.log. Live run online-verse-daemons-20260602-002116 / VerseCaptures/20260602-002124 showed all four local workers alive, Recorder receiving 520 observations, 479 body refs, Nightwing observations flowing after manual restart, and capture-page stdout summaries with sampleCount/body-redacted."} +{"ts": "2026-06-02T00:46:28+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02 live shakedown: Added provisional clock-domain edge-fit corrections through MimirSourceTimingCorrection, actual oldest-sample bounds for synchronized frame planning, Well clock-domain diagnostics, and mimir.well_stream_pressure.v1 snapshot telemetry for poll/publish pressure. Hot-swapped active run online-verse-daemons-20260602-002116 to the new build; latest VerseCaptures/20260602-002124 snapshot showed complete 6/6 Ready slices over Scarlett ASIO, Kiyo Pro, and Leap with zero slice distance. The same telemetry exposed the streaming-organ scar: ASIO had about 20 ms of actual buffered history against a five-second retention target, and WebSocket JSON capture-page publishing reached multi-MB messages with >100 ms publish spikes. Next transport cut should split CultMesh typed control/cursor state from CultCache page/shard body streaming and dashboard latest-state lanes."} diff --git a/state/map.yaml b/state/map.yaml index 643a386..666bb2f 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -424,10 +424,23 @@ invariants: gestures with Realtek plus opportunistic ASIO music evidence, and `Mimir.Well` with capture-page/body-paging enabled. This owns a first local paged-recording ABI, not the final CultCache shard engine yet. The live - timestamp-domain scar is explicit: ASIO device time - and camera driver time are not in one canonical timeline, so sync frame - publication is degraded until a camera-clock/global-time correction owner - exists. + timestamp-domain scar is now instrumented: `Mimir.Well` publishes + `clockDomains` diagnostics in `mimir.well_snapshot.v1`, and + `MimirSynchronizationHub` feeds provisional clock-domain edge-fit + corrections into the existing `MimirSourceTimingCorrection` primitive so + camera/ASIO clocks can be compared on one canonical timeline while the + evidence remains explicitly provisional. The synchronized-frame planner now + uses actual oldest buffered sample timestamps, not just the configured + retention boundary, so a five-second retention target cannot pretend to + contain five seconds of samples under publish pressure. Snapshots expose + `OldestSampleTimestampNs`, `WindowStartNs`, and `streamPressure` + (`mimir.well_stream_pressure.v1`) so the dashboard can distinguish clock + fit, real buffered span, JSON/WebSocket publish cost, and body-lane + backpressure. Live shakedown after this cut produced a complete 6/6 Ready + frame over Scarlett ASIO, Kiyo Pro, and Leap with zero slice distance, while + also showing the stream-organ scar plainly: ASIO had only about 20 ms of + actual buffered history despite a five-second retention target because the + current WebSocket JSON capture-page path is still the hose, not the organ. The Eve live-stats provider now reads the latest `mimir.well_snapshot.v1` records from the observation ledger and the latest `mimir.online_move_music_sync_frame.v1` trace from runtime artifacts. Its