Expose Well stream pressure metrics
This commit is contained in:
parent
0cf790f1d3
commit
1474f14c41
7 changed files with 391 additions and 34 deletions
|
|
@ -594,11 +594,21 @@ a named invariant that the native runtime cannot protect yet.
|
|||
Well profile with capture-page/body-paging enabled. This is the first local
|
||||
paged-recording ABI for offline reconstruction, not the final CultCache shard
|
||||
engine. GPU-handle-only samples remain metadata/body-handle evidence until a
|
||||
Fensalir/native program-output recorder owns rendered composite bodies. The
|
||||
current live scar is explicit: ASIO device timestamps and
|
||||
camera driver timestamps are different domains, so `Mimir.Well` publishes
|
||||
degraded sync status until a camera-clock/global-time correction owner maps
|
||||
those domains.
|
||||
Fensalir/native program-output recorder owns rendered composite bodies.
|
||||
`Mimir.Well` now publishes the first stream-organ pressure surface inside
|
||||
`mimir.well_snapshot.v1`: `clockDomains` reports per-domain live edges,
|
||||
actual overlap, and provisional offsets to the ASIO reference, while
|
||||
`streamPressure` (`mimir.well_stream_pressure.v1`) reports poll and publish
|
||||
pressure for the transitional WebSocket/JSON path. The planner feeds
|
||||
provisional clock-domain edge fits through `MimirSourceTimingCorrection`,
|
||||
then uses actual oldest buffered sample timestamps instead of the configured
|
||||
retention boundary when choosing a synchronized window. This makes the live
|
||||
metric honest: a five-second buffer target is no longer allowed to masquerade
|
||||
as five seconds of resident samples when body serialization/backpressure
|
||||
starves a source. The next coherent cut is a CultMesh streaming organ with a
|
||||
compact typed control lane, a CultCache page/shard append body lane, explicit
|
||||
cursors/backpressure, and a lossy latest-state dashboard lane; do not keep
|
||||
expanding inline base64 capture pages as if they were the final transport.
|
||||
Eve now follows the same rule for its own device body:
|
||||
`MimirDistributedWitnessConfigurations.EveLocalSensors` and the
|
||||
`eve-local-sensors` Perfect Machine profile make Eve responsible for local
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ public sealed class MimirRollingStreamBuffer
|
|||
|
||||
public long WindowStartNs => Math.Max(0, EdgeNs - DurationNs);
|
||||
|
||||
public long OldestSampleTimestampNs => samples.Count == 0 ? 0L : samples.Peek().TimestampNs;
|
||||
|
||||
public int Count => samples.Count;
|
||||
|
||||
public MimirStreamSample? Latest { get; private set; }
|
||||
|
|
|
|||
|
|
@ -67,10 +67,14 @@ public sealed class MimirSynchronizationHub : IDisposable
|
|||
{
|
||||
ObjectDisposedException.ThrowIf(disposed, this);
|
||||
var delay = presentationDelay ?? TimeSpan.FromTicks(Settings.BufferDuration.Ticks / 2);
|
||||
var timingCorrections = MimirSynchronizedBufferPlanner
|
||||
.CorrectionsFromClockDomainEdges(Buffers.Buffers, Settings.Audio.ReferenceSourceId)
|
||||
.Concat(MimirSynchronizedBufferPlanner.CorrectionsFromAudioStates(audioSynchronizationState.States, Buffers.Buffers))
|
||||
.ToArray();
|
||||
return new MimirSynchronizedBufferPlanner().BuildFrame(
|
||||
Buffers.Buffers,
|
||||
delay,
|
||||
MimirSynchronizedBufferPlanner.CorrectionsFromAudioStates(audioSynchronizationState.States, Buffers.Buffers));
|
||||
timingCorrections);
|
||||
}
|
||||
|
||||
public void AddSource(IMimirStreamSource source)
|
||||
|
|
|
|||
|
|
@ -98,6 +98,49 @@ public sealed class MimirSynchronizedBufferPlanner
|
|||
return corrections;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<MimirSourceTimingCorrection> CorrectionsFromClockDomainEdges(
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers,
|
||||
string referenceSourceId,
|
||||
double confidence = 0.25)
|
||||
{
|
||||
var activeBuffers = buffers
|
||||
.Where(static buffer => buffer.Latest.HasValue)
|
||||
.ToArray();
|
||||
var referenceBuffer = activeBuffers.FirstOrDefault(buffer =>
|
||||
string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal));
|
||||
var referenceEdgeNs = referenceBuffer?.Latest?.TimestampNs ?? 0L;
|
||||
if (referenceEdgeNs <= 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var referenceClockDomainId = referenceBuffer?.Descriptor.EffectiveClockDomainId ?? "";
|
||||
var corrections = new List<MimirSourceTimingCorrection>();
|
||||
foreach (var group in activeBuffers.GroupBy(static buffer => buffer.Descriptor.EffectiveClockDomainId, StringComparer.Ordinal))
|
||||
{
|
||||
var domainEdgeNs = group
|
||||
.Select(static buffer => buffer.Latest?.TimestampNs ?? 0L)
|
||||
.Where(static timestamp => timestamp > 0)
|
||||
.DefaultIfEmpty(0L)
|
||||
.Max();
|
||||
if (domainEdgeNs <= 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var offsetNs = checked(referenceEdgeNs - domainEdgeNs);
|
||||
var isReference = string.Equals(group.Key, referenceClockDomainId, StringComparison.Ordinal);
|
||||
corrections.Add(new MimirSourceTimingCorrection(
|
||||
isReference ? referenceSourceId : "",
|
||||
group.Key,
|
||||
isReference ? 0L : offsetNs,
|
||||
isReference ? 1.0 : Math.Clamp(confidence, 0.0, 1.0),
|
||||
isReference ? "reference-clock-domain" : "provisional-clock-domain-edge-fit"));
|
||||
}
|
||||
|
||||
return corrections;
|
||||
}
|
||||
|
||||
public MimirSynchronizedBufferFrame BuildFrame(
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers,
|
||||
TimeSpan presentationDelay,
|
||||
|
|
@ -113,7 +156,10 @@ public sealed class MimirSynchronizedBufferPlanner
|
|||
}
|
||||
|
||||
var corrections = BuildCorrectionMap(timingCorrections);
|
||||
var windowStartNs = activeBuffers.Max(buffer => Correct(corrections, buffer.Descriptor, buffer.WindowStartNs));
|
||||
var windowStartNs = activeBuffers.Max(buffer => Correct(
|
||||
corrections,
|
||||
buffer.Descriptor,
|
||||
buffer.OldestSampleTimestampNs > 0 ? buffer.OldestSampleTimestampNs : buffer.WindowStartNs));
|
||||
var windowEndNs = activeBuffers.Min(buffer =>
|
||||
{
|
||||
var latest = buffer.Latest!.Value;
|
||||
|
|
@ -210,15 +256,20 @@ public sealed class MimirSynchronizedBufferPlanner
|
|||
}
|
||||
|
||||
MimirStreamSample? best = null;
|
||||
var bestIndex = -1;
|
||||
MimirSynchronizedSliceStatus status = MimirSynchronizedSliceStatus.Missing;
|
||||
long bestDistance = long.MaxValue;
|
||||
foreach (var sample in samples)
|
||||
for (var index = 0; index < samples.Count; index++)
|
||||
{
|
||||
var sample = samples[index];
|
||||
var previous = index > 0 ? samples[index - 1] : (MimirStreamSample?)null;
|
||||
var next = index < samples.Count - 1 ? samples[index + 1] : (MimirStreamSample?)null;
|
||||
var startNs = correction.ToCanonicalNs(sample.TimestampNs);
|
||||
var endNs = SampleCanonicalEndNs(sample, correction);
|
||||
var endNs = SampleCanonicalEndNs(sample, correction, previous, next);
|
||||
if (presentationTimeNs >= startNs && presentationTimeNs < endNs)
|
||||
{
|
||||
best = sample;
|
||||
bestIndex = index;
|
||||
status = MimirSynchronizedSliceStatus.Ready;
|
||||
bestDistance = 0;
|
||||
break;
|
||||
|
|
@ -230,6 +281,7 @@ public sealed class MimirSynchronizedBufferPlanner
|
|||
if (distance < bestDistance)
|
||||
{
|
||||
best = sample;
|
||||
bestIndex = index;
|
||||
bestDistance = distance;
|
||||
status = presentationTimeNs >= endNs
|
||||
? MimirSynchronizedSliceStatus.HoldingPreviousSample
|
||||
|
|
@ -239,7 +291,11 @@ public sealed class MimirSynchronizedBufferPlanner
|
|||
|
||||
var selected = best!.Value;
|
||||
var selectedStartNs = correction.ToCanonicalNs(selected.TimestampNs);
|
||||
var selectedEndNs = SampleCanonicalEndNs(selected, correction);
|
||||
var selectedPrevious = bestIndex > 0 ? samples[bestIndex - 1] : (MimirStreamSample?)null;
|
||||
var selectedNext = bestIndex >= 0 && bestIndex < samples.Count - 1
|
||||
? samples[bestIndex + 1]
|
||||
: (MimirStreamSample?)null;
|
||||
var selectedEndNs = SampleCanonicalEndNs(selected, correction, selectedPrevious, selectedNext);
|
||||
return new MimirSynchronizedStreamSlice(
|
||||
buffer.Descriptor.SourceId,
|
||||
buffer.Descriptor.Kind,
|
||||
|
|
@ -256,20 +312,42 @@ public sealed class MimirSynchronizedBufferPlanner
|
|||
correction.EvidenceKind);
|
||||
}
|
||||
|
||||
private static long SampleCanonicalEndNs(MimirStreamSample sample, MimirSourceTimingCorrection correction)
|
||||
private static long SampleCanonicalEndNs(
|
||||
MimirStreamSample sample,
|
||||
MimirSourceTimingCorrection correction,
|
||||
MimirStreamSample? previousSample = null,
|
||||
MimirStreamSample? nextSample = null)
|
||||
{
|
||||
var startNs = correction.ToCanonicalNs(sample.TimestampNs);
|
||||
var durationNs = SampleDurationNs(sample);
|
||||
var durationNs = SampleDurationNs(sample, previousSample, nextSample);
|
||||
return checked(startNs + Math.Max(1, durationNs));
|
||||
}
|
||||
|
||||
private static long SampleDurationNs(MimirStreamSample sample)
|
||||
private static long SampleDurationNs(
|
||||
MimirStreamSample sample,
|
||||
MimirStreamSample? previousSample = null,
|
||||
MimirStreamSample? nextSample = null)
|
||||
{
|
||||
if (sample.AudioBlock is { SampleRate: > 0, FrameCount: > 0 } block)
|
||||
{
|
||||
return checked((long)Math.Ceiling(block.FrameCount * 1_000_000_000.0 / block.SampleRate));
|
||||
}
|
||||
|
||||
if (sample.VideoFrame is not null)
|
||||
{
|
||||
if (nextSample.HasValue && nextSample.Value.TimestampNs > sample.TimestampNs)
|
||||
{
|
||||
return nextSample.Value.TimestampNs - sample.TimestampNs;
|
||||
}
|
||||
|
||||
if (previousSample.HasValue && sample.TimestampNs > previousSample.Value.TimestampNs)
|
||||
{
|
||||
return sample.TimestampNs - previousSample.Value.TimestampNs;
|
||||
}
|
||||
|
||||
return 33_333_334L;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using System.Net.WebSockets;
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Mimir.Runtime.Synchronization;
|
||||
|
|
@ -54,6 +55,9 @@ var nextPublish = DateTimeOffset.MinValue;
|
|||
var nextCapture = DateTimeOffset.MinValue;
|
||||
var nextSync = DateTimeOffset.MinValue;
|
||||
var nextVisualCalibration = DateTimeOffset.MinValue;
|
||||
var frameDegradedCount = 0L;
|
||||
var nextFrameDegradedLog = DateTimeOffset.MinValue;
|
||||
var streamTelemetry = new MimirWellStreamTelemetry();
|
||||
var presentation = new MimirPresentationControlState();
|
||||
var exposureController = new MimirCameraExposureController(new MimirCameraExposureControlOptions(
|
||||
options.VisualCalibrationEnabled,
|
||||
|
|
@ -66,7 +70,10 @@ Console.Error.WriteLine($"Mimir Well sources={runtimeConfig.SourceFactories.Coun
|
|||
|
||||
while (!stopping.IsCancellationRequested)
|
||||
{
|
||||
hub.PollSources(options.MaxSamplesPerSource);
|
||||
var pollStopwatch = Stopwatch.StartNew();
|
||||
var consumedSamples = hub.PollSources(options.MaxSamplesPerSource);
|
||||
pollStopwatch.Stop();
|
||||
streamTelemetry.ObservePoll(consumedSamples, pollStopwatch.Elapsed);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (now >= nextSync)
|
||||
{
|
||||
|
|
@ -95,28 +102,43 @@ while (!stopping.IsCancellationRequested)
|
|||
if (now >= nextPublish)
|
||||
{
|
||||
presentation.SyncFromBuffers(hub.Buffers.Buffers);
|
||||
var frame = BuildFrameOrEmpty(hub, TimeSpan.FromMilliseconds(options.PresentationDelayMs));
|
||||
var frameResult = BuildFrameOrEmpty(hub, TimeSpan.FromMilliseconds(options.PresentationDelayMs));
|
||||
if (!string.IsNullOrWhiteSpace(frameResult.DegradedReason))
|
||||
{
|
||||
frameDegradedCount++;
|
||||
if (now >= nextFrameDegradedLog)
|
||||
{
|
||||
Console.Error.WriteLine(
|
||||
$"mimir-well synchronized-frame degraded count={frameDegradedCount} reason={frameResult.DegradedReason}");
|
||||
nextFrameDegradedLog = now + TimeSpan.FromSeconds(5);
|
||||
}
|
||||
}
|
||||
|
||||
var frame = frameResult.Frame;
|
||||
var snapshot = MimirWellSnapshot.Build(
|
||||
options,
|
||||
runtimeConfig,
|
||||
hub,
|
||||
presentation,
|
||||
frame,
|
||||
frameResult,
|
||||
exposureController.Statuses,
|
||||
sourceErrors,
|
||||
streamTelemetry.Snapshot(now, subscribersAreExternal: true),
|
||||
++sequence,
|
||||
startedAt);
|
||||
await publisher.PublishAsync(snapshot, stopping.Token).ConfigureAwait(false);
|
||||
streamTelemetry.ObservePublish(
|
||||
await publisher.PublishAsync(snapshot, "mimir.well_snapshot.v1", stopping.Token).ConfigureAwait(false));
|
||||
if (options.CapturePagesEnabled && now >= nextCapture)
|
||||
{
|
||||
var capturePage = MimirWellCapturePage.Build(
|
||||
options,
|
||||
presentation,
|
||||
frame,
|
||||
frameResult,
|
||||
hub.Buffers.Buffers,
|
||||
++captureSequence,
|
||||
startedAt);
|
||||
await publisher.PublishAsync(capturePage, stopping.Token).ConfigureAwait(false);
|
||||
streamTelemetry.ObservePublish(
|
||||
await publisher.PublishAsync(capturePage, "mimir.well_capture_page.v1", stopping.Token).ConfigureAwait(false));
|
||||
nextCapture = now + TimeSpan.FromMilliseconds(options.CaptureIntervalMs);
|
||||
}
|
||||
|
||||
|
|
@ -143,24 +165,117 @@ Console.Error.WriteLine($"Mimir Well complete sequence={sequence} ingested={hub.
|
|||
|
||||
static long NowNs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L;
|
||||
|
||||
static MimirSynchronizedBufferFrame BuildFrameOrEmpty(MimirSynchronizationHub hub, TimeSpan presentationDelay)
|
||||
static MimirWellFrameBuildResult BuildFrameOrEmpty(MimirSynchronizationHub hub, TimeSpan presentationDelay)
|
||||
{
|
||||
try
|
||||
{
|
||||
return hub.BuildSynchronizedBufferFrame(presentationDelay);
|
||||
return new MimirWellFrameBuildResult(
|
||||
hub.BuildSynchronizedBufferFrame(presentationDelay),
|
||||
"",
|
||||
"");
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
Console.Error.WriteLine($"mimir-well synchronized-frame degraded: {ex.Message}");
|
||||
return new MimirSynchronizedBufferFrame(0, 0, 0, presentationDelay, []);
|
||||
return new MimirWellFrameBuildResult(
|
||||
new MimirSynchronizedBufferFrame(0, 0, 0, presentationDelay, []),
|
||||
ex.GetType().Name,
|
||||
ex.Message);
|
||||
}
|
||||
catch (OverflowException ex)
|
||||
{
|
||||
Console.Error.WriteLine($"mimir-well synchronized-frame degraded: {ex.Message}");
|
||||
return new MimirSynchronizedBufferFrame(0, 0, 0, presentationDelay, []);
|
||||
return new MimirWellFrameBuildResult(
|
||||
new MimirSynchronizedBufferFrame(0, 0, 0, presentationDelay, []),
|
||||
ex.GetType().Name,
|
||||
ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record MimirWellFrameBuildResult(
|
||||
MimirSynchronizedBufferFrame Frame,
|
||||
string DegradedKind,
|
||||
string DegradedReason);
|
||||
|
||||
internal sealed record MimirWellPublishReceipt(
|
||||
string Document,
|
||||
int ByteLength,
|
||||
double SerializeMilliseconds,
|
||||
double SendMilliseconds,
|
||||
double TotalMilliseconds);
|
||||
|
||||
internal sealed class MimirWellStreamTelemetry
|
||||
{
|
||||
private long pollIterations;
|
||||
private long consumedSamples;
|
||||
private long zeroPollIterations;
|
||||
private double maxPollMilliseconds;
|
||||
private double totalPollMilliseconds;
|
||||
private long publishedDocuments;
|
||||
private long publishedBytes;
|
||||
private double maxPublishMilliseconds;
|
||||
private double totalPublishMilliseconds;
|
||||
private string lastPublishedDocument = "";
|
||||
private int lastPublishedBytes;
|
||||
private double lastPublishMilliseconds;
|
||||
|
||||
public void ObservePoll(int consumed, TimeSpan elapsed)
|
||||
{
|
||||
pollIterations++;
|
||||
consumedSamples += Math.Max(0, consumed);
|
||||
if (consumed <= 0)
|
||||
{
|
||||
zeroPollIterations++;
|
||||
}
|
||||
|
||||
var milliseconds = elapsed.TotalMilliseconds;
|
||||
totalPollMilliseconds += milliseconds;
|
||||
maxPollMilliseconds = Math.Max(maxPollMilliseconds, milliseconds);
|
||||
}
|
||||
|
||||
public void ObservePublish(MimirWellPublishReceipt receipt)
|
||||
{
|
||||
publishedDocuments++;
|
||||
publishedBytes += receipt.ByteLength;
|
||||
totalPublishMilliseconds += receipt.TotalMilliseconds;
|
||||
maxPublishMilliseconds = Math.Max(maxPublishMilliseconds, receipt.TotalMilliseconds);
|
||||
lastPublishedDocument = receipt.Document;
|
||||
lastPublishedBytes = receipt.ByteLength;
|
||||
lastPublishMilliseconds = receipt.TotalMilliseconds;
|
||||
}
|
||||
|
||||
public object Snapshot(DateTimeOffset now, bool subscribersAreExternal) => new
|
||||
{
|
||||
document = "mimir.well_stream_pressure.v1",
|
||||
status = "transitional-websocket-json",
|
||||
authority = "Mimir.Well observes producer pressure; CultMesh streaming organ should own durable body lanes.",
|
||||
wallClockUtc = now.ToString("O"),
|
||||
poll = new
|
||||
{
|
||||
iterations = pollIterations,
|
||||
consumedSamples,
|
||||
zeroPollIterations,
|
||||
averageMilliseconds = pollIterations == 0 ? 0.0 : totalPollMilliseconds / pollIterations,
|
||||
maxMilliseconds = maxPollMilliseconds,
|
||||
},
|
||||
publish = new
|
||||
{
|
||||
documents = publishedDocuments,
|
||||
bytes = publishedBytes,
|
||||
averageMilliseconds = publishedDocuments == 0 ? 0.0 : totalPublishMilliseconds / publishedDocuments,
|
||||
maxMilliseconds = maxPublishMilliseconds,
|
||||
lastDocument = lastPublishedDocument,
|
||||
lastBytes = lastPublishedBytes,
|
||||
lastMilliseconds = lastPublishMilliseconds,
|
||||
subscribersAreExternal,
|
||||
},
|
||||
nextOrgan = new
|
||||
{
|
||||
controlLane = "CultMesh typed state and compact stream cursors",
|
||||
bodyLane = "CultCache page/shard append stream with refs, hashes, and backpressure",
|
||||
realtimeLane = "loss-aware latest-state observation stream for dashboards",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
internal sealed record MimirWellOptions(
|
||||
Uri PublishUrl,
|
||||
string NodeId,
|
||||
|
|
@ -255,11 +370,24 @@ internal sealed class MimirWellPublisher(Uri url) : IAsyncDisposable
|
|||
await socket.ConnectAsync(url, stopping).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task PublishAsync(object document, CancellationToken stopping)
|
||||
public async Task<MimirWellPublishReceipt> PublishAsync(
|
||||
object document,
|
||||
string documentName,
|
||||
CancellationToken stopping)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var json = JsonSerializer.Serialize(document);
|
||||
var bytes = Encoding.UTF8.GetBytes(json);
|
||||
var serializeMilliseconds = stopwatch.Elapsed.TotalMilliseconds;
|
||||
stopwatch.Restart();
|
||||
await socket.SendAsync(bytes, WebSocketMessageType.Text, endOfMessage: true, stopping).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
return new MimirWellPublishReceipt(
|
||||
documentName,
|
||||
bytes.Length,
|
||||
serializeMilliseconds,
|
||||
stopwatch.Elapsed.TotalMilliseconds,
|
||||
serializeMilliseconds + stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
|
|
@ -285,11 +413,14 @@ internal static class MimirWellCapturePage
|
|||
public static object Build(
|
||||
MimirWellOptions options,
|
||||
MimirPresentationControlState presentation,
|
||||
MimirSynchronizedBufferFrame frame,
|
||||
MimirWellFrameBuildResult frameResult,
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers,
|
||||
long captureSequence,
|
||||
DateTimeOffset startedAt) => new
|
||||
DateTimeOffset startedAt)
|
||||
{
|
||||
var frame = frameResult.Frame;
|
||||
return new
|
||||
{
|
||||
type = "cultmesh-observation",
|
||||
document = "mimir.well_capture_page.v1",
|
||||
sourceId = "mimir-well",
|
||||
|
|
@ -304,6 +435,8 @@ internal static class MimirWellCapturePage
|
|||
frame.WindowEndNs,
|
||||
PresentationDelayMs = frame.PresentationDelay.TotalMilliseconds,
|
||||
frame.IsComplete,
|
||||
degradedKind = frameResult.DegradedKind,
|
||||
degradedReason = frameResult.DegradedReason,
|
||||
},
|
||||
storage = new
|
||||
{
|
||||
|
|
@ -340,6 +473,7 @@ internal static class MimirWellCapturePage
|
|||
samples = CaptureSamples(options, captureSequence, frame, buffers)
|
||||
.ToArray(),
|
||||
};
|
||||
}
|
||||
|
||||
private static IEnumerable<object> CaptureSamples(
|
||||
MimirWellOptions options,
|
||||
|
|
@ -484,6 +618,108 @@ internal static class MimirWellCapturePage
|
|||
}
|
||||
}
|
||||
|
||||
internal static class MimirWellClockDiagnostics
|
||||
{
|
||||
public static object Build(
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers,
|
||||
string referenceSourceId,
|
||||
MimirWellFrameBuildResult frameResult)
|
||||
{
|
||||
var activeBuffers = buffers
|
||||
.Where(static buffer => buffer.Latest.HasValue)
|
||||
.ToArray();
|
||||
var referenceBuffer = activeBuffers.FirstOrDefault(buffer =>
|
||||
string.Equals(buffer.Descriptor.SourceId, referenceSourceId, StringComparison.Ordinal));
|
||||
var referenceEdgeNs = referenceBuffer?.Latest?.TimestampNs ?? 0L;
|
||||
var referenceClockDomain = referenceBuffer?.Descriptor.EffectiveClockDomainId ?? "";
|
||||
|
||||
var domains = activeBuffers
|
||||
.GroupBy(static buffer => buffer.Descriptor.EffectiveClockDomainId, StringComparer.Ordinal)
|
||||
.Select(group => BuildDomain(group.Key, group.ToArray(), referenceClockDomain, referenceEdgeNs))
|
||||
.OrderBy(domain => domain.ClockDomainId, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return new
|
||||
{
|
||||
status = frameResult.Frame.Slices.Count > 0 ? "frame-built" : "degraded",
|
||||
frameResult.DegradedKind,
|
||||
frameResult.DegradedReason,
|
||||
referenceSourceId,
|
||||
referenceClockDomain,
|
||||
referenceEdgeNs,
|
||||
domainCount = domains.Length,
|
||||
activeSourceCount = activeBuffers.Length,
|
||||
domains,
|
||||
};
|
||||
}
|
||||
|
||||
private static ClockDomainDiagnostics BuildDomain(
|
||||
string clockDomainId,
|
||||
IReadOnlyList<MimirRollingStreamBuffer> buffers,
|
||||
string referenceClockDomain,
|
||||
long referenceEdgeNs)
|
||||
{
|
||||
var latestEdges = buffers
|
||||
.Select(buffer => buffer.Latest?.TimestampNs ?? 0L)
|
||||
.Where(static value => value > 0)
|
||||
.ToArray();
|
||||
var windowStarts = buffers
|
||||
.Select(static buffer => buffer.OldestSampleTimestampNs > 0 ? buffer.OldestSampleTimestampNs : buffer.WindowStartNs)
|
||||
.Where(static value => value > 0)
|
||||
.ToArray();
|
||||
var minLatest = latestEdges.Length == 0 ? 0L : latestEdges.Min();
|
||||
var maxLatest = latestEdges.Length == 0 ? 0L : latestEdges.Max();
|
||||
var maxWindowStart = windowStarts.Length == 0 ? 0L : windowStarts.Max();
|
||||
var minWindowEnd = minLatest;
|
||||
var overlapNs = minWindowEnd > 0 && maxWindowStart > 0
|
||||
? minWindowEnd - maxWindowStart
|
||||
: 0L;
|
||||
var domainEdgeNs = maxLatest;
|
||||
var provisionalOffsetToReferenceNs = referenceEdgeNs > 0 && domainEdgeNs > 0
|
||||
? referenceEdgeNs - domainEdgeNs
|
||||
: 0L;
|
||||
|
||||
return new ClockDomainDiagnostics(
|
||||
clockDomainId,
|
||||
buffers.Count,
|
||||
buffers
|
||||
.OrderBy(static buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal)
|
||||
.Select(buffer => new
|
||||
{
|
||||
buffer.Descriptor.SourceId,
|
||||
Kind = buffer.Descriptor.Kind.ToString(),
|
||||
Origin = buffer.Descriptor.Origin.ToString(),
|
||||
buffer.Count,
|
||||
buffer.WindowStartNs,
|
||||
buffer.OldestSampleTimestampNs,
|
||||
EdgeNs = buffer.Latest?.TimestampNs ?? 0L,
|
||||
ByteLength = buffer.Latest?.ByteLength ?? 0,
|
||||
})
|
||||
.ToArray(),
|
||||
maxWindowStart,
|
||||
minLatest,
|
||||
maxLatest,
|
||||
overlapNs,
|
||||
overlapNs >= 0,
|
||||
string.Equals(clockDomainId, referenceClockDomain, StringComparison.Ordinal),
|
||||
provisionalOffsetToReferenceNs,
|
||||
provisionalOffsetToReferenceNs / 1_000_000.0);
|
||||
}
|
||||
|
||||
private sealed record ClockDomainDiagnostics(
|
||||
string ClockDomainId,
|
||||
int SourceCount,
|
||||
object[] Sources,
|
||||
long MaxWindowStartNs,
|
||||
long MinLatestEdgeNs,
|
||||
long MaxLatestEdgeNs,
|
||||
long OverlapNs,
|
||||
bool HasLocalOverlap,
|
||||
bool IsReferenceDomain,
|
||||
long ProvisionalOffsetToReferenceNs,
|
||||
double ProvisionalOffsetToReferenceMs);
|
||||
}
|
||||
|
||||
internal static class MimirWellSnapshot
|
||||
{
|
||||
public static object Build(
|
||||
|
|
@ -491,12 +727,16 @@ internal static class MimirWellSnapshot
|
|||
MimirRuntimeConfiguration runtimeConfig,
|
||||
MimirSynchronizationHub hub,
|
||||
MimirPresentationControlState presentation,
|
||||
MimirSynchronizedBufferFrame frame,
|
||||
MimirWellFrameBuildResult frameResult,
|
||||
IReadOnlyList<MimirCameraExposureControlStatus> visualCalibration,
|
||||
IReadOnlyList<object> sourceErrors,
|
||||
object streamPressure,
|
||||
long sequence,
|
||||
DateTimeOffset startedAt) => new
|
||||
DateTimeOffset startedAt)
|
||||
{
|
||||
var frame = frameResult.Frame;
|
||||
return new
|
||||
{
|
||||
type = "cultmesh-observation",
|
||||
document = "mimir.well_snapshot.v1",
|
||||
sourceId = "mimir-well",
|
||||
|
|
@ -516,8 +756,15 @@ internal static class MimirWellSnapshot
|
|||
frame.WindowEndNs,
|
||||
PresentationDelayMs = frame.PresentationDelay.TotalMilliseconds,
|
||||
frame.IsComplete,
|
||||
degradedKind = frameResult.DegradedKind,
|
||||
degradedReason = frameResult.DegradedReason,
|
||||
slices = frame.Slices.Select(SliceSnapshot).ToArray(),
|
||||
},
|
||||
clockDomains = MimirWellClockDiagnostics.Build(
|
||||
hub.Buffers.Buffers,
|
||||
runtimeConfig.Settings.Audio.ReferenceSourceId,
|
||||
frameResult),
|
||||
streamPressure,
|
||||
audioSync = new
|
||||
{
|
||||
referenceSourceId = runtimeConfig.Settings.Audio.ReferenceSourceId,
|
||||
|
|
@ -622,6 +869,7 @@ internal static class MimirWellSnapshot
|
|||
}).ToArray(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static object BufferSnapshot(MimirRollingStreamBuffer buffer) => new
|
||||
{
|
||||
|
|
@ -632,6 +880,7 @@ internal static class MimirWellSnapshot
|
|||
buffer.Descriptor.EffectiveClockDomainId,
|
||||
buffer.Count,
|
||||
buffer.EdgeNs,
|
||||
buffer.OldestSampleTimestampNs,
|
||||
buffer.WindowStartNs,
|
||||
hasLatest = buffer.Latest.HasValue,
|
||||
latest = buffer.Latest.HasValue ? SampleSnapshot(buffer.Latest.Value) : null,
|
||||
|
|
|
|||
|
|
@ -347,3 +347,4 @@
|
|||
{"ts": "2026-06-02T00:06:12+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Added the first Well-to-Recorder archival payload path. Mimir.Well now publishes bounded mimir.well_capture_page.v1 documents beside the dashboard snapshot; capture pages carry synchronized frame-slice timing, configured composite state, and inline base64 sample bodies when CPU payload bytes exist below the configured limit. Mimir.VerseRecorder strips those inline bodies into paged binary files under bodies/page-*.bin, replaces them with mimir.recorder_body_ref.v1 refs in observations.jsonl, and appends mimir.recorder_body_index.v1 records to bodies/index.jsonl. Verified with dotnet build for Mimir.Well and Mimir.VerseRecorder, supervisor dry-runs, and a relay/recorder synthetic capture-page smoke that wrote a 4-byte page body with the expected SHA-256 ref."}
|
||||
{"ts": "2026-06-02T00:06:13+01:00", "type": "verification", "status": "degraded", "note": "2026-06-02: A short local online-verse daemon run proved recorder startup and session/body directory creation, but live Mimir.Well synchronized-frame planning degraded repeatedly with mixed timestamp-domain range errors such as camera/ASIO timestamps being compared before a shared canonical clock owner exists. Treat this as the known camera-clock/global-time correction scar, not a recorder storage failure."}
|
||||
{"ts": "2026-06-02T00:25:02+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02 shakedown: Fixed two data-loss/disk-pressure scars while leaving the long-running Well stack active. The supervisor now passes --seconds 0 to Mimir.Well for indefinite runs instead of a one-year CancelAfter delay that .NET rejects, and waits for relay /health before starting subscribers/producers so Nightwing does not race dotnet cold start. Mimir.Well capture pages now fall back to latest per-buffer samples with TimingEvidenceKind=unsynchronized-fallback when synchronized-frame planning degrades, so Recorder pages raw CPU bodies even before the canonical clock owner exists. Mimir.EveSensorReceiver still broadcasts full capture pages to subscribers but redacts capture-page bodies from diagnostic stdout to avoid duplicating the paged body store in verse-relay.out.log. Live run online-verse-daemons-20260602-002116 / VerseCaptures/20260602-002124 showed all four local workers alive, Recorder receiving 520 observations, 479 body refs, Nightwing observations flowing after manual restart, and capture-page stdout summaries with sampleCount/body-redacted."}
|
||||
{"ts": "2026-06-02T00:46:28+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02 live shakedown: Added provisional clock-domain edge-fit corrections through MimirSourceTimingCorrection, actual oldest-sample bounds for synchronized frame planning, Well clock-domain diagnostics, and mimir.well_stream_pressure.v1 snapshot telemetry for poll/publish pressure. Hot-swapped active run online-verse-daemons-20260602-002116 to the new build; latest VerseCaptures/20260602-002124 snapshot showed complete 6/6 Ready slices over Scarlett ASIO, Kiyo Pro, and Leap with zero slice distance. The same telemetry exposed the streaming-organ scar: ASIO had about 20 ms of actual buffered history against a five-second retention target, and WebSocket JSON capture-page publishing reached multi-MB messages with >100 ms publish spikes. Next transport cut should split CultMesh typed control/cursor state from CultCache page/shard body streaming and dashboard latest-state lanes."}
|
||||
|
|
|
|||
|
|
@ -424,10 +424,23 @@ invariants:
|
|||
gestures with Realtek plus opportunistic ASIO music evidence, and
|
||||
`Mimir.Well` with capture-page/body-paging enabled. This owns a first local
|
||||
paged-recording ABI, not the final CultCache shard engine yet. The live
|
||||
timestamp-domain scar is explicit: ASIO device time
|
||||
and camera driver time are not in one canonical timeline, so sync frame
|
||||
publication is degraded until a camera-clock/global-time correction owner
|
||||
exists.
|
||||
timestamp-domain scar is now instrumented: `Mimir.Well` publishes
|
||||
`clockDomains` diagnostics in `mimir.well_snapshot.v1`, and
|
||||
`MimirSynchronizationHub` feeds provisional clock-domain edge-fit
|
||||
corrections into the existing `MimirSourceTimingCorrection` primitive so
|
||||
camera/ASIO clocks can be compared on one canonical timeline while the
|
||||
evidence remains explicitly provisional. The synchronized-frame planner now
|
||||
uses actual oldest buffered sample timestamps, not just the configured
|
||||
retention boundary, so a five-second retention target cannot pretend to
|
||||
contain five seconds of samples under publish pressure. Snapshots expose
|
||||
`OldestSampleTimestampNs`, `WindowStartNs`, and `streamPressure`
|
||||
(`mimir.well_stream_pressure.v1`) so the dashboard can distinguish clock
|
||||
fit, real buffered span, JSON/WebSocket publish cost, and body-lane
|
||||
backpressure. Live shakedown after this cut produced a complete 6/6 Ready
|
||||
frame over Scarlett ASIO, Kiyo Pro, and Leap with zero slice distance, while
|
||||
also showing the stream-organ scar plainly: ASIO had only about 20 ms of
|
||||
actual buffered history despite a five-second retention target because the
|
||||
current WebSocket JSON capture-page path is still the hose, not the organ.
|
||||
The Eve live-stats provider now reads the latest `mimir.well_snapshot.v1`
|
||||
records from the observation ledger and the latest
|
||||
`mimir.online_move_music_sync_frame.v1` trace from runtime artifacts. Its
|
||||
|
|
|
|||
Loading…
Reference in a new issue