diff --git a/src/Mimir.EveDashboard/Program.cs b/src/Mimir.EveDashboard/Program.cs index d9e281c..eeec818 100644 --- a/src/Mimir.EveDashboard/Program.cs +++ b/src/Mimir.EveDashboard/Program.cs @@ -366,16 +366,23 @@ internal sealed class EveDashboardServer(int port, EveDashboardProviderCatalog p element.AssetUri, element.BindNodeId, element.CommandId, - element.Layout == null - ? null - : new MimirEveDashboardUiLayoutSnapshot( - element.Layout.Direction, - element.Layout.Width, - element.Layout.Height, - element.Layout.Grow, - element.Layout.Gap, - element.Layout.Padding, - element.Layout.Overflow), + element.Layout == null + ? null + : new MimirEveDashboardUiLayoutSnapshot( + element.Layout.Direction, + element.Layout.Width, + element.Layout.Height, + element.Layout.Grow, + element.Layout.Gap, + element.Layout.Padding, + element.Layout.Overflow, + element.Layout.MinWidth, + element.Layout.MinHeight, + element.Layout.PreferredWidth, + element.Layout.PreferredHeight, + element.Layout.Priority, + element.Layout.Density, + element.Layout.ViewportMode), element.Style == null ? null : new MimirEveDashboardUiStyleSnapshot(element.Style.Variant, element.Style.Tone), @@ -899,8 +906,8 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider StatBar("audio-rms", "max channel RMS", NormalizeRms(rms), "warm", $"{rms:0.000000}"), StatBar("observation-liveness", "device stream liveness", snapshot.ObservationStreams.Count == 0 ? 0.0 : liveObservationCount / (double)snapshot.ObservationStreams.Count, "cool"), ]), - BuildMoveMusicPane(snapshot.MoveMusic), BuildWellPane(snapshot.Well), + BuildMoveMusicPane(snapshot.MoveMusic), ]), UiElement.Container( "mimir-live-row-runtime", @@ -930,7 +937,21 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider Root = UiElement.Container( "mimir-live-stats-root", "dashboard", - new DashboardUiLayout { Direction = "vertical", Gap = 6, Padding = 6, Overflow = "scroll" }, + new DashboardUiLayout + { + Direction = "vertical", + Gap = 6, + Padding = 6, + Overflow = "scroll", + Grow = 4, + MinWidth = 96, + MinHeight = 56, + PreferredWidth = 156, + PreferredHeight = 128, + Priority = -80, + Density = "continuous", + ViewportMode = "continuous-metrics", + }, children), }; } @@ -1085,6 +1106,19 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider UiElement.Text("mimir-move-score", $"score {Bar(music.ScoreEnvelopeMax, 18)} max env {music.ScoreEnvelopeMax:0.000} max rgb {music.MaxRgb}", "caption"), UiElement.Text("mimir-move-sources", string.Join(" | ", music.Sources.Take(4)), "caption"), }; + children.AddRange(music.Voices + .Take(5) + .Select(voice => UiElement.Text( + $"mimir-move-voice-{StableId(voice.SourceId)}", + $"{voice.SourceId}: {voice.NoteName} {voice.FrequencyHz:0.0}Hz c={voice.Confidence:0.000} {voice.Role}", + "caption"))); + children.Add(UiElement.Text("mimir-move-piano-roll", music.PianoRoll, "mono")); + children.AddRange(music.MoveTargets + .Take(5) + .Select(target => UiElement.Text( + $"mimir-move-target-{target.MoveIndex}", + $"move {target.MoveIndex}: {target.NoteName} {target.SourceId} priority {target.CalibrationPriority:0.000}", + "caption"))); return UiElement.Pane("mimir-move-music-pane", "Move Score", children); } @@ -1096,12 +1130,47 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider return UiElement.Pane("mimir-well-pane", "Well", [UiElement.Text("mimir-well-empty", "no Well snapshot yet", "caption")]); } - return UiElement.Pane("mimir-well-pane", "Well", [ - UiElement.Text("mimir-well-summary", $"seq {well.Sequence} {well.ElapsedSeconds:0}s ingested {well.IngestedSamples:0} sources {well.LiveSources}/{well.ConfiguredSources}", "mono"), + var children = new List + { + UiElement.Text("mimir-well-summary", $"seq {well.Sequence} {well.ElapsedSeconds:0}s ingested {well.IngestedSamples:0} sources {well.LiveSources}/{well.ConfiguredSources} source errors {well.SourceErrorCount}", "mono"), StatBar("mimir-well-completeness", "frame completeness", well.SynchronizedFrameComplete ? 1.0 : 0.0, well.SynchronizedFrameComplete ? "cool" : "danger"), - UiElement.Text("mimir-well-buffers", $"buffers {well.Buffers.Count} audio {well.AudioBuffers} video {well.VideoBuffers} ready {well.ReadySlices}/{well.TotalSlices}", "caption"), + UiElement.Text("mimir-well-frame", $"frame ready {well.ReadySlices}/{well.TotalSlices} delay {well.PresentationDelayMs:0}ms degraded {well.FrameDegradedKind} {well.FrameDegradedReason}", "caption"), + UiElement.Text("mimir-well-latency", $"latency {well.LatencyCurrentMs:0}ms floor {well.LatencyFloorMs:0} ceiling {well.LatencyCeilingMs:0} overlap {well.LatencyRetainedOverlapMs:0}ms skew {well.LatencyEdgeSkewMs:0.0}ms {well.LatencyReason}", "caption"), + UiElement.Text("mimir-well-pressure", $"poll {well.PollAverageMs:0.000}/{well.PollMaxMs:0.000}ms zero {well.ZeroPollIterations} publish {well.PublishAverageMs:0.000}/{well.PublishMaxMs:0.000}ms bytes {well.PublishedBytes:0}", "caption"), + UiElement.Text("mimir-well-buffers", $"buffers {well.Buffers.Count} audio {well.AudioBuffers} video {well.VideoBuffers} capture bodies {well.CaptureInlineBodies}", "caption"), + UiElement.Text("mimir-well-clock-maps", $"canonical clocks {well.CanonicalClockMaps.Count} max offset {well.MaxCanonicalClockOffsetMs:0.000}ms", "caption"), + UiElement.Text("mimir-well-features", $"video features stable {well.FeatureStableTracks} c={well.FeatureMeanConfidence:0.000} motion {well.FeatureMeanMotionPixelsPerSecond:0.0}px/s faust signals {well.FeatureSignals.Count}", "caption"), UiElement.Text("mimir-well-probe", well.Probe == null ? "probe: none" : $"probe emit={well.Probe.ShouldEmit} sync={well.Probe.SyncConfidence:0.000} freq={well.Probe.FrequencyConfidence:0.000} {well.Probe.Reason}", "caption"), - ]); + }; + children.AddRange(well.FeatureSignals + .OrderByDescending(static signal => signal.Confidence) + .ThenBy(static signal => signal.SourceId, StringComparer.Ordinal) + .Take(5) + .Select(signal => UiElement.Text( + $"mimir-well-feature-{StableId(signal.SourceId)}", + $"{signal.SourceId}: tracks {signal.StableTrackCount} c={signal.Confidence:0.000} motion {signal.MotionEnergy:0.000} centroid {signal.CentroidX:0.00},{signal.CentroidY:0.00}", + "caption"))); + children.AddRange(well.ClockDomains + .Take(6) + .Select(domain => UiElement.Text( + $"mimir-well-clock-{StableId(domain.ClockDomainId)}", + $"{domain.ClockDomainId}: {domain.SourceCount} src overlap {domain.OverlapMs:0.000}ms offset {domain.OffsetMs:0.000}ms {(domain.IsReference ? "ref" : "")}", + "caption"))); + children.AddRange(well.CanonicalClockMaps + .OrderByDescending(static map => Math.Abs(map.OffsetMs)) + .ThenBy(static map => map.StreamKey, StringComparer.Ordinal) + .Take(4) + .Select(map => UiElement.Text( + $"mimir-well-clock-map-{StableId(map.StreamKey)}", + $"{map.StreamKey}: ingress offset {map.OffsetMs:0.000}ms samples {map.SampleCount}", + "caption"))); + children.AddRange(well.VisualCalibration + .Take(6) + .Select(camera => UiElement.Text( + $"mimir-well-cal-{StableId(camera.SourceId)}", + $"{camera.SourceId}: score {camera.BestScore:0.000} leds {camera.BestDetectedLedCount} exp {camera.BestExposure:0} gain {camera.BestGain:0} {camera.State}", + "caption"))); + return UiElement.Pane("mimir-well-pane", "Well", children); } private static DashboardUiElement BuildCalibrationPane(MimirLiveStatsSnapshot snapshot) @@ -1225,7 +1294,7 @@ internal sealed class MimirLiveStatsSnapshot ActuatorCommands = LatestBySource(telemetryLines.Select(ParseActuatorCommand).Where(static value => value != null)!), ObservationStreams = LatestObservationStreams(observationLines), Well = LatestWellSnapshot(observationLines), - MoveMusic = LatestMoveMusicTrace(), + MoveMusic = LatestMoveMusicTrace(telemetrySource, telemetryLines), }; } @@ -1361,21 +1430,103 @@ internal sealed class MimirLiveStatsSnapshot JsonNumberAny(probeElement, "AggregateFrequencyResponseConfidence", "aggregateFrequencyResponseConfidence"), JsonStringAny(probeElement, "reason") ?? "unknown") : null; + var streamPressure = TryGetAny(root, "streamPressure"); + var pollPressure = TryGetAny(streamPressure, "poll"); + var publishPressure = TryGetAny(streamPressure, "publish"); + var latency = TryGetAny(root, "latency"); + var featureSignals = TryGetAny(root, "featureSignals"); + var featureSignalStats = ArrayItems(TryGetAny(featureSignals, "signals")) + .Select(static signal => new MimirWellFeatureSignalStat( + JsonStringAny(signal, "SourceId", "sourceId") ?? "unknown", + (int)JsonNumberAny(signal, "StableTrackCount", "stableTrackCount"), + JsonNumberAny(signal, "Confidence", "confidence"), + JsonNumberAny(signal, "MeanMotionPixelsPerSecond", "meanMotionPixelsPerSecond"), + JsonNumberAny(signal, "MotionEnergy", "motionEnergy"), + JsonNumberAny(signal, "NormalizedCentroidX", "normalizedCentroidX"), + JsonNumberAny(signal, "NormalizedCentroidY", "normalizedCentroidY"))) + .ToArray(); + var canonicalClockMaps = ArrayItems(TryGetAny(root, "canonicalClockMaps")) + .Select(static map => new MimirWellCanonicalClockMapStat( + JsonStringAny(map, "StreamKey", "streamKey") ?? "unknown", + JsonNumberAny(map, "offsetMs"), + (long)JsonNumberAny(map, "SampleCount", "sampleCount"))) + .ToArray(); + var clockDomains = ArrayItems(TryGetAny(TryGetAny(root, "clockDomains"), "domains")) + .Select(static domain => new MimirWellClockDomainStat( + JsonStringAny(domain, "ClockDomainId", "clockDomainId") ?? "unknown", + (int)JsonNumberAny(domain, "SourceCount", "sourceCount"), + JsonNumberAny(domain, "OverlapNs", "overlapNs") / 1_000_000.0, + JsonBoolAny(domain, "HasLocalOverlap", "hasLocalOverlap"), + JsonBoolAny(domain, "IsReferenceDomain", "isReferenceDomain"), + JsonNumberAny(domain, "ProvisionalOffsetToReferenceMs", "provisionalOffsetToReferenceMs"))) + .ToArray(); + var visualCalibration = ArrayItems(TryGetAny(TryGetAny(root, "visualCalibration"), "cameras")) + .Select(static camera => new MimirWellVisualCalibrationStat( + JsonStringAny(camera, "SourceId", "sourceId") ?? "unknown", + JsonStringAny(camera, "State", "state") ?? "unknown", + JsonNumberAny(camera, "BestScore", "bestScore"), + (int)JsonNumberAny(camera, "BestDetectedLedCount", "bestDetectedLedCount"), + JsonBoolAny(camera, "BestUsableForCalibration", "bestUsableForCalibration"), + JsonNumberAny(camera, "BestExposure", "bestExposure"), + JsonNumberAny(camera, "BestGain", "bestGain"))) + .ToArray(); + var captureStorage = TryGetAny(TryGetAny(root, "capture"), "storage"); + var publication = TryGetAny(root, "publication"); + var stems = ArrayItems(TryGetAny(publication, "stems")).ToArray(); return new MimirWellStat( (long)JsonNumberAny(root, "sequence"), JsonNumberAny(root, "elapsedSeconds"), JsonNumberAny(root, "ingestedSamples"), (int)JsonNumberAny(root, "configuredSources"), (int)JsonNumberAny(root, "liveSources"), + ArrayItems(TryGetAny(root, "sourceErrors")).Count(), JsonBoolAny(frame, "IsComplete", "isComplete"), + JsonNumberAny(frame, "PresentationDelayMs", "presentationDelayMs"), + JsonStringAny(frame, "degradedKind") ?? "", + JsonStringAny(frame, "degradedReason") ?? "", slices.Count(static slice => string.Equals(JsonStringAny(slice, "Status", "status"), "Ready", StringComparison.OrdinalIgnoreCase)), slices.Length, + JsonNumberAny(latency, "currentDelayMs"), + JsonNumberAny(latency, "ceilingDelayMs"), + JsonNumberAny(latency, "floorDelayMs"), + JsonNumberAny(latency, "retainedOverlapMs"), + JsonNumberAny(latency, "edgeSkewMs"), + JsonStringAny(latency, "Reason", "reason") ?? "", + JsonNumberAny(pollPressure, "averageMilliseconds"), + JsonNumberAny(pollPressure, "maxMilliseconds"), + (long)JsonNumberAny(pollPressure, "zeroPollIterations"), + JsonNumberAny(publishPressure, "averageMilliseconds"), + JsonNumberAny(publishPressure, "maxMilliseconds"), + (long)JsonNumberAny(publishPressure, "bytes"), + JsonStringAny(captureStorage, "bodyTransport") ?? "unknown", + stems.Length, buffers, + clockDomains, states, + visualCalibration, + JsonNumberAny(featureSignals, "MeanConfidence", "meanConfidence"), + JsonNumberAny(featureSignals, "MeanMotionPixelsPerSecond", "meanMotionPixelsPerSecond"), + (int)JsonNumberAny(featureSignals, "StableTrackCount", "stableTrackCount"), + featureSignalStats, + canonicalClockMaps, probe); } - private static MimirMoveMusicStat? LatestMoveMusicTrace() + private static MimirMoveMusicStat? LatestMoveMusicTrace(string telemetrySource, IReadOnlyList telemetryLines) + { + var configuredLine = telemetryLines + .Reverse() + .FirstOrDefault(static line => line.Contains("\"live_score\"", StringComparison.OrdinalIgnoreCase) || + line.Contains("\"score_gesture_envelopes\"", StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrWhiteSpace(configuredLine)) + { + return ParseMoveMusicTrace(telemetrySource, configuredLine); + } + + return LatestMoveMusicTraceFromRuntime(); + } + + private static MimirMoveMusicStat? LatestMoveMusicTraceFromRuntime() { var runtimeRoot = Path.Combine(AppContext.BaseDirectory, "artifacts", "runtime"); if (!Directory.Exists(runtimeRoot)) @@ -1406,6 +1557,11 @@ internal sealed class MimirLiveStatsSnapshot return null; } + return ParseMoveMusicTrace(trace.FullName, line); + } + + private static MimirMoveMusicStat? ParseMoveMusicTrace(string sourcePath, string line) + { try { using var document = JsonDocument.Parse(line); @@ -1424,6 +1580,30 @@ internal sealed class MimirLiveStatsSnapshot } var envelopes = ArrayItems(TryGetAny(root, "score_gesture_envelopes")).Select(JsonNumber).DefaultIfEmpty(0.0).ToArray(); + var liveScore = TryGetAny(root, "live_score"); + var voices = ArrayItems(TryGetAny(liveScore, "voices")) + .Select(static voice => new MimirMoveMusicVoiceStat( + JsonStringAny(voice, "source") ?? "unknown", + JsonStringAny(voice, "role") ?? "voice", + JsonStringAny(voice, "note_name") ?? "?", + JsonNumberAny(voice, "frequency_hz"), + JsonNumberAny(voice, "midi"), + JsonNumberAny(voice, "confidence"), + JsonNumberAny(voice, "strength"), + JsonBoolAny(voice, "active"))) + .OrderByDescending(static voice => voice.Active) + .ThenByDescending(static voice => voice.Confidence) + .ToArray(); + var targets = ArrayItems(TryGetAny(liveScore, "move_targets")) + .Select(static target => new MimirMoveMusicTargetStat( + (int)JsonNumberAny(target, "move_index"), + JsonStringAny(target, "source") ?? "unknown", + JsonStringAny(target, "note_name") ?? "?", + JsonNumberAny(target, "target_note"), + JsonNumberAny(target, "confidence"), + JsonNumberAny(target, "calibration_priority"))) + .OrderBy(static target => target.MoveIndex) + .ToArray(); var sources = ArrayItems(TryGetAny(root, "music_sources")) .Select(static source => { @@ -1438,7 +1618,7 @@ internal sealed class MimirLiveStatsSnapshot }) .ToArray(); return new MimirMoveMusicStat( - trace.FullName, + sourcePath, JsonNumberAny(root, "bpm"), JsonNumberAny(root, "bpm_confidence"), JsonNumberAny(root, "loudness_gate"), @@ -1448,7 +1628,10 @@ internal sealed class MimirLiveStatsSnapshot JsonStringAny(root, "chord_name") ?? "?", envelopes.Max(), maxRgb, - sources); + sources, + voices, + targets, + BuildPianoRoll(voices, targets)); } catch (JsonException) { @@ -1456,6 +1639,43 @@ internal sealed class MimirLiveStatsSnapshot } } + private static string BuildPianoRoll( + IReadOnlyList voices, + IReadOnlyList targets) + { + var marks = new char[36]; + Array.Fill(marks, '.'); + foreach (var voice in voices.Where(static voice => voice.Active)) + { + var index = PianoIndex(voice.MidiNote); + if (index >= 0) + { + marks[index] = voice.Confidence >= 0.65 ? '#' : '+'; + } + } + + foreach (var target in targets) + { + var index = PianoIndex(target.TargetNote); + if (index >= 0 && marks[index] == '.') + { + marks[index] = 'o'; + } + } + + return $"C3 {new string(marks)} C6"; + } + + private static int PianoIndex(double midiNote) + { + if (midiNote <= 0.0) + { + return -1; + } + + return (int)Math.Clamp(Math.Round(midiNote) - 48, 0, 35); + } + private static IReadOnlyList LatestObservationStreams(IEnumerable lines) { var byStream = new Dictionary(StringComparer.Ordinal); @@ -1844,16 +2064,45 @@ internal sealed record MimirWellStat( double IngestedSamples, int ConfiguredSources, int LiveSources, + int SourceErrorCount, bool SynchronizedFrameComplete, + double PresentationDelayMs, + string FrameDegradedKind, + string FrameDegradedReason, int ReadySlices, int TotalSlices, + double LatencyCurrentMs, + double LatencyCeilingMs, + double LatencyFloorMs, + double LatencyRetainedOverlapMs, + double LatencyEdgeSkewMs, + string LatencyReason, + double PollAverageMs, + double PollMaxMs, + long ZeroPollIterations, + double PublishAverageMs, + double PublishMaxMs, + long PublishedBytes, + string CaptureInlineBodies, + int PublicationStemCount, IReadOnlyList Buffers, + IReadOnlyList ClockDomains, IReadOnlyList AudioSyncStates, + IReadOnlyList VisualCalibration, + double FeatureMeanConfidence, + double FeatureMeanMotionPixelsPerSecond, + int FeatureStableTracks, + IReadOnlyList FeatureSignals, + IReadOnlyList CanonicalClockMaps, MimirWellProbeStat? Probe) { public int AudioBuffers => Buffers.Count(static buffer => string.Equals(buffer.Kind, "Audio", StringComparison.OrdinalIgnoreCase)); public int VideoBuffers => Buffers.Count(static buffer => string.Equals(buffer.Kind, "Video", StringComparison.OrdinalIgnoreCase)); + + public double MaxCanonicalClockOffsetMs => CanonicalClockMaps.Count == 0 + ? 0.0 + : CanonicalClockMaps.Max(static map => Math.Abs(map.OffsetMs)); } internal sealed record MimirWellBufferStat( @@ -1869,6 +2118,14 @@ internal sealed record MimirWellBufferStat( internal sealed record MimirWellAudioSyncStateStat(string SourceId, string ReferenceSourceId, double DelaySamples, double SroPpm, double Confidence); +internal sealed record MimirWellClockDomainStat(string ClockDomainId, int SourceCount, double OverlapMs, bool HasLocalOverlap, bool IsReference, double OffsetMs); + +internal sealed record MimirWellVisualCalibrationStat(string SourceId, string State, double BestScore, int BestDetectedLedCount, bool BestUsableForCalibration, double BestExposure, double BestGain); + +internal sealed record MimirWellFeatureSignalStat(string SourceId, int StableTrackCount, double Confidence, double MeanMotionPixelsPerSecond, double MotionEnergy, double CentroidX, double CentroidY); + +internal sealed record MimirWellCanonicalClockMapStat(string StreamKey, double OffsetMs, long SampleCount); + internal sealed record MimirWellProbeStat(bool ShouldEmit, double SyncConfidence, double FrequencyConfidence, string Reason); internal sealed record MimirMoveMusicStat( @@ -1882,7 +2139,14 @@ internal sealed record MimirMoveMusicStat( string ChordName, double ScoreEnvelopeMax, int MaxRgb, - IReadOnlyList Sources); + IReadOnlyList Sources, + IReadOnlyList Voices, + IReadOnlyList MoveTargets, + string PianoRoll); + +internal sealed record MimirMoveMusicVoiceStat(string SourceId, string Role, string NoteName, double FrequencyHz, double MidiNote, double Confidence, double Strength, bool Active); + +internal sealed record MimirMoveMusicTargetStat(int MoveIndex, string SourceId, string NoteName, double TargetNote, double Confidence, double CalibrationPriority); internal sealed class MimirStreamLayoutProvider : MutableDashboardProvider { @@ -2016,6 +2280,7 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider var summary = TryGet(root, "summary"); var controls = TryGet(root, "controls"); var cultMesh = TryGet(root, "cultMesh"); + var orchestrator = TryGet(root, "orchestrator"); var participants = ArrayItems(TryGet(root, "participants")).ToArray(); var upcoming = ArrayItems(TryGet(root, "upcomingTurns")).ToArray(); @@ -2070,7 +2335,7 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider SelectedNodeId = selectedNodeId, Nodes = nodes, }; - state.Surface = BuildVoidBotSurface(state, summary, cultMesh, controls, upcoming, participants, selectedAgent); + state.Surface = BuildVoidBotSurface(state, summary, cultMesh, controls, orchestrator, upcoming, participants, selectedAgent); return state; } @@ -2079,6 +2344,7 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider JsonElement summary, JsonElement cultMesh, JsonElement controls, + JsonElement orchestrator, IReadOnlyList upcoming, IReadOnlyList participants, JsonElement selectedAgent) @@ -2096,13 +2362,35 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider Root = UiElement.Container( "voidbot-cockpit", "cockpit", - new DashboardUiLayout { Direction = "vertical", Gap = 10, Padding = 10 }, + new DashboardUiLayout + { + Direction = "vertical", + Gap = 8, + Padding = 8, + Overflow = "scroll", + Grow = 5, + MinWidth = 112, + MinHeight = 30, + PreferredWidth = 156, + PreferredHeight = 58, + Priority = -30, + Density = "continuous", + ViewportMode = "continuous-ops", + }, [ BuildTurnRail(upcoming), + UiElement.Container( + "voidbot-ops-row", + "ops-row", + new DashboardUiLayout { Direction = "horizontal", Gap = 8, Grow = 1 }, + [ + BuildUpcomingFacesPane(upcoming), + BuildWatchdogPane(orchestrator), + ]), UiElement.Container( "voidbot-workspace", "workspace", - new DashboardUiLayout { Direction = "horizontal", Gap = 10, Grow = 1 }, + new DashboardUiLayout { Direction = "horizontal", Gap = 8, Grow = 2 }, [ BuildSelectedFacePane(summary, cultMesh, selectedAgent, cadence, generatedAt), BuildStateGraphPane(leaves), @@ -2123,7 +2411,7 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider private DashboardUiElement BuildTurnRail(IReadOnlyList upcoming) { - var count = Math.Min(8, upcoming.Count); + var count = Math.Min(14, upcoming.Count); var cards = new List(); for (var index = 0; index < count; index++) { @@ -2152,6 +2440,70 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider cards); } + private DashboardUiElement BuildUpcomingFacesPane(IReadOnlyList upcoming) + { + var rows = upcoming + .Take(10) + .Select((turn, index) => + { + var identityId = StringValue(turn, "identityId") ?? $"turn-{index}"; + var active = !string.IsNullOrWhiteSpace(StringValue(turn, "activeJobId")); + var mentionCount = NumberValue(turn, "pendingMentionCount") ?? 0; + var rest = TryGet(turn, "restState"); + var napping = BoolValue(rest, "isNapping") == true; + var state = active ? "active" : mentionCount > 0 ? "mention" : napping ? "nap" : Minutes(NumberValue(turn, "nextTurnInMinutes")); + var heat = NumberValue(turn, "heat") ?? 0; + var speed = NumberValue(turn, "effectiveSpeed") ?? 0; + return UiElement.Text( + $"upcoming-face-{index}-{identityId}", + $"{index + 1,2}. {(StringValue(turn, "displayName") ?? identityId),-10} {state,-8} {StringValue(turn, "repoName") ?? "repo"} spd {speed:0.###} heat {heat:0.##}", + "mono"); + }) + .ToArray(); + + return UiElement.Pane( + "upcoming-faces-pane", + "Next Faces", + rows.Length > 0 + ? rows + : [UiElement.Text("upcoming-faces-empty", "no upcoming Face turns", "caption")]); + } + + private DashboardUiElement BuildWatchdogPane(JsonElement orchestrator) + { + var state = StringValue(orchestrator, "state") ?? "unknown"; + var organs = ArrayItems(TryGet(orchestrator, "organs")).ToArray(); + var watchdog = organs.FirstOrDefault(organ => string.Equals(StringValue(organ, "id"), "voidbot-operations-watchdog", StringComparison.OrdinalIgnoreCase)); + var children = new List + { + UiElement.Text("watchdog-summary", $"orchestrator {state} organs {organs.Length}", "mono"), + }; + + if (watchdog.ValueKind != JsonValueKind.Undefined) + { + children.Add(UiElement.Text( + "watchdog-status", + $"watchdog {StringValue(watchdog, "lastStatus") ?? "unknown"} exit {(NumberValue(watchdog, "lastExitCode") ?? 0):0}\nlast {ShortIso(StringValue(watchdog, "lastFinishedAt") ?? StringValue(watchdog, "lastStartedAt"))}", + "strong")); + } + else + { + children.Add(UiElement.Text("watchdog-missing", "watchdog organ not present in snapshot", "caption")); + } + + children.AddRange(organs + .Where(static organ => organ.ValueKind != JsonValueKind.Undefined) + .OrderBy(organ => string.Equals(StringValue(organ, "id"), "voidbot-operations-watchdog", StringComparison.OrdinalIgnoreCase) ? 0 : 1) + .ThenBy(organ => StringValue(organ, "id")) + .Take(7) + .Select(organ => UiElement.Text( + $"watchdog-organ-{StableId(StringValue(organ, "id") ?? "organ")}", + $"{StringValue(organ, "label") ?? StringValue(organ, "id") ?? "organ"}: {StringValue(organ, "lastStatus") ?? "unknown"}", + "caption"))); + + return UiElement.Pane("voidbot-watchdog-pane", "VoidBot Watchdog", children); + } + private DashboardUiElement BuildSelectedFacePane(JsonElement summary, JsonElement cultMesh, JsonElement selectedAgent, double cadence, string generatedAt) { var identityId = StringValue(selectedAgent, "identityId") ?? ""; @@ -2447,6 +2799,29 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider private static string Truncate(string value, int max) => value.Length <= max ? value : value[..Math.Max(0, max - 1)] + "..."; + + private static string StableId(string value) + { + var builder = new StringBuilder(value.Length); + foreach (var ch in value.ToLowerInvariant()) + { + builder.Append(char.IsLetterOrDigit(ch) ? ch : '-'); + } + + return builder.ToString().Trim('-'); + } + + private static string ShortIso(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "unknown"; + } + + return DateTimeOffset.TryParse(value, out var timestamp) + ? timestamp.ToLocalTime().ToString("MM-dd HH:mm") + : value; + } } internal sealed record VoidBotStateLeaf(string Label, string Path, string Preview, string Detail); @@ -2647,6 +3022,20 @@ internal sealed class DashboardUiLayout public double? Padding { get; set; } public string? Overflow { get; set; } + + public double? MinWidth { get; set; } + + public double? MinHeight { get; set; } + + public double? PreferredWidth { get; set; } + + public double? PreferredHeight { get; set; } + + public double? Priority { get; set; } + + public string? Density { get; set; } + + public string? ViewportMode { get; set; } } internal sealed class DashboardUiStyle diff --git a/src/Mimir.Runtime/Synchronization/IMimirStreamSource.cs b/src/Mimir.Runtime/Synchronization/IMimirStreamSource.cs index c06d392..fa65318 100644 --- a/src/Mimir.Runtime/Synchronization/IMimirStreamSource.cs +++ b/src/Mimir.Runtime/Synchronization/IMimirStreamSource.cs @@ -8,3 +8,8 @@ public interface IMimirStreamSource : IDisposable bool TryRead(out MimirStreamSample sample); } + +public interface IMimirMultiplexedStreamSource : IMimirStreamSource +{ + int LogicalStreamCount { get; } +} diff --git a/src/Mimir.Runtime/Synchronization/MimirAsioStreamSource.cs b/src/Mimir.Runtime/Synchronization/MimirAsioStreamSource.cs index 9d93b53..e60921d 100644 --- a/src/Mimir.Runtime/Synchronization/MimirAsioStreamSource.cs +++ b/src/Mimir.Runtime/Synchronization/MimirAsioStreamSource.cs @@ -11,7 +11,7 @@ public sealed record MimirAsioStreamSourceOptions( IReadOnlyList SourceIds); [SupportedOSPlatform("windows")] -public sealed class MimirAsioStreamSource : IMimirStreamSource +public sealed class MimirAsioStreamSource : IMimirMultiplexedStreamSource { private const string DefaultClsid = "{AC4D0455-50D7-4498-B3CD-9A41D130B759}"; private const int MaxQueuedBlocks = 32768; @@ -57,6 +57,8 @@ public sealed class MimirAsioStreamSource : IMimirStreamSource public bool ExposesDescriptorBuffer => false; + public int LogicalStreamCount => Math.Max(1, sourceIds.Length); + public bool TryRead(out MimirStreamSample sample) { sample = default; diff --git a/src/Mimir.Runtime/Synchronization/MimirCanonicalClockMapper.cs b/src/Mimir.Runtime/Synchronization/MimirCanonicalClockMapper.cs new file mode 100644 index 0000000..72dd81a --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirCanonicalClockMapper.cs @@ -0,0 +1,72 @@ +namespace Mimir.Runtime.Synchronization; + +public sealed class MimirCanonicalClockMapper +{ + private readonly Dictionary maps = new(StringComparer.Ordinal); + + public IReadOnlyList Snapshots => + maps + .OrderBy(pair => pair.Key, StringComparer.Ordinal) + .Select(pair => new MimirCanonicalClockMapSnapshot( + pair.Key, + pair.Value.OffsetNs, + pair.Value.FirstSourceTimestampNs, + pair.Value.FirstArrivalNs, + pair.Value.LatestSourceTimestampNs, + pair.Value.LatestCanonicalTimestampNs, + pair.Value.SampleCount)) + .ToArray(); + + public MimirStreamSample ToCanonical(MimirStreamSample sample) + { + var sourceTimestampNs = sample.TimestampNs; + if (sourceTimestampNs <= 0) + { + sourceTimestampNs = sample.ArrivalNs > 0 ? sample.ArrivalNs : NowNs(); + } + + var arrivalNs = sample.ArrivalNs > 0 ? sample.ArrivalNs : NowNs(); + var key = $"{sample.Kind}:{sample.Origin}:{sample.SourceId}"; + if (!maps.TryGetValue(key, out var map)) + { + map = new ClockMap(checked(arrivalNs - sourceTimestampNs), sourceTimestampNs, arrivalNs); + maps.Add(key, map); + } + + var canonicalTimestampNs = checked(sourceTimestampNs + map.OffsetNs); + map.LatestSourceTimestampNs = sourceTimestampNs; + map.LatestCanonicalTimestampNs = canonicalTimestampNs; + map.SampleCount++; + return sample with + { + TimestampNs = canonicalTimestampNs, + ArrivalNs = arrivalNs, + }; + } + + private static long NowNs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L; + + private sealed class ClockMap(long offsetNs, long firstSourceTimestampNs, long firstArrivalNs) + { + public long OffsetNs { get; } = offsetNs; + + public long FirstSourceTimestampNs { get; } = firstSourceTimestampNs; + + public long FirstArrivalNs { get; } = firstArrivalNs; + + public long LatestSourceTimestampNs { get; set; } = firstSourceTimestampNs; + + public long LatestCanonicalTimestampNs { get; set; } = firstArrivalNs; + + public long SampleCount { get; set; } + } +} + +public sealed record MimirCanonicalClockMapSnapshot( + string StreamKey, + long OffsetNs, + long FirstSourceTimestampNs, + long FirstArrivalNs, + long LatestSourceTimestampNs, + long LatestCanonicalTimestampNs, + long SampleCount); diff --git a/src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs b/src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs index 983dc15..d5da154 100644 --- a/src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs +++ b/src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs @@ -198,7 +198,14 @@ public sealed record MimirEveDashboardUiLayoutSnapshot( [property: Key(3)] double? Grow, [property: Key(4)] double? Gap, [property: Key(5)] double? Padding, - [property: Key(6)] string? Overflow); + [property: Key(6)] string? Overflow, + [property: Key(7)] double? MinWidth = null, + [property: Key(8)] double? MinHeight = null, + [property: Key(9)] double? PreferredWidth = null, + [property: Key(10)] double? PreferredHeight = null, + [property: Key(11)] double? Priority = null, + [property: Key(12)] string? Density = null, + [property: Key(13)] string? ViewportMode = null); [MessagePackObject] public sealed record MimirEveDashboardUiStyleSnapshot( diff --git a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs index 99ef348..6e4f6c6 100644 --- a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs +++ b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs @@ -7,6 +7,7 @@ public sealed class MimirSynchronizationHub : IDisposable private readonly MimirComplexContourRuntimeAnalyzer? complexContourSynchronization; private readonly MimirAudioSynchronizationStateTracker audioSynchronizationState = new(); private readonly MimirBioacousticProbeScheduler bioacousticProbeScheduler = new(); + private readonly MimirCanonicalClockMapper clockMapper = new(); private readonly Dictionary audioSynchronizationReports = new(StringComparer.Ordinal); private readonly Dictionary complexContourReports = new(StringComparer.Ordinal); private readonly MimirChirpBinCalibrationModel? chirpBinCalibrationModel; @@ -44,6 +45,9 @@ public sealed class MimirSynchronizationHub : IDisposable public IReadOnlyList AudioSynchronizationStates => audioSynchronizationState.States; + public IReadOnlyList CanonicalClockMaps => + clockMapper.Snapshots; + public IReadOnlyList AudioSynchronizationReports => audioSynchronizationReports.Values.OrderBy(report => report.SourceId, StringComparer.Ordinal).ToArray(); @@ -110,9 +114,13 @@ public sealed class MimirSynchronizationHub : IDisposable var consumed = 0; foreach (var source in sources) { - for (var index = 0; index < maxSamplesPerSource && source.TryRead(out var sample); index++) + var logicalStreamCount = source is IMimirMultiplexedStreamSource multiplexed + ? Math.Max(1, multiplexed.LogicalStreamCount) + : 1; + var sourceBudget = checked(Math.Max(1, maxSamplesPerSource) * logicalStreamCount); + for (var index = 0; index < sourceBudget && source.TryRead(out var sample); index++) { - Buffers.Append(sample); + Buffers.Append(clockMapper.ToCanonical(sample)); consumed++; ingestedSamples++; } diff --git a/src/Mimir.Well/Program.cs b/src/Mimir.Well/Program.cs index 43f56df..df4f1f2 100644 --- a/src/Mimir.Well/Program.cs +++ b/src/Mimir.Well/Program.cs @@ -58,6 +58,8 @@ var nextVisualCalibration = DateTimeOffset.MinValue; var frameDegradedCount = 0L; var nextFrameDegradedLog = DateTimeOffset.MinValue; var streamTelemetry = new MimirWellStreamTelemetry(); +var latencyController = new MimirWellLatencyController(options); +var featureSignals = new MimirWellFeatureSignalBank(options); var presentation = new MimirPresentationControlState(); var exposureController = new MimirCameraExposureController(new MimirCameraExposureControlOptions( options.VisualCalibrationEnabled, @@ -102,7 +104,10 @@ while (!stopping.IsCancellationRequested) if (now >= nextPublish) { presentation.SyncFromBuffers(hub.Buffers.Buffers); - var frameResult = BuildFrameOrEmpty(hub, TimeSpan.FromMilliseconds(options.PresentationDelayMs)); + var latencyDecision = latencyController.Decide(hub, now); + var frameResult = BuildFrameOrEmpty(hub, latencyDecision.PresentationDelay); + latencyController.ObserveFrame(frameResult); + var featureSignalFrame = featureSignals.Update(hub.Buffers.Buffers, frameResult.Frame); if (!string.IsNullOrWhiteSpace(frameResult.DegradedReason)) { frameDegradedCount++; @@ -121,6 +126,8 @@ while (!stopping.IsCancellationRequested) hub, presentation, frameResult, + latencyDecision, + featureSignalFrame, exposureController.Statuses, sourceErrors, streamTelemetry.Snapshot(now, subscribersAreExternal: true), @@ -130,6 +137,21 @@ while (!stopping.IsCancellationRequested) await publisher.PublishAsync(snapshot, "mimir.well_snapshot.v1", stopping.Token).ConfigureAwait(false)); if (options.CapturePagesEnabled && now >= nextCapture) { + var nextStreamSequence = captureSequence + 1; + if (options.StreamFramesEnabled) + { + foreach (var streamFrame in MimirWellCapturePage.BuildStreamFrames( + options, + frameResult, + hub.Buffers.Buffers, + nextStreamSequence, + startedAt)) + { + streamTelemetry.ObservePublish( + await publisher.PublishAsync(streamFrame, "mimir.cultmesh_stream_frame.v1", stopping.Token).ConfigureAwait(false)); + } + } + var capturePage = MimirWellCapturePage.Build( options, presentation, @@ -195,6 +217,386 @@ internal sealed record MimirWellFrameBuildResult( string DegradedKind, string DegradedReason); +internal sealed record MimirWellLatencyDecision( + TimeSpan PresentationDelay, + TimeSpan CeilingDelay, + TimeSpan FloorDelay, + TimeSpan RetainedOverlap, + TimeSpan EdgeSkew, + double ReadinessConfidence, + double SyncConfidence, + int ActiveBufferCount, + int ReadySliceCount, + int TotalSliceCount, + string Reason); + +internal sealed class MimirWellLatencyController +{ + private readonly MimirWellOptions options; + private readonly TimeSpan floor; + private TimeSpan current; + private int completeStreak; + private int degradedStreak; + private MimirWellFrameBuildResult? lastFrame; + + public MimirWellLatencyController(MimirWellOptions options) + { + this.options = options; + floor = TimeSpan.FromMilliseconds(Math.Clamp(options.MinPresentationDelayMs, 0, options.PresentationDelayMs)); + current = TimeSpan.FromMilliseconds(Math.Clamp( + options.TargetPresentationDelayMs, + options.MinPresentationDelayMs, + options.PresentationDelayMs)); + } + + public MimirWellLatencyDecision Decide(MimirSynchronizationHub hub, DateTimeOffset now) + { + var active = hub.Buffers.Buffers + .Where(static buffer => buffer.Latest.HasValue) + .ToArray(); + if (active.Length == 0) + { + current = TimeSpan.FromMilliseconds(options.PresentationDelayMs); + return Decision(TimeSpan.Zero, TimeSpan.Zero, 0.0, 0.0, 0, 0, 0, "waiting-for-sources"); + } + + var latestEdges = active + .Select(static buffer => buffer.Latest?.TimestampNs ?? 0L) + .Where(static value => value > 0) + .ToArray(); + var starts = active + .Select(static buffer => buffer.OldestSampleTimestampNs > 0 ? buffer.OldestSampleTimestampNs : buffer.WindowStartNs) + .Where(static value => value > 0) + .ToArray(); + var minEdge = latestEdges.Length == 0 ? 0L : latestEdges.Min(); + var maxEdge = latestEdges.Length == 0 ? 0L : latestEdges.Max(); + var maxStart = starts.Length == 0 ? 0L : starts.Max(); + var retainedOverlap = minEdge > 0 && maxStart > 0 ? TimeSpan.FromTicks(Math.Max(0L, (minEdge - maxStart) / 100L)) : TimeSpan.Zero; + var edgeSkew = minEdge > 0 && maxEdge > minEdge ? TimeSpan.FromTicks((maxEdge - minEdge) / 100L) : TimeSpan.Zero; + var syncConfidence = hub.AudioSynchronizationStates.Count == 0 + ? 0.0 + : hub.AudioSynchronizationStates.Average(static state => Math.Clamp(state.Confidence, 0.0, 1.0)); + var previousReady = lastFrame?.Frame.Slices.Count(static slice => slice.Status == MimirSynchronizedSliceStatus.Ready) ?? 0; + var previousTotal = lastFrame?.Frame.Slices.Count ?? 0; + var readiness = previousTotal == 0 ? 0.0 : previousReady / (double)previousTotal; + + var ceiling = TimeSpan.FromMilliseconds(options.PresentationDelayMs); + var target = TimeSpan.FromMilliseconds(Math.Clamp( + Math.Max(options.MinPresentationDelayMs, edgeSkew.TotalMilliseconds + options.LatencyGuardMs), + options.MinPresentationDelayMs, + options.PresentationDelayMs)); + var reason = "edge-skew-plus-guard"; + if (retainedOverlap > TimeSpan.Zero) + { + var overlapBound = retainedOverlap - TimeSpan.FromMilliseconds(options.LatencyGuardMs); + if (overlapBound > floor) + { + target = target < overlapBound ? target : overlapBound; + } + else + { + target = floor; + reason = "overlap-thin"; + } + } + + if (lastFrame != null && !lastFrame.Frame.IsComplete) + { + target += TimeSpan.FromMilliseconds(options.LatencyStepMs * Math.Max(1, degradedStreak)); + reason = "backoff-after-incomplete-frame"; + } + else if (completeStreak >= Math.Max(1, options.LatencyConvergenceFrames) && readiness >= options.LatencyReadinessTarget) + { + target -= TimeSpan.FromMilliseconds(options.LatencyStepMs); + reason = "complete-frame-convergence"; + } + + current = ClampDelay(Smooth(current, target), floor, ceiling); + return Decision(retainedOverlap, edgeSkew, readiness, syncConfidence, active.Length, previousReady, previousTotal, reason); + } + + public void ObserveFrame(MimirWellFrameBuildResult result) + { + lastFrame = result; + if (result.Frame.IsComplete) + { + completeStreak++; + degradedStreak = 0; + } + else + { + degradedStreak++; + completeStreak = 0; + } + } + + private MimirWellLatencyDecision Decision( + TimeSpan retainedOverlap, + TimeSpan edgeSkew, + double readiness, + double syncConfidence, + int activeBuffers, + int readySlices, + int totalSlices, + string reason) => + new( + current, + TimeSpan.FromMilliseconds(options.PresentationDelayMs), + floor, + retainedOverlap, + edgeSkew, + readiness, + syncConfidence, + activeBuffers, + readySlices, + totalSlices, + reason); + + private static TimeSpan Smooth(TimeSpan current, TimeSpan target) + { + var alpha = target > current ? 0.65 : 0.20; + return TimeSpan.FromTicks((long)Math.Round(current.Ticks + (target.Ticks - current.Ticks) * alpha)); + } + + private static TimeSpan ClampDelay(TimeSpan value, TimeSpan floor, TimeSpan ceiling) => + value < floor ? floor : value > ceiling ? ceiling : value; +} + +internal sealed record MimirWellFeatureSignalFrame( + string Document, + long Sequence, + IReadOnlyList Signals, + double MeanConfidence, + double MeanMotionPixelsPerSecond, + int StableTrackCount, + string FaustSignalContract); + +internal sealed record MimirWellFeatureSignal( + string SourceId, + long TimestampNs, + int Width, + int Height, + int StableTrackCount, + double Confidence, + double MeanMotionPixelsPerSecond, + double MotionEnergy, + double NormalizedCentroidX, + double NormalizedCentroidY, + IReadOnlyDictionary FaustControls); + +internal sealed class MimirWellFeatureSignalBank +{ + private readonly MimirWellOptions options; + private readonly Dictionary trackers = new(StringComparer.Ordinal); + private readonly Dictionary lastSequences = new(StringComparer.Ordinal); + private long sequence; + + public MimirWellFeatureSignalBank(MimirWellOptions options) + { + this.options = options; + } + + public MimirWellFeatureSignalFrame Update( + IEnumerable buffers, + MimirSynchronizedBufferFrame frame) + { + if (!options.VideoFeatureSignalsEnabled) + { + return Empty(); + } + + var signals = frame.VideoSlices + .Where(static slice => slice.Sample.HasValue) + .Select(slice => BuildSignal(slice.Sample!.Value)) + .Where(static signal => signal != null) + .Select(static signal => signal!) + .ToArray(); + if (signals.Length == 0) + { + signals = buffers + .Where(static buffer => buffer.Descriptor.Kind == MimirStreamKind.Video && buffer.Latest.HasValue) + .Select(buffer => BuildSignal(buffer.Latest!.Value)) + .Where(static signal => signal != null) + .Select(static signal => signal!) + .ToArray(); + } + + if (signals.Length == 0) + { + return Empty(); + } + + return new MimirWellFeatureSignalFrame( + "mimir.well_feature_signals.v1", + ++sequence, + signals, + signals.Average(static signal => signal.Confidence), + signals.Average(static signal => signal.MeanMotionPixelsPerSecond), + signals.Sum(static signal => signal.StableTrackCount), + "Faust controls are normalized scalar signals under video//{motion_energy,confidence,centroid_x,centroid_y}; Mimir estimates, Faust/native DSP consumes."); + } + + private MimirWellFeatureSignalFrame Empty() => + new("mimir.well_feature_signals.v1", sequence, [], 0.0, 0.0, 0, "no feature-bearing CPU video slice available"); + + private MimirWellFeatureSignal? BuildSignal(MimirStreamSample sample) + { + if (sample.VideoFrame is not { } video || + sample.Data.IsEmpty || + sample.Sequence == (lastSequences.TryGetValue(sample.SourceId, out var previous) ? previous : ulong.MaxValue)) + { + return null; + } + + var luma = ExtractLuma(video, sample.Data.Span); + if (luma.Length == 0) + { + return null; + } + + lastSequences[sample.SourceId] = sample.Sequence; + if (!trackers.TryGetValue(sample.SourceId, out var tracker)) + { + tracker = new MimirSparseFeatureTracker(new MimirSparseFeatureTrackerOptions( + MaxFeatures: options.VideoFeatureMaxTracks, + CellSizePixels: options.VideoFeatureCellSizePixels, + SearchRadiusPixels: options.VideoFeatureSearchRadiusPixels)); + trackers.Add(sample.SourceId, tracker); + } + + var tracked = tracker.Update( + sample.SourceId, + video.Width, + video.Height, + luma, + sample.TimestampNs); + if (tracked.Tracks.Count == 0) + { + return new MimirWellFeatureSignal( + sample.SourceId, + sample.TimestampNs, + video.Width, + video.Height, + 0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + FaustControls(sample.SourceId, 0.0, 0.0, 0.5, 0.5)); + } + + var stableTracks = tracked.Tracks.Where(track => track.AgeFrames >= 3).ToArray(); + var points = stableTracks.Length == 0 ? tracked.Tracks : stableTracks; + var centroidX = points.Average(static track => track.ImageX) / Math.Max(1.0, video.Width); + var centroidY = points.Average(static track => track.ImageY) / Math.Max(1.0, video.Height); + var motionEnergy = Math.Clamp(tracked.MeanSpeedPixelsPerSecond / Math.Max(1.0, Math.Max(video.Width, video.Height)), 0.0, 1.0); + return new MimirWellFeatureSignal( + sample.SourceId, + sample.TimestampNs, + video.Width, + video.Height, + tracked.StableTrackCount, + tracked.Confidence, + tracked.MeanSpeedPixelsPerSecond, + motionEnergy, + Math.Clamp(centroidX, 0.0, 1.0), + Math.Clamp(centroidY, 0.0, 1.0), + FaustControls(sample.SourceId, tracked.Confidence, motionEnergy, centroidX, centroidY)); + } + + private static IReadOnlyDictionary FaustControls( + string sourceId, + double confidence, + double motionEnergy, + double centroidX, + double centroidY) + { + var prefix = $"video/{sourceId}"; + return new Dictionary(StringComparer.Ordinal) + { + [$"{prefix}/confidence"] = (float)Math.Clamp(confidence, 0.0, 1.0), + [$"{prefix}/motion_energy"] = (float)Math.Clamp(motionEnergy, 0.0, 1.0), + [$"{prefix}/centroid_x"] = (float)Math.Clamp(centroidX, 0.0, 1.0), + [$"{prefix}/centroid_y"] = (float)Math.Clamp(centroidY, 0.0, 1.0), + }; + } + + private static byte[] ExtractLuma(MimirVideoFrameDescriptor video, ReadOnlySpan data) => + video.PixelFormat switch + { + MimirVideoPixelFormat.Gray8 or MimirVideoPixelFormat.R8 or MimirVideoPixelFormat.Bayer8 => + CopySinglePlane(video.Width, video.Height, Math.Max(video.Width, video.StrideBytes), data), + MimirVideoPixelFormat.LeapStereoIr or MimirVideoPixelFormat.Rg8 => + CopyInterleavedLuma(video.Width, video.Height, Math.Max(video.Width * 2, video.StrideBytes), 2, data), + MimirVideoPixelFormat.Yuy2 => + CopyInterleavedLuma(video.Width, video.Height, Math.Max(video.Width * 2, video.StrideBytes), 2, data), + MimirVideoPixelFormat.Bgra8 => + CopyBgraLuma(video.Width, video.Height, Math.Max(video.Width * 4, video.StrideBytes), data), + _ => [], + }; + + private static byte[] CopySinglePlane(int width, int height, int stride, ReadOnlySpan data) + { + if (width <= 0 || height <= 0 || data.Length < stride * Math.Max(0, height - 1) + width) + { + return []; + } + + var luma = new byte[width * height]; + for (var y = 0; y < height; y++) + { + data.Slice(y * stride, width).CopyTo(luma.AsSpan(y * width, width)); + } + + return luma; + } + + private static byte[] CopyInterleavedLuma(int width, int height, int stride, int step, ReadOnlySpan data) + { + if (width <= 0 || height <= 0 || step <= 0 || data.Length < stride * Math.Max(0, height - 1) + width * step) + { + return []; + } + + var luma = new byte[width * height]; + for (var y = 0; y < height; y++) + { + var row = data.Slice(y * stride, width * step); + for (var x = 0; x < width; x++) + { + luma[y * width + x] = row[x * step]; + } + } + + return luma; + } + + private static byte[] CopyBgraLuma(int width, int height, int stride, ReadOnlySpan data) + { + if (width <= 0 || height <= 0 || data.Length < stride * Math.Max(0, height - 1) + width * 4) + { + return []; + } + + var luma = new byte[width * height]; + for (var y = 0; y < height; y++) + { + var row = data.Slice(y * stride, width * 4); + for (var x = 0; x < width; x++) + { + var offset = x * 4; + luma[y * width + x] = (byte)Math.Clamp( + (int)Math.Round(row[offset] * 0.114 + row[offset + 1] * 0.587 + row[offset + 2] * 0.299), + 0, + 255); + } + } + + return luma; + } +} + internal sealed record MimirWellPublishReceipt( string Document, int ByteLength, @@ -284,8 +686,18 @@ internal sealed record MimirWellOptions( int PublishIntervalMs, int SyncIntervalMs, int PresentationDelayMs, + int TargetPresentationDelayMs, + int MinPresentationDelayMs, + int LatencyGuardMs, + int LatencyStepMs, + int LatencyConvergenceFrames, + double LatencyReadinessTarget, int MaxSamplesPerSource, int SyncCandidatesPerStep, + bool VideoFeatureSignalsEnabled, + int VideoFeatureMaxTracks, + int VideoFeatureCellSizePixels, + int VideoFeatureSearchRadiusPixels, bool VisualCalibrationEnabled, int VisualCalibrationIntervalMs, int VisualExpectedLedCount, @@ -296,6 +708,8 @@ internal sealed record MimirWellOptions( int CaptureIntervalMs, int CaptureMaxBodyBytes, bool CaptureInlineBodies, + bool StreamFramesEnabled, + bool StreamFrameInlineBodies, int MeterEvery) { public static MimirWellOptions Parse(string[] args) => new( @@ -306,8 +720,18 @@ internal sealed record MimirWellOptions( ParseInt(args, "--publish-ms", 250), ParseInt(args, "--sync-ms", 250), ParseInt(args, "--presentation-delay-ms", 2500), + ParseInt(args, "--target-presentation-delay-ms", 750), + ParseInt(args, "--min-presentation-delay-ms", 40), + ParseInt(args, "--latency-guard-ms", 35), + ParseInt(args, "--latency-step-ms", 25), + ParseInt(args, "--latency-convergence-frames", 8), + ParseDouble(args, "--latency-readiness-target", 0.98), ParseInt(args, "--max-samples-per-source", 4), ParseInt(args, "--sync-candidates", 1), + ParseBool(args, "--video-feature-signals", true), + ParseInt(args, "--video-feature-max-tracks", 96), + ParseInt(args, "--video-feature-cell-size", 12), + ParseInt(args, "--video-feature-search-radius", 18), ParseBool(args, "--visual-calibration", true), ParseInt(args, "--visual-calibration-ms", 250), ParseInt(args, "--visual-expected-leds", 38), @@ -318,6 +742,8 @@ internal sealed record MimirWellOptions( ParseInt(args, "--capture-ms", 250), ParseInt(args, "--capture-max-body-bytes", 4 * 1024 * 1024), ParseBool(args, "--capture-inline-bodies", true), + ParseBool(args, "--stream-frames", true), + ParseBool(args, "--stream-frame-inline-bodies", true), ParseInt(args, "--meter-every", 20)); private static string ParseString(IReadOnlyList args, string name, string fallback) @@ -475,6 +901,29 @@ internal static class MimirWellCapturePage }; } + public static IEnumerable BuildStreamFrames( + MimirWellOptions options, + MimirWellFrameBuildResult frameResult, + IEnumerable buffers, + long captureSequence, + DateTimeOffset startedAt) + { + var frame = frameResult.Frame; + var sliceFrames = frame.Slices + .Where(slice => slice.Sample.HasValue) + .Select(slice => CaptureStreamFrame(options, captureSequence, startedAt, slice)) + .ToArray(); + if (sliceFrames.Length > 0) + { + return sliceFrames; + } + + return buffers + .Where(buffer => buffer.Latest.HasValue) + .Select(buffer => CaptureLatestFallbackStreamFrame(options, captureSequence, startedAt, buffer)) + .ToArray(); + } + private static IEnumerable CaptureSamples( MimirWellOptions options, long captureSequence, @@ -521,7 +970,7 @@ internal static class MimirWellCapturePage slice.TimingEvidenceKind, }, sample = SampleMetadata(sample), - body = CaptureBody(options, sample), + body = CaptureBody(options, sample, options.CaptureInlineBodies), }; } @@ -551,7 +1000,115 @@ internal static class MimirWellCapturePage TimingEvidenceKind = "unsynchronized-fallback", }, sample = SampleMetadata(sample), - body = CaptureBody(options, sample), + body = CaptureBody(options, sample, options.CaptureInlineBodies), + }; + } + + private static object CaptureStreamFrame( + MimirWellOptions options, + long captureSequence, + DateTimeOffset startedAt, + MimirSynchronizedStreamSlice slice) + { + var sample = slice.Sample!.Value; + return CaptureStreamFrame( + options, + captureSequence, + startedAt, + sample, + new + { + slice.SourceId, + Kind = slice.Kind.ToString(), + Origin = slice.Origin.ToString(), + Status = slice.Status.ToString(), + slice.SourceTimestampNs, + slice.CanonicalStartNs, + slice.CanonicalEndNs, + slice.PresentationTimeNs, + slice.TimingOffsetNs, + slice.DistanceFromPresentationNs, + slice.TimingConfidence, + slice.TimingEvidenceKind, + }); + } + + private static object CaptureLatestFallbackStreamFrame( + MimirWellOptions options, + long captureSequence, + DateTimeOffset startedAt, + MimirRollingStreamBuffer buffer) + { + var sample = buffer.Latest!.Value; + return CaptureStreamFrame( + options, + captureSequence, + startedAt, + sample, + new + { + sample.SourceId, + Kind = sample.Kind.ToString(), + Origin = sample.Origin.ToString(), + Status = "LatestUnalignedFallback", + SourceTimestampNs = sample.TimestampNs, + CanonicalStartNs = sample.TimestampNs, + CanonicalEndNs = sample.TimestampNs, + PresentationTimeNs = 0L, + TimingOffsetNs = 0L, + DistanceFromPresentationNs = 0L, + TimingConfidence = 0.0, + TimingEvidenceKind = "unsynchronized-fallback", + }); + } + + private static object CaptureStreamFrame( + MimirWellOptions options, + long captureSequence, + DateTimeOffset startedAt, + MimirStreamSample sample, + object slice) + { + var bodyId = $"{options.NodeId}:{captureSequence}:{sample.SourceId}:{sample.Sequence}"; + var transport = PreferredBodyTransport(sample); + return new + { + type = "cultmesh-observation", + document = "mimir.cultmesh_stream_frame.v1", + sourceId = "mimir-well", + nodeId = options.NodeId, + captureSequence, + wallClockUtc = DateTimeOffset.UtcNow.ToString("O"), + elapsedSeconds = (DateTimeOffset.UtcNow - startedAt).TotalSeconds, + bodyId, + stream = new + { + streamId = sample.SourceId, + verseId = "mimir-live", + ownerPeerId = options.NodeId, + kind = sample.Kind.ToString(), + origin = sample.Origin.ToString(), + preferredTransports = StreamTransports(sample), + clockDomainId = sample.SourceId, + }, + frame = new + { + streamId = sample.SourceId, + sequence = sample.Sequence, + timestampNs = sample.TimestampNs, + arrivalNs = sample.ArrivalNs, + byteLength = sample.ByteLength, + bodyTransport = transport, + nativeHandle = NativeHandle(sample), + nativeHandleKind = NativeHandleKind(sample), + resourceKey = sample.VideoFrame?.ResourceKey, + producerFenceHandle = sample.VideoFrame?.ProducerFenceHandle, + producerFenceValue = sample.VideoFrame?.ProducerFenceValue ?? 0UL, + unavoidableCopyCount = sample.VideoFrame?.UnavoidableCopyCount ?? 0, + }, + slice, + sample = SampleMetadata(sample), + body = CaptureBody(options, sample, options.StreamFrameInlineBodies), }; } @@ -591,9 +1148,43 @@ internal static class MimirWellCapturePage }, }; - private static object CaptureBody(MimirWellOptions options, MimirStreamSample sample) + private static string PreferredBodyTransport(MimirStreamSample sample) { - if (!options.CaptureInlineBodies) + var kind = NativeHandleKind(sample); + if (!string.IsNullOrWhiteSpace(kind)) + { + return kind.Contains("d3d12", StringComparison.OrdinalIgnoreCase) + ? "shared-d3d12-texture" + : kind.Contains("d3d11", StringComparison.OrdinalIgnoreCase) + ? "shared-d3d11-texture" + : "native-handle"; + } + + return sample.Data.IsEmpty ? "metadata-only" : "inline-bytes"; + } + + private static string[] StreamTransports(MimirStreamSample sample) + { + var transport = PreferredBodyTransport(sample); + return transport switch + { + "shared-d3d12-texture" => ["shared-d3d12-texture", "mimir-recorder-page", "cultcache-page"], + "shared-d3d11-texture" => ["shared-d3d11-texture", "mimir-recorder-page", "cultcache-page"], + "native-handle" => ["native-handle", "mimir-recorder-page", "cultcache-page"], + "inline-bytes" => ["mimir-recorder-page", "inline-bytes", "cultcache-page"], + _ => ["metadata-only"], + }; + } + + private static ulong NativeHandle(MimirStreamSample sample) => + sample.VideoFrame?.NativeHandle ?? sample.AudioBlock?.NativeHandle ?? 0UL; + + private static string NativeHandleKind(MimirStreamSample sample) => + sample.VideoFrame?.NativeHandleKind ?? sample.AudioBlock?.NativeHandleKind ?? ""; + + private static object CaptureBody(MimirWellOptions options, MimirStreamSample sample, bool inlineBodies) + { + if (!inlineBodies) { return new { status = "not-inline", sample.ByteLength }; } @@ -728,6 +1319,8 @@ internal static class MimirWellSnapshot MimirSynchronizationHub hub, MimirPresentationControlState presentation, MimirWellFrameBuildResult frameResult, + MimirWellLatencyDecision latencyDecision, + MimirWellFeatureSignalFrame featureSignals, IReadOnlyList visualCalibration, IReadOnlyList sourceErrors, object streamPressure, @@ -760,11 +1353,66 @@ internal static class MimirWellSnapshot degradedReason = frameResult.DegradedReason, slices = frame.Slices.Select(SliceSnapshot).ToArray(), }, + latency = new + { + document = "mimir.well_latency_policy.v1", + mode = "adaptive-bounded-ceiling", + currentDelayMs = latencyDecision.PresentationDelay.TotalMilliseconds, + ceilingDelayMs = latencyDecision.CeilingDelay.TotalMilliseconds, + floorDelayMs = latencyDecision.FloorDelay.TotalMilliseconds, + retainedOverlapMs = latencyDecision.RetainedOverlap.TotalMilliseconds, + edgeSkewMs = latencyDecision.EdgeSkew.TotalMilliseconds, + latencyDecision.ReadinessConfidence, + latencyDecision.SyncConfidence, + latencyDecision.ActiveBufferCount, + latencyDecision.ReadySliceCount, + latencyDecision.TotalSliceCount, + latencyDecision.Reason, + invariant = "Five seconds is a ceiling; the Well lowers requested presentation delay when overlap, slice readiness, and sync evidence allow it.", + }, clockDomains = MimirWellClockDiagnostics.Build( hub.Buffers.Buffers, runtimeConfig.Settings.Audio.ReferenceSourceId, frameResult), + canonicalClockMaps = hub.CanonicalClockMaps.Select(map => new + { + map.StreamKey, + map.OffsetNs, + offsetMs = map.OffsetNs / 1_000_000.0, + map.FirstSourceTimestampNs, + map.FirstArrivalNs, + map.LatestSourceTimestampNs, + map.LatestCanonicalTimestampNs, + map.SampleCount, + }).ToArray(), streamPressure, + featureSignals = new + { + featureSignals.Document, + featureSignals.Sequence, + featureSignals.MeanConfidence, + featureSignals.MeanMotionPixelsPerSecond, + featureSignals.StableTrackCount, + featureSignals.FaustSignalContract, + signals = featureSignals.Signals.Select(signal => new + { + signal.SourceId, + signal.TimestampNs, + signal.Width, + signal.Height, + signal.StableTrackCount, + signal.Confidence, + signal.MeanMotionPixelsPerSecond, + signal.MotionEnergy, + signal.NormalizedCentroidX, + signal.NormalizedCentroidY, + faustControls = signal.FaustControls.Select(pair => new + { + path = pair.Key, + value = pair.Value, + }).ToArray(), + }).ToArray(), + }, audioSync = new { referenceSourceId = runtimeConfig.Settings.Audio.ReferenceSourceId, diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 07c373c..dc2d7cc 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -354,3 +354,4 @@ {"ts": "2026-06-02T14:13:56+01:00", "type": "decision", "status": "ok", "note": "Starfire USB PS Move shakedown: Mimir.PsMoveProbe sees three Sony VID_054C/PID_03D5 HID collections and can write report 0x06 to col01. Added --rgb/--pulse-ms/--pulse-count/--interval-ms/--event-log; proof wrote readable mimir.psmove_usb_pulse_event.v1 receipts and 49-byte on/off reports. This is a local fast visual timing witness primitive; precision scheduling should move into a persistent/native worker because CLI pulse proof showed requested 60ms pulse producing about 221ms on/off gap."} {"ts": "2026-06-02T14:41:46+01:00", "type": "verification", "status": "ok", "note": "Starfire USB PS Move paired chirp train now derives audio samples and visual LED pulses from one event schedule. 2026-06-02 run artifacts/runtime/starfire-usb-move-paired-20260602-144135 emitted three WASAPI chirps and three Move LED pulses with shared event ids; receipt showed planned/observed LED onsets 150/400/650 ms at millisecond resolution after high-resolution wait cut."} {"ts": "2026-06-02T15:09:31+01:00", "type": "verification", "status": "ok", "note": "mimir_well_shakedown.py now verifies paired PS Move chirp receipts: scheduled events must have nonzero audio samples, render completion, LED on/off phases, no orphan visual events, and LED schedule error within a configurable ceiling. Good receipt starfire-usb-move-paired-20260602-144135 passes under 5 ms; earlier receipt starfire-usb-move-paired-20260602-144003 fails with 67 ms max error and exit 2 under --fail-on-paired-pulse-error."} +{"ts": "2026-06-02T15:12:44+01:00", "type": "verification", "status": "ok", "note": "Adaptive Well/runtime dashboard cut builds clean: Mimir.Runtime, Mimir.Well, and Mimir.EveDashboard compile after canonical clock mapping, ASIO multiplex drain, adaptive latency telemetry, feature-signal mirror, and richer Eve dashboard lowering. Rebooted lightweight Eve broker/Odin projection on 2026-06-02; health ok at 127.0.0.1:8795 and 8797 without restarting heavy Well capture."}