Add adaptive Well timing telemetry
This commit is contained in:
Meta Vens
parent
05be4b2e44
commit
ec91c58408
8 changed files with 1166 additions and 34 deletions
|
|
@ -366,16 +366,23 @@ internal sealed class EveDashboardServer(int port, EveDashboardProviderCatalog p
|
|||
element.AssetUri,
|
||||
element.BindNodeId,
|
||||
element.CommandId,
|
||||
element.Layout == null
|
||||
? null
|
||||
: new MimirEveDashboardUiLayoutSnapshot(
|
||||
element.Layout.Direction,
|
||||
element.Layout.Width,
|
||||
element.Layout.Height,
|
||||
element.Layout.Grow,
|
||||
element.Layout.Gap,
|
||||
element.Layout.Padding,
|
||||
element.Layout.Overflow),
|
||||
element.Layout == null
|
||||
? null
|
||||
: new MimirEveDashboardUiLayoutSnapshot(
|
||||
element.Layout.Direction,
|
||||
element.Layout.Width,
|
||||
element.Layout.Height,
|
||||
element.Layout.Grow,
|
||||
element.Layout.Gap,
|
||||
element.Layout.Padding,
|
||||
element.Layout.Overflow,
|
||||
element.Layout.MinWidth,
|
||||
element.Layout.MinHeight,
|
||||
element.Layout.PreferredWidth,
|
||||
element.Layout.PreferredHeight,
|
||||
element.Layout.Priority,
|
||||
element.Layout.Density,
|
||||
element.Layout.ViewportMode),
|
||||
element.Style == null
|
||||
? null
|
||||
: new MimirEveDashboardUiStyleSnapshot(element.Style.Variant, element.Style.Tone),
|
||||
|
|
@ -899,8 +906,8 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
StatBar("audio-rms", "max channel RMS", NormalizeRms(rms), "warm", $"{rms:0.000000}"),
|
||||
StatBar("observation-liveness", "device stream liveness", snapshot.ObservationStreams.Count == 0 ? 0.0 : liveObservationCount / (double)snapshot.ObservationStreams.Count, "cool"),
|
||||
]),
|
||||
BuildMoveMusicPane(snapshot.MoveMusic),
|
||||
BuildWellPane(snapshot.Well),
|
||||
BuildMoveMusicPane(snapshot.MoveMusic),
|
||||
]),
|
||||
UiElement.Container(
|
||||
"mimir-live-row-runtime",
|
||||
|
|
@ -930,7 +937,21 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
Root = UiElement.Container(
|
||||
"mimir-live-stats-root",
|
||||
"dashboard",
|
||||
new DashboardUiLayout { Direction = "vertical", Gap = 6, Padding = 6, Overflow = "scroll" },
|
||||
new DashboardUiLayout
|
||||
{
|
||||
Direction = "vertical",
|
||||
Gap = 6,
|
||||
Padding = 6,
|
||||
Overflow = "scroll",
|
||||
Grow = 4,
|
||||
MinWidth = 96,
|
||||
MinHeight = 56,
|
||||
PreferredWidth = 156,
|
||||
PreferredHeight = 128,
|
||||
Priority = -80,
|
||||
Density = "continuous",
|
||||
ViewportMode = "continuous-metrics",
|
||||
},
|
||||
children),
|
||||
};
|
||||
}
|
||||
|
|
@ -1085,6 +1106,19 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
UiElement.Text("mimir-move-score", $"score {Bar(music.ScoreEnvelopeMax, 18)} max env {music.ScoreEnvelopeMax:0.000} max rgb {music.MaxRgb}", "caption"),
|
||||
UiElement.Text("mimir-move-sources", string.Join(" | ", music.Sources.Take(4)), "caption"),
|
||||
};
|
||||
children.AddRange(music.Voices
|
||||
.Take(5)
|
||||
.Select(voice => UiElement.Text(
|
||||
$"mimir-move-voice-{StableId(voice.SourceId)}",
|
||||
$"{voice.SourceId}: {voice.NoteName} {voice.FrequencyHz:0.0}Hz c={voice.Confidence:0.000} {voice.Role}",
|
||||
"caption")));
|
||||
children.Add(UiElement.Text("mimir-move-piano-roll", music.PianoRoll, "mono"));
|
||||
children.AddRange(music.MoveTargets
|
||||
.Take(5)
|
||||
.Select(target => UiElement.Text(
|
||||
$"mimir-move-target-{target.MoveIndex}",
|
||||
$"move {target.MoveIndex}: {target.NoteName} {target.SourceId} priority {target.CalibrationPriority:0.000}",
|
||||
"caption")));
|
||||
|
||||
return UiElement.Pane("mimir-move-music-pane", "Move Score", children);
|
||||
}
|
||||
|
|
@ -1096,12 +1130,47 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
return UiElement.Pane("mimir-well-pane", "Well", [UiElement.Text("mimir-well-empty", "no Well snapshot yet", "caption")]);
|
||||
}
|
||||
|
||||
return UiElement.Pane("mimir-well-pane", "Well", [
|
||||
UiElement.Text("mimir-well-summary", $"seq {well.Sequence} {well.ElapsedSeconds:0}s ingested {well.IngestedSamples:0} sources {well.LiveSources}/{well.ConfiguredSources}", "mono"),
|
||||
var children = new List<DashboardUiElement>
|
||||
{
|
||||
UiElement.Text("mimir-well-summary", $"seq {well.Sequence} {well.ElapsedSeconds:0}s ingested {well.IngestedSamples:0} sources {well.LiveSources}/{well.ConfiguredSources} source errors {well.SourceErrorCount}", "mono"),
|
||||
StatBar("mimir-well-completeness", "frame completeness", well.SynchronizedFrameComplete ? 1.0 : 0.0, well.SynchronizedFrameComplete ? "cool" : "danger"),
|
||||
UiElement.Text("mimir-well-buffers", $"buffers {well.Buffers.Count} audio {well.AudioBuffers} video {well.VideoBuffers} ready {well.ReadySlices}/{well.TotalSlices}", "caption"),
|
||||
UiElement.Text("mimir-well-frame", $"frame ready {well.ReadySlices}/{well.TotalSlices} delay {well.PresentationDelayMs:0}ms degraded {well.FrameDegradedKind} {well.FrameDegradedReason}", "caption"),
|
||||
UiElement.Text("mimir-well-latency", $"latency {well.LatencyCurrentMs:0}ms floor {well.LatencyFloorMs:0} ceiling {well.LatencyCeilingMs:0} overlap {well.LatencyRetainedOverlapMs:0}ms skew {well.LatencyEdgeSkewMs:0.0}ms {well.LatencyReason}", "caption"),
|
||||
UiElement.Text("mimir-well-pressure", $"poll {well.PollAverageMs:0.000}/{well.PollMaxMs:0.000}ms zero {well.ZeroPollIterations} publish {well.PublishAverageMs:0.000}/{well.PublishMaxMs:0.000}ms bytes {well.PublishedBytes:0}", "caption"),
|
||||
UiElement.Text("mimir-well-buffers", $"buffers {well.Buffers.Count} audio {well.AudioBuffers} video {well.VideoBuffers} capture bodies {well.CaptureInlineBodies}", "caption"),
|
||||
UiElement.Text("mimir-well-clock-maps", $"canonical clocks {well.CanonicalClockMaps.Count} max offset {well.MaxCanonicalClockOffsetMs:0.000}ms", "caption"),
|
||||
UiElement.Text("mimir-well-features", $"video features stable {well.FeatureStableTracks} c={well.FeatureMeanConfidence:0.000} motion {well.FeatureMeanMotionPixelsPerSecond:0.0}px/s faust signals {well.FeatureSignals.Count}", "caption"),
|
||||
UiElement.Text("mimir-well-probe", well.Probe == null ? "probe: none" : $"probe emit={well.Probe.ShouldEmit} sync={well.Probe.SyncConfidence:0.000} freq={well.Probe.FrequencyConfidence:0.000} {well.Probe.Reason}", "caption"),
|
||||
]);
|
||||
};
|
||||
children.AddRange(well.FeatureSignals
|
||||
.OrderByDescending(static signal => signal.Confidence)
|
||||
.ThenBy(static signal => signal.SourceId, StringComparer.Ordinal)
|
||||
.Take(5)
|
||||
.Select(signal => UiElement.Text(
|
||||
$"mimir-well-feature-{StableId(signal.SourceId)}",
|
||||
$"{signal.SourceId}: tracks {signal.StableTrackCount} c={signal.Confidence:0.000} motion {signal.MotionEnergy:0.000} centroid {signal.CentroidX:0.00},{signal.CentroidY:0.00}",
|
||||
"caption")));
|
||||
children.AddRange(well.ClockDomains
|
||||
.Take(6)
|
||||
.Select(domain => UiElement.Text(
|
||||
$"mimir-well-clock-{StableId(domain.ClockDomainId)}",
|
||||
$"{domain.ClockDomainId}: {domain.SourceCount} src overlap {domain.OverlapMs:0.000}ms offset {domain.OffsetMs:0.000}ms {(domain.IsReference ? "ref" : "")}",
|
||||
"caption")));
|
||||
children.AddRange(well.CanonicalClockMaps
|
||||
.OrderByDescending(static map => Math.Abs(map.OffsetMs))
|
||||
.ThenBy(static map => map.StreamKey, StringComparer.Ordinal)
|
||||
.Take(4)
|
||||
.Select(map => UiElement.Text(
|
||||
$"mimir-well-clock-map-{StableId(map.StreamKey)}",
|
||||
$"{map.StreamKey}: ingress offset {map.OffsetMs:0.000}ms samples {map.SampleCount}",
|
||||
"caption")));
|
||||
children.AddRange(well.VisualCalibration
|
||||
.Take(6)
|
||||
.Select(camera => UiElement.Text(
|
||||
$"mimir-well-cal-{StableId(camera.SourceId)}",
|
||||
$"{camera.SourceId}: score {camera.BestScore:0.000} leds {camera.BestDetectedLedCount} exp {camera.BestExposure:0} gain {camera.BestGain:0} {camera.State}",
|
||||
"caption")));
|
||||
return UiElement.Pane("mimir-well-pane", "Well", children);
|
||||
}
|
||||
|
||||
private static DashboardUiElement BuildCalibrationPane(MimirLiveStatsSnapshot snapshot)
|
||||
|
|
@ -1225,7 +1294,7 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
ActuatorCommands = LatestBySource(telemetryLines.Select(ParseActuatorCommand).Where(static value => value != null)!),
|
||||
ObservationStreams = LatestObservationStreams(observationLines),
|
||||
Well = LatestWellSnapshot(observationLines),
|
||||
MoveMusic = LatestMoveMusicTrace(),
|
||||
MoveMusic = LatestMoveMusicTrace(telemetrySource, telemetryLines),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -1361,21 +1430,103 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
JsonNumberAny(probeElement, "AggregateFrequencyResponseConfidence", "aggregateFrequencyResponseConfidence"),
|
||||
JsonStringAny(probeElement, "reason") ?? "unknown")
|
||||
: null;
|
||||
var streamPressure = TryGetAny(root, "streamPressure");
|
||||
var pollPressure = TryGetAny(streamPressure, "poll");
|
||||
var publishPressure = TryGetAny(streamPressure, "publish");
|
||||
var latency = TryGetAny(root, "latency");
|
||||
var featureSignals = TryGetAny(root, "featureSignals");
|
||||
var featureSignalStats = ArrayItems(TryGetAny(featureSignals, "signals"))
|
||||
.Select(static signal => new MimirWellFeatureSignalStat(
|
||||
JsonStringAny(signal, "SourceId", "sourceId") ?? "unknown",
|
||||
(int)JsonNumberAny(signal, "StableTrackCount", "stableTrackCount"),
|
||||
JsonNumberAny(signal, "Confidence", "confidence"),
|
||||
JsonNumberAny(signal, "MeanMotionPixelsPerSecond", "meanMotionPixelsPerSecond"),
|
||||
JsonNumberAny(signal, "MotionEnergy", "motionEnergy"),
|
||||
JsonNumberAny(signal, "NormalizedCentroidX", "normalizedCentroidX"),
|
||||
JsonNumberAny(signal, "NormalizedCentroidY", "normalizedCentroidY")))
|
||||
.ToArray();
|
||||
var canonicalClockMaps = ArrayItems(TryGetAny(root, "canonicalClockMaps"))
|
||||
.Select(static map => new MimirWellCanonicalClockMapStat(
|
||||
JsonStringAny(map, "StreamKey", "streamKey") ?? "unknown",
|
||||
JsonNumberAny(map, "offsetMs"),
|
||||
(long)JsonNumberAny(map, "SampleCount", "sampleCount")))
|
||||
.ToArray();
|
||||
var clockDomains = ArrayItems(TryGetAny(TryGetAny(root, "clockDomains"), "domains"))
|
||||
.Select(static domain => new MimirWellClockDomainStat(
|
||||
JsonStringAny(domain, "ClockDomainId", "clockDomainId") ?? "unknown",
|
||||
(int)JsonNumberAny(domain, "SourceCount", "sourceCount"),
|
||||
JsonNumberAny(domain, "OverlapNs", "overlapNs") / 1_000_000.0,
|
||||
JsonBoolAny(domain, "HasLocalOverlap", "hasLocalOverlap"),
|
||||
JsonBoolAny(domain, "IsReferenceDomain", "isReferenceDomain"),
|
||||
JsonNumberAny(domain, "ProvisionalOffsetToReferenceMs", "provisionalOffsetToReferenceMs")))
|
||||
.ToArray();
|
||||
var visualCalibration = ArrayItems(TryGetAny(TryGetAny(root, "visualCalibration"), "cameras"))
|
||||
.Select(static camera => new MimirWellVisualCalibrationStat(
|
||||
JsonStringAny(camera, "SourceId", "sourceId") ?? "unknown",
|
||||
JsonStringAny(camera, "State", "state") ?? "unknown",
|
||||
JsonNumberAny(camera, "BestScore", "bestScore"),
|
||||
(int)JsonNumberAny(camera, "BestDetectedLedCount", "bestDetectedLedCount"),
|
||||
JsonBoolAny(camera, "BestUsableForCalibration", "bestUsableForCalibration"),
|
||||
JsonNumberAny(camera, "BestExposure", "bestExposure"),
|
||||
JsonNumberAny(camera, "BestGain", "bestGain")))
|
||||
.ToArray();
|
||||
var captureStorage = TryGetAny(TryGetAny(root, "capture"), "storage");
|
||||
var publication = TryGetAny(root, "publication");
|
||||
var stems = ArrayItems(TryGetAny(publication, "stems")).ToArray();
|
||||
return new MimirWellStat(
|
||||
(long)JsonNumberAny(root, "sequence"),
|
||||
JsonNumberAny(root, "elapsedSeconds"),
|
||||
JsonNumberAny(root, "ingestedSamples"),
|
||||
(int)JsonNumberAny(root, "configuredSources"),
|
||||
(int)JsonNumberAny(root, "liveSources"),
|
||||
ArrayItems(TryGetAny(root, "sourceErrors")).Count(),
|
||||
JsonBoolAny(frame, "IsComplete", "isComplete"),
|
||||
JsonNumberAny(frame, "PresentationDelayMs", "presentationDelayMs"),
|
||||
JsonStringAny(frame, "degradedKind") ?? "",
|
||||
JsonStringAny(frame, "degradedReason") ?? "",
|
||||
slices.Count(static slice => string.Equals(JsonStringAny(slice, "Status", "status"), "Ready", StringComparison.OrdinalIgnoreCase)),
|
||||
slices.Length,
|
||||
JsonNumberAny(latency, "currentDelayMs"),
|
||||
JsonNumberAny(latency, "ceilingDelayMs"),
|
||||
JsonNumberAny(latency, "floorDelayMs"),
|
||||
JsonNumberAny(latency, "retainedOverlapMs"),
|
||||
JsonNumberAny(latency, "edgeSkewMs"),
|
||||
JsonStringAny(latency, "Reason", "reason") ?? "",
|
||||
JsonNumberAny(pollPressure, "averageMilliseconds"),
|
||||
JsonNumberAny(pollPressure, "maxMilliseconds"),
|
||||
(long)JsonNumberAny(pollPressure, "zeroPollIterations"),
|
||||
JsonNumberAny(publishPressure, "averageMilliseconds"),
|
||||
JsonNumberAny(publishPressure, "maxMilliseconds"),
|
||||
(long)JsonNumberAny(publishPressure, "bytes"),
|
||||
JsonStringAny(captureStorage, "bodyTransport") ?? "unknown",
|
||||
stems.Length,
|
||||
buffers,
|
||||
clockDomains,
|
||||
states,
|
||||
visualCalibration,
|
||||
JsonNumberAny(featureSignals, "MeanConfidence", "meanConfidence"),
|
||||
JsonNumberAny(featureSignals, "MeanMotionPixelsPerSecond", "meanMotionPixelsPerSecond"),
|
||||
(int)JsonNumberAny(featureSignals, "StableTrackCount", "stableTrackCount"),
|
||||
featureSignalStats,
|
||||
canonicalClockMaps,
|
||||
probe);
|
||||
}
|
||||
|
||||
private static MimirMoveMusicStat? LatestMoveMusicTrace()
|
||||
private static MimirMoveMusicStat? LatestMoveMusicTrace(string telemetrySource, IReadOnlyList<string> telemetryLines)
|
||||
{
|
||||
var configuredLine = telemetryLines
|
||||
.Reverse()
|
||||
.FirstOrDefault(static line => line.Contains("\"live_score\"", StringComparison.OrdinalIgnoreCase) ||
|
||||
line.Contains("\"score_gesture_envelopes\"", StringComparison.OrdinalIgnoreCase));
|
||||
if (!string.IsNullOrWhiteSpace(configuredLine))
|
||||
{
|
||||
return ParseMoveMusicTrace(telemetrySource, configuredLine);
|
||||
}
|
||||
|
||||
return LatestMoveMusicTraceFromRuntime();
|
||||
}
|
||||
|
||||
private static MimirMoveMusicStat? LatestMoveMusicTraceFromRuntime()
|
||||
{
|
||||
var runtimeRoot = Path.Combine(AppContext.BaseDirectory, "artifacts", "runtime");
|
||||
if (!Directory.Exists(runtimeRoot))
|
||||
|
|
@ -1406,6 +1557,11 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
return null;
|
||||
}
|
||||
|
||||
return ParseMoveMusicTrace(trace.FullName, line);
|
||||
}
|
||||
|
||||
private static MimirMoveMusicStat? ParseMoveMusicTrace(string sourcePath, string line)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(line);
|
||||
|
|
@ -1424,6 +1580,30 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
}
|
||||
|
||||
var envelopes = ArrayItems(TryGetAny(root, "score_gesture_envelopes")).Select(JsonNumber).DefaultIfEmpty(0.0).ToArray();
|
||||
var liveScore = TryGetAny(root, "live_score");
|
||||
var voices = ArrayItems(TryGetAny(liveScore, "voices"))
|
||||
.Select(static voice => new MimirMoveMusicVoiceStat(
|
||||
JsonStringAny(voice, "source") ?? "unknown",
|
||||
JsonStringAny(voice, "role") ?? "voice",
|
||||
JsonStringAny(voice, "note_name") ?? "?",
|
||||
JsonNumberAny(voice, "frequency_hz"),
|
||||
JsonNumberAny(voice, "midi"),
|
||||
JsonNumberAny(voice, "confidence"),
|
||||
JsonNumberAny(voice, "strength"),
|
||||
JsonBoolAny(voice, "active")))
|
||||
.OrderByDescending(static voice => voice.Active)
|
||||
.ThenByDescending(static voice => voice.Confidence)
|
||||
.ToArray();
|
||||
var targets = ArrayItems(TryGetAny(liveScore, "move_targets"))
|
||||
.Select(static target => new MimirMoveMusicTargetStat(
|
||||
(int)JsonNumberAny(target, "move_index"),
|
||||
JsonStringAny(target, "source") ?? "unknown",
|
||||
JsonStringAny(target, "note_name") ?? "?",
|
||||
JsonNumberAny(target, "target_note"),
|
||||
JsonNumberAny(target, "confidence"),
|
||||
JsonNumberAny(target, "calibration_priority")))
|
||||
.OrderBy(static target => target.MoveIndex)
|
||||
.ToArray();
|
||||
var sources = ArrayItems(TryGetAny(root, "music_sources"))
|
||||
.Select(static source =>
|
||||
{
|
||||
|
|
@ -1438,7 +1618,7 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
})
|
||||
.ToArray();
|
||||
return new MimirMoveMusicStat(
|
||||
trace.FullName,
|
||||
sourcePath,
|
||||
JsonNumberAny(root, "bpm"),
|
||||
JsonNumberAny(root, "bpm_confidence"),
|
||||
JsonNumberAny(root, "loudness_gate"),
|
||||
|
|
@ -1448,7 +1628,10 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
JsonStringAny(root, "chord_name") ?? "?",
|
||||
envelopes.Max(),
|
||||
maxRgb,
|
||||
sources);
|
||||
sources,
|
||||
voices,
|
||||
targets,
|
||||
BuildPianoRoll(voices, targets));
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
|
|
@ -1456,6 +1639,43 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
}
|
||||
}
|
||||
|
||||
private static string BuildPianoRoll(
|
||||
IReadOnlyList<MimirMoveMusicVoiceStat> voices,
|
||||
IReadOnlyList<MimirMoveMusicTargetStat> targets)
|
||||
{
|
||||
var marks = new char[36];
|
||||
Array.Fill(marks, '.');
|
||||
foreach (var voice in voices.Where(static voice => voice.Active))
|
||||
{
|
||||
var index = PianoIndex(voice.MidiNote);
|
||||
if (index >= 0)
|
||||
{
|
||||
marks[index] = voice.Confidence >= 0.65 ? '#' : '+';
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var target in targets)
|
||||
{
|
||||
var index = PianoIndex(target.TargetNote);
|
||||
if (index >= 0 && marks[index] == '.')
|
||||
{
|
||||
marks[index] = 'o';
|
||||
}
|
||||
}
|
||||
|
||||
return $"C3 {new string(marks)} C6";
|
||||
}
|
||||
|
||||
private static int PianoIndex(double midiNote)
|
||||
{
|
||||
if (midiNote <= 0.0)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
return (int)Math.Clamp(Math.Round(midiNote) - 48, 0, 35);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<MimirObservationStreamStat> LatestObservationStreams(IEnumerable<string> lines)
|
||||
{
|
||||
var byStream = new Dictionary<string, MimirObservationStreamStat>(StringComparer.Ordinal);
|
||||
|
|
@ -1844,16 +2064,45 @@ internal sealed record MimirWellStat(
|
|||
double IngestedSamples,
|
||||
int ConfiguredSources,
|
||||
int LiveSources,
|
||||
int SourceErrorCount,
|
||||
bool SynchronizedFrameComplete,
|
||||
double PresentationDelayMs,
|
||||
string FrameDegradedKind,
|
||||
string FrameDegradedReason,
|
||||
int ReadySlices,
|
||||
int TotalSlices,
|
||||
double LatencyCurrentMs,
|
||||
double LatencyCeilingMs,
|
||||
double LatencyFloorMs,
|
||||
double LatencyRetainedOverlapMs,
|
||||
double LatencyEdgeSkewMs,
|
||||
string LatencyReason,
|
||||
double PollAverageMs,
|
||||
double PollMaxMs,
|
||||
long ZeroPollIterations,
|
||||
double PublishAverageMs,
|
||||
double PublishMaxMs,
|
||||
long PublishedBytes,
|
||||
string CaptureInlineBodies,
|
||||
int PublicationStemCount,
|
||||
IReadOnlyList<MimirWellBufferStat> Buffers,
|
||||
IReadOnlyList<MimirWellClockDomainStat> ClockDomains,
|
||||
IReadOnlyList<MimirWellAudioSyncStateStat> AudioSyncStates,
|
||||
IReadOnlyList<MimirWellVisualCalibrationStat> VisualCalibration,
|
||||
double FeatureMeanConfidence,
|
||||
double FeatureMeanMotionPixelsPerSecond,
|
||||
int FeatureStableTracks,
|
||||
IReadOnlyList<MimirWellFeatureSignalStat> FeatureSignals,
|
||||
IReadOnlyList<MimirWellCanonicalClockMapStat> CanonicalClockMaps,
|
||||
MimirWellProbeStat? Probe)
|
||||
{
|
||||
public int AudioBuffers => Buffers.Count(static buffer => string.Equals(buffer.Kind, "Audio", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public int VideoBuffers => Buffers.Count(static buffer => string.Equals(buffer.Kind, "Video", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public double MaxCanonicalClockOffsetMs => CanonicalClockMaps.Count == 0
|
||||
? 0.0
|
||||
: CanonicalClockMaps.Max(static map => Math.Abs(map.OffsetMs));
|
||||
}
|
||||
|
||||
internal sealed record MimirWellBufferStat(
|
||||
|
|
@ -1869,6 +2118,14 @@ internal sealed record MimirWellBufferStat(
|
|||
|
||||
internal sealed record MimirWellAudioSyncStateStat(string SourceId, string ReferenceSourceId, double DelaySamples, double SroPpm, double Confidence);
|
||||
|
||||
internal sealed record MimirWellClockDomainStat(string ClockDomainId, int SourceCount, double OverlapMs, bool HasLocalOverlap, bool IsReference, double OffsetMs);
|
||||
|
||||
internal sealed record MimirWellVisualCalibrationStat(string SourceId, string State, double BestScore, int BestDetectedLedCount, bool BestUsableForCalibration, double BestExposure, double BestGain);
|
||||
|
||||
internal sealed record MimirWellFeatureSignalStat(string SourceId, int StableTrackCount, double Confidence, double MeanMotionPixelsPerSecond, double MotionEnergy, double CentroidX, double CentroidY);
|
||||
|
||||
internal sealed record MimirWellCanonicalClockMapStat(string StreamKey, double OffsetMs, long SampleCount);
|
||||
|
||||
internal sealed record MimirWellProbeStat(bool ShouldEmit, double SyncConfidence, double FrequencyConfidence, string Reason);
|
||||
|
||||
internal sealed record MimirMoveMusicStat(
|
||||
|
|
@ -1882,7 +2139,14 @@ internal sealed record MimirMoveMusicStat(
|
|||
string ChordName,
|
||||
double ScoreEnvelopeMax,
|
||||
int MaxRgb,
|
||||
IReadOnlyList<string> Sources);
|
||||
IReadOnlyList<string> Sources,
|
||||
IReadOnlyList<MimirMoveMusicVoiceStat> Voices,
|
||||
IReadOnlyList<MimirMoveMusicTargetStat> MoveTargets,
|
||||
string PianoRoll);
|
||||
|
||||
internal sealed record MimirMoveMusicVoiceStat(string SourceId, string Role, string NoteName, double FrequencyHz, double MidiNote, double Confidence, double Strength, bool Active);
|
||||
|
||||
internal sealed record MimirMoveMusicTargetStat(int MoveIndex, string SourceId, string NoteName, double TargetNote, double Confidence, double CalibrationPriority);
|
||||
|
||||
internal sealed class MimirStreamLayoutProvider : MutableDashboardProvider
|
||||
{
|
||||
|
|
@ -2016,6 +2280,7 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider
|
|||
var summary = TryGet(root, "summary");
|
||||
var controls = TryGet(root, "controls");
|
||||
var cultMesh = TryGet(root, "cultMesh");
|
||||
var orchestrator = TryGet(root, "orchestrator");
|
||||
var participants = ArrayItems(TryGet(root, "participants")).ToArray();
|
||||
var upcoming = ArrayItems(TryGet(root, "upcomingTurns")).ToArray();
|
||||
|
||||
|
|
@ -2070,7 +2335,7 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider
|
|||
SelectedNodeId = selectedNodeId,
|
||||
Nodes = nodes,
|
||||
};
|
||||
state.Surface = BuildVoidBotSurface(state, summary, cultMesh, controls, upcoming, participants, selectedAgent);
|
||||
state.Surface = BuildVoidBotSurface(state, summary, cultMesh, controls, orchestrator, upcoming, participants, selectedAgent);
|
||||
return state;
|
||||
}
|
||||
|
||||
|
|
@ -2079,6 +2344,7 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider
|
|||
JsonElement summary,
|
||||
JsonElement cultMesh,
|
||||
JsonElement controls,
|
||||
JsonElement orchestrator,
|
||||
IReadOnlyList<JsonElement> upcoming,
|
||||
IReadOnlyList<JsonElement> participants,
|
||||
JsonElement selectedAgent)
|
||||
|
|
@ -2096,13 +2362,35 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider
|
|||
Root = UiElement.Container(
|
||||
"voidbot-cockpit",
|
||||
"cockpit",
|
||||
new DashboardUiLayout { Direction = "vertical", Gap = 10, Padding = 10 },
|
||||
new DashboardUiLayout
|
||||
{
|
||||
Direction = "vertical",
|
||||
Gap = 8,
|
||||
Padding = 8,
|
||||
Overflow = "scroll",
|
||||
Grow = 5,
|
||||
MinWidth = 112,
|
||||
MinHeight = 30,
|
||||
PreferredWidth = 156,
|
||||
PreferredHeight = 58,
|
||||
Priority = -30,
|
||||
Density = "continuous",
|
||||
ViewportMode = "continuous-ops",
|
||||
},
|
||||
[
|
||||
BuildTurnRail(upcoming),
|
||||
UiElement.Container(
|
||||
"voidbot-ops-row",
|
||||
"ops-row",
|
||||
new DashboardUiLayout { Direction = "horizontal", Gap = 8, Grow = 1 },
|
||||
[
|
||||
BuildUpcomingFacesPane(upcoming),
|
||||
BuildWatchdogPane(orchestrator),
|
||||
]),
|
||||
UiElement.Container(
|
||||
"voidbot-workspace",
|
||||
"workspace",
|
||||
new DashboardUiLayout { Direction = "horizontal", Gap = 10, Grow = 1 },
|
||||
new DashboardUiLayout { Direction = "horizontal", Gap = 8, Grow = 2 },
|
||||
[
|
||||
BuildSelectedFacePane(summary, cultMesh, selectedAgent, cadence, generatedAt),
|
||||
BuildStateGraphPane(leaves),
|
||||
|
|
@ -2123,7 +2411,7 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider
|
|||
|
||||
private DashboardUiElement BuildTurnRail(IReadOnlyList<JsonElement> upcoming)
|
||||
{
|
||||
var count = Math.Min(8, upcoming.Count);
|
||||
var count = Math.Min(14, upcoming.Count);
|
||||
var cards = new List<DashboardUiElement>();
|
||||
for (var index = 0; index < count; index++)
|
||||
{
|
||||
|
|
@ -2152,6 +2440,70 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider
|
|||
cards);
|
||||
}
|
||||
|
||||
private DashboardUiElement BuildUpcomingFacesPane(IReadOnlyList<JsonElement> upcoming)
|
||||
{
|
||||
var rows = upcoming
|
||||
.Take(10)
|
||||
.Select((turn, index) =>
|
||||
{
|
||||
var identityId = StringValue(turn, "identityId") ?? $"turn-{index}";
|
||||
var active = !string.IsNullOrWhiteSpace(StringValue(turn, "activeJobId"));
|
||||
var mentionCount = NumberValue(turn, "pendingMentionCount") ?? 0;
|
||||
var rest = TryGet(turn, "restState");
|
||||
var napping = BoolValue(rest, "isNapping") == true;
|
||||
var state = active ? "active" : mentionCount > 0 ? "mention" : napping ? "nap" : Minutes(NumberValue(turn, "nextTurnInMinutes"));
|
||||
var heat = NumberValue(turn, "heat") ?? 0;
|
||||
var speed = NumberValue(turn, "effectiveSpeed") ?? 0;
|
||||
return UiElement.Text(
|
||||
$"upcoming-face-{index}-{identityId}",
|
||||
$"{index + 1,2}. {(StringValue(turn, "displayName") ?? identityId),-10} {state,-8} {StringValue(turn, "repoName") ?? "repo"} spd {speed:0.###} heat {heat:0.##}",
|
||||
"mono");
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
return UiElement.Pane(
|
||||
"upcoming-faces-pane",
|
||||
"Next Faces",
|
||||
rows.Length > 0
|
||||
? rows
|
||||
: [UiElement.Text("upcoming-faces-empty", "no upcoming Face turns", "caption")]);
|
||||
}
|
||||
|
||||
private DashboardUiElement BuildWatchdogPane(JsonElement orchestrator)
|
||||
{
|
||||
var state = StringValue(orchestrator, "state") ?? "unknown";
|
||||
var organs = ArrayItems(TryGet(orchestrator, "organs")).ToArray();
|
||||
var watchdog = organs.FirstOrDefault(organ => string.Equals(StringValue(organ, "id"), "voidbot-operations-watchdog", StringComparison.OrdinalIgnoreCase));
|
||||
var children = new List<DashboardUiElement>
|
||||
{
|
||||
UiElement.Text("watchdog-summary", $"orchestrator {state} organs {organs.Length}", "mono"),
|
||||
};
|
||||
|
||||
if (watchdog.ValueKind != JsonValueKind.Undefined)
|
||||
{
|
||||
children.Add(UiElement.Text(
|
||||
"watchdog-status",
|
||||
$"watchdog {StringValue(watchdog, "lastStatus") ?? "unknown"} exit {(NumberValue(watchdog, "lastExitCode") ?? 0):0}\nlast {ShortIso(StringValue(watchdog, "lastFinishedAt") ?? StringValue(watchdog, "lastStartedAt"))}",
|
||||
"strong"));
|
||||
}
|
||||
else
|
||||
{
|
||||
children.Add(UiElement.Text("watchdog-missing", "watchdog organ not present in snapshot", "caption"));
|
||||
}
|
||||
|
||||
children.AddRange(organs
|
||||
.Where(static organ => organ.ValueKind != JsonValueKind.Undefined)
|
||||
.OrderBy(organ => string.Equals(StringValue(organ, "id"), "voidbot-operations-watchdog", StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(organ => StringValue(organ, "id"))
|
||||
.Take(7)
|
||||
.Select(organ => UiElement.Text(
|
||||
$"watchdog-organ-{StableId(StringValue(organ, "id") ?? "organ")}",
|
||||
$"{StringValue(organ, "label") ?? StringValue(organ, "id") ?? "organ"}: {StringValue(organ, "lastStatus") ?? "unknown"}",
|
||||
"caption")));
|
||||
|
||||
return UiElement.Pane("voidbot-watchdog-pane", "VoidBot Watchdog", children);
|
||||
}
|
||||
|
||||
private DashboardUiElement BuildSelectedFacePane(JsonElement summary, JsonElement cultMesh, JsonElement selectedAgent, double cadence, string generatedAt)
|
||||
{
|
||||
var identityId = StringValue(selectedAgent, "identityId") ?? "";
|
||||
|
|
@ -2447,6 +2799,29 @@ internal sealed class VoidBotSwarmProvider : IDashboardProvider
|
|||
|
||||
private static string Truncate(string value, int max) =>
|
||||
value.Length <= max ? value : value[..Math.Max(0, max - 1)] + "...";
|
||||
|
||||
private static string StableId(string value)
|
||||
{
|
||||
var builder = new StringBuilder(value.Length);
|
||||
foreach (var ch in value.ToLowerInvariant())
|
||||
{
|
||||
builder.Append(char.IsLetterOrDigit(ch) ? ch : '-');
|
||||
}
|
||||
|
||||
return builder.ToString().Trim('-');
|
||||
}
|
||||
|
||||
private static string ShortIso(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
return DateTimeOffset.TryParse(value, out var timestamp)
|
||||
? timestamp.ToLocalTime().ToString("MM-dd HH:mm")
|
||||
: value;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record VoidBotStateLeaf(string Label, string Path, string Preview, string Detail);
|
||||
|
|
@ -2647,6 +3022,20 @@ internal sealed class DashboardUiLayout
|
|||
public double? Padding { get; set; }
|
||||
|
||||
public string? Overflow { get; set; }
|
||||
|
||||
public double? MinWidth { get; set; }
|
||||
|
||||
public double? MinHeight { get; set; }
|
||||
|
||||
public double? PreferredWidth { get; set; }
|
||||
|
||||
public double? PreferredHeight { get; set; }
|
||||
|
||||
public double? Priority { get; set; }
|
||||
|
||||
public string? Density { get; set; }
|
||||
|
||||
public string? ViewportMode { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class DashboardUiStyle
|
||||
|
|
|
|||
|
|
@ -8,3 +8,8 @@ public interface IMimirStreamSource : IDisposable
|
|||
|
||||
bool TryRead(out MimirStreamSample sample);
|
||||
}
|
||||
|
||||
public interface IMimirMultiplexedStreamSource : IMimirStreamSource
|
||||
{
|
||||
int LogicalStreamCount { get; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ public sealed record MimirAsioStreamSourceOptions(
|
|||
IReadOnlyList<string> SourceIds);
|
||||
|
||||
[SupportedOSPlatform("windows")]
|
||||
public sealed class MimirAsioStreamSource : IMimirStreamSource
|
||||
public sealed class MimirAsioStreamSource : IMimirMultiplexedStreamSource
|
||||
{
|
||||
private const string DefaultClsid = "{AC4D0455-50D7-4498-B3CD-9A41D130B759}";
|
||||
private const int MaxQueuedBlocks = 32768;
|
||||
|
|
@ -57,6 +57,8 @@ public sealed class MimirAsioStreamSource : IMimirStreamSource
|
|||
|
||||
public bool ExposesDescriptorBuffer => false;
|
||||
|
||||
public int LogicalStreamCount => Math.Max(1, sourceIds.Length);
|
||||
|
||||
public bool TryRead(out MimirStreamSample sample)
|
||||
{
|
||||
sample = default;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
namespace Mimir.Runtime.Synchronization;
|
||||
|
||||
public sealed class MimirCanonicalClockMapper
|
||||
{
|
||||
private readonly Dictionary<string, ClockMap> maps = new(StringComparer.Ordinal);
|
||||
|
||||
public IReadOnlyList<MimirCanonicalClockMapSnapshot> Snapshots =>
|
||||
maps
|
||||
.OrderBy(pair => pair.Key, StringComparer.Ordinal)
|
||||
.Select(pair => new MimirCanonicalClockMapSnapshot(
|
||||
pair.Key,
|
||||
pair.Value.OffsetNs,
|
||||
pair.Value.FirstSourceTimestampNs,
|
||||
pair.Value.FirstArrivalNs,
|
||||
pair.Value.LatestSourceTimestampNs,
|
||||
pair.Value.LatestCanonicalTimestampNs,
|
||||
pair.Value.SampleCount))
|
||||
.ToArray();
|
||||
|
||||
public MimirStreamSample ToCanonical(MimirStreamSample sample)
|
||||
{
|
||||
var sourceTimestampNs = sample.TimestampNs;
|
||||
if (sourceTimestampNs <= 0)
|
||||
{
|
||||
sourceTimestampNs = sample.ArrivalNs > 0 ? sample.ArrivalNs : NowNs();
|
||||
}
|
||||
|
||||
var arrivalNs = sample.ArrivalNs > 0 ? sample.ArrivalNs : NowNs();
|
||||
var key = $"{sample.Kind}:{sample.Origin}:{sample.SourceId}";
|
||||
if (!maps.TryGetValue(key, out var map))
|
||||
{
|
||||
map = new ClockMap(checked(arrivalNs - sourceTimestampNs), sourceTimestampNs, arrivalNs);
|
||||
maps.Add(key, map);
|
||||
}
|
||||
|
||||
var canonicalTimestampNs = checked(sourceTimestampNs + map.OffsetNs);
|
||||
map.LatestSourceTimestampNs = sourceTimestampNs;
|
||||
map.LatestCanonicalTimestampNs = canonicalTimestampNs;
|
||||
map.SampleCount++;
|
||||
return sample with
|
||||
{
|
||||
TimestampNs = canonicalTimestampNs,
|
||||
ArrivalNs = arrivalNs,
|
||||
};
|
||||
}
|
||||
|
||||
private static long NowNs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L;
|
||||
|
||||
private sealed class ClockMap(long offsetNs, long firstSourceTimestampNs, long firstArrivalNs)
|
||||
{
|
||||
public long OffsetNs { get; } = offsetNs;
|
||||
|
||||
public long FirstSourceTimestampNs { get; } = firstSourceTimestampNs;
|
||||
|
||||
public long FirstArrivalNs { get; } = firstArrivalNs;
|
||||
|
||||
public long LatestSourceTimestampNs { get; set; } = firstSourceTimestampNs;
|
||||
|
||||
public long LatestCanonicalTimestampNs { get; set; } = firstArrivalNs;
|
||||
|
||||
public long SampleCount { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record MimirCanonicalClockMapSnapshot(
|
||||
string StreamKey,
|
||||
long OffsetNs,
|
||||
long FirstSourceTimestampNs,
|
||||
long FirstArrivalNs,
|
||||
long LatestSourceTimestampNs,
|
||||
long LatestCanonicalTimestampNs,
|
||||
long SampleCount);
|
||||
|
|
@ -198,7 +198,14 @@ public sealed record MimirEveDashboardUiLayoutSnapshot(
|
|||
[property: Key(3)] double? Grow,
|
||||
[property: Key(4)] double? Gap,
|
||||
[property: Key(5)] double? Padding,
|
||||
[property: Key(6)] string? Overflow);
|
||||
[property: Key(6)] string? Overflow,
|
||||
[property: Key(7)] double? MinWidth = null,
|
||||
[property: Key(8)] double? MinHeight = null,
|
||||
[property: Key(9)] double? PreferredWidth = null,
|
||||
[property: Key(10)] double? PreferredHeight = null,
|
||||
[property: Key(11)] double? Priority = null,
|
||||
[property: Key(12)] string? Density = null,
|
||||
[property: Key(13)] string? ViewportMode = null);
|
||||
|
||||
[MessagePackObject]
|
||||
public sealed record MimirEveDashboardUiStyleSnapshot(
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ public sealed class MimirSynchronizationHub : IDisposable
|
|||
private readonly MimirComplexContourRuntimeAnalyzer? complexContourSynchronization;
|
||||
private readonly MimirAudioSynchronizationStateTracker audioSynchronizationState = new();
|
||||
private readonly MimirBioacousticProbeScheduler bioacousticProbeScheduler = new();
|
||||
private readonly MimirCanonicalClockMapper clockMapper = new();
|
||||
private readonly Dictionary<string, MimirAudioSynchronizationReport> audioSynchronizationReports = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, MimirAudioSynchronizationReport> complexContourReports = new(StringComparer.Ordinal);
|
||||
private readonly MimirChirpBinCalibrationModel? chirpBinCalibrationModel;
|
||||
|
|
@ -44,6 +45,9 @@ public sealed class MimirSynchronizationHub : IDisposable
|
|||
public IReadOnlyList<MimirAudioSynchronizationState> AudioSynchronizationStates =>
|
||||
audioSynchronizationState.States;
|
||||
|
||||
public IReadOnlyList<MimirCanonicalClockMapSnapshot> CanonicalClockMaps =>
|
||||
clockMapper.Snapshots;
|
||||
|
||||
public IReadOnlyList<MimirAudioSynchronizationReport> AudioSynchronizationReports =>
|
||||
audioSynchronizationReports.Values.OrderBy(report => report.SourceId, StringComparer.Ordinal).ToArray();
|
||||
|
||||
|
|
@ -110,9 +114,13 @@ public sealed class MimirSynchronizationHub : IDisposable
|
|||
var consumed = 0;
|
||||
foreach (var source in sources)
|
||||
{
|
||||
for (var index = 0; index < maxSamplesPerSource && source.TryRead(out var sample); index++)
|
||||
var logicalStreamCount = source is IMimirMultiplexedStreamSource multiplexed
|
||||
? Math.Max(1, multiplexed.LogicalStreamCount)
|
||||
: 1;
|
||||
var sourceBudget = checked(Math.Max(1, maxSamplesPerSource) * logicalStreamCount);
|
||||
for (var index = 0; index < sourceBudget && source.TryRead(out var sample); index++)
|
||||
{
|
||||
Buffers.Append(sample);
|
||||
Buffers.Append(clockMapper.ToCanonical(sample));
|
||||
consumed++;
|
||||
ingestedSamples++;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ var nextVisualCalibration = DateTimeOffset.MinValue;
|
|||
var frameDegradedCount = 0L;
|
||||
var nextFrameDegradedLog = DateTimeOffset.MinValue;
|
||||
var streamTelemetry = new MimirWellStreamTelemetry();
|
||||
var latencyController = new MimirWellLatencyController(options);
|
||||
var featureSignals = new MimirWellFeatureSignalBank(options);
|
||||
var presentation = new MimirPresentationControlState();
|
||||
var exposureController = new MimirCameraExposureController(new MimirCameraExposureControlOptions(
|
||||
options.VisualCalibrationEnabled,
|
||||
|
|
@ -102,7 +104,10 @@ while (!stopping.IsCancellationRequested)
|
|||
if (now >= nextPublish)
|
||||
{
|
||||
presentation.SyncFromBuffers(hub.Buffers.Buffers);
|
||||
var frameResult = BuildFrameOrEmpty(hub, TimeSpan.FromMilliseconds(options.PresentationDelayMs));
|
||||
var latencyDecision = latencyController.Decide(hub, now);
|
||||
var frameResult = BuildFrameOrEmpty(hub, latencyDecision.PresentationDelay);
|
||||
latencyController.ObserveFrame(frameResult);
|
||||
var featureSignalFrame = featureSignals.Update(hub.Buffers.Buffers, frameResult.Frame);
|
||||
if (!string.IsNullOrWhiteSpace(frameResult.DegradedReason))
|
||||
{
|
||||
frameDegradedCount++;
|
||||
|
|
@ -121,6 +126,8 @@ while (!stopping.IsCancellationRequested)
|
|||
hub,
|
||||
presentation,
|
||||
frameResult,
|
||||
latencyDecision,
|
||||
featureSignalFrame,
|
||||
exposureController.Statuses,
|
||||
sourceErrors,
|
||||
streamTelemetry.Snapshot(now, subscribersAreExternal: true),
|
||||
|
|
@ -130,6 +137,21 @@ while (!stopping.IsCancellationRequested)
|
|||
await publisher.PublishAsync(snapshot, "mimir.well_snapshot.v1", stopping.Token).ConfigureAwait(false));
|
||||
if (options.CapturePagesEnabled && now >= nextCapture)
|
||||
{
|
||||
var nextStreamSequence = captureSequence + 1;
|
||||
if (options.StreamFramesEnabled)
|
||||
{
|
||||
foreach (var streamFrame in MimirWellCapturePage.BuildStreamFrames(
|
||||
options,
|
||||
frameResult,
|
||||
hub.Buffers.Buffers,
|
||||
nextStreamSequence,
|
||||
startedAt))
|
||||
{
|
||||
streamTelemetry.ObservePublish(
|
||||
await publisher.PublishAsync(streamFrame, "mimir.cultmesh_stream_frame.v1", stopping.Token).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
|
||||
var capturePage = MimirWellCapturePage.Build(
|
||||
options,
|
||||
presentation,
|
||||
|
|
@ -195,6 +217,386 @@ internal sealed record MimirWellFrameBuildResult(
|
|||
string DegradedKind,
|
||||
string DegradedReason);
|
||||
|
||||
internal sealed record MimirWellLatencyDecision(
|
||||
TimeSpan PresentationDelay,
|
||||
TimeSpan CeilingDelay,
|
||||
TimeSpan FloorDelay,
|
||||
TimeSpan RetainedOverlap,
|
||||
TimeSpan EdgeSkew,
|
||||
double ReadinessConfidence,
|
||||
double SyncConfidence,
|
||||
int ActiveBufferCount,
|
||||
int ReadySliceCount,
|
||||
int TotalSliceCount,
|
||||
string Reason);
|
||||
|
||||
internal sealed class MimirWellLatencyController
|
||||
{
|
||||
private readonly MimirWellOptions options;
|
||||
private readonly TimeSpan floor;
|
||||
private TimeSpan current;
|
||||
private int completeStreak;
|
||||
private int degradedStreak;
|
||||
private MimirWellFrameBuildResult? lastFrame;
|
||||
|
||||
public MimirWellLatencyController(MimirWellOptions options)
|
||||
{
|
||||
this.options = options;
|
||||
floor = TimeSpan.FromMilliseconds(Math.Clamp(options.MinPresentationDelayMs, 0, options.PresentationDelayMs));
|
||||
current = TimeSpan.FromMilliseconds(Math.Clamp(
|
||||
options.TargetPresentationDelayMs,
|
||||
options.MinPresentationDelayMs,
|
||||
options.PresentationDelayMs));
|
||||
}
|
||||
|
||||
public MimirWellLatencyDecision Decide(MimirSynchronizationHub hub, DateTimeOffset now)
|
||||
{
|
||||
var active = hub.Buffers.Buffers
|
||||
.Where(static buffer => buffer.Latest.HasValue)
|
||||
.ToArray();
|
||||
if (active.Length == 0)
|
||||
{
|
||||
current = TimeSpan.FromMilliseconds(options.PresentationDelayMs);
|
||||
return Decision(TimeSpan.Zero, TimeSpan.Zero, 0.0, 0.0, 0, 0, 0, "waiting-for-sources");
|
||||
}
|
||||
|
||||
var latestEdges = active
|
||||
.Select(static buffer => buffer.Latest?.TimestampNs ?? 0L)
|
||||
.Where(static value => value > 0)
|
||||
.ToArray();
|
||||
var starts = active
|
||||
.Select(static buffer => buffer.OldestSampleTimestampNs > 0 ? buffer.OldestSampleTimestampNs : buffer.WindowStartNs)
|
||||
.Where(static value => value > 0)
|
||||
.ToArray();
|
||||
var minEdge = latestEdges.Length == 0 ? 0L : latestEdges.Min();
|
||||
var maxEdge = latestEdges.Length == 0 ? 0L : latestEdges.Max();
|
||||
var maxStart = starts.Length == 0 ? 0L : starts.Max();
|
||||
var retainedOverlap = minEdge > 0 && maxStart > 0 ? TimeSpan.FromTicks(Math.Max(0L, (minEdge - maxStart) / 100L)) : TimeSpan.Zero;
|
||||
var edgeSkew = minEdge > 0 && maxEdge > minEdge ? TimeSpan.FromTicks((maxEdge - minEdge) / 100L) : TimeSpan.Zero;
|
||||
var syncConfidence = hub.AudioSynchronizationStates.Count == 0
|
||||
? 0.0
|
||||
: hub.AudioSynchronizationStates.Average(static state => Math.Clamp(state.Confidence, 0.0, 1.0));
|
||||
var previousReady = lastFrame?.Frame.Slices.Count(static slice => slice.Status == MimirSynchronizedSliceStatus.Ready) ?? 0;
|
||||
var previousTotal = lastFrame?.Frame.Slices.Count ?? 0;
|
||||
var readiness = previousTotal == 0 ? 0.0 : previousReady / (double)previousTotal;
|
||||
|
||||
var ceiling = TimeSpan.FromMilliseconds(options.PresentationDelayMs);
|
||||
var target = TimeSpan.FromMilliseconds(Math.Clamp(
|
||||
Math.Max(options.MinPresentationDelayMs, edgeSkew.TotalMilliseconds + options.LatencyGuardMs),
|
||||
options.MinPresentationDelayMs,
|
||||
options.PresentationDelayMs));
|
||||
var reason = "edge-skew-plus-guard";
|
||||
if (retainedOverlap > TimeSpan.Zero)
|
||||
{
|
||||
var overlapBound = retainedOverlap - TimeSpan.FromMilliseconds(options.LatencyGuardMs);
|
||||
if (overlapBound > floor)
|
||||
{
|
||||
target = target < overlapBound ? target : overlapBound;
|
||||
}
|
||||
else
|
||||
{
|
||||
target = floor;
|
||||
reason = "overlap-thin";
|
||||
}
|
||||
}
|
||||
|
||||
if (lastFrame != null && !lastFrame.Frame.IsComplete)
|
||||
{
|
||||
target += TimeSpan.FromMilliseconds(options.LatencyStepMs * Math.Max(1, degradedStreak));
|
||||
reason = "backoff-after-incomplete-frame";
|
||||
}
|
||||
else if (completeStreak >= Math.Max(1, options.LatencyConvergenceFrames) && readiness >= options.LatencyReadinessTarget)
|
||||
{
|
||||
target -= TimeSpan.FromMilliseconds(options.LatencyStepMs);
|
||||
reason = "complete-frame-convergence";
|
||||
}
|
||||
|
||||
current = ClampDelay(Smooth(current, target), floor, ceiling);
|
||||
return Decision(retainedOverlap, edgeSkew, readiness, syncConfidence, active.Length, previousReady, previousTotal, reason);
|
||||
}
|
||||
|
||||
public void ObserveFrame(MimirWellFrameBuildResult result)
|
||||
{
|
||||
lastFrame = result;
|
||||
if (result.Frame.IsComplete)
|
||||
{
|
||||
completeStreak++;
|
||||
degradedStreak = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
degradedStreak++;
|
||||
completeStreak = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private MimirWellLatencyDecision Decision(
|
||||
TimeSpan retainedOverlap,
|
||||
TimeSpan edgeSkew,
|
||||
double readiness,
|
||||
double syncConfidence,
|
||||
int activeBuffers,
|
||||
int readySlices,
|
||||
int totalSlices,
|
||||
string reason) =>
|
||||
new(
|
||||
current,
|
||||
TimeSpan.FromMilliseconds(options.PresentationDelayMs),
|
||||
floor,
|
||||
retainedOverlap,
|
||||
edgeSkew,
|
||||
readiness,
|
||||
syncConfidence,
|
||||
activeBuffers,
|
||||
readySlices,
|
||||
totalSlices,
|
||||
reason);
|
||||
|
||||
private static TimeSpan Smooth(TimeSpan current, TimeSpan target)
|
||||
{
|
||||
var alpha = target > current ? 0.65 : 0.20;
|
||||
return TimeSpan.FromTicks((long)Math.Round(current.Ticks + (target.Ticks - current.Ticks) * alpha));
|
||||
}
|
||||
|
||||
private static TimeSpan ClampDelay(TimeSpan value, TimeSpan floor, TimeSpan ceiling) =>
|
||||
value < floor ? floor : value > ceiling ? ceiling : value;
|
||||
}
|
||||
|
||||
internal sealed record MimirWellFeatureSignalFrame(
|
||||
string Document,
|
||||
long Sequence,
|
||||
IReadOnlyList<MimirWellFeatureSignal> Signals,
|
||||
double MeanConfidence,
|
||||
double MeanMotionPixelsPerSecond,
|
||||
int StableTrackCount,
|
||||
string FaustSignalContract);
|
||||
|
||||
internal sealed record MimirWellFeatureSignal(
|
||||
string SourceId,
|
||||
long TimestampNs,
|
||||
int Width,
|
||||
int Height,
|
||||
int StableTrackCount,
|
||||
double Confidence,
|
||||
double MeanMotionPixelsPerSecond,
|
||||
double MotionEnergy,
|
||||
double NormalizedCentroidX,
|
||||
double NormalizedCentroidY,
|
||||
IReadOnlyDictionary<string, float> FaustControls);
|
||||
|
||||
internal sealed class MimirWellFeatureSignalBank
|
||||
{
|
||||
private readonly MimirWellOptions options;
|
||||
private readonly Dictionary<string, MimirSparseFeatureTracker> trackers = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, ulong> lastSequences = new(StringComparer.Ordinal);
|
||||
private long sequence;
|
||||
|
||||
public MimirWellFeatureSignalBank(MimirWellOptions options)
|
||||
{
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public MimirWellFeatureSignalFrame Update(
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers,
|
||||
MimirSynchronizedBufferFrame frame)
|
||||
{
|
||||
if (!options.VideoFeatureSignalsEnabled)
|
||||
{
|
||||
return Empty();
|
||||
}
|
||||
|
||||
var signals = frame.VideoSlices
|
||||
.Where(static slice => slice.Sample.HasValue)
|
||||
.Select(slice => BuildSignal(slice.Sample!.Value))
|
||||
.Where(static signal => signal != null)
|
||||
.Select(static signal => signal!)
|
||||
.ToArray();
|
||||
if (signals.Length == 0)
|
||||
{
|
||||
signals = buffers
|
||||
.Where(static buffer => buffer.Descriptor.Kind == MimirStreamKind.Video && buffer.Latest.HasValue)
|
||||
.Select(buffer => BuildSignal(buffer.Latest!.Value))
|
||||
.Where(static signal => signal != null)
|
||||
.Select(static signal => signal!)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
if (signals.Length == 0)
|
||||
{
|
||||
return Empty();
|
||||
}
|
||||
|
||||
return new MimirWellFeatureSignalFrame(
|
||||
"mimir.well_feature_signals.v1",
|
||||
++sequence,
|
||||
signals,
|
||||
signals.Average(static signal => signal.Confidence),
|
||||
signals.Average(static signal => signal.MeanMotionPixelsPerSecond),
|
||||
signals.Sum(static signal => signal.StableTrackCount),
|
||||
"Faust controls are normalized scalar signals under video/<source>/{motion_energy,confidence,centroid_x,centroid_y}; Mimir estimates, Faust/native DSP consumes.");
|
||||
}
|
||||
|
||||
private MimirWellFeatureSignalFrame Empty() =>
|
||||
new("mimir.well_feature_signals.v1", sequence, [], 0.0, 0.0, 0, "no feature-bearing CPU video slice available");
|
||||
|
||||
private MimirWellFeatureSignal? BuildSignal(MimirStreamSample sample)
|
||||
{
|
||||
if (sample.VideoFrame is not { } video ||
|
||||
sample.Data.IsEmpty ||
|
||||
sample.Sequence == (lastSequences.TryGetValue(sample.SourceId, out var previous) ? previous : ulong.MaxValue))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var luma = ExtractLuma(video, sample.Data.Span);
|
||||
if (luma.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
lastSequences[sample.SourceId] = sample.Sequence;
|
||||
if (!trackers.TryGetValue(sample.SourceId, out var tracker))
|
||||
{
|
||||
tracker = new MimirSparseFeatureTracker(new MimirSparseFeatureTrackerOptions(
|
||||
MaxFeatures: options.VideoFeatureMaxTracks,
|
||||
CellSizePixels: options.VideoFeatureCellSizePixels,
|
||||
SearchRadiusPixels: options.VideoFeatureSearchRadiusPixels));
|
||||
trackers.Add(sample.SourceId, tracker);
|
||||
}
|
||||
|
||||
var tracked = tracker.Update(
|
||||
sample.SourceId,
|
||||
video.Width,
|
||||
video.Height,
|
||||
luma,
|
||||
sample.TimestampNs);
|
||||
if (tracked.Tracks.Count == 0)
|
||||
{
|
||||
return new MimirWellFeatureSignal(
|
||||
sample.SourceId,
|
||||
sample.TimestampNs,
|
||||
video.Width,
|
||||
video.Height,
|
||||
0,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
0.5,
|
||||
0.5,
|
||||
FaustControls(sample.SourceId, 0.0, 0.0, 0.5, 0.5));
|
||||
}
|
||||
|
||||
var stableTracks = tracked.Tracks.Where(track => track.AgeFrames >= 3).ToArray();
|
||||
var points = stableTracks.Length == 0 ? tracked.Tracks : stableTracks;
|
||||
var centroidX = points.Average(static track => track.ImageX) / Math.Max(1.0, video.Width);
|
||||
var centroidY = points.Average(static track => track.ImageY) / Math.Max(1.0, video.Height);
|
||||
var motionEnergy = Math.Clamp(tracked.MeanSpeedPixelsPerSecond / Math.Max(1.0, Math.Max(video.Width, video.Height)), 0.0, 1.0);
|
||||
return new MimirWellFeatureSignal(
|
||||
sample.SourceId,
|
||||
sample.TimestampNs,
|
||||
video.Width,
|
||||
video.Height,
|
||||
tracked.StableTrackCount,
|
||||
tracked.Confidence,
|
||||
tracked.MeanSpeedPixelsPerSecond,
|
||||
motionEnergy,
|
||||
Math.Clamp(centroidX, 0.0, 1.0),
|
||||
Math.Clamp(centroidY, 0.0, 1.0),
|
||||
FaustControls(sample.SourceId, tracked.Confidence, motionEnergy, centroidX, centroidY));
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, float> FaustControls(
|
||||
string sourceId,
|
||||
double confidence,
|
||||
double motionEnergy,
|
||||
double centroidX,
|
||||
double centroidY)
|
||||
{
|
||||
var prefix = $"video/{sourceId}";
|
||||
return new Dictionary<string, float>(StringComparer.Ordinal)
|
||||
{
|
||||
[$"{prefix}/confidence"] = (float)Math.Clamp(confidence, 0.0, 1.0),
|
||||
[$"{prefix}/motion_energy"] = (float)Math.Clamp(motionEnergy, 0.0, 1.0),
|
||||
[$"{prefix}/centroid_x"] = (float)Math.Clamp(centroidX, 0.0, 1.0),
|
||||
[$"{prefix}/centroid_y"] = (float)Math.Clamp(centroidY, 0.0, 1.0),
|
||||
};
|
||||
}
|
||||
|
||||
private static byte[] ExtractLuma(MimirVideoFrameDescriptor video, ReadOnlySpan<byte> data) =>
|
||||
video.PixelFormat switch
|
||||
{
|
||||
MimirVideoPixelFormat.Gray8 or MimirVideoPixelFormat.R8 or MimirVideoPixelFormat.Bayer8 =>
|
||||
CopySinglePlane(video.Width, video.Height, Math.Max(video.Width, video.StrideBytes), data),
|
||||
MimirVideoPixelFormat.LeapStereoIr or MimirVideoPixelFormat.Rg8 =>
|
||||
CopyInterleavedLuma(video.Width, video.Height, Math.Max(video.Width * 2, video.StrideBytes), 2, data),
|
||||
MimirVideoPixelFormat.Yuy2 =>
|
||||
CopyInterleavedLuma(video.Width, video.Height, Math.Max(video.Width * 2, video.StrideBytes), 2, data),
|
||||
MimirVideoPixelFormat.Bgra8 =>
|
||||
CopyBgraLuma(video.Width, video.Height, Math.Max(video.Width * 4, video.StrideBytes), data),
|
||||
_ => [],
|
||||
};
|
||||
|
||||
private static byte[] CopySinglePlane(int width, int height, int stride, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (width <= 0 || height <= 0 || data.Length < stride * Math.Max(0, height - 1) + width)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var luma = new byte[width * height];
|
||||
for (var y = 0; y < height; y++)
|
||||
{
|
||||
data.Slice(y * stride, width).CopyTo(luma.AsSpan(y * width, width));
|
||||
}
|
||||
|
||||
return luma;
|
||||
}
|
||||
|
||||
private static byte[] CopyInterleavedLuma(int width, int height, int stride, int step, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (width <= 0 || height <= 0 || step <= 0 || data.Length < stride * Math.Max(0, height - 1) + width * step)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var luma = new byte[width * height];
|
||||
for (var y = 0; y < height; y++)
|
||||
{
|
||||
var row = data.Slice(y * stride, width * step);
|
||||
for (var x = 0; x < width; x++)
|
||||
{
|
||||
luma[y * width + x] = row[x * step];
|
||||
}
|
||||
}
|
||||
|
||||
return luma;
|
||||
}
|
||||
|
||||
private static byte[] CopyBgraLuma(int width, int height, int stride, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (width <= 0 || height <= 0 || data.Length < stride * Math.Max(0, height - 1) + width * 4)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var luma = new byte[width * height];
|
||||
for (var y = 0; y < height; y++)
|
||||
{
|
||||
var row = data.Slice(y * stride, width * 4);
|
||||
for (var x = 0; x < width; x++)
|
||||
{
|
||||
var offset = x * 4;
|
||||
luma[y * width + x] = (byte)Math.Clamp(
|
||||
(int)Math.Round(row[offset] * 0.114 + row[offset + 1] * 0.587 + row[offset + 2] * 0.299),
|
||||
0,
|
||||
255);
|
||||
}
|
||||
}
|
||||
|
||||
return luma;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record MimirWellPublishReceipt(
|
||||
string Document,
|
||||
int ByteLength,
|
||||
|
|
@ -284,8 +686,18 @@ internal sealed record MimirWellOptions(
|
|||
int PublishIntervalMs,
|
||||
int SyncIntervalMs,
|
||||
int PresentationDelayMs,
|
||||
int TargetPresentationDelayMs,
|
||||
int MinPresentationDelayMs,
|
||||
int LatencyGuardMs,
|
||||
int LatencyStepMs,
|
||||
int LatencyConvergenceFrames,
|
||||
double LatencyReadinessTarget,
|
||||
int MaxSamplesPerSource,
|
||||
int SyncCandidatesPerStep,
|
||||
bool VideoFeatureSignalsEnabled,
|
||||
int VideoFeatureMaxTracks,
|
||||
int VideoFeatureCellSizePixels,
|
||||
int VideoFeatureSearchRadiusPixels,
|
||||
bool VisualCalibrationEnabled,
|
||||
int VisualCalibrationIntervalMs,
|
||||
int VisualExpectedLedCount,
|
||||
|
|
@ -296,6 +708,8 @@ internal sealed record MimirWellOptions(
|
|||
int CaptureIntervalMs,
|
||||
int CaptureMaxBodyBytes,
|
||||
bool CaptureInlineBodies,
|
||||
bool StreamFramesEnabled,
|
||||
bool StreamFrameInlineBodies,
|
||||
int MeterEvery)
|
||||
{
|
||||
public static MimirWellOptions Parse(string[] args) => new(
|
||||
|
|
@ -306,8 +720,18 @@ internal sealed record MimirWellOptions(
|
|||
ParseInt(args, "--publish-ms", 250),
|
||||
ParseInt(args, "--sync-ms", 250),
|
||||
ParseInt(args, "--presentation-delay-ms", 2500),
|
||||
ParseInt(args, "--target-presentation-delay-ms", 750),
|
||||
ParseInt(args, "--min-presentation-delay-ms", 40),
|
||||
ParseInt(args, "--latency-guard-ms", 35),
|
||||
ParseInt(args, "--latency-step-ms", 25),
|
||||
ParseInt(args, "--latency-convergence-frames", 8),
|
||||
ParseDouble(args, "--latency-readiness-target", 0.98),
|
||||
ParseInt(args, "--max-samples-per-source", 4),
|
||||
ParseInt(args, "--sync-candidates", 1),
|
||||
ParseBool(args, "--video-feature-signals", true),
|
||||
ParseInt(args, "--video-feature-max-tracks", 96),
|
||||
ParseInt(args, "--video-feature-cell-size", 12),
|
||||
ParseInt(args, "--video-feature-search-radius", 18),
|
||||
ParseBool(args, "--visual-calibration", true),
|
||||
ParseInt(args, "--visual-calibration-ms", 250),
|
||||
ParseInt(args, "--visual-expected-leds", 38),
|
||||
|
|
@ -318,6 +742,8 @@ internal sealed record MimirWellOptions(
|
|||
ParseInt(args, "--capture-ms", 250),
|
||||
ParseInt(args, "--capture-max-body-bytes", 4 * 1024 * 1024),
|
||||
ParseBool(args, "--capture-inline-bodies", true),
|
||||
ParseBool(args, "--stream-frames", true),
|
||||
ParseBool(args, "--stream-frame-inline-bodies", true),
|
||||
ParseInt(args, "--meter-every", 20));
|
||||
|
||||
private static string ParseString(IReadOnlyList<string> args, string name, string fallback)
|
||||
|
|
@ -475,6 +901,29 @@ internal static class MimirWellCapturePage
|
|||
};
|
||||
}
|
||||
|
||||
public static IEnumerable<object> BuildStreamFrames(
|
||||
MimirWellOptions options,
|
||||
MimirWellFrameBuildResult frameResult,
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers,
|
||||
long captureSequence,
|
||||
DateTimeOffset startedAt)
|
||||
{
|
||||
var frame = frameResult.Frame;
|
||||
var sliceFrames = frame.Slices
|
||||
.Where(slice => slice.Sample.HasValue)
|
||||
.Select(slice => CaptureStreamFrame(options, captureSequence, startedAt, slice))
|
||||
.ToArray();
|
||||
if (sliceFrames.Length > 0)
|
||||
{
|
||||
return sliceFrames;
|
||||
}
|
||||
|
||||
return buffers
|
||||
.Where(buffer => buffer.Latest.HasValue)
|
||||
.Select(buffer => CaptureLatestFallbackStreamFrame(options, captureSequence, startedAt, buffer))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IEnumerable<object> CaptureSamples(
|
||||
MimirWellOptions options,
|
||||
long captureSequence,
|
||||
|
|
@ -521,7 +970,7 @@ internal static class MimirWellCapturePage
|
|||
slice.TimingEvidenceKind,
|
||||
},
|
||||
sample = SampleMetadata(sample),
|
||||
body = CaptureBody(options, sample),
|
||||
body = CaptureBody(options, sample, options.CaptureInlineBodies),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -551,7 +1000,115 @@ internal static class MimirWellCapturePage
|
|||
TimingEvidenceKind = "unsynchronized-fallback",
|
||||
},
|
||||
sample = SampleMetadata(sample),
|
||||
body = CaptureBody(options, sample),
|
||||
body = CaptureBody(options, sample, options.CaptureInlineBodies),
|
||||
};
|
||||
}
|
||||
|
||||
private static object CaptureStreamFrame(
|
||||
MimirWellOptions options,
|
||||
long captureSequence,
|
||||
DateTimeOffset startedAt,
|
||||
MimirSynchronizedStreamSlice slice)
|
||||
{
|
||||
var sample = slice.Sample!.Value;
|
||||
return CaptureStreamFrame(
|
||||
options,
|
||||
captureSequence,
|
||||
startedAt,
|
||||
sample,
|
||||
new
|
||||
{
|
||||
slice.SourceId,
|
||||
Kind = slice.Kind.ToString(),
|
||||
Origin = slice.Origin.ToString(),
|
||||
Status = slice.Status.ToString(),
|
||||
slice.SourceTimestampNs,
|
||||
slice.CanonicalStartNs,
|
||||
slice.CanonicalEndNs,
|
||||
slice.PresentationTimeNs,
|
||||
slice.TimingOffsetNs,
|
||||
slice.DistanceFromPresentationNs,
|
||||
slice.TimingConfidence,
|
||||
slice.TimingEvidenceKind,
|
||||
});
|
||||
}
|
||||
|
||||
private static object CaptureLatestFallbackStreamFrame(
|
||||
MimirWellOptions options,
|
||||
long captureSequence,
|
||||
DateTimeOffset startedAt,
|
||||
MimirRollingStreamBuffer buffer)
|
||||
{
|
||||
var sample = buffer.Latest!.Value;
|
||||
return CaptureStreamFrame(
|
||||
options,
|
||||
captureSequence,
|
||||
startedAt,
|
||||
sample,
|
||||
new
|
||||
{
|
||||
sample.SourceId,
|
||||
Kind = sample.Kind.ToString(),
|
||||
Origin = sample.Origin.ToString(),
|
||||
Status = "LatestUnalignedFallback",
|
||||
SourceTimestampNs = sample.TimestampNs,
|
||||
CanonicalStartNs = sample.TimestampNs,
|
||||
CanonicalEndNs = sample.TimestampNs,
|
||||
PresentationTimeNs = 0L,
|
||||
TimingOffsetNs = 0L,
|
||||
DistanceFromPresentationNs = 0L,
|
||||
TimingConfidence = 0.0,
|
||||
TimingEvidenceKind = "unsynchronized-fallback",
|
||||
});
|
||||
}
|
||||
|
||||
private static object CaptureStreamFrame(
|
||||
MimirWellOptions options,
|
||||
long captureSequence,
|
||||
DateTimeOffset startedAt,
|
||||
MimirStreamSample sample,
|
||||
object slice)
|
||||
{
|
||||
var bodyId = $"{options.NodeId}:{captureSequence}:{sample.SourceId}:{sample.Sequence}";
|
||||
var transport = PreferredBodyTransport(sample);
|
||||
return new
|
||||
{
|
||||
type = "cultmesh-observation",
|
||||
document = "mimir.cultmesh_stream_frame.v1",
|
||||
sourceId = "mimir-well",
|
||||
nodeId = options.NodeId,
|
||||
captureSequence,
|
||||
wallClockUtc = DateTimeOffset.UtcNow.ToString("O"),
|
||||
elapsedSeconds = (DateTimeOffset.UtcNow - startedAt).TotalSeconds,
|
||||
bodyId,
|
||||
stream = new
|
||||
{
|
||||
streamId = sample.SourceId,
|
||||
verseId = "mimir-live",
|
||||
ownerPeerId = options.NodeId,
|
||||
kind = sample.Kind.ToString(),
|
||||
origin = sample.Origin.ToString(),
|
||||
preferredTransports = StreamTransports(sample),
|
||||
clockDomainId = sample.SourceId,
|
||||
},
|
||||
frame = new
|
||||
{
|
||||
streamId = sample.SourceId,
|
||||
sequence = sample.Sequence,
|
||||
timestampNs = sample.TimestampNs,
|
||||
arrivalNs = sample.ArrivalNs,
|
||||
byteLength = sample.ByteLength,
|
||||
bodyTransport = transport,
|
||||
nativeHandle = NativeHandle(sample),
|
||||
nativeHandleKind = NativeHandleKind(sample),
|
||||
resourceKey = sample.VideoFrame?.ResourceKey,
|
||||
producerFenceHandle = sample.VideoFrame?.ProducerFenceHandle,
|
||||
producerFenceValue = sample.VideoFrame?.ProducerFenceValue ?? 0UL,
|
||||
unavoidableCopyCount = sample.VideoFrame?.UnavoidableCopyCount ?? 0,
|
||||
},
|
||||
slice,
|
||||
sample = SampleMetadata(sample),
|
||||
body = CaptureBody(options, sample, options.StreamFrameInlineBodies),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -591,9 +1148,43 @@ internal static class MimirWellCapturePage
|
|||
},
|
||||
};
|
||||
|
||||
private static object CaptureBody(MimirWellOptions options, MimirStreamSample sample)
|
||||
private static string PreferredBodyTransport(MimirStreamSample sample)
|
||||
{
|
||||
if (!options.CaptureInlineBodies)
|
||||
var kind = NativeHandleKind(sample);
|
||||
if (!string.IsNullOrWhiteSpace(kind))
|
||||
{
|
||||
return kind.Contains("d3d12", StringComparison.OrdinalIgnoreCase)
|
||||
? "shared-d3d12-texture"
|
||||
: kind.Contains("d3d11", StringComparison.OrdinalIgnoreCase)
|
||||
? "shared-d3d11-texture"
|
||||
: "native-handle";
|
||||
}
|
||||
|
||||
return sample.Data.IsEmpty ? "metadata-only" : "inline-bytes";
|
||||
}
|
||||
|
||||
private static string[] StreamTransports(MimirStreamSample sample)
|
||||
{
|
||||
var transport = PreferredBodyTransport(sample);
|
||||
return transport switch
|
||||
{
|
||||
"shared-d3d12-texture" => ["shared-d3d12-texture", "mimir-recorder-page", "cultcache-page"],
|
||||
"shared-d3d11-texture" => ["shared-d3d11-texture", "mimir-recorder-page", "cultcache-page"],
|
||||
"native-handle" => ["native-handle", "mimir-recorder-page", "cultcache-page"],
|
||||
"inline-bytes" => ["mimir-recorder-page", "inline-bytes", "cultcache-page"],
|
||||
_ => ["metadata-only"],
|
||||
};
|
||||
}
|
||||
|
||||
private static ulong NativeHandle(MimirStreamSample sample) =>
|
||||
sample.VideoFrame?.NativeHandle ?? sample.AudioBlock?.NativeHandle ?? 0UL;
|
||||
|
||||
private static string NativeHandleKind(MimirStreamSample sample) =>
|
||||
sample.VideoFrame?.NativeHandleKind ?? sample.AudioBlock?.NativeHandleKind ?? "";
|
||||
|
||||
private static object CaptureBody(MimirWellOptions options, MimirStreamSample sample, bool inlineBodies)
|
||||
{
|
||||
if (!inlineBodies)
|
||||
{
|
||||
return new { status = "not-inline", sample.ByteLength };
|
||||
}
|
||||
|
|
@ -728,6 +1319,8 @@ internal static class MimirWellSnapshot
|
|||
MimirSynchronizationHub hub,
|
||||
MimirPresentationControlState presentation,
|
||||
MimirWellFrameBuildResult frameResult,
|
||||
MimirWellLatencyDecision latencyDecision,
|
||||
MimirWellFeatureSignalFrame featureSignals,
|
||||
IReadOnlyList<MimirCameraExposureControlStatus> visualCalibration,
|
||||
IReadOnlyList<object> sourceErrors,
|
||||
object streamPressure,
|
||||
|
|
@ -760,11 +1353,66 @@ internal static class MimirWellSnapshot
|
|||
degradedReason = frameResult.DegradedReason,
|
||||
slices = frame.Slices.Select(SliceSnapshot).ToArray(),
|
||||
},
|
||||
latency = new
|
||||
{
|
||||
document = "mimir.well_latency_policy.v1",
|
||||
mode = "adaptive-bounded-ceiling",
|
||||
currentDelayMs = latencyDecision.PresentationDelay.TotalMilliseconds,
|
||||
ceilingDelayMs = latencyDecision.CeilingDelay.TotalMilliseconds,
|
||||
floorDelayMs = latencyDecision.FloorDelay.TotalMilliseconds,
|
||||
retainedOverlapMs = latencyDecision.RetainedOverlap.TotalMilliseconds,
|
||||
edgeSkewMs = latencyDecision.EdgeSkew.TotalMilliseconds,
|
||||
latencyDecision.ReadinessConfidence,
|
||||
latencyDecision.SyncConfidence,
|
||||
latencyDecision.ActiveBufferCount,
|
||||
latencyDecision.ReadySliceCount,
|
||||
latencyDecision.TotalSliceCount,
|
||||
latencyDecision.Reason,
|
||||
invariant = "Five seconds is a ceiling; the Well lowers requested presentation delay when overlap, slice readiness, and sync evidence allow it.",
|
||||
},
|
||||
clockDomains = MimirWellClockDiagnostics.Build(
|
||||
hub.Buffers.Buffers,
|
||||
runtimeConfig.Settings.Audio.ReferenceSourceId,
|
||||
frameResult),
|
||||
canonicalClockMaps = hub.CanonicalClockMaps.Select(map => new
|
||||
{
|
||||
map.StreamKey,
|
||||
map.OffsetNs,
|
||||
offsetMs = map.OffsetNs / 1_000_000.0,
|
||||
map.FirstSourceTimestampNs,
|
||||
map.FirstArrivalNs,
|
||||
map.LatestSourceTimestampNs,
|
||||
map.LatestCanonicalTimestampNs,
|
||||
map.SampleCount,
|
||||
}).ToArray(),
|
||||
streamPressure,
|
||||
featureSignals = new
|
||||
{
|
||||
featureSignals.Document,
|
||||
featureSignals.Sequence,
|
||||
featureSignals.MeanConfidence,
|
||||
featureSignals.MeanMotionPixelsPerSecond,
|
||||
featureSignals.StableTrackCount,
|
||||
featureSignals.FaustSignalContract,
|
||||
signals = featureSignals.Signals.Select(signal => new
|
||||
{
|
||||
signal.SourceId,
|
||||
signal.TimestampNs,
|
||||
signal.Width,
|
||||
signal.Height,
|
||||
signal.StableTrackCount,
|
||||
signal.Confidence,
|
||||
signal.MeanMotionPixelsPerSecond,
|
||||
signal.MotionEnergy,
|
||||
signal.NormalizedCentroidX,
|
||||
signal.NormalizedCentroidY,
|
||||
faustControls = signal.FaustControls.Select(pair => new
|
||||
{
|
||||
path = pair.Key,
|
||||
value = pair.Value,
|
||||
}).ToArray(),
|
||||
}).ToArray(),
|
||||
},
|
||||
audioSync = new
|
||||
{
|
||||
referenceSourceId = runtimeConfig.Settings.Audio.ReferenceSourceId,
|
||||
|
|
|
|||
|
|
@ -354,3 +354,4 @@
|
|||
{"ts": "2026-06-02T14:13:56+01:00", "type": "decision", "status": "ok", "note": "Starfire USB PS Move shakedown: Mimir.PsMoveProbe sees three Sony VID_054C/PID_03D5 HID collections and can write report 0x06 to col01. Added --rgb/--pulse-ms/--pulse-count/--interval-ms/--event-log; proof wrote readable mimir.psmove_usb_pulse_event.v1 receipts and 49-byte on/off reports. This is a local fast visual timing witness primitive; precision scheduling should move into a persistent/native worker because CLI pulse proof showed requested 60ms pulse producing about 221ms on/off gap."}
|
||||
{"ts": "2026-06-02T14:41:46+01:00", "type": "verification", "status": "ok", "note": "Starfire USB PS Move paired chirp train now derives audio samples and visual LED pulses from one event schedule. 2026-06-02 run artifacts/runtime/starfire-usb-move-paired-20260602-144135 emitted three WASAPI chirps and three Move LED pulses with shared event ids; receipt showed planned/observed LED onsets 150/400/650 ms at millisecond resolution after high-resolution wait cut."}
|
||||
{"ts": "2026-06-02T15:09:31+01:00", "type": "verification", "status": "ok", "note": "mimir_well_shakedown.py now verifies paired PS Move chirp receipts: scheduled events must have nonzero audio samples, render completion, LED on/off phases, no orphan visual events, and LED schedule error within a configurable ceiling. Good receipt starfire-usb-move-paired-20260602-144135 passes under 5 ms; earlier receipt starfire-usb-move-paired-20260602-144003 fails with 67 ms max error and exit 2 under --fail-on-paired-pulse-error."}
|
||||
{"ts": "2026-06-02T15:12:44+01:00", "type": "verification", "status": "ok", "note": "Adaptive Well/runtime dashboard cut builds clean: Mimir.Runtime, Mimir.Well, and Mimir.EveDashboard compile after canonical clock mapping, ASIO multiplex drain, adaptive latency telemetry, feature-signal mirror, and richer Eve dashboard lowering. Rebooted lightweight Eve broker/Odin projection on 2026-06-02; health ok at 127.0.0.1:8795 and 8797 without restarting heavy Well capture."}
|
||||
|
|
|
|||
Loading…
Reference in a new issue