Expose Well score state on Eve dashboard
This commit is contained in:
parent
2e897f5879
commit
1ccf99486d
5 changed files with 479 additions and 22 deletions
|
|
@ -177,9 +177,18 @@ if (-not $SkipMoves) {
|
|||
"--hit-loudness-threshold", "0.84",
|
||||
"--loudness-floor", "0.006",
|
||||
"--loudness-exponent", "2.6",
|
||||
"--quiet-brightness", "0.008",
|
||||
"--max-brightness", "0.72",
|
||||
"--brightness-exponent", "0.9",
|
||||
"--quiet-brightness", "0.0",
|
||||
"--max-brightness", "0.16",
|
||||
"--brightness-exponent", "1.2",
|
||||
"--score-subdivisions", "8",
|
||||
"--score-instrument-spacing", "3",
|
||||
"--score-release", "0.50",
|
||||
"--score-loudness-threshold", "0.97",
|
||||
"--score-min-loudness-gate", "0.18",
|
||||
"--score-loudness-exponent", "3.0",
|
||||
"--score-min-accent", "0.04",
|
||||
"--score-max-envelope", "0.40",
|
||||
"--score-min-tempo-confidence", "0.0",
|
||||
"--debruijn-polyrhythm",
|
||||
"--harmonic-base", "1.5",
|
||||
"--microtonal-cents", "17"
|
||||
|
|
|
|||
|
|
@ -842,6 +842,7 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
var confidence = snapshot.SyncStates.Count == 0
|
||||
? snapshot.SyncReports.Select(static report => report.Confidence)
|
||||
.Concat(snapshot.SyncDecodeAttempts.Select(static attempt => attempt.Confidence))
|
||||
.Concat(snapshot.Well?.AudioSyncStates.Select(static state => state.Confidence) ?? [])
|
||||
.DefaultIfEmpty(0.0)
|
||||
.Max()
|
||||
: snapshot.SyncStates.Average(static state => state.Confidence);
|
||||
|
|
@ -860,7 +861,7 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
},
|
||||
new("mimir-audio-rms", $"Audio RMS\n{rms:0.000000}", "audio-rms", 0.0, -0.12, 0.28, 0.16, ToneFor(NormalizeRms(rms)))
|
||||
{
|
||||
Detail = $"{snapshot.Spectra.Count} spectrum lanes",
|
||||
Detail = $"{snapshot.Spectra.Count} spectrum lanes; Well audio buffers {snapshot.Well?.AudioBuffers ?? 0}",
|
||||
},
|
||||
new("mimir-observation-streams", $"Device Streams\n{liveObservationCount}/{snapshot.ObservationStreams.Count} active", "observation-streams", 0.44, -0.12, 0.28, 0.16, liveObservationCount > 0 ? "active" : "waiting")
|
||||
{
|
||||
|
|
@ -886,18 +887,39 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
var pulse = Pulse();
|
||||
var children = new List<DashboardUiElement>
|
||||
{
|
||||
UiElement.Pane("mimir-live-overview", "Mimir Live Stats", [
|
||||
UiElement.Container(
|
||||
"mimir-live-row-overview",
|
||||
"row",
|
||||
new DashboardUiLayout { Direction = "horizontal", Gap = 8 },
|
||||
[
|
||||
UiElement.Pane("mimir-live-overview", "Mimir Live Stats", [
|
||||
UiElement.Text("mimir-live-summary", $"{pulse} {snapshot.Summary}", "strong"),
|
||||
UiElement.Text("mimir-live-sources", $"telemetry: {snapshot.TelemetrySource}\nobservations: {snapshot.ObservationSource}", "caption"),
|
||||
StatBar("tracking-confidence", "tracking confidence", confidence, "cool"),
|
||||
StatBar("audio-rms", "max channel RMS", NormalizeRms(rms), "warm", $"{rms:0.000000}"),
|
||||
StatBar("observation-liveness", "device stream liveness", snapshot.ObservationStreams.Count == 0 ? 0.0 : liveObservationCount / (double)snapshot.ObservationStreams.Count, "cool"),
|
||||
]),
|
||||
BuildAudioPane(snapshot.Spectra),
|
||||
BuildSyncPane(snapshot),
|
||||
BuildVideoPane(snapshot),
|
||||
BuildObservationPane(snapshot.ObservationStreams),
|
||||
BuildActuatorPane(snapshot.ActuatorCommands),
|
||||
]),
|
||||
BuildMoveMusicPane(snapshot.MoveMusic),
|
||||
BuildWellPane(snapshot.Well),
|
||||
]),
|
||||
UiElement.Container(
|
||||
"mimir-live-row-runtime",
|
||||
"row",
|
||||
new DashboardUiLayout { Direction = "horizontal", Gap = 8 },
|
||||
[
|
||||
BuildAudioPane(snapshot.Spectra, snapshot.Well),
|
||||
BuildSyncPane(snapshot),
|
||||
BuildVideoPane(snapshot),
|
||||
]),
|
||||
UiElement.Container(
|
||||
"mimir-live-row-witness",
|
||||
"row",
|
||||
new DashboardUiLayout { Direction = "horizontal", Gap = 8 },
|
||||
[
|
||||
BuildObservationPane(snapshot.ObservationStreams),
|
||||
BuildActuatorPane(snapshot.ActuatorCommands),
|
||||
BuildCalibrationPane(snapshot),
|
||||
]),
|
||||
};
|
||||
|
||||
return new DashboardSurface
|
||||
|
|
@ -908,14 +930,23 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
Root = UiElement.Container(
|
||||
"mimir-live-stats-root",
|
||||
"dashboard",
|
||||
new DashboardUiLayout { Direction = "vertical", Gap = 8, Padding = 8, Overflow = "scroll" },
|
||||
new DashboardUiLayout { Direction = "vertical", Gap = 6, Padding = 6, Overflow = "scroll" },
|
||||
children),
|
||||
};
|
||||
}
|
||||
|
||||
private static DashboardUiElement BuildAudioPane(IReadOnlyList<MimirSpectrumStat> spectra)
|
||||
private static DashboardUiElement BuildAudioPane(IReadOnlyList<MimirSpectrumStat> spectra, MimirWellStat? well)
|
||||
{
|
||||
var items = spectra.Count == 0
|
||||
var wellAudio = well?.Buffers
|
||||
.Where(static buffer => string.Equals(buffer.Kind, "Audio", StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(static buffer => buffer.SourceId, StringComparer.Ordinal)
|
||||
.Take(8)
|
||||
.Select(buffer => UiElement.Text(
|
||||
$"well-audio-{StableId(buffer.SourceId)}",
|
||||
$"{buffer.SourceId}: {buffer.Count} blocks edge {NsAge(buffer.EdgeNs)}",
|
||||
"caption"))
|
||||
.ToArray() ?? [];
|
||||
var items = spectra.Count == 0 && wellAudio.Length == 0
|
||||
? [UiElement.Text("mimir-spectrum-empty", "no spectrum telemetry yet", "caption")]
|
||||
: spectra
|
||||
.OrderBy(static spectrum => spectrum.SourceId, StringComparer.Ordinal)
|
||||
|
|
@ -930,9 +961,10 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
UiElement.Text($"spectrum-{StableId(spectrum.SourceId)}-detail", $"rms {spectrum.Rms:0.000000} peak {spectrum.Peak:0.000000} floor {spectrum.NoiseFloorDb:0.0} dB", "caption"),
|
||||
UiElement.Text($"spectrum-{StableId(spectrum.SourceId)}-peaks", spectrum.Peaks, "caption"),
|
||||
]))
|
||||
.Concat(wellAudio)
|
||||
.ToArray();
|
||||
|
||||
return UiElement.Pane("mimir-audio-pane", "Per-Channel RMS", items);
|
||||
return UiElement.Pane("mimir-audio-pane", "Audio", items);
|
||||
}
|
||||
|
||||
private static DashboardUiElement BuildSyncPane(MimirLiveStatsSnapshot snapshot)
|
||||
|
|
@ -1039,6 +1071,64 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
return UiElement.Pane("mimir-actuator-pane", "Alignment Actuator", children);
|
||||
}
|
||||
|
||||
private static DashboardUiElement BuildMoveMusicPane(MimirMoveMusicStat? music)
|
||||
{
|
||||
if (music == null)
|
||||
{
|
||||
return UiElement.Pane("mimir-move-music-pane", "Move Score", [UiElement.Text("mimir-move-music-empty", "no Move score trace yet", "caption")]);
|
||||
}
|
||||
|
||||
var children = new List<DashboardUiElement>
|
||||
{
|
||||
UiElement.Text("mimir-move-music-head", $"bpm {music.Bpm:0.0} c={music.BpmConfidence:0.00} {music.KeyName} {music.KeyMode} chord {music.ChordName}", "mono"),
|
||||
StatBar("mimir-move-loudness", "loudness gate", music.LoudnessGate, "warm", $"{music.LoudnessGate:0.000}"),
|
||||
UiElement.Text("mimir-move-score", $"score {Bar(music.ScoreEnvelopeMax, 18)} max env {music.ScoreEnvelopeMax:0.000} max rgb {music.MaxRgb}", "caption"),
|
||||
UiElement.Text("mimir-move-sources", string.Join(" | ", music.Sources.Take(4)), "caption"),
|
||||
};
|
||||
|
||||
return UiElement.Pane("mimir-move-music-pane", "Move Score", children);
|
||||
}
|
||||
|
||||
private static DashboardUiElement BuildWellPane(MimirWellStat? well)
|
||||
{
|
||||
if (well == null)
|
||||
{
|
||||
return UiElement.Pane("mimir-well-pane", "Well", [UiElement.Text("mimir-well-empty", "no Well snapshot yet", "caption")]);
|
||||
}
|
||||
|
||||
return UiElement.Pane("mimir-well-pane", "Well", [
|
||||
UiElement.Text("mimir-well-summary", $"seq {well.Sequence} {well.ElapsedSeconds:0}s ingested {well.IngestedSamples:0} sources {well.LiveSources}/{well.ConfiguredSources}", "mono"),
|
||||
StatBar("mimir-well-completeness", "frame completeness", well.SynchronizedFrameComplete ? 1.0 : 0.0, well.SynchronizedFrameComplete ? "cool" : "danger"),
|
||||
UiElement.Text("mimir-well-buffers", $"buffers {well.Buffers.Count} audio {well.AudioBuffers} video {well.VideoBuffers} ready {well.ReadySlices}/{well.TotalSlices}", "caption"),
|
||||
UiElement.Text("mimir-well-probe", well.Probe == null ? "probe: none" : $"probe emit={well.Probe.ShouldEmit} sync={well.Probe.SyncConfidence:0.000} freq={well.Probe.FrequencyConfidence:0.000} {well.Probe.Reason}", "caption"),
|
||||
]);
|
||||
}
|
||||
|
||||
private static DashboardUiElement BuildCalibrationPane(MimirLiveStatsSnapshot snapshot)
|
||||
{
|
||||
var leap = snapshot.Well?.Buffers.FirstOrDefault(static buffer => buffer.SourceId.Contains("leap", StringComparison.OrdinalIgnoreCase));
|
||||
var eyeStreams = snapshot.ObservationStreams
|
||||
.Where(static stream => stream.StreamId.Contains("eye", StringComparison.OrdinalIgnoreCase) || stream.StreamId.Contains("video2", StringComparison.OrdinalIgnoreCase) || stream.StreamId.Contains("video3", StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(static stream => stream.StreamId, StringComparer.Ordinal)
|
||||
.Take(4)
|
||||
.ToArray();
|
||||
var moveStreams = snapshot.ObservationStreams
|
||||
.Where(static stream => stream.Kind.Contains("move", StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(static stream => stream.StreamId, StringComparer.Ordinal)
|
||||
.Take(4)
|
||||
.ToArray();
|
||||
|
||||
var children = new List<DashboardUiElement>
|
||||
{
|
||||
UiElement.Text("mimir-calibration-leap", leap == null ? "Leap: no Well buffer" : $"Leap: {leap.Count} frames {leap.PixelFormat} {leap.Width:0}x{leap.Height:0} edge {NsAge(leap.EdgeNs)}", "mono"),
|
||||
UiElement.Text("mimir-calibration-eyes", $"Eyes: {eyeStreams.Count(static stream => stream.State == "active")}/{eyeStreams.Length} active", "caption"),
|
||||
UiElement.Text("mimir-calibration-moves", $"Moves: {moveStreams.Count(static stream => stream.State == "active")}/{moveStreams.Length} active", "caption"),
|
||||
};
|
||||
children.AddRange(eyeStreams.Select(stream => UiElement.Text($"mimir-eye-{StableId(stream.StreamId)}", $"{stream.StreamId} {stream.Shape} age {stream.AgeSeconds:0}s", "caption")));
|
||||
children.AddRange(moveStreams.Select(stream => UiElement.Text($"mimir-move-{StableId(stream.StreamId)}", $"{stream.StreamId} {stream.Shape} age {stream.AgeSeconds:0}s", "caption")));
|
||||
return UiElement.Pane("mimir-calibration-pane", "Leap / Eyes / Moves", children);
|
||||
}
|
||||
|
||||
private static DashboardUiElement StatBar(string id, string label, double value, string tone, string? displayValue = null) =>
|
||||
UiElement.Card(
|
||||
id,
|
||||
|
|
@ -1079,6 +1169,18 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider
|
|||
private static string ToneFor(double value) =>
|
||||
value >= 0.70 ? "cool" : value >= 0.25 ? "warm" : "danger";
|
||||
|
||||
private static string NsAge(double timestampNs)
|
||||
{
|
||||
if (timestampNs <= 0)
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var nowNs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000.0;
|
||||
var seconds = Math.Max(0.0, (nowNs - timestampNs) / 1_000_000_000.0);
|
||||
return seconds < 1.0 ? $"{seconds * 1000.0:0}ms" : $"{seconds:0.0}s";
|
||||
}
|
||||
|
||||
private static string StableId(string value) =>
|
||||
string.Join("-", value.ToLowerInvariant().Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries))
|
||||
.Replace(":", "-", StringComparison.Ordinal)
|
||||
|
|
@ -1098,8 +1200,12 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
public IReadOnlyList<MimirVideoBufferStat> VideoBuffers { get; init; } = [];
|
||||
public IReadOnlyList<MimirActuatorCommandStat> ActuatorCommands { get; init; } = [];
|
||||
public IReadOnlyList<MimirObservationStreamStat> ObservationStreams { get; init; } = [];
|
||||
public bool HasAnyData => Telemetry != null || Spectra.Count > 0 || SyncStates.Count > 0 || VideoBuffers.Count > 0 || ObservationStreams.Count > 0;
|
||||
public string Summary => $"{Spectra.Count} audio lanes / {SyncStates.Count} sync states / {SyncDecodeAttempts.Count} decode attempts / {VideoBuffers.Count} video buffers / {ObservationStreams.Count} device streams";
|
||||
public MimirWellStat? Well { get; init; }
|
||||
public MimirMoveMusicStat? MoveMusic { get; init; }
|
||||
public bool HasAnyData => Telemetry != null || Well != null || MoveMusic != null || Spectra.Count > 0 || SyncStates.Count > 0 || VideoBuffers.Count > 0 || ObservationStreams.Count > 0;
|
||||
public string Summary => Well == null
|
||||
? $"{Spectra.Count} audio lanes / {SyncStates.Count} sync states / {SyncDecodeAttempts.Count} decode attempts / {VideoBuffers.Count} video buffers / {ObservationStreams.Count} device streams"
|
||||
: $"Well {Well.LiveSources}/{Well.ConfiguredSources} sources / {Well.AudioBuffers} audio / {Well.VideoBuffers} video / {ObservationStreams.Count} device streams";
|
||||
|
||||
public static MimirLiveStatsSnapshot Load(string telemetryLogPath, string observationLogPath)
|
||||
{
|
||||
|
|
@ -1118,6 +1224,8 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
VideoBuffers = LatestBySource(telemetryLines.Select(ParseVideoBuffer).Where(static value => value != null)!),
|
||||
ActuatorCommands = LatestBySource(telemetryLines.Select(ParseActuatorCommand).Where(static value => value != null)!),
|
||||
ObservationStreams = LatestObservationStreams(observationLines),
|
||||
Well = LatestWellSnapshot(observationLines),
|
||||
MoveMusic = LatestMoveMusicTrace(),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -1192,6 +1300,152 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
.ToArray();
|
||||
}
|
||||
|
||||
private static MimirWellStat? LatestWellSnapshot(IEnumerable<string> lines)
|
||||
{
|
||||
MimirWellStat? latest = null;
|
||||
foreach (var line in lines)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(line);
|
||||
var root = document.RootElement;
|
||||
if (string.Equals(JsonStringAny(root, "document"), "mimir.well_snapshot.v1", StringComparison.Ordinal))
|
||||
{
|
||||
latest = ParseWellSnapshot(root);
|
||||
}
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return latest;
|
||||
}
|
||||
|
||||
private static MimirWellStat ParseWellSnapshot(JsonElement root)
|
||||
{
|
||||
var buffers = ArrayItems(TryGetAny(root, "buffers"))
|
||||
.Select(static buffer =>
|
||||
{
|
||||
var latest = TryGetAny(buffer, "latest");
|
||||
var video = TryGetAny(latest, "video");
|
||||
var audio = TryGetAny(latest, "audio");
|
||||
return new MimirWellBufferStat(
|
||||
JsonStringAny(buffer, "SourceId", "sourceId") ?? "unknown",
|
||||
JsonStringAny(buffer, "Kind", "kind") ?? "unknown",
|
||||
(int)JsonNumberAny(buffer, "Count", "count"),
|
||||
JsonNumberAny(buffer, "EdgeNs", "edgeNs"),
|
||||
(int)JsonNumberAny(video, "Width", "width"),
|
||||
(int)JsonNumberAny(video, "Height", "height"),
|
||||
JsonStringAny(video, "PixelFormat", "pixelFormat") ?? "",
|
||||
(int)JsonNumberAny(audio, "SampleRate", "sampleRate"),
|
||||
(int)JsonNumberAny(audio, "Channels", "channels"));
|
||||
})
|
||||
.ToArray();
|
||||
var frame = TryGetAny(root, "synchronizedFrame");
|
||||
var slices = ArrayItems(TryGetAny(frame, "slices")).ToArray();
|
||||
var audioSync = TryGetAny(root, "audioSync");
|
||||
var states = ArrayItems(TryGetAny(audioSync, "states"))
|
||||
.Select(static state => new MimirWellAudioSyncStateStat(
|
||||
JsonStringAny(state, "SourceId", "sourceId") ?? "unknown",
|
||||
JsonStringAny(state, "ReferenceSourceId", "referenceSourceId") ?? "unknown",
|
||||
JsonNumberAny(state, "SmoothedDelaySamples", "smoothedDelaySamples"),
|
||||
JsonNumberAny(state, "SamplingRateOffsetPpm", "samplingRateOffsetPpm"),
|
||||
JsonNumberAny(state, "Confidence", "confidence")))
|
||||
.ToArray();
|
||||
var probeElement = TryGetAny(audioSync, "probe");
|
||||
var probe = probeElement.ValueKind == JsonValueKind.Object
|
||||
? new MimirWellProbeStat(
|
||||
JsonBoolAny(probeElement, "ShouldEmit", "shouldEmit"),
|
||||
JsonNumberAny(probeElement, "AggregateSyncConfidence", "aggregateSyncConfidence"),
|
||||
JsonNumberAny(probeElement, "AggregateFrequencyResponseConfidence", "aggregateFrequencyResponseConfidence"),
|
||||
JsonStringAny(probeElement, "reason") ?? "unknown")
|
||||
: null;
|
||||
return new MimirWellStat(
|
||||
(long)JsonNumberAny(root, "sequence"),
|
||||
JsonNumberAny(root, "elapsedSeconds"),
|
||||
JsonNumberAny(root, "ingestedSamples"),
|
||||
(int)JsonNumberAny(root, "configuredSources"),
|
||||
(int)JsonNumberAny(root, "liveSources"),
|
||||
JsonBoolAny(frame, "IsComplete", "isComplete"),
|
||||
slices.Count(static slice => string.Equals(JsonStringAny(slice, "Status", "status"), "Ready", StringComparison.OrdinalIgnoreCase)),
|
||||
slices.Length,
|
||||
buffers,
|
||||
states,
|
||||
probe);
|
||||
}
|
||||
|
||||
private static MimirMoveMusicStat? LatestMoveMusicTrace()
|
||||
{
|
||||
var runtimeRoot = Path.Combine(AppContext.BaseDirectory, "artifacts", "runtime");
|
||||
if (!Directory.Exists(runtimeRoot))
|
||||
{
|
||||
runtimeRoot = @"E:\Projects\Mimir\artifacts\runtime";
|
||||
}
|
||||
|
||||
if (!Directory.Exists(runtimeRoot))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trace = Directory.EnumerateFiles(runtimeRoot, "online-sync.jsonl", SearchOption.AllDirectories)
|
||||
.Select(static path => new FileInfo(path))
|
||||
.Where(static info => info.DirectoryName?.Contains("move", StringComparison.OrdinalIgnoreCase) == true)
|
||||
.OrderByDescending(static info => info.LastWriteTimeUtc)
|
||||
.FirstOrDefault();
|
||||
if (trace == null || trace.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var line = TailTextFile(trace.FullName, 64 * 1024)
|
||||
.Split("\n", StringSplitOptions.RemoveEmptyEntries)
|
||||
.LastOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(line);
|
||||
var root = document.RootElement;
|
||||
var maxRgb = 0;
|
||||
var moves = TryGetAny(root, "moves");
|
||||
if (moves.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
foreach (var move in moves.EnumerateObject())
|
||||
{
|
||||
if (move.Value.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
maxRgb = Math.Max(maxRgb, move.Value.EnumerateArray().Select(JsonNumber).Select(static value => (int)value).DefaultIfEmpty(0).Max());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var envelopes = ArrayItems(TryGetAny(root, "score_gesture_envelopes")).Select(JsonNumber).DefaultIfEmpty(0.0).ToArray();
|
||||
var sources = ArrayItems(TryGetAny(root, "music_sources"))
|
||||
.Select(static source => $"{JsonStringAny(source, "source") ?? "source"} {JsonNumberAny(source, "bpm"):0.0}/{JsonNumberAny(source, "bpm_confidence"):0.00}")
|
||||
.ToArray();
|
||||
return new MimirMoveMusicStat(
|
||||
trace.FullName,
|
||||
JsonNumberAny(root, "bpm"),
|
||||
JsonNumberAny(root, "bpm_confidence"),
|
||||
JsonNumberAny(root, "loudness_gate"),
|
||||
JsonNumberAny(root, "loudness_percentile"),
|
||||
JsonStringAny(root, "key_name") ?? "?",
|
||||
JsonStringAny(root, "key_mode") ?? "?",
|
||||
JsonStringAny(root, "chord_name") ?? "?",
|
||||
envelopes.Max(),
|
||||
maxRgb,
|
||||
sources);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<MimirObservationStreamStat> LatestObservationStreams(IEnumerable<string> lines)
|
||||
{
|
||||
var byStream = new Dictionary<string, MimirObservationStreamStat>(StringComparer.Ordinal);
|
||||
|
|
@ -1476,6 +1730,66 @@ internal sealed class MimirLiveStatsSnapshot
|
|||
private static double JsonNumber(JsonElement root, string property) =>
|
||||
root.TryGetProperty(property, out var child) && child.ValueKind == JsonValueKind.Number && child.TryGetDouble(out var value) ? value : 0.0;
|
||||
|
||||
private static JsonElement TryGetAny(JsonElement root, params string[] properties)
|
||||
{
|
||||
if (root.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return default;
|
||||
}
|
||||
|
||||
foreach (var property in properties)
|
||||
{
|
||||
if (root.TryGetProperty(property, out var child))
|
||||
{
|
||||
return child;
|
||||
}
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
|
||||
private static IEnumerable<JsonElement> ArrayItems(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var item in element.EnumerateArray())
|
||||
{
|
||||
yield return item;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? JsonStringAny(JsonElement root, params string[] properties)
|
||||
{
|
||||
var child = TryGetAny(root, properties);
|
||||
return child.ValueKind == JsonValueKind.String ? child.GetString() : null;
|
||||
}
|
||||
|
||||
private static double JsonNumberAny(JsonElement root, params string[] properties) => JsonNumber(TryGetAny(root, properties));
|
||||
|
||||
private static double JsonNumber(JsonElement element) =>
|
||||
element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value) ? value : 0.0;
|
||||
|
||||
private static bool JsonBoolAny(JsonElement root, params string[] properties)
|
||||
{
|
||||
var child = TryGetAny(root, properties);
|
||||
return child.ValueKind == JsonValueKind.True || (child.ValueKind == JsonValueKind.Number && child.TryGetDouble(out var value) && value != 0.0);
|
||||
}
|
||||
|
||||
private static string NsAge(double timestampNs)
|
||||
{
|
||||
if (timestampNs <= 0)
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var nowNs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000.0;
|
||||
var seconds = Math.Max(0.0, (nowNs - timestampNs) / 1_000_000_000.0);
|
||||
return seconds < 1.0 ? $"{seconds * 1000.0:0}ms" : $"{seconds:0.0}s";
|
||||
}
|
||||
|
||||
private static string TailTextFile(string filePath, int maxBytes)
|
||||
{
|
||||
var info = new FileInfo(filePath);
|
||||
|
|
@ -1514,6 +1828,52 @@ internal sealed record MimirActuatorCommandStat(string SourceId, double DelaySam
|
|||
|
||||
internal sealed record MimirObservationStreamStat(string DeviceId, string StreamId, string Kind, double Sequence, string State, double AgeSeconds, string Shape);
|
||||
|
||||
internal sealed record MimirWellStat(
|
||||
long Sequence,
|
||||
double ElapsedSeconds,
|
||||
double IngestedSamples,
|
||||
int ConfiguredSources,
|
||||
int LiveSources,
|
||||
bool SynchronizedFrameComplete,
|
||||
int ReadySlices,
|
||||
int TotalSlices,
|
||||
IReadOnlyList<MimirWellBufferStat> Buffers,
|
||||
IReadOnlyList<MimirWellAudioSyncStateStat> AudioSyncStates,
|
||||
MimirWellProbeStat? Probe)
|
||||
{
|
||||
public int AudioBuffers => Buffers.Count(static buffer => string.Equals(buffer.Kind, "Audio", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
public int VideoBuffers => Buffers.Count(static buffer => string.Equals(buffer.Kind, "Video", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
internal sealed record MimirWellBufferStat(
|
||||
string SourceId,
|
||||
string Kind,
|
||||
int Count,
|
||||
double EdgeNs,
|
||||
int Width,
|
||||
int Height,
|
||||
string PixelFormat,
|
||||
int SampleRate,
|
||||
int Channels);
|
||||
|
||||
internal sealed record MimirWellAudioSyncStateStat(string SourceId, string ReferenceSourceId, double DelaySamples, double SroPpm, double Confidence);
|
||||
|
||||
internal sealed record MimirWellProbeStat(bool ShouldEmit, double SyncConfidence, double FrequencyConfidence, string Reason);
|
||||
|
||||
internal sealed record MimirMoveMusicStat(
|
||||
string TracePath,
|
||||
double Bpm,
|
||||
double BpmConfidence,
|
||||
double LoudnessGate,
|
||||
double LoudnessPercentile,
|
||||
string KeyName,
|
||||
string KeyMode,
|
||||
string ChordName,
|
||||
double ScoreEnvelopeMax,
|
||||
int MaxRgb,
|
||||
IReadOnlyList<string> Sources);
|
||||
|
||||
internal sealed class MimirStreamLayoutProvider : MutableDashboardProvider
|
||||
{
|
||||
public const string ProviderId = "mimir.stream.layout";
|
||||
|
|
|
|||
|
|
@ -338,3 +338,4 @@
|
|||
{"ts": "2026-06-01T21:06:07+01:00", "type": "implementation", "status": "ok", "note": "Added the first online Verse observation daemon stack: Mimir.EveSensorReceiver now serves a /eve/periwinkle/subscribe WebSocket lane, Mimir.VerseRecorder subscribes and writes session.json plus observations.jsonl under C:\\Users\\Meta\\Videos\\Mimir\\VerseCaptures, and scripts/start-online-verse-daemons.ps1 supervises relay, recorder, Nightwing Eyes/Move witness publication, music-synced Move gestures, and a local BufferSmoke runtime capture worker. Live run online-verse-daemons-20260601-210106 had relay, recorder, Move sync, BufferSmoke, and Nightwing witness running, with observations.jsonl past 1000 records and growing. This captures typed witness observations; raw aligned source windows/stems still need a MimirSynchronizationHub Verse publisher."}
|
||||
{"ts": "2026-06-01T21:27:15+01:00", "type": "implementation", "status": "partial", "note": "Added Mimir.Well as the long-running Starfire Well daemon and broad config/mimir-runtime.well.local.json. Live run online-verse-daemons-20260601-212502 started relay, recorder, Mimir.Well, music-synced de-Bruijn/microtonal Move gestures, and Nightwing witness PID 1116285. Recorder C:\\Users\\Meta\\Videos\\Mimir\\VerseCaptures\\20260601-212509 reached 292 records and included mimir.well_snapshot.v1, Nightwing builtin camera/mic witness records, and ps3-eye-0/1 Move observations. Well snapshots showed live Scarlett ASIO, Leap stereo IR, Kiyo Pro RGB, and Kiyo Basic RGB buffers; Raven and Eve lanes were configured but empty because their producers were not feeding. Sync frame remains degraded because ASIO device timestamps and camera driver timestamps are not yet mapped into one canonical timeline."}
|
||||
{"ts": "2026-06-01T21:41:07+01:00", "type": "decision", "status": "ok", "note": "2026-06-01 online Move music sync demoted spectral flux from brightness authority: Realtek loopback remains primary, optional Scarlett ASIO channels add auxiliary onset/tempo evidence, chroma estimates key/chord, and recent loudness percentile gates all Move brightness so quiet/steady-state music stays dark."}
|
||||
{"ts": "2026-06-01T22:09:13+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: online Move music sync now places visual gestures through a score scheduler instead of continuous envelope brightness; raw level/hit/flux/color are inputs, not RGB owners. Live score worker move-music-score-20260601-220807 showed max_env=0.4, max_rgb=4, and 9 lit frames in an 824-frame poll. Eve Live Stats now reads typed Well snapshots and latest Move score traces, exposing Well buffers/sync/probe, Leap/Eyes/Moves evidence, and Move music key/tempo/loudness/envelopes. AquaSynth Syrinx is identified as the correct Realtek voice target but is not yet wired into the live emitter."}
|
||||
|
|
|
|||
|
|
@ -226,6 +226,21 @@ invariants:
|
|||
own autocorrelation tempo estimates, chroma scoring estimates key and chord
|
||||
candidates, and a recent loudness-percentile gate owns whether the Moves
|
||||
are allowed to brighten. Flux alone no longer has authority to shout.
|
||||
The current score cut adds `ScoreGestureScheduler` as the only owner of
|
||||
Move visual gesture openings. Raw level, hit, flux, and color balance are
|
||||
inputs; they no longer emit continuous RGB by themselves. The scheduler
|
||||
divides the beat into score slots, assigns slots across Moves with
|
||||
de-Bruijn/polyrhythmic identity, opens short envelopes only when recent
|
||||
broadband spectral rise reaches the adaptive loudness gate, and maps the
|
||||
top of that gate to the configured maximum envelope. Quiet frames are
|
||||
black. The 2026-06-01 live worker
|
||||
`artifacts/runtime/move-music-score-20260601-220807/online-sync.jsonl`
|
||||
showed `max_env=0.4`, `max_rgb=4`, and only 9 lit frames in an 824-frame
|
||||
short-window poll, which is the intended conservative instrument behavior.
|
||||
AquaSynth Syrinx voices are the correct Realtek audio-signal target, but
|
||||
the live emitter is still the Mimir bioacoustic/Move script until the
|
||||
Realtek path is explicitly routed through AquaSynth.Faust/Syrinx patch
|
||||
compilation.
|
||||
`tools/move_latency_probe.py` is the diagnostic PS Move latency receipt:
|
||||
PS3 Eyes are the default fast visual witnesses, Scarlett cardioid is the
|
||||
default chirp timing witness, and the event train uses an order-3 de Bruijn
|
||||
|
|
@ -332,6 +347,14 @@ invariants:
|
|||
and camera driver time are not in one canonical timeline, so sync frame
|
||||
publication is degraded until a camera-clock/global-time correction owner
|
||||
exists.
|
||||
The Eve live-stats provider now reads the latest `mimir.well_snapshot.v1`
|
||||
records from the observation ledger and the latest
|
||||
`mimir.online_move_music_sync_frame.v1` trace from runtime artifacts. Its
|
||||
compact retained surface subdivides into overview, Move score, Well,
|
||||
audio, sync, video, Leap/Eyes/Moves calibration evidence, and actuator
|
||||
panes. Leap usefulness is reported as live Well buffer health and edge age,
|
||||
while Nightwing Eyes/Moves remain compact typed witness observations; the
|
||||
dashboard observes those numbers and does not become placement authority.
|
||||
Eve follows the same distributed-witness rule for its own body:
|
||||
`eve-local-sensor-witness` / `eve-local-sensors` owns Eve camera, Eve mic,
|
||||
local flash/chirp decode, feature tracks, visual markers, and local frustum
|
||||
|
|
|
|||
|
|
@ -450,6 +450,53 @@ def fuse_music_sources(primary: dict[str, object], source_states: list[dict[str,
|
|||
return primary
|
||||
|
||||
|
||||
class ScoreGestureScheduler:
|
||||
def __init__(self, move_count: int, args: argparse.Namespace) -> None:
|
||||
self.args = args
|
||||
self.envelopes = [0.0 for _ in range(move_count)]
|
||||
self.last_slots = [-1 for _ in range(move_count)]
|
||||
|
||||
def update(self, state: dict[str, object]) -> tuple[float, ...]:
|
||||
beat = float(state.get("beat_phase", 0.0))
|
||||
slot_count = max(1, int(self.args.score_subdivisions))
|
||||
slot = int(math.floor((beat % 1.0) * slot_count))
|
||||
loudness_rank = float(state.get("loudness_percentile", 0.0))
|
||||
loudness_gate = float(state.get("loudness_gate", 0.0))
|
||||
hit = float(state.get("hit", 0.0))
|
||||
confidence = float(state.get("bpm_confidence", 0.0))
|
||||
can_play = (
|
||||
loudness_rank >= self.args.score_loudness_threshold
|
||||
and loudness_gate >= self.args.score_min_loudness_gate
|
||||
and confidence >= self.args.score_min_tempo_confidence
|
||||
)
|
||||
loud_span = max(1e-6, 1.0 - self.args.score_loudness_threshold)
|
||||
loud_accent = max(0.0, min(1.0, (loudness_rank - self.args.score_loudness_threshold) / loud_span))
|
||||
rise_accent = max(loud_accent, loudness_gate)
|
||||
onset_accent = 0.5 + 0.5 * min(1.0, hit + loudness_gate * self.args.score_loudness_weight)
|
||||
for index in range(len(self.envelopes)):
|
||||
self.envelopes[index] *= self.args.score_release
|
||||
if slot == self.last_slots[index]:
|
||||
continue
|
||||
self.last_slots[index] = slot
|
||||
if not can_play:
|
||||
continue
|
||||
if not self._slot_belongs_to_instrument(slot, index):
|
||||
continue
|
||||
accent = (rise_accent ** self.args.score_loudness_exponent) * onset_accent
|
||||
if accent < self.args.score_min_accent:
|
||||
continue
|
||||
self.envelopes[index] = max(self.envelopes[index], min(self.args.score_max_envelope, accent))
|
||||
return tuple(float(value) for value in self.envelopes)
|
||||
|
||||
def _slot_belongs_to_instrument(self, slot: int, move_index: int) -> bool:
|
||||
if self.args.debruijn_polyrhythm:
|
||||
bit_index = (slot + move_index * 3) % len(DEBRUIJN_2_3)
|
||||
if DEBRUIJN_2_3[bit_index] == "0":
|
||||
return False
|
||||
spacing = max(1, int(self.args.score_instrument_spacing))
|
||||
return (slot + move_index) % spacing == 0
|
||||
|
||||
|
||||
DEBRUIJN_2_3 = "00010111"
|
||||
|
||||
|
||||
|
|
@ -463,6 +510,10 @@ def debruijn_accent(frame_phase: float, move_index: int, args: argparse.Namespac
|
|||
|
||||
|
||||
def move_rgb(move: Move, move_index: int, move_count: int, state: dict[str, object], args: argparse.Namespace) -> tuple[int, int, int]:
|
||||
envelopes = state.get("score_gesture_envelopes", ())
|
||||
score_envelope = float(envelopes[move_index]) if isinstance(envelopes, tuple) and move_index < len(envelopes) else 0.0
|
||||
if score_envelope <= 0.0005:
|
||||
return (0, 0, 0)
|
||||
level = float(state["level"])
|
||||
beat = float(state["beat_phase"])
|
||||
hit = float(state["hit"])
|
||||
|
|
@ -476,7 +527,7 @@ def move_rgb(move: Move, move_index: int, move_count: int, state: dict[str, obje
|
|||
phase = (beat * harmonic * micro + move_index / max(1, move_count)) % 1.0
|
||||
poly = debruijn_accent(beat, move_index, args)
|
||||
gesture = poly * (0.42 + 0.58 * (0.5 + 0.5 * math.sin(2.0 * math.pi * phase)) ** 2.1)
|
||||
accent = min(1.0, loudness_gate * (level * gesture + hit * (0.34 + 0.12 * poly)))
|
||||
accent = min(1.0, score_envelope * loudness_gate * (level * gesture + hit * (0.22 + 0.08 * poly)))
|
||||
hue = (base_h + args.hue_bend * math.sin(2.0 * math.pi * phase) + math.log2(harmonic * micro) * 0.0833) % 1.0
|
||||
chord_confidence = float(state.get("chord_confidence", 0.0))
|
||||
chord_root = int(state.get("chord_root", 0))
|
||||
|
|
@ -497,7 +548,7 @@ def move_rgb(move: Move, move_index: int, move_count: int, state: dict[str, obje
|
|||
r = rr * (0.45 + 0.55 * float(spectral[0]))
|
||||
g = gg * (0.45 + 0.55 * float(spectral[1]))
|
||||
b = bb * (0.45 + 0.55 * float(spectral[2]))
|
||||
white = loudness_gate * max(0.0, hit - 0.94) / 0.06
|
||||
white = score_envelope * loudness_gate * max(0.0, hit - 0.97) / 0.03
|
||||
return (
|
||||
int(min(255, 255 * r + 18 * white)),
|
||||
int(min(255, 255 * g + 18 * white)),
|
||||
|
|
@ -534,9 +585,19 @@ def main() -> int:
|
|||
parser.add_argument("--hit-loudness-threshold", type=float, default=0.82)
|
||||
parser.add_argument("--loudness-floor", type=float, default=0.006)
|
||||
parser.add_argument("--loudness-exponent", type=float, default=2.4)
|
||||
parser.add_argument("--quiet-brightness", type=float, default=0.012)
|
||||
parser.add_argument("--max-brightness", type=float, default=0.76)
|
||||
parser.add_argument("--brightness-exponent", type=float, default=0.82)
|
||||
parser.add_argument("--quiet-brightness", type=float, default=0.0)
|
||||
parser.add_argument("--max-brightness", type=float, default=0.18)
|
||||
parser.add_argument("--brightness-exponent", type=float, default=1.15)
|
||||
parser.add_argument("--score-subdivisions", type=int, default=8)
|
||||
parser.add_argument("--score-instrument-spacing", type=int, default=3)
|
||||
parser.add_argument("--score-release", type=float, default=0.54)
|
||||
parser.add_argument("--score-loudness-threshold", type=float, default=0.965)
|
||||
parser.add_argument("--score-min-loudness-gate", type=float, default=0.16)
|
||||
parser.add_argument("--score-loudness-exponent", type=float, default=2.8)
|
||||
parser.add_argument("--score-loudness-weight", type=float, default=0.28)
|
||||
parser.add_argument("--score-min-accent", type=float, default=0.035)
|
||||
parser.add_argument("--score-max-envelope", type=float, default=0.45)
|
||||
parser.add_argument("--score-min-tempo-confidence", type=float, default=0.0)
|
||||
parser.add_argument("--onset-decay", type=float, default=0.18)
|
||||
parser.add_argument("--onset-threshold", type=float, default=0.62)
|
||||
parser.add_argument("--onset-exponent", type=float, default=2.0)
|
||||
|
|
@ -601,6 +662,7 @@ def main() -> int:
|
|||
)
|
||||
assert wasapi.stdout is not None and ssh.stdin is not None
|
||||
analyzer = OnlineAnalyzer(args.sample_rate, args.fps, args.fft_size, args)
|
||||
score_scheduler = ScoreGestureScheduler(len(moves), args)
|
||||
asio_reader = None
|
||||
asio_tracker = None
|
||||
if args.asio_music_channels.strip():
|
||||
|
|
@ -664,6 +726,7 @@ def main() -> int:
|
|||
asio_reader = None
|
||||
asio_tracker = None
|
||||
state = fuse_music_sources(state, auxiliary_states)
|
||||
state["score_gesture_envelopes"] = score_scheduler.update(state)
|
||||
rgbs = {}
|
||||
for index, move in enumerate(moves):
|
||||
rgb = move_rgb(move, index, len(moves), state, args)
|
||||
|
|
@ -680,6 +743,7 @@ def main() -> int:
|
|||
"color_balance": list(state["color_balance"]), # type: ignore[index]
|
||||
"chroma": list(state["chroma"]), # type: ignore[index]
|
||||
"music_sources": state["music_sources"],
|
||||
"score_gesture_envelopes": list(state["score_gesture_envelopes"]), # type: ignore[index]
|
||||
"moves": {name: list(rgb) for name, rgb in rgbs.items()},
|
||||
}
|
||||
trace.write(json.dumps(record, sort_keys=True) + "\n")
|
||||
|
|
|
|||
Loading…
Reference in a new issue