From 1ccf99486d0cd4f30ed00c3aba8a9567c364bab1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 1 Jun 2026 22:11:58 +0100 Subject: [PATCH] Expose Well score state on Eve dashboard --- scripts/start-online-verse-daemons.ps1 | 15 +- src/Mimir.EveDashboard/Program.cs | 388 ++++++++++++++++++++++++- state/evidence.jsonl | 1 + state/map.yaml | 23 ++ tools/online_move_music_sync.py | 74 ++++- 5 files changed, 479 insertions(+), 22 deletions(-) diff --git a/scripts/start-online-verse-daemons.ps1 b/scripts/start-online-verse-daemons.ps1 index 78bf157..60fbc99 100644 --- a/scripts/start-online-verse-daemons.ps1 +++ b/scripts/start-online-verse-daemons.ps1 @@ -177,9 +177,18 @@ if (-not $SkipMoves) { "--hit-loudness-threshold", "0.84", "--loudness-floor", "0.006", "--loudness-exponent", "2.6", - "--quiet-brightness", "0.008", - "--max-brightness", "0.72", - "--brightness-exponent", "0.9", + "--quiet-brightness", "0.0", + "--max-brightness", "0.16", + "--brightness-exponent", "1.2", + "--score-subdivisions", "8", + "--score-instrument-spacing", "3", + "--score-release", "0.50", + "--score-loudness-threshold", "0.97", + "--score-min-loudness-gate", "0.18", + "--score-loudness-exponent", "3.0", + "--score-min-accent", "0.04", + "--score-max-envelope", "0.40", + "--score-min-tempo-confidence", "0.0", "--debruijn-polyrhythm", "--harmonic-base", "1.5", "--microtonal-cents", "17" diff --git a/src/Mimir.EveDashboard/Program.cs b/src/Mimir.EveDashboard/Program.cs index d6d6e79..e8037a9 100644 --- a/src/Mimir.EveDashboard/Program.cs +++ b/src/Mimir.EveDashboard/Program.cs @@ -842,6 +842,7 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider var confidence = snapshot.SyncStates.Count == 0 ? snapshot.SyncReports.Select(static report => report.Confidence) .Concat(snapshot.SyncDecodeAttempts.Select(static attempt => attempt.Confidence)) + .Concat(snapshot.Well?.AudioSyncStates.Select(static state => state.Confidence) ?? []) .DefaultIfEmpty(0.0) .Max() : snapshot.SyncStates.Average(static state => state.Confidence); @@ -860,7 +861,7 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider }, new("mimir-audio-rms", $"Audio RMS\n{rms:0.000000}", "audio-rms", 0.0, -0.12, 0.28, 0.16, ToneFor(NormalizeRms(rms))) { - Detail = $"{snapshot.Spectra.Count} spectrum lanes", + Detail = $"{snapshot.Spectra.Count} spectrum lanes; Well audio buffers {snapshot.Well?.AudioBuffers ?? 0}", }, new("mimir-observation-streams", $"Device Streams\n{liveObservationCount}/{snapshot.ObservationStreams.Count} active", "observation-streams", 0.44, -0.12, 0.28, 0.16, liveObservationCount > 0 ? "active" : "waiting") { @@ -886,18 +887,39 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider var pulse = Pulse(); var children = new List { - UiElement.Pane("mimir-live-overview", "Mimir Live Stats", [ + UiElement.Container( + "mimir-live-row-overview", + "row", + new DashboardUiLayout { Direction = "horizontal", Gap = 8 }, + [ + UiElement.Pane("mimir-live-overview", "Mimir Live Stats", [ UiElement.Text("mimir-live-summary", $"{pulse} {snapshot.Summary}", "strong"), UiElement.Text("mimir-live-sources", $"telemetry: {snapshot.TelemetrySource}\nobservations: {snapshot.ObservationSource}", "caption"), StatBar("tracking-confidence", "tracking confidence", confidence, "cool"), StatBar("audio-rms", "max channel RMS", NormalizeRms(rms), "warm", $"{rms:0.000000}"), StatBar("observation-liveness", "device stream liveness", snapshot.ObservationStreams.Count == 0 ? 0.0 : liveObservationCount / (double)snapshot.ObservationStreams.Count, "cool"), - ]), - BuildAudioPane(snapshot.Spectra), - BuildSyncPane(snapshot), - BuildVideoPane(snapshot), - BuildObservationPane(snapshot.ObservationStreams), - BuildActuatorPane(snapshot.ActuatorCommands), + ]), + BuildMoveMusicPane(snapshot.MoveMusic), + BuildWellPane(snapshot.Well), + ]), + UiElement.Container( + "mimir-live-row-runtime", + "row", + new DashboardUiLayout { Direction = "horizontal", Gap = 8 }, + [ + BuildAudioPane(snapshot.Spectra, snapshot.Well), + BuildSyncPane(snapshot), + BuildVideoPane(snapshot), + ]), + UiElement.Container( + "mimir-live-row-witness", + "row", + new DashboardUiLayout { Direction = "horizontal", Gap = 8 }, + [ + BuildObservationPane(snapshot.ObservationStreams), + BuildActuatorPane(snapshot.ActuatorCommands), + BuildCalibrationPane(snapshot), + ]), }; return new DashboardSurface @@ -908,14 +930,23 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider Root = UiElement.Container( "mimir-live-stats-root", "dashboard", - new DashboardUiLayout { Direction = "vertical", Gap = 8, Padding = 8, Overflow = "scroll" }, + new DashboardUiLayout { Direction = "vertical", Gap = 6, Padding = 6, Overflow = "scroll" }, children), }; } - private static DashboardUiElement BuildAudioPane(IReadOnlyList spectra) + private static DashboardUiElement BuildAudioPane(IReadOnlyList spectra, MimirWellStat? well) { - var items = spectra.Count == 0 + var wellAudio = well?.Buffers + .Where(static buffer => string.Equals(buffer.Kind, "Audio", StringComparison.OrdinalIgnoreCase)) + .OrderBy(static buffer => buffer.SourceId, StringComparer.Ordinal) + .Take(8) + .Select(buffer => UiElement.Text( + $"well-audio-{StableId(buffer.SourceId)}", + $"{buffer.SourceId}: {buffer.Count} blocks edge {NsAge(buffer.EdgeNs)}", + "caption")) + .ToArray() ?? []; + var items = spectra.Count == 0 && wellAudio.Length == 0 ? [UiElement.Text("mimir-spectrum-empty", "no spectrum telemetry yet", "caption")] : spectra .OrderBy(static spectrum => spectrum.SourceId, StringComparer.Ordinal) @@ -930,9 +961,10 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider UiElement.Text($"spectrum-{StableId(spectrum.SourceId)}-detail", $"rms {spectrum.Rms:0.000000} peak {spectrum.Peak:0.000000} floor {spectrum.NoiseFloorDb:0.0} dB", "caption"), UiElement.Text($"spectrum-{StableId(spectrum.SourceId)}-peaks", spectrum.Peaks, "caption"), ])) + .Concat(wellAudio) .ToArray(); - return UiElement.Pane("mimir-audio-pane", "Per-Channel RMS", items); + return UiElement.Pane("mimir-audio-pane", "Audio", items); } private static DashboardUiElement BuildSyncPane(MimirLiveStatsSnapshot snapshot) @@ -1039,6 +1071,64 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider return UiElement.Pane("mimir-actuator-pane", "Alignment Actuator", children); } + private static DashboardUiElement BuildMoveMusicPane(MimirMoveMusicStat? music) + { + if (music == null) + { + return UiElement.Pane("mimir-move-music-pane", "Move Score", [UiElement.Text("mimir-move-music-empty", "no Move score trace yet", "caption")]); + } + + var children = new List + { + UiElement.Text("mimir-move-music-head", $"bpm {music.Bpm:0.0} c={music.BpmConfidence:0.00} {music.KeyName} {music.KeyMode} chord {music.ChordName}", "mono"), + StatBar("mimir-move-loudness", "loudness gate", music.LoudnessGate, "warm", $"{music.LoudnessGate:0.000}"), + UiElement.Text("mimir-move-score", $"score {Bar(music.ScoreEnvelopeMax, 18)} max env {music.ScoreEnvelopeMax:0.000} max rgb {music.MaxRgb}", "caption"), + UiElement.Text("mimir-move-sources", string.Join(" | ", music.Sources.Take(4)), "caption"), + }; + + return UiElement.Pane("mimir-move-music-pane", "Move Score", children); + } + + private static DashboardUiElement BuildWellPane(MimirWellStat? well) + { + if (well == null) + { + return UiElement.Pane("mimir-well-pane", "Well", [UiElement.Text("mimir-well-empty", "no Well snapshot yet", "caption")]); + } + + return UiElement.Pane("mimir-well-pane", "Well", [ + UiElement.Text("mimir-well-summary", $"seq {well.Sequence} {well.ElapsedSeconds:0}s ingested {well.IngestedSamples:0} sources {well.LiveSources}/{well.ConfiguredSources}", "mono"), + StatBar("mimir-well-completeness", "frame completeness", well.SynchronizedFrameComplete ? 1.0 : 0.0, well.SynchronizedFrameComplete ? "cool" : "danger"), + UiElement.Text("mimir-well-buffers", $"buffers {well.Buffers.Count} audio {well.AudioBuffers} video {well.VideoBuffers} ready {well.ReadySlices}/{well.TotalSlices}", "caption"), + UiElement.Text("mimir-well-probe", well.Probe == null ? "probe: none" : $"probe emit={well.Probe.ShouldEmit} sync={well.Probe.SyncConfidence:0.000} freq={well.Probe.FrequencyConfidence:0.000} {well.Probe.Reason}", "caption"), + ]); + } + + private static DashboardUiElement BuildCalibrationPane(MimirLiveStatsSnapshot snapshot) + { + var leap = snapshot.Well?.Buffers.FirstOrDefault(static buffer => buffer.SourceId.Contains("leap", StringComparison.OrdinalIgnoreCase)); + var eyeStreams = snapshot.ObservationStreams + .Where(static stream => stream.StreamId.Contains("eye", StringComparison.OrdinalIgnoreCase) || stream.StreamId.Contains("video2", StringComparison.OrdinalIgnoreCase) || stream.StreamId.Contains("video3", StringComparison.OrdinalIgnoreCase)) + .OrderBy(static stream => stream.StreamId, StringComparer.Ordinal) + .Take(4) + .ToArray(); + var moveStreams = snapshot.ObservationStreams + .Where(static stream => stream.Kind.Contains("move", StringComparison.OrdinalIgnoreCase)) + .OrderBy(static stream => stream.StreamId, StringComparer.Ordinal) + .Take(4) + .ToArray(); + + var children = new List + { + UiElement.Text("mimir-calibration-leap", leap == null ? "Leap: no Well buffer" : $"Leap: {leap.Count} frames {leap.PixelFormat} {leap.Width:0}x{leap.Height:0} edge {NsAge(leap.EdgeNs)}", "mono"), + UiElement.Text("mimir-calibration-eyes", $"Eyes: {eyeStreams.Count(static stream => stream.State == "active")}/{eyeStreams.Length} active", "caption"), + UiElement.Text("mimir-calibration-moves", $"Moves: {moveStreams.Count(static stream => stream.State == "active")}/{moveStreams.Length} active", "caption"), + }; + children.AddRange(eyeStreams.Select(stream => UiElement.Text($"mimir-eye-{StableId(stream.StreamId)}", $"{stream.StreamId} {stream.Shape} age {stream.AgeSeconds:0}s", "caption"))); + children.AddRange(moveStreams.Select(stream => UiElement.Text($"mimir-move-{StableId(stream.StreamId)}", $"{stream.StreamId} {stream.Shape} age {stream.AgeSeconds:0}s", "caption"))); + return UiElement.Pane("mimir-calibration-pane", "Leap / Eyes / Moves", children); + } + private static DashboardUiElement StatBar(string id, string label, double value, string tone, string? displayValue = null) => UiElement.Card( id, @@ -1079,6 +1169,18 @@ internal sealed class MimirLiveStatsProvider : IDashboardProvider private static string ToneFor(double value) => value >= 0.70 ? "cool" : value >= 0.25 ? "warm" : "danger"; + private static string NsAge(double timestampNs) + { + if (timestampNs <= 0) + { + return "unknown"; + } + + var nowNs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000.0; + var seconds = Math.Max(0.0, (nowNs - timestampNs) / 1_000_000_000.0); + return seconds < 1.0 ? $"{seconds * 1000.0:0}ms" : $"{seconds:0.0}s"; + } + private static string StableId(string value) => string.Join("-", value.ToLowerInvariant().Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries)) .Replace(":", "-", StringComparison.Ordinal) @@ -1098,8 +1200,12 @@ internal sealed class MimirLiveStatsSnapshot public IReadOnlyList VideoBuffers { get; init; } = []; public IReadOnlyList ActuatorCommands { get; init; } = []; public IReadOnlyList ObservationStreams { get; init; } = []; - public bool HasAnyData => Telemetry != null || Spectra.Count > 0 || SyncStates.Count > 0 || VideoBuffers.Count > 0 || ObservationStreams.Count > 0; - public string Summary => $"{Spectra.Count} audio lanes / {SyncStates.Count} sync states / {SyncDecodeAttempts.Count} decode attempts / {VideoBuffers.Count} video buffers / {ObservationStreams.Count} device streams"; + public MimirWellStat? Well { get; init; } + public MimirMoveMusicStat? MoveMusic { get; init; } + public bool HasAnyData => Telemetry != null || Well != null || MoveMusic != null || Spectra.Count > 0 || SyncStates.Count > 0 || VideoBuffers.Count > 0 || ObservationStreams.Count > 0; + public string Summary => Well == null + ? $"{Spectra.Count} audio lanes / {SyncStates.Count} sync states / {SyncDecodeAttempts.Count} decode attempts / {VideoBuffers.Count} video buffers / {ObservationStreams.Count} device streams" + : $"Well {Well.LiveSources}/{Well.ConfiguredSources} sources / {Well.AudioBuffers} audio / {Well.VideoBuffers} video / {ObservationStreams.Count} device streams"; public static MimirLiveStatsSnapshot Load(string telemetryLogPath, string observationLogPath) { @@ -1118,6 +1224,8 @@ internal sealed class MimirLiveStatsSnapshot VideoBuffers = LatestBySource(telemetryLines.Select(ParseVideoBuffer).Where(static value => value != null)!), ActuatorCommands = LatestBySource(telemetryLines.Select(ParseActuatorCommand).Where(static value => value != null)!), ObservationStreams = LatestObservationStreams(observationLines), + Well = LatestWellSnapshot(observationLines), + MoveMusic = LatestMoveMusicTrace(), }; } @@ -1192,6 +1300,152 @@ internal sealed class MimirLiveStatsSnapshot .ToArray(); } + private static MimirWellStat? LatestWellSnapshot(IEnumerable lines) + { + MimirWellStat? latest = null; + foreach (var line in lines) + { + try + { + using var document = JsonDocument.Parse(line); + var root = document.RootElement; + if (string.Equals(JsonStringAny(root, "document"), "mimir.well_snapshot.v1", StringComparison.Ordinal)) + { + latest = ParseWellSnapshot(root); + } + } + catch (JsonException) + { + } + } + + return latest; + } + + private static MimirWellStat ParseWellSnapshot(JsonElement root) + { + var buffers = ArrayItems(TryGetAny(root, "buffers")) + .Select(static buffer => + { + var latest = TryGetAny(buffer, "latest"); + var video = TryGetAny(latest, "video"); + var audio = TryGetAny(latest, "audio"); + return new MimirWellBufferStat( + JsonStringAny(buffer, "SourceId", "sourceId") ?? "unknown", + JsonStringAny(buffer, "Kind", "kind") ?? "unknown", + (int)JsonNumberAny(buffer, "Count", "count"), + JsonNumberAny(buffer, "EdgeNs", "edgeNs"), + (int)JsonNumberAny(video, "Width", "width"), + (int)JsonNumberAny(video, "Height", "height"), + JsonStringAny(video, "PixelFormat", "pixelFormat") ?? "", + (int)JsonNumberAny(audio, "SampleRate", "sampleRate"), + (int)JsonNumberAny(audio, "Channels", "channels")); + }) + .ToArray(); + var frame = TryGetAny(root, "synchronizedFrame"); + var slices = ArrayItems(TryGetAny(frame, "slices")).ToArray(); + var audioSync = TryGetAny(root, "audioSync"); + var states = ArrayItems(TryGetAny(audioSync, "states")) + .Select(static state => new MimirWellAudioSyncStateStat( + JsonStringAny(state, "SourceId", "sourceId") ?? "unknown", + JsonStringAny(state, "ReferenceSourceId", "referenceSourceId") ?? "unknown", + JsonNumberAny(state, "SmoothedDelaySamples", "smoothedDelaySamples"), + JsonNumberAny(state, "SamplingRateOffsetPpm", "samplingRateOffsetPpm"), + JsonNumberAny(state, "Confidence", "confidence"))) + .ToArray(); + var probeElement = TryGetAny(audioSync, "probe"); + var probe = probeElement.ValueKind == JsonValueKind.Object + ? new MimirWellProbeStat( + JsonBoolAny(probeElement, "ShouldEmit", "shouldEmit"), + JsonNumberAny(probeElement, "AggregateSyncConfidence", "aggregateSyncConfidence"), + JsonNumberAny(probeElement, "AggregateFrequencyResponseConfidence", "aggregateFrequencyResponseConfidence"), + JsonStringAny(probeElement, "reason") ?? "unknown") + : null; + return new MimirWellStat( + (long)JsonNumberAny(root, "sequence"), + JsonNumberAny(root, "elapsedSeconds"), + JsonNumberAny(root, "ingestedSamples"), + (int)JsonNumberAny(root, "configuredSources"), + (int)JsonNumberAny(root, "liveSources"), + JsonBoolAny(frame, "IsComplete", "isComplete"), + slices.Count(static slice => string.Equals(JsonStringAny(slice, "Status", "status"), "Ready", StringComparison.OrdinalIgnoreCase)), + slices.Length, + buffers, + states, + probe); + } + + private static MimirMoveMusicStat? LatestMoveMusicTrace() + { + var runtimeRoot = Path.Combine(AppContext.BaseDirectory, "artifacts", "runtime"); + if (!Directory.Exists(runtimeRoot)) + { + runtimeRoot = @"E:\Projects\Mimir\artifacts\runtime"; + } + + if (!Directory.Exists(runtimeRoot)) + { + return null; + } + + var trace = Directory.EnumerateFiles(runtimeRoot, "online-sync.jsonl", SearchOption.AllDirectories) + .Select(static path => new FileInfo(path)) + .Where(static info => info.DirectoryName?.Contains("move", StringComparison.OrdinalIgnoreCase) == true) + .OrderByDescending(static info => info.LastWriteTimeUtc) + .FirstOrDefault(); + if (trace == null || trace.Length == 0) + { + return null; + } + + var line = TailTextFile(trace.FullName, 64 * 1024) + .Split("\n", StringSplitOptions.RemoveEmptyEntries) + .LastOrDefault(); + if (string.IsNullOrWhiteSpace(line)) + { + return null; + } + + try + { + using var document = JsonDocument.Parse(line); + var root = document.RootElement; + var maxRgb = 0; + var moves = TryGetAny(root, "moves"); + if (moves.ValueKind == JsonValueKind.Object) + { + foreach (var move in moves.EnumerateObject()) + { + if (move.Value.ValueKind == JsonValueKind.Array) + { + maxRgb = Math.Max(maxRgb, move.Value.EnumerateArray().Select(JsonNumber).Select(static value => (int)value).DefaultIfEmpty(0).Max()); + } + } + } + + var envelopes = ArrayItems(TryGetAny(root, "score_gesture_envelopes")).Select(JsonNumber).DefaultIfEmpty(0.0).ToArray(); + var sources = ArrayItems(TryGetAny(root, "music_sources")) + .Select(static source => $"{JsonStringAny(source, "source") ?? "source"} {JsonNumberAny(source, "bpm"):0.0}/{JsonNumberAny(source, "bpm_confidence"):0.00}") + .ToArray(); + return new MimirMoveMusicStat( + trace.FullName, + JsonNumberAny(root, "bpm"), + JsonNumberAny(root, "bpm_confidence"), + JsonNumberAny(root, "loudness_gate"), + JsonNumberAny(root, "loudness_percentile"), + JsonStringAny(root, "key_name") ?? "?", + JsonStringAny(root, "key_mode") ?? "?", + JsonStringAny(root, "chord_name") ?? "?", + envelopes.Max(), + maxRgb, + sources); + } + catch (JsonException) + { + return null; + } + } + private static IReadOnlyList LatestObservationStreams(IEnumerable lines) { var byStream = new Dictionary(StringComparer.Ordinal); @@ -1476,6 +1730,66 @@ internal sealed class MimirLiveStatsSnapshot private static double JsonNumber(JsonElement root, string property) => root.TryGetProperty(property, out var child) && child.ValueKind == JsonValueKind.Number && child.TryGetDouble(out var value) ? value : 0.0; + private static JsonElement TryGetAny(JsonElement root, params string[] properties) + { + if (root.ValueKind != JsonValueKind.Object) + { + return default; + } + + foreach (var property in properties) + { + if (root.TryGetProperty(property, out var child)) + { + return child; + } + } + + return default; + } + + private static IEnumerable ArrayItems(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Array) + { + yield break; + } + + foreach (var item in element.EnumerateArray()) + { + yield return item; + } + } + + private static string? JsonStringAny(JsonElement root, params string[] properties) + { + var child = TryGetAny(root, properties); + return child.ValueKind == JsonValueKind.String ? child.GetString() : null; + } + + private static double JsonNumberAny(JsonElement root, params string[] properties) => JsonNumber(TryGetAny(root, properties)); + + private static double JsonNumber(JsonElement element) => + element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value) ? value : 0.0; + + private static bool JsonBoolAny(JsonElement root, params string[] properties) + { + var child = TryGetAny(root, properties); + return child.ValueKind == JsonValueKind.True || (child.ValueKind == JsonValueKind.Number && child.TryGetDouble(out var value) && value != 0.0); + } + + private static string NsAge(double timestampNs) + { + if (timestampNs <= 0) + { + return "unknown"; + } + + var nowNs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000.0; + var seconds = Math.Max(0.0, (nowNs - timestampNs) / 1_000_000_000.0); + return seconds < 1.0 ? $"{seconds * 1000.0:0}ms" : $"{seconds:0.0}s"; + } + private static string TailTextFile(string filePath, int maxBytes) { var info = new FileInfo(filePath); @@ -1514,6 +1828,52 @@ internal sealed record MimirActuatorCommandStat(string SourceId, double DelaySam internal sealed record MimirObservationStreamStat(string DeviceId, string StreamId, string Kind, double Sequence, string State, double AgeSeconds, string Shape); +internal sealed record MimirWellStat( + long Sequence, + double ElapsedSeconds, + double IngestedSamples, + int ConfiguredSources, + int LiveSources, + bool SynchronizedFrameComplete, + int ReadySlices, + int TotalSlices, + IReadOnlyList Buffers, + IReadOnlyList AudioSyncStates, + MimirWellProbeStat? Probe) +{ + public int AudioBuffers => Buffers.Count(static buffer => string.Equals(buffer.Kind, "Audio", StringComparison.OrdinalIgnoreCase)); + + public int VideoBuffers => Buffers.Count(static buffer => string.Equals(buffer.Kind, "Video", StringComparison.OrdinalIgnoreCase)); +} + +internal sealed record MimirWellBufferStat( + string SourceId, + string Kind, + int Count, + double EdgeNs, + int Width, + int Height, + string PixelFormat, + int SampleRate, + int Channels); + +internal sealed record MimirWellAudioSyncStateStat(string SourceId, string ReferenceSourceId, double DelaySamples, double SroPpm, double Confidence); + +internal sealed record MimirWellProbeStat(bool ShouldEmit, double SyncConfidence, double FrequencyConfidence, string Reason); + +internal sealed record MimirMoveMusicStat( + string TracePath, + double Bpm, + double BpmConfidence, + double LoudnessGate, + double LoudnessPercentile, + string KeyName, + string KeyMode, + string ChordName, + double ScoreEnvelopeMax, + int MaxRgb, + IReadOnlyList Sources); + internal sealed class MimirStreamLayoutProvider : MutableDashboardProvider { public const string ProviderId = "mimir.stream.layout"; diff --git a/state/evidence.jsonl b/state/evidence.jsonl index eec862b..931d1e8 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -338,3 +338,4 @@ {"ts": "2026-06-01T21:06:07+01:00", "type": "implementation", "status": "ok", "note": "Added the first online Verse observation daemon stack: Mimir.EveSensorReceiver now serves a /eve/periwinkle/subscribe WebSocket lane, Mimir.VerseRecorder subscribes and writes session.json plus observations.jsonl under C:\\Users\\Meta\\Videos\\Mimir\\VerseCaptures, and scripts/start-online-verse-daemons.ps1 supervises relay, recorder, Nightwing Eyes/Move witness publication, music-synced Move gestures, and a local BufferSmoke runtime capture worker. Live run online-verse-daemons-20260601-210106 had relay, recorder, Move sync, BufferSmoke, and Nightwing witness running, with observations.jsonl past 1000 records and growing. This captures typed witness observations; raw aligned source windows/stems still need a MimirSynchronizationHub Verse publisher."} {"ts": "2026-06-01T21:27:15+01:00", "type": "implementation", "status": "partial", "note": "Added Mimir.Well as the long-running Starfire Well daemon and broad config/mimir-runtime.well.local.json. Live run online-verse-daemons-20260601-212502 started relay, recorder, Mimir.Well, music-synced de-Bruijn/microtonal Move gestures, and Nightwing witness PID 1116285. Recorder C:\\Users\\Meta\\Videos\\Mimir\\VerseCaptures\\20260601-212509 reached 292 records and included mimir.well_snapshot.v1, Nightwing builtin camera/mic witness records, and ps3-eye-0/1 Move observations. Well snapshots showed live Scarlett ASIO, Leap stereo IR, Kiyo Pro RGB, and Kiyo Basic RGB buffers; Raven and Eve lanes were configured but empty because their producers were not feeding. Sync frame remains degraded because ASIO device timestamps and camera driver timestamps are not yet mapped into one canonical timeline."} {"ts": "2026-06-01T21:41:07+01:00", "type": "decision", "status": "ok", "note": "2026-06-01 online Move music sync demoted spectral flux from brightness authority: Realtek loopback remains primary, optional Scarlett ASIO channels add auxiliary onset/tempo evidence, chroma estimates key/chord, and recent loudness percentile gates all Move brightness so quiet/steady-state music stays dark."} +{"ts": "2026-06-01T22:09:13+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: online Move music sync now places visual gestures through a score scheduler instead of continuous envelope brightness; raw level/hit/flux/color are inputs, not RGB owners. Live score worker move-music-score-20260601-220807 showed max_env=0.4, max_rgb=4, and 9 lit frames in an 824-frame poll. Eve Live Stats now reads typed Well snapshots and latest Move score traces, exposing Well buffers/sync/probe, Leap/Eyes/Moves evidence, and Move music key/tempo/loudness/envelopes. AquaSynth Syrinx is identified as the correct Realtek voice target but is not yet wired into the live emitter."} diff --git a/state/map.yaml b/state/map.yaml index e096017..79b6820 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -226,6 +226,21 @@ invariants: own autocorrelation tempo estimates, chroma scoring estimates key and chord candidates, and a recent loudness-percentile gate owns whether the Moves are allowed to brighten. Flux alone no longer has authority to shout. + The current score cut adds `ScoreGestureScheduler` as the only owner of + Move visual gesture openings. Raw level, hit, flux, and color balance are + inputs; they no longer emit continuous RGB by themselves. The scheduler + divides the beat into score slots, assigns slots across Moves with + de-Bruijn/polyrhythmic identity, opens short envelopes only when recent + broadband spectral rise reaches the adaptive loudness gate, and maps the + top of that gate to the configured maximum envelope. Quiet frames are + black. The 2026-06-01 live worker + `artifacts/runtime/move-music-score-20260601-220807/online-sync.jsonl` + showed `max_env=0.4`, `max_rgb=4`, and only 9 lit frames in an 824-frame + short-window poll, which is the intended conservative instrument behavior. + AquaSynth Syrinx voices are the correct Realtek audio-signal target, but + the live emitter is still the Mimir bioacoustic/Move script until the + Realtek path is explicitly routed through AquaSynth.Faust/Syrinx patch + compilation. `tools/move_latency_probe.py` is the diagnostic PS Move latency receipt: PS3 Eyes are the default fast visual witnesses, Scarlett cardioid is the default chirp timing witness, and the event train uses an order-3 de Bruijn @@ -332,6 +347,14 @@ invariants: and camera driver time are not in one canonical timeline, so sync frame publication is degraded until a camera-clock/global-time correction owner exists. + The Eve live-stats provider now reads the latest `mimir.well_snapshot.v1` + records from the observation ledger and the latest + `mimir.online_move_music_sync_frame.v1` trace from runtime artifacts. Its + compact retained surface subdivides into overview, Move score, Well, + audio, sync, video, Leap/Eyes/Moves calibration evidence, and actuator + panes. Leap usefulness is reported as live Well buffer health and edge age, + while Nightwing Eyes/Moves remain compact typed witness observations; the + dashboard observes those numbers and does not become placement authority. Eve follows the same distributed-witness rule for its own body: `eve-local-sensor-witness` / `eve-local-sensors` owns Eve camera, Eve mic, local flash/chirp decode, feature tracks, visual markers, and local frustum diff --git a/tools/online_move_music_sync.py b/tools/online_move_music_sync.py index 28e41c9..3a64f48 100644 --- a/tools/online_move_music_sync.py +++ b/tools/online_move_music_sync.py @@ -450,6 +450,53 @@ def fuse_music_sources(primary: dict[str, object], source_states: list[dict[str, return primary +class ScoreGestureScheduler: + def __init__(self, move_count: int, args: argparse.Namespace) -> None: + self.args = args + self.envelopes = [0.0 for _ in range(move_count)] + self.last_slots = [-1 for _ in range(move_count)] + + def update(self, state: dict[str, object]) -> tuple[float, ...]: + beat = float(state.get("beat_phase", 0.0)) + slot_count = max(1, int(self.args.score_subdivisions)) + slot = int(math.floor((beat % 1.0) * slot_count)) + loudness_rank = float(state.get("loudness_percentile", 0.0)) + loudness_gate = float(state.get("loudness_gate", 0.0)) + hit = float(state.get("hit", 0.0)) + confidence = float(state.get("bpm_confidence", 0.0)) + can_play = ( + loudness_rank >= self.args.score_loudness_threshold + and loudness_gate >= self.args.score_min_loudness_gate + and confidence >= self.args.score_min_tempo_confidence + ) + loud_span = max(1e-6, 1.0 - self.args.score_loudness_threshold) + loud_accent = max(0.0, min(1.0, (loudness_rank - self.args.score_loudness_threshold) / loud_span)) + rise_accent = max(loud_accent, loudness_gate) + onset_accent = 0.5 + 0.5 * min(1.0, hit + loudness_gate * self.args.score_loudness_weight) + for index in range(len(self.envelopes)): + self.envelopes[index] *= self.args.score_release + if slot == self.last_slots[index]: + continue + self.last_slots[index] = slot + if not can_play: + continue + if not self._slot_belongs_to_instrument(slot, index): + continue + accent = (rise_accent ** self.args.score_loudness_exponent) * onset_accent + if accent < self.args.score_min_accent: + continue + self.envelopes[index] = max(self.envelopes[index], min(self.args.score_max_envelope, accent)) + return tuple(float(value) for value in self.envelopes) + + def _slot_belongs_to_instrument(self, slot: int, move_index: int) -> bool: + if self.args.debruijn_polyrhythm: + bit_index = (slot + move_index * 3) % len(DEBRUIJN_2_3) + if DEBRUIJN_2_3[bit_index] == "0": + return False + spacing = max(1, int(self.args.score_instrument_spacing)) + return (slot + move_index) % spacing == 0 + + DEBRUIJN_2_3 = "00010111" @@ -463,6 +510,10 @@ def debruijn_accent(frame_phase: float, move_index: int, args: argparse.Namespac def move_rgb(move: Move, move_index: int, move_count: int, state: dict[str, object], args: argparse.Namespace) -> tuple[int, int, int]: + envelopes = state.get("score_gesture_envelopes", ()) + score_envelope = float(envelopes[move_index]) if isinstance(envelopes, tuple) and move_index < len(envelopes) else 0.0 + if score_envelope <= 0.0005: + return (0, 0, 0) level = float(state["level"]) beat = float(state["beat_phase"]) hit = float(state["hit"]) @@ -476,7 +527,7 @@ def move_rgb(move: Move, move_index: int, move_count: int, state: dict[str, obje phase = (beat * harmonic * micro + move_index / max(1, move_count)) % 1.0 poly = debruijn_accent(beat, move_index, args) gesture = poly * (0.42 + 0.58 * (0.5 + 0.5 * math.sin(2.0 * math.pi * phase)) ** 2.1) - accent = min(1.0, loudness_gate * (level * gesture + hit * (0.34 + 0.12 * poly))) + accent = min(1.0, score_envelope * loudness_gate * (level * gesture + hit * (0.22 + 0.08 * poly))) hue = (base_h + args.hue_bend * math.sin(2.0 * math.pi * phase) + math.log2(harmonic * micro) * 0.0833) % 1.0 chord_confidence = float(state.get("chord_confidence", 0.0)) chord_root = int(state.get("chord_root", 0)) @@ -497,7 +548,7 @@ def move_rgb(move: Move, move_index: int, move_count: int, state: dict[str, obje r = rr * (0.45 + 0.55 * float(spectral[0])) g = gg * (0.45 + 0.55 * float(spectral[1])) b = bb * (0.45 + 0.55 * float(spectral[2])) - white = loudness_gate * max(0.0, hit - 0.94) / 0.06 + white = score_envelope * loudness_gate * max(0.0, hit - 0.97) / 0.03 return ( int(min(255, 255 * r + 18 * white)), int(min(255, 255 * g + 18 * white)), @@ -534,9 +585,19 @@ def main() -> int: parser.add_argument("--hit-loudness-threshold", type=float, default=0.82) parser.add_argument("--loudness-floor", type=float, default=0.006) parser.add_argument("--loudness-exponent", type=float, default=2.4) - parser.add_argument("--quiet-brightness", type=float, default=0.012) - parser.add_argument("--max-brightness", type=float, default=0.76) - parser.add_argument("--brightness-exponent", type=float, default=0.82) + parser.add_argument("--quiet-brightness", type=float, default=0.0) + parser.add_argument("--max-brightness", type=float, default=0.18) + parser.add_argument("--brightness-exponent", type=float, default=1.15) + parser.add_argument("--score-subdivisions", type=int, default=8) + parser.add_argument("--score-instrument-spacing", type=int, default=3) + parser.add_argument("--score-release", type=float, default=0.54) + parser.add_argument("--score-loudness-threshold", type=float, default=0.965) + parser.add_argument("--score-min-loudness-gate", type=float, default=0.16) + parser.add_argument("--score-loudness-exponent", type=float, default=2.8) + parser.add_argument("--score-loudness-weight", type=float, default=0.28) + parser.add_argument("--score-min-accent", type=float, default=0.035) + parser.add_argument("--score-max-envelope", type=float, default=0.45) + parser.add_argument("--score-min-tempo-confidence", type=float, default=0.0) parser.add_argument("--onset-decay", type=float, default=0.18) parser.add_argument("--onset-threshold", type=float, default=0.62) parser.add_argument("--onset-exponent", type=float, default=2.0) @@ -601,6 +662,7 @@ def main() -> int: ) assert wasapi.stdout is not None and ssh.stdin is not None analyzer = OnlineAnalyzer(args.sample_rate, args.fps, args.fft_size, args) + score_scheduler = ScoreGestureScheduler(len(moves), args) asio_reader = None asio_tracker = None if args.asio_music_channels.strip(): @@ -664,6 +726,7 @@ def main() -> int: asio_reader = None asio_tracker = None state = fuse_music_sources(state, auxiliary_states) + state["score_gesture_envelopes"] = score_scheduler.update(state) rgbs = {} for index, move in enumerate(moves): rgb = move_rgb(move, index, len(moves), state, args) @@ -680,6 +743,7 @@ def main() -> int: "color_balance": list(state["color_balance"]), # type: ignore[index] "chroma": list(state["chroma"]), # type: ignore[index] "music_sources": state["music_sources"], + "score_gesture_envelopes": list(state["score_gesture_envelopes"]), # type: ignore[index] "moves": {name: list(rgb) for name, rgb in rgbs.items()}, } trace.write(json.dumps(record, sort_keys=True) + "\n")