From 92bc5d24f1e69fd11ba9dc926050c261cdeb38c1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 2 Jun 2026 19:18:55 +0100 Subject: [PATCH] Expose online daemon status to Odin --- scripts/start-online-verse-daemons.ps1 | 14 +-- src/Mimir.FensalirDaemon/Program.cs | 92 +++++++++++++++++++ .../MimirCanonicalClockMapper.cs | 11 ++- .../MimirSynchronizationHub.cs | 7 +- state/evidence.jsonl | 1 + 5 files changed, 112 insertions(+), 13 deletions(-) diff --git a/scripts/start-online-verse-daemons.ps1 b/scripts/start-online-verse-daemons.ps1 index dd0cd74..b941abd 100644 --- a/scripts/start-online-verse-daemons.ps1 +++ b/scripts/start-online-verse-daemons.ps1 @@ -1,9 +1,10 @@ param( [string]$NightwingHost = "nightwing", - [string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle", - [string]$SubscribeUrl = "ws://127.0.0.1:8796/eve/periwinkle/subscribe", + [int]$VerseRelayPort = 8798, + [string]$WitnessUrl = "ws://192.168.1.66:8798/eve/periwinkle", + [string]$SubscribeUrl = "ws://127.0.0.1:8798/eve/periwinkle/subscribe", [string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.well.local.json", - [string]$WellPublishUrl = "ws://127.0.0.1:8796/eve/periwinkle", + [string]$WellPublishUrl = "ws://127.0.0.1:8798/eve/periwinkle", [string]$RecorderRoot = "C:\Users\Meta\Videos\Mimir\VerseCaptures", [int]$FensalirPort = 8799, [int]$FensalirWorkerMs = 2, @@ -103,6 +104,7 @@ $manifest = [ordered]@{ witnessUrl = $WitnessUrl subscribeUrl = $SubscribeUrl wellPublishUrl = $WellPublishUrl + verseRelayPort = $VerseRelayPort recorderRoot = $RecorderRoot fensalirPort = $FensalirPort configPath = $ConfigPath @@ -117,7 +119,7 @@ Write-Host "Online Verse daemon run: $runDir" if (-not $DryRun) { Get-CimInstance Win32_Process | Where-Object { - ($_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796") -or + ($_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port $VerseRelayPort") -or ($_.CommandLine -match "Mimir.VerseRecorder") -or ($_.CommandLine -match "Mimir.FensalirDaemon") -or ($_.CommandLine -match "Mimir.Well") -or @@ -136,7 +138,7 @@ $receiver = Start-LoggedProcess ` "--project", (Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"), "--", - "--port", "8796", + "--port", "$VerseRelayPort", "--path", "/eve/periwinkle", "--subscribe-path", "/eve/periwinkle/subscribe", "--source-id", "nightwing-witness", @@ -144,7 +146,7 @@ $receiver = Start-LoggedProcess ` ) if ($receiver) { $manifest.processes["verseRelayPid"] = $receiver.Id } -Wait-HttpHealth -Url "http://127.0.0.1:8796/health" +Wait-HttpHealth -Url "http://127.0.0.1:$VerseRelayPort/health" $recorderArgs = @( "run", diff --git a/src/Mimir.FensalirDaemon/Program.cs b/src/Mimir.FensalirDaemon/Program.cs index 5c9e8ba..2bb2994 100644 --- a/src/Mimir.FensalirDaemon/Program.cs +++ b/src/Mimir.FensalirDaemon/Program.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Diagnostics; using System.Net; using System.Net.Sockets; using System.Security.Cryptography; @@ -1040,6 +1041,7 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen var readiness = doc.TotalSlices <= 0 ? 0.0 : doc.ReadySlices / (double)doc.TotalSlices; var streamPressure = Math.Clamp(Math.Max(doc.PollAverageMs / 16.0, doc.PublishAverageMs / 16.0), 0.0, 1.0); var workerPressure = Math.Clamp(worker.GpuQueueMs / 1000.0, 0.0, 1.0); + var observedDaemons = FensalirDaemonObserver.Collect(options); var nodes = new List { new("daemon", $"Fensalir Daemon\n{doc.Status}", "daemon", 0.0, -0.42, 0.70, 0.18, doc.Status), @@ -1047,6 +1049,7 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen new("capture", $"Capture Pages\nseq {doc.LastCaptureSequence}", "cultcache", 0.0, -0.08, 0.32, 0.18, doc.LastCaptureSequence >= 0 ? "paged" : "waiting"), new("budget", $"Budget\nGPU {doc.GpuBudget:0.00} CPU {doc.CpuBudget:0.00}", "budget", 0.38, -0.08, 0.32, 0.18, "configured"), new("worker", $"Reservoir Worker\n{worker.Status}", "worker", 0.0, 0.26, 0.54, 0.18, worker.Status), + new("online-daemons", $"Online Daemons\n{observedDaemons.Count(static daemon => daemon.Status == "online")}/{observedDaemons.Count}", "daemon-status", -0.38, 0.48, 0.70, 0.18, "observed"), }; return new DashboardState { @@ -1105,6 +1108,17 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen Ui.Text("notes", string.Join("\n", doc.Notes.Take(4)), "caption", Bind("/notes")), ]), ]), + Ui.Row("daemon-row-observed", + [ + Ui.Pane("observed-daemon-pane", "Observed Daemons", + observedDaemons.Select(static daemon => + Ui.Text( + $"observed-{daemon.Id}", + $"{daemon.Name}: {daemon.Status} pid {daemon.Pid} {daemon.Detail}", + daemon.Status == "online" ? "mono" : "caption")) + .Cast() + .ToArray()), + ]), ]), }, }; @@ -1245,6 +1259,84 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen } } +internal static class FensalirDaemonObserver +{ + private static readonly (string Id, string Name, string[] PidFiles)[] LocalDaemons = + [ + ("verse-relay", "Verse relay", ["verse-relay-direct.pid", "verse-relay.pid"]), + ("verse-recorder", "Verse recorder", ["verse-recorder.pid"]), + ("mimir-well", "Mimir Well", ["mimir-well-rebased.pid", "mimir-well-fixed.pid", "mimir-well.pid"]), + ("move-sync", "Move sync", ["move-sync-lean.pid", "move-sync.pid"]), + ("fensalir", "Fensalir", ["fensalir-daemon.pid"]), + ]; + + public static IReadOnlyList Collect(FensalirDaemonOptions options) + { + var runDir = Path.GetDirectoryName(options.CultCachePath) ?? ""; + var statuses = new List + { + new("fensalir-self", "Fensalir self", Environment.ProcessId, "online", "eve/cultmesh provider") + }; + if (string.IsNullOrWhiteSpace(runDir) || !Directory.Exists(runDir)) + { + statuses.Add(new("supervisor-run", "Supervisor run", 0, "missing", "run directory missing")); + return statuses; + } + + foreach (var daemon in LocalDaemons) + { + statuses.Add(ObserveLocal(runDir, daemon.Id, daemon.Name, daemon.PidFiles)); + } + + return statuses + .GroupBy(static status => status.Id, StringComparer.Ordinal) + .Select(static group => group.First()) + .OrderBy(static status => status.Name, StringComparer.Ordinal) + .ToArray(); + } + + private static ObservedDaemonStatus ObserveLocal(string runDir, string id, string name, IReadOnlyList pidFiles) + { + foreach (var pidFile in pidFiles) + { + var path = Path.Combine(runDir, pidFile); + if (!File.Exists(path)) + { + continue; + } + + var raw = File.ReadAllText(path).Trim(); + if (!int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var pid) || pid <= 0) + { + return new ObservedDaemonStatus(id, name, 0, "invalid", pidFile); + } + + try + { + using var process = Process.GetProcessById(pid); + return new ObservedDaemonStatus(id, name, pid, process.HasExited ? "stopped" : "online", pidFile); + } + catch (ArgumentException) + { + return new ObservedDaemonStatus(id, name, pid, "stopped", pidFile); + } + catch (InvalidOperationException) + { + return new ObservedDaemonStatus(id, name, pid, "stopped", pidFile); + } + } + + return new ObservedDaemonStatus(id, name, 0, "missing", "no pid file"); + } +} + +internal sealed record ObservedDaemonStatus( + string Id, + string Name, + int Pid, + string Status, + string Detail); + internal sealed class WellLogTailState { private readonly string path; diff --git a/src/Mimir.Runtime/Synchronization/MimirCanonicalClockMapper.cs b/src/Mimir.Runtime/Synchronization/MimirCanonicalClockMapper.cs index 72dd81a..23ebc60 100644 --- a/src/Mimir.Runtime/Synchronization/MimirCanonicalClockMapper.cs +++ b/src/Mimir.Runtime/Synchronization/MimirCanonicalClockMapper.cs @@ -2,6 +2,7 @@ namespace Mimir.Runtime.Synchronization; public sealed class MimirCanonicalClockMapper { + private const long MaxArrivalDriftBeforeRebaseNs = 10_000_000_000L; private readonly Dictionary maps = new(StringComparer.Ordinal); public IReadOnlyList Snapshots => @@ -32,6 +33,14 @@ public sealed class MimirCanonicalClockMapper map = new ClockMap(checked(arrivalNs - sourceTimestampNs), sourceTimestampNs, arrivalNs); maps.Add(key, map); } + else + { + var projectedArrivalNs = checked(sourceTimestampNs + map.OffsetNs); + if (Math.Abs(projectedArrivalNs - arrivalNs) > MaxArrivalDriftBeforeRebaseNs) + { + map.OffsetNs = checked(arrivalNs - sourceTimestampNs); + } + } var canonicalTimestampNs = checked(sourceTimestampNs + map.OffsetNs); map.LatestSourceTimestampNs = sourceTimestampNs; @@ -48,7 +57,7 @@ public sealed class MimirCanonicalClockMapper private sealed class ClockMap(long offsetNs, long firstSourceTimestampNs, long firstArrivalNs) { - public long OffsetNs { get; } = offsetNs; + public long OffsetNs { get; set; } = offsetNs; public long FirstSourceTimestampNs { get; } = firstSourceTimestampNs; diff --git a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs index 6e4f6c6..4b40172 100644 --- a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs +++ b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs @@ -71,14 +71,9 @@ public sealed class MimirSynchronizationHub : IDisposable { ObjectDisposedException.ThrowIf(disposed, this); var delay = presentationDelay ?? TimeSpan.FromTicks(Settings.BufferDuration.Ticks / 2); - var timingCorrections = MimirSynchronizedBufferPlanner - .CorrectionsFromClockDomainEdges(Buffers.Buffers, Settings.Audio.ReferenceSourceId) - .Concat(MimirSynchronizedBufferPlanner.CorrectionsFromAudioStates(audioSynchronizationState.States, Buffers.Buffers)) - .ToArray(); return new MimirSynchronizedBufferPlanner().BuildFrame( Buffers.Buffers, - delay, - timingCorrections); + delay); } public void AddSource(IMimirStreamSource source) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 6dfd5a7..1485a64 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -364,3 +364,4 @@ {"ts": "2026-06-02T18:14:37+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Wired Fensalir daemon into the online Verse supervisor. VerseRecorder now accepts --run-id, start-online-verse-daemons passes the supervisor runId, and FensalirDaemon starts against the exact VerseCaptures//observations.jsonl with a run-local CultCache instead of artifact discovery. Dry-run verified relay, recorder, and Fensalir command shapes; VerseRecorder and FensalirDaemon builds passed."} {"ts": "2026-06-02T18:17:56+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Fensalir daemon now consumes mimir.cultmesh_stream_frame.v1 as a fine-grained reservoir job lane in addition to well_capture_page batches. Timed frames with usable body/native/shared transport enqueue jobs; metadata-only or low-confidence frames drop with reasons. Isolated build passed and a stream-frame fixture on port 8802 completed one worker job with droppedJobs=0."} {"ts": "2026-06-02T18:23:46+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Added mimir.fensalir_reservoir_pressure.v1. Fensalir daemon now persists pressure cause metrics alongside worker state: Well poll/publish ms, capture bytes, ready-slice ratio, timing confidence/distance, GPU queue ms, and missing-body/fence-unavailable/timing-degraded drop counters. Fixture on port 8803 completed one stream-frame job and health advertised worker + pressure documents."} +{"ts": "2026-06-02T19:17:54+01:00", "type": "decision", "status": "ok", "note": "Online Verse shakedown: canonical timestamps are owned by MimirCanonicalClockMapper, which now rebases streams whose projected source clock drifts more than 10s from arrival after ASIO/device resets; MimirSynchronizationHub no longer reapplies raw clock-domain corrections to already-canonical buffers. Fensalir Eve surface now mirrors sibling daemon PID liveness so Odin can lower relay, recorder, Well, Move sync, and Fensalir statuses without taking control authority."}