Expose online daemon status to Odin

This commit is contained in:
Meta Vens 2026-06-02 19:18:55 +01:00
commit 92bc5d24f1
5 changed files with 112 additions and 13 deletions

View file

@ -1,9 +1,10 @@
param(
[string]$NightwingHost = "nightwing",
[string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle",
[string]$SubscribeUrl = "ws://127.0.0.1:8796/eve/periwinkle/subscribe",
[int]$VerseRelayPort = 8798,
[string]$WitnessUrl = "ws://192.168.1.66:8798/eve/periwinkle",
[string]$SubscribeUrl = "ws://127.0.0.1:8798/eve/periwinkle/subscribe",
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.well.local.json",
[string]$WellPublishUrl = "ws://127.0.0.1:8796/eve/periwinkle",
[string]$WellPublishUrl = "ws://127.0.0.1:8798/eve/periwinkle",
[string]$RecorderRoot = "C:\Users\Meta\Videos\Mimir\VerseCaptures",
[int]$FensalirPort = 8799,
[int]$FensalirWorkerMs = 2,
@ -103,6 +104,7 @@ $manifest = [ordered]@{
witnessUrl = $WitnessUrl
subscribeUrl = $SubscribeUrl
wellPublishUrl = $WellPublishUrl
verseRelayPort = $VerseRelayPort
recorderRoot = $RecorderRoot
fensalirPort = $FensalirPort
configPath = $ConfigPath
@ -117,7 +119,7 @@ Write-Host "Online Verse daemon run: $runDir"
if (-not $DryRun) {
Get-CimInstance Win32_Process |
Where-Object {
($_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796") -or
($_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port $VerseRelayPort") -or
($_.CommandLine -match "Mimir.VerseRecorder") -or
($_.CommandLine -match "Mimir.FensalirDaemon") -or
($_.CommandLine -match "Mimir.Well") -or
@ -136,7 +138,7 @@ $receiver = Start-LoggedProcess `
"--project",
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
"--",
"--port", "8796",
"--port", "$VerseRelayPort",
"--path", "/eve/periwinkle",
"--subscribe-path", "/eve/periwinkle/subscribe",
"--source-id", "nightwing-witness",
@ -144,7 +146,7 @@ $receiver = Start-LoggedProcess `
)
if ($receiver) { $manifest.processes["verseRelayPid"] = $receiver.Id }
Wait-HttpHealth -Url "http://127.0.0.1:8796/health"
Wait-HttpHealth -Url "http://127.0.0.1:$VerseRelayPort/health"
$recorderArgs = @(
"run",

View file

@ -1,4 +1,5 @@
using System.Globalization;
using System.Diagnostics;
using System.Net;
using System.Net.Sockets;
using System.Security.Cryptography;
@ -1040,6 +1041,7 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen
var readiness = doc.TotalSlices <= 0 ? 0.0 : doc.ReadySlices / (double)doc.TotalSlices;
var streamPressure = Math.Clamp(Math.Max(doc.PollAverageMs / 16.0, doc.PublishAverageMs / 16.0), 0.0, 1.0);
var workerPressure = Math.Clamp(worker.GpuQueueMs / 1000.0, 0.0, 1.0);
var observedDaemons = FensalirDaemonObserver.Collect(options);
var nodes = new List<DashboardNode>
{
new("daemon", $"Fensalir Daemon\n{doc.Status}", "daemon", 0.0, -0.42, 0.70, 0.18, doc.Status),
@ -1047,6 +1049,7 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen
new("capture", $"Capture Pages\nseq {doc.LastCaptureSequence}", "cultcache", 0.0, -0.08, 0.32, 0.18, doc.LastCaptureSequence >= 0 ? "paged" : "waiting"),
new("budget", $"Budget\nGPU {doc.GpuBudget:0.00} CPU {doc.CpuBudget:0.00}", "budget", 0.38, -0.08, 0.32, 0.18, "configured"),
new("worker", $"Reservoir Worker\n{worker.Status}", "worker", 0.0, 0.26, 0.54, 0.18, worker.Status),
new("online-daemons", $"Online Daemons\n{observedDaemons.Count(static daemon => daemon.Status == "online")}/{observedDaemons.Count}", "daemon-status", -0.38, 0.48, 0.70, 0.18, "observed"),
};
return new DashboardState
{
@ -1105,6 +1108,17 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen
Ui.Text("notes", string.Join("\n", doc.Notes.Take(4)), "caption", Bind("/notes")),
]),
]),
Ui.Row("daemon-row-observed",
[
Ui.Pane("observed-daemon-pane", "Observed Daemons",
observedDaemons.Select(static daemon =>
Ui.Text(
$"observed-{daemon.Id}",
$"{daemon.Name}: {daemon.Status} pid {daemon.Pid} {daemon.Detail}",
daemon.Status == "online" ? "mono" : "caption"))
.Cast<UiElement>()
.ToArray()),
]),
]),
},
};
@ -1245,6 +1259,84 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen
}
}
internal static class FensalirDaemonObserver
{
private static readonly (string Id, string Name, string[] PidFiles)[] LocalDaemons =
[
("verse-relay", "Verse relay", ["verse-relay-direct.pid", "verse-relay.pid"]),
("verse-recorder", "Verse recorder", ["verse-recorder.pid"]),
("mimir-well", "Mimir Well", ["mimir-well-rebased.pid", "mimir-well-fixed.pid", "mimir-well.pid"]),
("move-sync", "Move sync", ["move-sync-lean.pid", "move-sync.pid"]),
("fensalir", "Fensalir", ["fensalir-daemon.pid"]),
];
public static IReadOnlyList<ObservedDaemonStatus> Collect(FensalirDaemonOptions options)
{
var runDir = Path.GetDirectoryName(options.CultCachePath) ?? "";
var statuses = new List<ObservedDaemonStatus>
{
new("fensalir-self", "Fensalir self", Environment.ProcessId, "online", "eve/cultmesh provider")
};
if (string.IsNullOrWhiteSpace(runDir) || !Directory.Exists(runDir))
{
statuses.Add(new("supervisor-run", "Supervisor run", 0, "missing", "run directory missing"));
return statuses;
}
foreach (var daemon in LocalDaemons)
{
statuses.Add(ObserveLocal(runDir, daemon.Id, daemon.Name, daemon.PidFiles));
}
return statuses
.GroupBy(static status => status.Id, StringComparer.Ordinal)
.Select(static group => group.First())
.OrderBy(static status => status.Name, StringComparer.Ordinal)
.ToArray();
}
private static ObservedDaemonStatus ObserveLocal(string runDir, string id, string name, IReadOnlyList<string> pidFiles)
{
foreach (var pidFile in pidFiles)
{
var path = Path.Combine(runDir, pidFile);
if (!File.Exists(path))
{
continue;
}
var raw = File.ReadAllText(path).Trim();
if (!int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var pid) || pid <= 0)
{
return new ObservedDaemonStatus(id, name, 0, "invalid", pidFile);
}
try
{
using var process = Process.GetProcessById(pid);
return new ObservedDaemonStatus(id, name, pid, process.HasExited ? "stopped" : "online", pidFile);
}
catch (ArgumentException)
{
return new ObservedDaemonStatus(id, name, pid, "stopped", pidFile);
}
catch (InvalidOperationException)
{
return new ObservedDaemonStatus(id, name, pid, "stopped", pidFile);
}
}
return new ObservedDaemonStatus(id, name, 0, "missing", "no pid file");
}
}
internal sealed record ObservedDaemonStatus(
string Id,
string Name,
int Pid,
string Status,
string Detail);
internal sealed class WellLogTailState
{
private readonly string path;

View file

@ -2,6 +2,7 @@ namespace Mimir.Runtime.Synchronization;
public sealed class MimirCanonicalClockMapper
{
private const long MaxArrivalDriftBeforeRebaseNs = 10_000_000_000L;
private readonly Dictionary<string, ClockMap> maps = new(StringComparer.Ordinal);
public IReadOnlyList<MimirCanonicalClockMapSnapshot> Snapshots =>
@ -32,6 +33,14 @@ public sealed class MimirCanonicalClockMapper
map = new ClockMap(checked(arrivalNs - sourceTimestampNs), sourceTimestampNs, arrivalNs);
maps.Add(key, map);
}
else
{
var projectedArrivalNs = checked(sourceTimestampNs + map.OffsetNs);
if (Math.Abs(projectedArrivalNs - arrivalNs) > MaxArrivalDriftBeforeRebaseNs)
{
map.OffsetNs = checked(arrivalNs - sourceTimestampNs);
}
}
var canonicalTimestampNs = checked(sourceTimestampNs + map.OffsetNs);
map.LatestSourceTimestampNs = sourceTimestampNs;
@ -48,7 +57,7 @@ public sealed class MimirCanonicalClockMapper
private sealed class ClockMap(long offsetNs, long firstSourceTimestampNs, long firstArrivalNs)
{
public long OffsetNs { get; } = offsetNs;
public long OffsetNs { get; set; } = offsetNs;
public long FirstSourceTimestampNs { get; } = firstSourceTimestampNs;

View file

@ -71,14 +71,9 @@ public sealed class MimirSynchronizationHub : IDisposable
{
ObjectDisposedException.ThrowIf(disposed, this);
var delay = presentationDelay ?? TimeSpan.FromTicks(Settings.BufferDuration.Ticks / 2);
var timingCorrections = MimirSynchronizedBufferPlanner
.CorrectionsFromClockDomainEdges(Buffers.Buffers, Settings.Audio.ReferenceSourceId)
.Concat(MimirSynchronizedBufferPlanner.CorrectionsFromAudioStates(audioSynchronizationState.States, Buffers.Buffers))
.ToArray();
return new MimirSynchronizedBufferPlanner().BuildFrame(
Buffers.Buffers,
delay,
timingCorrections);
delay);
}
public void AddSource(IMimirStreamSource source)

View file

@ -364,3 +364,4 @@
{"ts": "2026-06-02T18:14:37+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Wired Fensalir daemon into the online Verse supervisor. VerseRecorder now accepts --run-id, start-online-verse-daemons passes the supervisor runId, and FensalirDaemon starts against the exact VerseCaptures/<runId>/observations.jsonl with a run-local CultCache instead of artifact discovery. Dry-run verified relay, recorder, and Fensalir command shapes; VerseRecorder and FensalirDaemon builds passed."}
{"ts": "2026-06-02T18:17:56+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Fensalir daemon now consumes mimir.cultmesh_stream_frame.v1 as a fine-grained reservoir job lane in addition to well_capture_page batches. Timed frames with usable body/native/shared transport enqueue jobs; metadata-only or low-confidence frames drop with reasons. Isolated build passed and a stream-frame fixture on port 8802 completed one worker job with droppedJobs=0."}
{"ts": "2026-06-02T18:23:46+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Added mimir.fensalir_reservoir_pressure.v1. Fensalir daemon now persists pressure cause metrics alongside worker state: Well poll/publish ms, capture bytes, ready-slice ratio, timing confidence/distance, GPU queue ms, and missing-body/fence-unavailable/timing-degraded drop counters. Fixture on port 8803 completed one stream-frame job and health advertised worker + pressure documents."}
{"ts": "2026-06-02T19:17:54+01:00", "type": "decision", "status": "ok", "note": "Online Verse shakedown: canonical timestamps are owned by MimirCanonicalClockMapper, which now rebases streams whose projected source clock drifts more than 10s from arrival after ASIO/device resets; MimirSynchronizationHub no longer reapplies raw clock-domain corrections to already-canonical buffers. Fensalir Eve surface now mirrors sibling daemon PID liveness so Odin can lower relay, recorder, Well, Move sync, and Fensalir statuses without taking control authority."}