Expose online daemon status to Odin
This commit is contained in:
parent
ba607f6028
commit
92bc5d24f1
5 changed files with 112 additions and 13 deletions
|
|
@ -1,9 +1,10 @@
|
|||
param(
|
||||
[string]$NightwingHost = "nightwing",
|
||||
[string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle",
|
||||
[string]$SubscribeUrl = "ws://127.0.0.1:8796/eve/periwinkle/subscribe",
|
||||
[int]$VerseRelayPort = 8798,
|
||||
[string]$WitnessUrl = "ws://192.168.1.66:8798/eve/periwinkle",
|
||||
[string]$SubscribeUrl = "ws://127.0.0.1:8798/eve/periwinkle/subscribe",
|
||||
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.well.local.json",
|
||||
[string]$WellPublishUrl = "ws://127.0.0.1:8796/eve/periwinkle",
|
||||
[string]$WellPublishUrl = "ws://127.0.0.1:8798/eve/periwinkle",
|
||||
[string]$RecorderRoot = "C:\Users\Meta\Videos\Mimir\VerseCaptures",
|
||||
[int]$FensalirPort = 8799,
|
||||
[int]$FensalirWorkerMs = 2,
|
||||
|
|
@ -103,6 +104,7 @@ $manifest = [ordered]@{
|
|||
witnessUrl = $WitnessUrl
|
||||
subscribeUrl = $SubscribeUrl
|
||||
wellPublishUrl = $WellPublishUrl
|
||||
verseRelayPort = $VerseRelayPort
|
||||
recorderRoot = $RecorderRoot
|
||||
fensalirPort = $FensalirPort
|
||||
configPath = $ConfigPath
|
||||
|
|
@ -117,7 +119,7 @@ Write-Host "Online Verse daemon run: $runDir"
|
|||
if (-not $DryRun) {
|
||||
Get-CimInstance Win32_Process |
|
||||
Where-Object {
|
||||
($_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796") -or
|
||||
($_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port $VerseRelayPort") -or
|
||||
($_.CommandLine -match "Mimir.VerseRecorder") -or
|
||||
($_.CommandLine -match "Mimir.FensalirDaemon") -or
|
||||
($_.CommandLine -match "Mimir.Well") -or
|
||||
|
|
@ -136,7 +138,7 @@ $receiver = Start-LoggedProcess `
|
|||
"--project",
|
||||
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
|
||||
"--",
|
||||
"--port", "8796",
|
||||
"--port", "$VerseRelayPort",
|
||||
"--path", "/eve/periwinkle",
|
||||
"--subscribe-path", "/eve/periwinkle/subscribe",
|
||||
"--source-id", "nightwing-witness",
|
||||
|
|
@ -144,7 +146,7 @@ $receiver = Start-LoggedProcess `
|
|||
)
|
||||
if ($receiver) { $manifest.processes["verseRelayPid"] = $receiver.Id }
|
||||
|
||||
Wait-HttpHealth -Url "http://127.0.0.1:8796/health"
|
||||
Wait-HttpHealth -Url "http://127.0.0.1:$VerseRelayPort/health"
|
||||
|
||||
$recorderArgs = @(
|
||||
"run",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using System.Globalization;
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Cryptography;
|
||||
|
|
@ -1040,6 +1041,7 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen
|
|||
var readiness = doc.TotalSlices <= 0 ? 0.0 : doc.ReadySlices / (double)doc.TotalSlices;
|
||||
var streamPressure = Math.Clamp(Math.Max(doc.PollAverageMs / 16.0, doc.PublishAverageMs / 16.0), 0.0, 1.0);
|
||||
var workerPressure = Math.Clamp(worker.GpuQueueMs / 1000.0, 0.0, 1.0);
|
||||
var observedDaemons = FensalirDaemonObserver.Collect(options);
|
||||
var nodes = new List<DashboardNode>
|
||||
{
|
||||
new("daemon", $"Fensalir Daemon\n{doc.Status}", "daemon", 0.0, -0.42, 0.70, 0.18, doc.Status),
|
||||
|
|
@ -1047,6 +1049,7 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen
|
|||
new("capture", $"Capture Pages\nseq {doc.LastCaptureSequence}", "cultcache", 0.0, -0.08, 0.32, 0.18, doc.LastCaptureSequence >= 0 ? "paged" : "waiting"),
|
||||
new("budget", $"Budget\nGPU {doc.GpuBudget:0.00} CPU {doc.CpuBudget:0.00}", "budget", 0.38, -0.08, 0.32, 0.18, "configured"),
|
||||
new("worker", $"Reservoir Worker\n{worker.Status}", "worker", 0.0, 0.26, 0.54, 0.18, worker.Status),
|
||||
new("online-daemons", $"Online Daemons\n{observedDaemons.Count(static daemon => daemon.Status == "online")}/{observedDaemons.Count}", "daemon-status", -0.38, 0.48, 0.70, 0.18, "observed"),
|
||||
};
|
||||
return new DashboardState
|
||||
{
|
||||
|
|
@ -1105,6 +1108,17 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen
|
|||
Ui.Text("notes", string.Join("\n", doc.Notes.Take(4)), "caption", Bind("/notes")),
|
||||
]),
|
||||
]),
|
||||
Ui.Row("daemon-row-observed",
|
||||
[
|
||||
Ui.Pane("observed-daemon-pane", "Observed Daemons",
|
||||
observedDaemons.Select(static daemon =>
|
||||
Ui.Text(
|
||||
$"observed-{daemon.Id}",
|
||||
$"{daemon.Name}: {daemon.Status} pid {daemon.Pid} {daemon.Detail}",
|
||||
daemon.Status == "online" ? "mono" : "caption"))
|
||||
.Cast<UiElement>()
|
||||
.ToArray()),
|
||||
]),
|
||||
]),
|
||||
},
|
||||
};
|
||||
|
|
@ -1245,6 +1259,84 @@ internal sealed class FensalirDaemonEveServer(FensalirDaemonOptions options, Fen
|
|||
}
|
||||
}
|
||||
|
||||
internal static class FensalirDaemonObserver
|
||||
{
|
||||
private static readonly (string Id, string Name, string[] PidFiles)[] LocalDaemons =
|
||||
[
|
||||
("verse-relay", "Verse relay", ["verse-relay-direct.pid", "verse-relay.pid"]),
|
||||
("verse-recorder", "Verse recorder", ["verse-recorder.pid"]),
|
||||
("mimir-well", "Mimir Well", ["mimir-well-rebased.pid", "mimir-well-fixed.pid", "mimir-well.pid"]),
|
||||
("move-sync", "Move sync", ["move-sync-lean.pid", "move-sync.pid"]),
|
||||
("fensalir", "Fensalir", ["fensalir-daemon.pid"]),
|
||||
];
|
||||
|
||||
public static IReadOnlyList<ObservedDaemonStatus> Collect(FensalirDaemonOptions options)
|
||||
{
|
||||
var runDir = Path.GetDirectoryName(options.CultCachePath) ?? "";
|
||||
var statuses = new List<ObservedDaemonStatus>
|
||||
{
|
||||
new("fensalir-self", "Fensalir self", Environment.ProcessId, "online", "eve/cultmesh provider")
|
||||
};
|
||||
if (string.IsNullOrWhiteSpace(runDir) || !Directory.Exists(runDir))
|
||||
{
|
||||
statuses.Add(new("supervisor-run", "Supervisor run", 0, "missing", "run directory missing"));
|
||||
return statuses;
|
||||
}
|
||||
|
||||
foreach (var daemon in LocalDaemons)
|
||||
{
|
||||
statuses.Add(ObserveLocal(runDir, daemon.Id, daemon.Name, daemon.PidFiles));
|
||||
}
|
||||
|
||||
return statuses
|
||||
.GroupBy(static status => status.Id, StringComparer.Ordinal)
|
||||
.Select(static group => group.First())
|
||||
.OrderBy(static status => status.Name, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static ObservedDaemonStatus ObserveLocal(string runDir, string id, string name, IReadOnlyList<string> pidFiles)
|
||||
{
|
||||
foreach (var pidFile in pidFiles)
|
||||
{
|
||||
var path = Path.Combine(runDir, pidFile);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var raw = File.ReadAllText(path).Trim();
|
||||
if (!int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var pid) || pid <= 0)
|
||||
{
|
||||
return new ObservedDaemonStatus(id, name, 0, "invalid", pidFile);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var process = Process.GetProcessById(pid);
|
||||
return new ObservedDaemonStatus(id, name, pid, process.HasExited ? "stopped" : "online", pidFile);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
return new ObservedDaemonStatus(id, name, pid, "stopped", pidFile);
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
return new ObservedDaemonStatus(id, name, pid, "stopped", pidFile);
|
||||
}
|
||||
}
|
||||
|
||||
return new ObservedDaemonStatus(id, name, 0, "missing", "no pid file");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record ObservedDaemonStatus(
|
||||
string Id,
|
||||
string Name,
|
||||
int Pid,
|
||||
string Status,
|
||||
string Detail);
|
||||
|
||||
internal sealed class WellLogTailState
|
||||
{
|
||||
private readonly string path;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ namespace Mimir.Runtime.Synchronization;
|
|||
|
||||
public sealed class MimirCanonicalClockMapper
|
||||
{
|
||||
private const long MaxArrivalDriftBeforeRebaseNs = 10_000_000_000L;
|
||||
private readonly Dictionary<string, ClockMap> maps = new(StringComparer.Ordinal);
|
||||
|
||||
public IReadOnlyList<MimirCanonicalClockMapSnapshot> Snapshots =>
|
||||
|
|
@ -32,6 +33,14 @@ public sealed class MimirCanonicalClockMapper
|
|||
map = new ClockMap(checked(arrivalNs - sourceTimestampNs), sourceTimestampNs, arrivalNs);
|
||||
maps.Add(key, map);
|
||||
}
|
||||
else
|
||||
{
|
||||
var projectedArrivalNs = checked(sourceTimestampNs + map.OffsetNs);
|
||||
if (Math.Abs(projectedArrivalNs - arrivalNs) > MaxArrivalDriftBeforeRebaseNs)
|
||||
{
|
||||
map.OffsetNs = checked(arrivalNs - sourceTimestampNs);
|
||||
}
|
||||
}
|
||||
|
||||
var canonicalTimestampNs = checked(sourceTimestampNs + map.OffsetNs);
|
||||
map.LatestSourceTimestampNs = sourceTimestampNs;
|
||||
|
|
@ -48,7 +57,7 @@ public sealed class MimirCanonicalClockMapper
|
|||
|
||||
private sealed class ClockMap(long offsetNs, long firstSourceTimestampNs, long firstArrivalNs)
|
||||
{
|
||||
public long OffsetNs { get; } = offsetNs;
|
||||
public long OffsetNs { get; set; } = offsetNs;
|
||||
|
||||
public long FirstSourceTimestampNs { get; } = firstSourceTimestampNs;
|
||||
|
||||
|
|
|
|||
|
|
@ -71,14 +71,9 @@ public sealed class MimirSynchronizationHub : IDisposable
|
|||
{
|
||||
ObjectDisposedException.ThrowIf(disposed, this);
|
||||
var delay = presentationDelay ?? TimeSpan.FromTicks(Settings.BufferDuration.Ticks / 2);
|
||||
var timingCorrections = MimirSynchronizedBufferPlanner
|
||||
.CorrectionsFromClockDomainEdges(Buffers.Buffers, Settings.Audio.ReferenceSourceId)
|
||||
.Concat(MimirSynchronizedBufferPlanner.CorrectionsFromAudioStates(audioSynchronizationState.States, Buffers.Buffers))
|
||||
.ToArray();
|
||||
return new MimirSynchronizedBufferPlanner().BuildFrame(
|
||||
Buffers.Buffers,
|
||||
delay,
|
||||
timingCorrections);
|
||||
delay);
|
||||
}
|
||||
|
||||
public void AddSource(IMimirStreamSource source)
|
||||
|
|
|
|||
|
|
@ -364,3 +364,4 @@
|
|||
{"ts": "2026-06-02T18:14:37+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Wired Fensalir daemon into the online Verse supervisor. VerseRecorder now accepts --run-id, start-online-verse-daemons passes the supervisor runId, and FensalirDaemon starts against the exact VerseCaptures/<runId>/observations.jsonl with a run-local CultCache instead of artifact discovery. Dry-run verified relay, recorder, and Fensalir command shapes; VerseRecorder and FensalirDaemon builds passed."}
|
||||
{"ts": "2026-06-02T18:17:56+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Fensalir daemon now consumes mimir.cultmesh_stream_frame.v1 as a fine-grained reservoir job lane in addition to well_capture_page batches. Timed frames with usable body/native/shared transport enqueue jobs; metadata-only or low-confidence frames drop with reasons. Isolated build passed and a stream-frame fixture on port 8802 completed one worker job with droppedJobs=0."}
|
||||
{"ts": "2026-06-02T18:23:46+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Added mimir.fensalir_reservoir_pressure.v1. Fensalir daemon now persists pressure cause metrics alongside worker state: Well poll/publish ms, capture bytes, ready-slice ratio, timing confidence/distance, GPU queue ms, and missing-body/fence-unavailable/timing-degraded drop counters. Fixture on port 8803 completed one stream-frame job and health advertised worker + pressure documents."}
|
||||
{"ts": "2026-06-02T19:17:54+01:00", "type": "decision", "status": "ok", "note": "Online Verse shakedown: canonical timestamps are owned by MimirCanonicalClockMapper, which now rebases streams whose projected source clock drifts more than 10s from arrival after ASIO/device resets; MimirSynchronizationHub no longer reapplies raw clock-domain corrections to already-canonical buffers. Fensalir Eve surface now mirrors sibling daemon PID liveness so Odin can lower relay, recorder, Well, Move sync, and Fensalir statuses without taking control authority."}
|
||||
|
|
|
|||
Loading…
Reference in a new issue