Make Gjallar health track snapshot freshness
This commit is contained in:
parent
e1ac821472
commit
6ab652b19c
2 changed files with 72 additions and 10 deletions
|
|
@ -111,9 +111,13 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
private int latestCatalogProviders;
|
||||
private int latestComposedProviders;
|
||||
private string lastReceiveStatus = "starting";
|
||||
private string lastReceiveAttemptStatus = "starting";
|
||||
private string lastReceiveError = "";
|
||||
private string lastProviderFetchError = "";
|
||||
private string lastProviderFetchUri = "";
|
||||
private DateTimeOffset lastSuccessfulCatalogReceiveAt = DateTimeOffset.MinValue;
|
||||
private string lastSuccessfulCatalogReceiveObservedAt = "";
|
||||
private int consecutiveCatalogReceiveFailures;
|
||||
private SceneCache? sceneCache;
|
||||
private FrameTimings lastTimings;
|
||||
private object[] lastPanelFontUsage = [];
|
||||
|
|
@ -344,6 +348,7 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
using var socket = new ClientWebSocket();
|
||||
await socket.ConnectAsync(target, token).ConfigureAwait(false);
|
||||
lastReceiveStatus = $"connected:{config.ProviderId}";
|
||||
lastReceiveAttemptStatus = lastReceiveStatus;
|
||||
lastReceiveError = "";
|
||||
await ReceiveLoopAsync(socket, token).ConfigureAwait(false);
|
||||
}
|
||||
|
|
@ -354,6 +359,7 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
catch (Exception error) when (!token.IsCancellationRequested)
|
||||
{
|
||||
lastReceiveStatus = $"connect-error:{config.ProviderId}";
|
||||
lastReceiveAttemptStatus = lastReceiveStatus;
|
||||
lastReceiveError = error.Message;
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), token).ConfigureAwait(false);
|
||||
}
|
||||
|
|
@ -396,10 +402,11 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
var (catalog, frames) = ReadCultNetSnapshot(transport);
|
||||
var observedAt = DateTimeOffset.UtcNow;
|
||||
latestCatalogProviders = catalog.Providers.Count;
|
||||
latestComposedProviders = frames.Count;
|
||||
Interlocked.Exchange(ref latestStateJson, BuildGjallarSurface(catalog, frames));
|
||||
lastReceiveStatus = "catalog-composed";
|
||||
MarkCatalogReceiveSuccess(observedAt);
|
||||
lastReceiveError = "";
|
||||
lastProviderFetchError = "";
|
||||
|
||||
|
|
@ -413,8 +420,7 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
}
|
||||
catch (Exception error) when (!token.IsCancellationRequested)
|
||||
{
|
||||
lastReceiveStatus = "catalog-error";
|
||||
lastReceiveError = error.Message;
|
||||
MarkCatalogReceiveFailure(error, DateTimeOffset.UtcNow);
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
|
@ -440,13 +446,12 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
|
||||
Interlocked.Exchange(ref latestStateJson, BuildGjallarSurface(catalog, states));
|
||||
latestComposedProviders = states.Count;
|
||||
lastReceiveStatus = "catalog-composed";
|
||||
MarkCatalogReceiveSuccess(DateTimeOffset.UtcNow);
|
||||
lastReceiveError = "";
|
||||
}
|
||||
catch (Exception error) when (!token.IsCancellationRequested)
|
||||
{
|
||||
lastReceiveStatus = "catalog-error";
|
||||
lastReceiveError = error.Message;
|
||||
MarkCatalogReceiveFailure(error, DateTimeOffset.UtcNow);
|
||||
// A missed catalog pass should make the display stale, not kill the horn.
|
||||
}
|
||||
|
||||
|
|
@ -454,6 +459,48 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
}
|
||||
}
|
||||
|
||||
private TimeSpan CatalogReceiveStaleAfter() =>
|
||||
TimeSpan.FromSeconds(Math.Max(10, config.CatalogRefreshSeconds * 3));
|
||||
|
||||
private string EffectiveReceiveStatus(DateTimeOffset observedAt)
|
||||
{
|
||||
if (!config.UsesOdinCultNetRudp && !string.IsNullOrWhiteSpace(config.ProviderId))
|
||||
{
|
||||
return lastReceiveStatus;
|
||||
}
|
||||
|
||||
if (Volatile.Read(ref latestStateJson) is null)
|
||||
{
|
||||
return lastReceiveAttemptStatus == "starting" ? "starting" : "catalog-error";
|
||||
}
|
||||
|
||||
if (lastSuccessfulCatalogReceiveAt == DateTimeOffset.MinValue)
|
||||
{
|
||||
return "catalog-error";
|
||||
}
|
||||
|
||||
return observedAt - lastSuccessfulCatalogReceiveAt <= CatalogReceiveStaleAfter()
|
||||
? "catalog-composed"
|
||||
: "catalog-stale";
|
||||
}
|
||||
|
||||
private void MarkCatalogReceiveSuccess(DateTimeOffset observedAt)
|
||||
{
|
||||
lastSuccessfulCatalogReceiveAt = observedAt;
|
||||
lastSuccessfulCatalogReceiveObservedAt = observedAt.ToString("O", CultureInfo.InvariantCulture);
|
||||
consecutiveCatalogReceiveFailures = 0;
|
||||
lastReceiveStatus = "catalog-composed";
|
||||
lastReceiveAttemptStatus = "catalog-composed";
|
||||
}
|
||||
|
||||
private void MarkCatalogReceiveFailure(Exception error, DateTimeOffset observedAt)
|
||||
{
|
||||
consecutiveCatalogReceiveFailures += 1;
|
||||
lastReceiveAttemptStatus = "catalog-error";
|
||||
lastReceiveError = error.Message;
|
||||
lastReceiveStatus = EffectiveReceiveStatus(observedAt);
|
||||
}
|
||||
|
||||
private async Task<ProviderCatalog> ReadProviderCatalogAsync(CancellationToken token)
|
||||
{
|
||||
using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(5) };
|
||||
|
|
@ -999,11 +1046,15 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
},
|
||||
receive = new
|
||||
{
|
||||
status = lastReceiveStatus,
|
||||
status = EffectiveReceiveStatus(started),
|
||||
lastAttemptStatus = lastReceiveAttemptStatus,
|
||||
transport = config.InputTransportId,
|
||||
error = lastReceiveError,
|
||||
providerFetchError = lastProviderFetchError,
|
||||
providerFetchUri = lastProviderFetchUri,
|
||||
lastSuccessfulAtUtc = lastSuccessfulCatalogReceiveObservedAt,
|
||||
staleAfterSeconds = (int)CatalogReceiveStaleAfter().TotalSeconds,
|
||||
consecutiveFailures = consecutiveCatalogReceiveFailures,
|
||||
catalogProviders = latestCatalogProviders,
|
||||
composedProviders = latestComposedProviders,
|
||||
stateBytes = Volatile.Read(ref latestStateJson)?.Length ?? 0,
|
||||
|
|
@ -1102,10 +1153,13 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
MeasuredFps = measuredFps,
|
||||
Timings = timings,
|
||||
ObservedAt = started,
|
||||
ReceiveStatus = lastReceiveStatus,
|
||||
ReceiveStatus = EffectiveReceiveStatus(started),
|
||||
ReceiveAttemptStatus = lastReceiveAttemptStatus,
|
||||
ReceiveError = lastReceiveError,
|
||||
ProviderFetchError = lastProviderFetchError,
|
||||
ProviderFetchUri = lastProviderFetchUri,
|
||||
LastSuccessfulReceiveAt = lastSuccessfulCatalogReceiveObservedAt,
|
||||
ConsecutiveReceiveFailures = consecutiveCatalogReceiveFailures,
|
||||
CatalogProviders = latestCatalogProviders,
|
||||
ComposedProviders = latestComposedProviders,
|
||||
StateBytes = Volatile.Read(ref latestStateJson)?.Length ?? 0,
|
||||
|
|
@ -1151,8 +1205,11 @@ internal sealed class GjallarRenderer : IDisposable
|
|||
double measuredFps,
|
||||
DateTimeOffset observedAt)
|
||||
{
|
||||
var state = string.Equals(status, "rendered", StringComparison.Ordinal) ? "healthy" : "active";
|
||||
var detail = $"Gjallar framebuffer {status}; frames={frames}; catalogProviders={latestCatalogProviders}; composedProviders={latestComposedProviders}; fps={Math.Round(measuredFps, 2)}; receive={lastReceiveStatus}";
|
||||
var receiveStatus = EffectiveReceiveStatus(observedAt);
|
||||
var state = string.Equals(status, "rendered", StringComparison.Ordinal)
|
||||
? (receiveStatus == "catalog-composed" ? "healthy" : "dependency-unavailable")
|
||||
: "active";
|
||||
var detail = $"Gjallar framebuffer {status}; frames={frames}; catalogProviders={latestCatalogProviders}; composedProviders={latestComposedProviders}; fps={Math.Round(measuredFps, 2)}; receive={receiveStatus}; attempt={lastReceiveAttemptStatus}; failures={consecutiveCatalogReceiveFailures}; lastSuccess={lastSuccessfulCatalogReceiveObservedAt}";
|
||||
return new IdunnDaemonHealthRecord
|
||||
{
|
||||
DaemonId = config.IdunnDaemon,
|
||||
|
|
|
|||
|
|
@ -200,6 +200,8 @@ internal sealed class GjallarVerseRuntime : IDisposable
|
|||
"nightwing-gjallar-transport",
|
||||
"Transport",
|
||||
TextNode("nightwing-gjallar-input-transport", $"input: {config.InputTransportId} ({pulse.ReceiveStatus})"),
|
||||
TextNode("nightwing-gjallar-input-attempt", $"attempt: {pulse.ReceiveAttemptStatus} / failures: {pulse.ConsecutiveReceiveFailures}"),
|
||||
TextNode("nightwing-gjallar-last-success", string.IsNullOrWhiteSpace(pulse.LastSuccessfulReceiveAt) ? "last success: none" : $"last success: {pulse.LastSuccessfulReceiveAt}"),
|
||||
TextNode("nightwing-gjallar-output-transport", "output: daemon-published-rudp-health + daemon-owned-cultcache-service-boundary"),
|
||||
TextNode("nightwing-gjallar-witness", $"witness: {config.CultCachePath}"),
|
||||
TextNode("nightwing-gjallar-status-path", $"status: {config.StatusPath}"),
|
||||
|
|
@ -401,9 +403,12 @@ internal sealed class GjallarVersePulse
|
|||
public FrameTimings Timings { get; set; }
|
||||
public DateTimeOffset ObservedAt { get; set; }
|
||||
public string ReceiveStatus { get; set; } = "starting";
|
||||
public string ReceiveAttemptStatus { get; set; } = "starting";
|
||||
public string ReceiveError { get; set; } = "";
|
||||
public string ProviderFetchError { get; set; } = "";
|
||||
public string ProviderFetchUri { get; set; } = "";
|
||||
public string LastSuccessfulReceiveAt { get; set; } = "";
|
||||
public int ConsecutiveReceiveFailures { get; set; }
|
||||
public int CatalogProviders { get; set; }
|
||||
public int ComposedProviders { get; set; }
|
||||
public int StateBytes { get; set; }
|
||||
|
|
|
|||
Reference in a new issue