Harden Well shakedown capture

This commit is contained in:
Meta Vens 2026-06-02 00:25:19 +01:00
commit 93986ebe0a
5 changed files with 127 additions and 10 deletions

View file

@ -65,6 +65,33 @@ function Start-LoggedProcess {
return $process
}
function Wait-HttpHealth {
param(
[string]$Url,
[int]$TimeoutSeconds = 45
)
if ($DryRun) {
Write-Host "DRY wait health: $Url"
return
}
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
do {
try {
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 3
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
Write-Host "health ok: $Url"
return
}
} catch {
Start-Sleep -Milliseconds 500
}
} while ((Get-Date) -lt $deadline)
throw "Timed out waiting for $Url"
}
$manifest = [ordered]@{
kind = "mimir.online_verse_daemon_supervisor.v1"
runId = $runId
@ -111,7 +138,7 @@ $receiver = Start-LoggedProcess `
)
if ($receiver) { $manifest.processes["verseRelayPid"] = $receiver.Id }
Start-Sleep -Milliseconds 900
Wait-HttpHealth -Url "http://127.0.0.1:8796/health"
$recorderArgs = @(
"run",
@ -263,7 +290,7 @@ if (-not $SkipRuntime) {
"--project",
(Join-Path $repo "src\Mimir.Well\Mimir.Well.csproj"),
"--",
"--seconds", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "31536000" }),
"--seconds", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "0" }),
"--poll-ms", "5",
"--publish-url", $WellPublishUrl,
"--node-id", "starfire",

View file

@ -114,7 +114,7 @@ internal sealed class EveSensorReceiver(EveSensorReceiverOptions options) : IDis
if (TryValidateCultMeshObservation(frame.Payload, out var normalizedObservation)
|| TryValidateCultMeshMediaObservation(frame.Payload, out normalizedObservation))
{
Console.WriteLine(normalizedObservation);
Console.WriteLine(ObservationStdoutLine(normalizedObservation));
Console.Out.Flush();
await BroadcastAsync(normalizedObservation).ConfigureAwait(false);
}
@ -130,13 +130,45 @@ internal sealed class EveSensorReceiver(EveSensorReceiverOptions options) : IDis
var text = Encoding.UTF8.GetString(frame.Payload);
if (TryValidateFrameEvent(text, out var normalized))
{
Console.WriteLine(normalized);
Console.WriteLine(ObservationStdoutLine(normalized));
Console.Out.Flush();
await BroadcastAsync(normalized).ConfigureAwait(false);
}
}
}
private static string ObservationStdoutLine(string normalized)
{
try
{
using var document = JsonDocument.Parse(normalized);
var root = document.RootElement;
if (root.TryGetProperty("document", out var documentKind)
&& string.Equals(documentKind.GetString(), "mimir.well_capture_page.v1", StringComparison.Ordinal))
{
var sampleCount = root.TryGetProperty("samples", out var samples)
&& samples.ValueKind == JsonValueKind.Array
? samples.GetArrayLength()
: 0;
return JsonSerializer.Serialize(new
{
type = root.GetProperty("type").GetString(),
document = "mimir.well_capture_page.v1",
sourceId = root.TryGetProperty("sourceId", out var sourceId) ? sourceId.GetString() : null,
nodeId = root.TryGetProperty("nodeId", out var nodeId) ? nodeId.GetString() : null,
captureSequence = root.TryGetProperty("captureSequence", out var sequence) ? sequence.GetInt64() : 0,
sampleCount,
stdoutBodyPolicy = "body-redacted",
});
}
}
catch (JsonException)
{
}
return normalized;
}
private async Task SubscribeAsync(NetworkStream stream)
{
var id = Interlocked.Increment(ref nextSubscriberId);

View file

@ -113,6 +113,7 @@ while (!stopping.IsCancellationRequested)
options,
presentation,
frame,
hub.Buffers.Buffers,
++captureSequence,
startedAt);
await publisher.PublishAsync(capturePage, stopping.Token).ConfigureAwait(false);
@ -285,6 +286,7 @@ internal static class MimirWellCapturePage
MimirWellOptions options,
MimirPresentationControlState presentation,
MimirSynchronizedBufferFrame frame,
IEnumerable<MimirRollingStreamBuffer> buffers,
long captureSequence,
DateTimeOffset startedAt) => new
{
@ -335,12 +337,30 @@ internal static class MimirWellCapturePage
reason = "Fensalir/OBS program-output capture is a separate producer; this page records the configured composite contract and synchronized source bodies.",
},
},
samples = frame.Slices
.Where(slice => slice.Sample.HasValue)
.Select(slice => CaptureSample(options, captureSequence, slice))
samples = CaptureSamples(options, captureSequence, frame, buffers)
.ToArray(),
};
private static IEnumerable<object> CaptureSamples(
MimirWellOptions options,
long captureSequence,
MimirSynchronizedBufferFrame frame,
IEnumerable<MimirRollingStreamBuffer> buffers)
{
var sliceSamples = frame.Slices
.Where(slice => slice.Sample.HasValue)
.Select(slice => CaptureSample(options, captureSequence, slice))
.ToArray();
if (sliceSamples.Length > 0)
{
return sliceSamples;
}
return buffers
.Where(buffer => buffer.Latest.HasValue)
.Select(buffer => CaptureLatestFallbackSample(options, captureSequence, buffer));
}
private static object CaptureSample(
MimirWellOptions options,
long captureSequence,
@ -371,6 +391,36 @@ internal static class MimirWellCapturePage
};
}
private static object CaptureLatestFallbackSample(
MimirWellOptions options,
long captureSequence,
MimirRollingStreamBuffer buffer)
{
var sample = buffer.Latest!.Value;
var bodyId = $"{options.NodeId}:{captureSequence}:{sample.SourceId}:{sample.Sequence}:latest";
return new
{
bodyId,
slice = new
{
sample.SourceId,
Kind = sample.Kind.ToString(),
Origin = sample.Origin.ToString(),
Status = "LatestUnalignedFallback",
SourceTimestampNs = sample.TimestampNs,
CanonicalStartNs = sample.TimestampNs,
CanonicalEndNs = sample.TimestampNs,
PresentationTimeNs = 0L,
TimingOffsetNs = 0L,
DistanceFromPresentationNs = 0L,
TimingConfidence = 0.0,
TimingEvidenceKind = "unsynchronized-fallback",
},
sample = SampleMetadata(sample),
body = CaptureBody(options, sample),
};
}
private static object SampleMetadata(MimirStreamSample sample) => new
{
sample.SourceId,

View file

@ -346,3 +346,4 @@
{"ts": "2026-06-01T23:50:56+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Mimir.Well now owns online camera exposure/gain optimization for visual sync. MimirCameraExposureController samples live Well video buffers, scores LED spline quality with MimirLedSplineFrameAnalyzer, cycles KS exposure/gain candidates, locks the best score, periodically resweeps, and publishes visualCalibration status in mimir.well_snapshot.v1. Verified with dotnet build src/Mimir.Well and supervisor dry-run options."}
{"ts": "2026-06-02T00:06:12+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Added the first Well-to-Recorder archival payload path. Mimir.Well now publishes bounded mimir.well_capture_page.v1 documents beside the dashboard snapshot; capture pages carry synchronized frame-slice timing, configured composite state, and inline base64 sample bodies when CPU payload bytes exist below the configured limit. Mimir.VerseRecorder strips those inline bodies into paged binary files under bodies/page-*.bin, replaces them with mimir.recorder_body_ref.v1 refs in observations.jsonl, and appends mimir.recorder_body_index.v1 records to bodies/index.jsonl. Verified with dotnet build for Mimir.Well and Mimir.VerseRecorder, supervisor dry-runs, and a relay/recorder synthetic capture-page smoke that wrote a 4-byte page body with the expected SHA-256 ref."}
{"ts": "2026-06-02T00:06:13+01:00", "type": "verification", "status": "degraded", "note": "2026-06-02: A short local online-verse daemon run proved recorder startup and session/body directory creation, but live Mimir.Well synchronized-frame planning degraded repeatedly with mixed timestamp-domain range errors such as camera/ASIO timestamps being compared before a shared canonical clock owner exists. Treat this as the known camera-clock/global-time correction scar, not a recorder storage failure."}
{"ts": "2026-06-02T00:25:02+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02 shakedown: Fixed two data-loss/disk-pressure scars while leaving the long-running Well stack active. The supervisor now passes --seconds 0 to Mimir.Well for indefinite runs instead of a one-year CancelAfter delay that .NET rejects, and waits for relay /health before starting subscribers/producers so Nightwing does not race dotnet cold start. Mimir.Well capture pages now fall back to latest per-buffer samples with TimingEvidenceKind=unsynchronized-fallback when synchronized-frame planning degrades, so Recorder pages raw CPU bodies even before the canonical clock owner exists. Mimir.EveSensorReceiver still broadcasts full capture pages to subscribers but redacts capture-page bodies from diagnostic stdout to avoid duplicating the paged body store in verse-relay.out.log. Live run online-verse-daemons-20260602-002116 / VerseCaptures/20260602-002124 showed all four local workers alive, Recorder receiving 520 observations, 479 body refs, Nightwing observations flowing after manual restart, and capture-page stdout summaries with sampleCount/body-redacted."}

View file

@ -409,9 +409,16 @@ invariants:
bounded `mimir.well_capture_page.v1` records for synchronized frame slices:
each page carries timing metadata, configured composite state, and inline
source sample bodies when a sample has CPU bytes and stays under the
configured body limit. GPU-handle-only samples remain metadata/body-handle
evidence until a Fensalir/native readback or direct program-output recorder
owns rendered composite bodies. The online supervisor
configured body limit. If synchronized-frame planning degrades because
source clocks are not yet in one canonical domain, capture pages fall back
to latest per-buffer samples with `TimingEvidenceKind =
unsynchronized-fallback`; this preserves raw evidence for offline
reconstruction without pretending the samples were synchronized.
GPU-handle-only samples remain metadata/body-handle evidence until a
Fensalir/native readback or direct program-output recorder owns rendered
composite bodies. The relay broadcasts full capture pages to subscribers
but redacts capture-page bodies from its diagnostic stdout so
`verse-relay.out.log` does not duplicate the body store. The online supervisor
now starts relay, recorder, Nightwing Eyes/Move/builtin-camera/builtin-mic
witness publication, conservative de-Bruijn/microtonal music-synced Move
gestures with Realtek plus opportunistic ASIO music evidence, and