From 93986ebe0a5dacb68dab6203a1087c49d30d06dd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 2 Jun 2026 00:25:19 +0100 Subject: [PATCH] Harden Well shakedown capture --- scripts/start-online-verse-daemons.ps1 | 31 +++++++++++++- src/Mimir.EveSensorReceiver/Program.cs | 36 ++++++++++++++++- src/Mimir.Well/Program.cs | 56 ++++++++++++++++++++++++-- state/evidence.jsonl | 1 + state/map.yaml | 13 ++++-- 5 files changed, 127 insertions(+), 10 deletions(-) diff --git a/scripts/start-online-verse-daemons.ps1 b/scripts/start-online-verse-daemons.ps1 index 679c4a1..c383b05 100644 --- a/scripts/start-online-verse-daemons.ps1 +++ b/scripts/start-online-verse-daemons.ps1 @@ -65,6 +65,33 @@ function Start-LoggedProcess { return $process } +function Wait-HttpHealth { + param( + [string]$Url, + [int]$TimeoutSeconds = 45 + ) + + if ($DryRun) { + Write-Host "DRY wait health: $Url" + return + } + + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + do { + try { + $response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 3 + if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { + Write-Host "health ok: $Url" + return + } + } catch { + Start-Sleep -Milliseconds 500 + } + } while ((Get-Date) -lt $deadline) + + throw "Timed out waiting for $Url" +} + $manifest = [ordered]@{ kind = "mimir.online_verse_daemon_supervisor.v1" runId = $runId @@ -111,7 +138,7 @@ $receiver = Start-LoggedProcess ` ) if ($receiver) { $manifest.processes["verseRelayPid"] = $receiver.Id } -Start-Sleep -Milliseconds 900 +Wait-HttpHealth -Url "http://127.0.0.1:8796/health" $recorderArgs = @( "run", @@ -263,7 +290,7 @@ if (-not $SkipRuntime) { "--project", (Join-Path $repo "src\Mimir.Well\Mimir.Well.csproj"), "--", - "--seconds", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "31536000" }), + "--seconds", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "0" }), "--poll-ms", "5", "--publish-url", $WellPublishUrl, "--node-id", "starfire", diff --git a/src/Mimir.EveSensorReceiver/Program.cs b/src/Mimir.EveSensorReceiver/Program.cs index 7cdfc78..d85d3ab 100644 --- a/src/Mimir.EveSensorReceiver/Program.cs +++ b/src/Mimir.EveSensorReceiver/Program.cs @@ -114,7 +114,7 @@ internal sealed class EveSensorReceiver(EveSensorReceiverOptions options) : IDis if (TryValidateCultMeshObservation(frame.Payload, out var normalizedObservation) || TryValidateCultMeshMediaObservation(frame.Payload, out normalizedObservation)) { - Console.WriteLine(normalizedObservation); + Console.WriteLine(ObservationStdoutLine(normalizedObservation)); Console.Out.Flush(); await BroadcastAsync(normalizedObservation).ConfigureAwait(false); } @@ -130,13 +130,45 @@ internal sealed class EveSensorReceiver(EveSensorReceiverOptions options) : IDis var text = Encoding.UTF8.GetString(frame.Payload); if (TryValidateFrameEvent(text, out var normalized)) { - Console.WriteLine(normalized); + Console.WriteLine(ObservationStdoutLine(normalized)); Console.Out.Flush(); await BroadcastAsync(normalized).ConfigureAwait(false); } } } + private static string ObservationStdoutLine(string normalized) + { + try + { + using var document = JsonDocument.Parse(normalized); + var root = document.RootElement; + if (root.TryGetProperty("document", out var documentKind) + && string.Equals(documentKind.GetString(), "mimir.well_capture_page.v1", StringComparison.Ordinal)) + { + var sampleCount = root.TryGetProperty("samples", out var samples) + && samples.ValueKind == JsonValueKind.Array + ? samples.GetArrayLength() + : 0; + return JsonSerializer.Serialize(new + { + type = root.GetProperty("type").GetString(), + document = "mimir.well_capture_page.v1", + sourceId = root.TryGetProperty("sourceId", out var sourceId) ? sourceId.GetString() : null, + nodeId = root.TryGetProperty("nodeId", out var nodeId) ? nodeId.GetString() : null, + captureSequence = root.TryGetProperty("captureSequence", out var sequence) ? sequence.GetInt64() : 0, + sampleCount, + stdoutBodyPolicy = "body-redacted", + }); + } + } + catch (JsonException) + { + } + + return normalized; + } + private async Task SubscribeAsync(NetworkStream stream) { var id = Interlocked.Increment(ref nextSubscriberId); diff --git a/src/Mimir.Well/Program.cs b/src/Mimir.Well/Program.cs index 723542c..6b6bd0a 100644 --- a/src/Mimir.Well/Program.cs +++ b/src/Mimir.Well/Program.cs @@ -113,6 +113,7 @@ while (!stopping.IsCancellationRequested) options, presentation, frame, + hub.Buffers.Buffers, ++captureSequence, startedAt); await publisher.PublishAsync(capturePage, stopping.Token).ConfigureAwait(false); @@ -285,6 +286,7 @@ internal static class MimirWellCapturePage MimirWellOptions options, MimirPresentationControlState presentation, MimirSynchronizedBufferFrame frame, + IEnumerable buffers, long captureSequence, DateTimeOffset startedAt) => new { @@ -335,12 +337,30 @@ internal static class MimirWellCapturePage reason = "Fensalir/OBS program-output capture is a separate producer; this page records the configured composite contract and synchronized source bodies.", }, }, - samples = frame.Slices - .Where(slice => slice.Sample.HasValue) - .Select(slice => CaptureSample(options, captureSequence, slice)) + samples = CaptureSamples(options, captureSequence, frame, buffers) .ToArray(), }; + private static IEnumerable CaptureSamples( + MimirWellOptions options, + long captureSequence, + MimirSynchronizedBufferFrame frame, + IEnumerable buffers) + { + var sliceSamples = frame.Slices + .Where(slice => slice.Sample.HasValue) + .Select(slice => CaptureSample(options, captureSequence, slice)) + .ToArray(); + if (sliceSamples.Length > 0) + { + return sliceSamples; + } + + return buffers + .Where(buffer => buffer.Latest.HasValue) + .Select(buffer => CaptureLatestFallbackSample(options, captureSequence, buffer)); + } + private static object CaptureSample( MimirWellOptions options, long captureSequence, @@ -371,6 +391,36 @@ internal static class MimirWellCapturePage }; } + private static object CaptureLatestFallbackSample( + MimirWellOptions options, + long captureSequence, + MimirRollingStreamBuffer buffer) + { + var sample = buffer.Latest!.Value; + var bodyId = $"{options.NodeId}:{captureSequence}:{sample.SourceId}:{sample.Sequence}:latest"; + return new + { + bodyId, + slice = new + { + sample.SourceId, + Kind = sample.Kind.ToString(), + Origin = sample.Origin.ToString(), + Status = "LatestUnalignedFallback", + SourceTimestampNs = sample.TimestampNs, + CanonicalStartNs = sample.TimestampNs, + CanonicalEndNs = sample.TimestampNs, + PresentationTimeNs = 0L, + TimingOffsetNs = 0L, + DistanceFromPresentationNs = 0L, + TimingConfidence = 0.0, + TimingEvidenceKind = "unsynchronized-fallback", + }, + sample = SampleMetadata(sample), + body = CaptureBody(options, sample), + }; + } + private static object SampleMetadata(MimirStreamSample sample) => new { sample.SourceId, diff --git a/state/evidence.jsonl b/state/evidence.jsonl index ddfff90..c31a8b0 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -346,3 +346,4 @@ {"ts": "2026-06-01T23:50:56+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Mimir.Well now owns online camera exposure/gain optimization for visual sync. MimirCameraExposureController samples live Well video buffers, scores LED spline quality with MimirLedSplineFrameAnalyzer, cycles KS exposure/gain candidates, locks the best score, periodically resweeps, and publishes visualCalibration status in mimir.well_snapshot.v1. Verified with dotnet build src/Mimir.Well and supervisor dry-run options."} {"ts": "2026-06-02T00:06:12+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Added the first Well-to-Recorder archival payload path. Mimir.Well now publishes bounded mimir.well_capture_page.v1 documents beside the dashboard snapshot; capture pages carry synchronized frame-slice timing, configured composite state, and inline base64 sample bodies when CPU payload bytes exist below the configured limit. Mimir.VerseRecorder strips those inline bodies into paged binary files under bodies/page-*.bin, replaces them with mimir.recorder_body_ref.v1 refs in observations.jsonl, and appends mimir.recorder_body_index.v1 records to bodies/index.jsonl. Verified with dotnet build for Mimir.Well and Mimir.VerseRecorder, supervisor dry-runs, and a relay/recorder synthetic capture-page smoke that wrote a 4-byte page body with the expected SHA-256 ref."} {"ts": "2026-06-02T00:06:13+01:00", "type": "verification", "status": "degraded", "note": "2026-06-02: A short local online-verse daemon run proved recorder startup and session/body directory creation, but live Mimir.Well synchronized-frame planning degraded repeatedly with mixed timestamp-domain range errors such as camera/ASIO timestamps being compared before a shared canonical clock owner exists. Treat this as the known camera-clock/global-time correction scar, not a recorder storage failure."} +{"ts": "2026-06-02T00:25:02+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02 shakedown: Fixed two data-loss/disk-pressure scars while leaving the long-running Well stack active. The supervisor now passes --seconds 0 to Mimir.Well for indefinite runs instead of a one-year CancelAfter delay that .NET rejects, and waits for relay /health before starting subscribers/producers so Nightwing does not race dotnet cold start. Mimir.Well capture pages now fall back to latest per-buffer samples with TimingEvidenceKind=unsynchronized-fallback when synchronized-frame planning degrades, so Recorder pages raw CPU bodies even before the canonical clock owner exists. Mimir.EveSensorReceiver still broadcasts full capture pages to subscribers but redacts capture-page bodies from diagnostic stdout to avoid duplicating the paged body store in verse-relay.out.log. Live run online-verse-daemons-20260602-002116 / VerseCaptures/20260602-002124 showed all four local workers alive, Recorder receiving 520 observations, 479 body refs, Nightwing observations flowing after manual restart, and capture-page stdout summaries with sampleCount/body-redacted."} diff --git a/state/map.yaml b/state/map.yaml index e635a42..643a386 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -409,9 +409,16 @@ invariants: bounded `mimir.well_capture_page.v1` records for synchronized frame slices: each page carries timing metadata, configured composite state, and inline source sample bodies when a sample has CPU bytes and stays under the - configured body limit. GPU-handle-only samples remain metadata/body-handle - evidence until a Fensalir/native readback or direct program-output recorder - owns rendered composite bodies. The online supervisor + configured body limit. If synchronized-frame planning degrades because + source clocks are not yet in one canonical domain, capture pages fall back + to latest per-buffer samples with `TimingEvidenceKind = + unsynchronized-fallback`; this preserves raw evidence for offline + reconstruction without pretending the samples were synchronized. + GPU-handle-only samples remain metadata/body-handle evidence until a + Fensalir/native readback or direct program-output recorder owns rendered + composite bodies. The relay broadcasts full capture pages to subscribers + but redacts capture-page bodies from its diagnostic stdout so + `verse-relay.out.log` does not duplicate the body store. The online supervisor now starts relay, recorder, Nightwing Eyes/Move/builtin-camera/builtin-mic witness publication, conservative de-Bruijn/microtonal music-synced Move gestures with Realtek plus opportunistic ASIO music evidence, and