Harden Well shakedown capture
This commit is contained in:
parent
71c436622b
commit
93986ebe0a
5 changed files with 127 additions and 10 deletions
|
|
@ -65,6 +65,33 @@ function Start-LoggedProcess {
|
|||
return $process
|
||||
}
|
||||
|
||||
function Wait-HttpHealth {
|
||||
param(
|
||||
[string]$Url,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host "DRY wait health: $Url"
|
||||
return
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 3
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
Write-Host "health ok: $Url"
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
$manifest = [ordered]@{
|
||||
kind = "mimir.online_verse_daemon_supervisor.v1"
|
||||
runId = $runId
|
||||
|
|
@ -111,7 +138,7 @@ $receiver = Start-LoggedProcess `
|
|||
)
|
||||
if ($receiver) { $manifest.processes["verseRelayPid"] = $receiver.Id }
|
||||
|
||||
Start-Sleep -Milliseconds 900
|
||||
Wait-HttpHealth -Url "http://127.0.0.1:8796/health"
|
||||
|
||||
$recorderArgs = @(
|
||||
"run",
|
||||
|
|
@ -263,7 +290,7 @@ if (-not $SkipRuntime) {
|
|||
"--project",
|
||||
(Join-Path $repo "src\Mimir.Well\Mimir.Well.csproj"),
|
||||
"--",
|
||||
"--seconds", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "31536000" }),
|
||||
"--seconds", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "0" }),
|
||||
"--poll-ms", "5",
|
||||
"--publish-url", $WellPublishUrl,
|
||||
"--node-id", "starfire",
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ internal sealed class EveSensorReceiver(EveSensorReceiverOptions options) : IDis
|
|||
if (TryValidateCultMeshObservation(frame.Payload, out var normalizedObservation)
|
||||
|| TryValidateCultMeshMediaObservation(frame.Payload, out normalizedObservation))
|
||||
{
|
||||
Console.WriteLine(normalizedObservation);
|
||||
Console.WriteLine(ObservationStdoutLine(normalizedObservation));
|
||||
Console.Out.Flush();
|
||||
await BroadcastAsync(normalizedObservation).ConfigureAwait(false);
|
||||
}
|
||||
|
|
@ -130,13 +130,45 @@ internal sealed class EveSensorReceiver(EveSensorReceiverOptions options) : IDis
|
|||
var text = Encoding.UTF8.GetString(frame.Payload);
|
||||
if (TryValidateFrameEvent(text, out var normalized))
|
||||
{
|
||||
Console.WriteLine(normalized);
|
||||
Console.WriteLine(ObservationStdoutLine(normalized));
|
||||
Console.Out.Flush();
|
||||
await BroadcastAsync(normalized).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string ObservationStdoutLine(string normalized)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(normalized);
|
||||
var root = document.RootElement;
|
||||
if (root.TryGetProperty("document", out var documentKind)
|
||||
&& string.Equals(documentKind.GetString(), "mimir.well_capture_page.v1", StringComparison.Ordinal))
|
||||
{
|
||||
var sampleCount = root.TryGetProperty("samples", out var samples)
|
||||
&& samples.ValueKind == JsonValueKind.Array
|
||||
? samples.GetArrayLength()
|
||||
: 0;
|
||||
return JsonSerializer.Serialize(new
|
||||
{
|
||||
type = root.GetProperty("type").GetString(),
|
||||
document = "mimir.well_capture_page.v1",
|
||||
sourceId = root.TryGetProperty("sourceId", out var sourceId) ? sourceId.GetString() : null,
|
||||
nodeId = root.TryGetProperty("nodeId", out var nodeId) ? nodeId.GetString() : null,
|
||||
captureSequence = root.TryGetProperty("captureSequence", out var sequence) ? sequence.GetInt64() : 0,
|
||||
sampleCount,
|
||||
stdoutBodyPolicy = "body-redacted",
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private async Task SubscribeAsync(NetworkStream stream)
|
||||
{
|
||||
var id = Interlocked.Increment(ref nextSubscriberId);
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ while (!stopping.IsCancellationRequested)
|
|||
options,
|
||||
presentation,
|
||||
frame,
|
||||
hub.Buffers.Buffers,
|
||||
++captureSequence,
|
||||
startedAt);
|
||||
await publisher.PublishAsync(capturePage, stopping.Token).ConfigureAwait(false);
|
||||
|
|
@ -285,6 +286,7 @@ internal static class MimirWellCapturePage
|
|||
MimirWellOptions options,
|
||||
MimirPresentationControlState presentation,
|
||||
MimirSynchronizedBufferFrame frame,
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers,
|
||||
long captureSequence,
|
||||
DateTimeOffset startedAt) => new
|
||||
{
|
||||
|
|
@ -335,12 +337,30 @@ internal static class MimirWellCapturePage
|
|||
reason = "Fensalir/OBS program-output capture is a separate producer; this page records the configured composite contract and synchronized source bodies.",
|
||||
},
|
||||
},
|
||||
samples = frame.Slices
|
||||
.Where(slice => slice.Sample.HasValue)
|
||||
.Select(slice => CaptureSample(options, captureSequence, slice))
|
||||
samples = CaptureSamples(options, captureSequence, frame, buffers)
|
||||
.ToArray(),
|
||||
};
|
||||
|
||||
private static IEnumerable<object> CaptureSamples(
|
||||
MimirWellOptions options,
|
||||
long captureSequence,
|
||||
MimirSynchronizedBufferFrame frame,
|
||||
IEnumerable<MimirRollingStreamBuffer> buffers)
|
||||
{
|
||||
var sliceSamples = frame.Slices
|
||||
.Where(slice => slice.Sample.HasValue)
|
||||
.Select(slice => CaptureSample(options, captureSequence, slice))
|
||||
.ToArray();
|
||||
if (sliceSamples.Length > 0)
|
||||
{
|
||||
return sliceSamples;
|
||||
}
|
||||
|
||||
return buffers
|
||||
.Where(buffer => buffer.Latest.HasValue)
|
||||
.Select(buffer => CaptureLatestFallbackSample(options, captureSequence, buffer));
|
||||
}
|
||||
|
||||
private static object CaptureSample(
|
||||
MimirWellOptions options,
|
||||
long captureSequence,
|
||||
|
|
@ -371,6 +391,36 @@ internal static class MimirWellCapturePage
|
|||
};
|
||||
}
|
||||
|
||||
private static object CaptureLatestFallbackSample(
|
||||
MimirWellOptions options,
|
||||
long captureSequence,
|
||||
MimirRollingStreamBuffer buffer)
|
||||
{
|
||||
var sample = buffer.Latest!.Value;
|
||||
var bodyId = $"{options.NodeId}:{captureSequence}:{sample.SourceId}:{sample.Sequence}:latest";
|
||||
return new
|
||||
{
|
||||
bodyId,
|
||||
slice = new
|
||||
{
|
||||
sample.SourceId,
|
||||
Kind = sample.Kind.ToString(),
|
||||
Origin = sample.Origin.ToString(),
|
||||
Status = "LatestUnalignedFallback",
|
||||
SourceTimestampNs = sample.TimestampNs,
|
||||
CanonicalStartNs = sample.TimestampNs,
|
||||
CanonicalEndNs = sample.TimestampNs,
|
||||
PresentationTimeNs = 0L,
|
||||
TimingOffsetNs = 0L,
|
||||
DistanceFromPresentationNs = 0L,
|
||||
TimingConfidence = 0.0,
|
||||
TimingEvidenceKind = "unsynchronized-fallback",
|
||||
},
|
||||
sample = SampleMetadata(sample),
|
||||
body = CaptureBody(options, sample),
|
||||
};
|
||||
}
|
||||
|
||||
private static object SampleMetadata(MimirStreamSample sample) => new
|
||||
{
|
||||
sample.SourceId,
|
||||
|
|
|
|||
|
|
@ -346,3 +346,4 @@
|
|||
{"ts": "2026-06-01T23:50:56+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Mimir.Well now owns online camera exposure/gain optimization for visual sync. MimirCameraExposureController samples live Well video buffers, scores LED spline quality with MimirLedSplineFrameAnalyzer, cycles KS exposure/gain candidates, locks the best score, periodically resweeps, and publishes visualCalibration status in mimir.well_snapshot.v1. Verified with dotnet build src/Mimir.Well and supervisor dry-run options."}
|
||||
{"ts": "2026-06-02T00:06:12+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02: Added the first Well-to-Recorder archival payload path. Mimir.Well now publishes bounded mimir.well_capture_page.v1 documents beside the dashboard snapshot; capture pages carry synchronized frame-slice timing, configured composite state, and inline base64 sample bodies when CPU payload bytes exist below the configured limit. Mimir.VerseRecorder strips those inline bodies into paged binary files under bodies/page-*.bin, replaces them with mimir.recorder_body_ref.v1 refs in observations.jsonl, and appends mimir.recorder_body_index.v1 records to bodies/index.jsonl. Verified with dotnet build for Mimir.Well and Mimir.VerseRecorder, supervisor dry-runs, and a relay/recorder synthetic capture-page smoke that wrote a 4-byte page body with the expected SHA-256 ref."}
|
||||
{"ts": "2026-06-02T00:06:13+01:00", "type": "verification", "status": "degraded", "note": "2026-06-02: A short local online-verse daemon run proved recorder startup and session/body directory creation, but live Mimir.Well synchronized-frame planning degraded repeatedly with mixed timestamp-domain range errors such as camera/ASIO timestamps being compared before a shared canonical clock owner exists. Treat this as the known camera-clock/global-time correction scar, not a recorder storage failure."}
|
||||
{"ts": "2026-06-02T00:25:02+01:00", "type": "implementation", "status": "ok", "note": "2026-06-02 shakedown: Fixed two data-loss/disk-pressure scars while leaving the long-running Well stack active. The supervisor now passes --seconds 0 to Mimir.Well for indefinite runs instead of a one-year CancelAfter delay that .NET rejects, and waits for relay /health before starting subscribers/producers so Nightwing does not race dotnet cold start. Mimir.Well capture pages now fall back to latest per-buffer samples with TimingEvidenceKind=unsynchronized-fallback when synchronized-frame planning degrades, so Recorder pages raw CPU bodies even before the canonical clock owner exists. Mimir.EveSensorReceiver still broadcasts full capture pages to subscribers but redacts capture-page bodies from diagnostic stdout to avoid duplicating the paged body store in verse-relay.out.log. Live run online-verse-daemons-20260602-002116 / VerseCaptures/20260602-002124 showed all four local workers alive, Recorder receiving 520 observations, 479 body refs, Nightwing observations flowing after manual restart, and capture-page stdout summaries with sampleCount/body-redacted."}
|
||||
|
|
|
|||
|
|
@ -409,9 +409,16 @@ invariants:
|
|||
bounded `mimir.well_capture_page.v1` records for synchronized frame slices:
|
||||
each page carries timing metadata, configured composite state, and inline
|
||||
source sample bodies when a sample has CPU bytes and stays under the
|
||||
configured body limit. GPU-handle-only samples remain metadata/body-handle
|
||||
evidence until a Fensalir/native readback or direct program-output recorder
|
||||
owns rendered composite bodies. The online supervisor
|
||||
configured body limit. If synchronized-frame planning degrades because
|
||||
source clocks are not yet in one canonical domain, capture pages fall back
|
||||
to latest per-buffer samples with `TimingEvidenceKind =
|
||||
unsynchronized-fallback`; this preserves raw evidence for offline
|
||||
reconstruction without pretending the samples were synchronized.
|
||||
GPU-handle-only samples remain metadata/body-handle evidence until a
|
||||
Fensalir/native readback or direct program-output recorder owns rendered
|
||||
composite bodies. The relay broadcasts full capture pages to subscribers
|
||||
but redacts capture-page bodies from its diagnostic stdout so
|
||||
`verse-relay.out.log` does not duplicate the body store. The online supervisor
|
||||
now starts relay, recorder, Nightwing Eyes/Move/builtin-camera/builtin-mic
|
||||
witness publication, conservative de-Bruijn/microtonal music-synced Move
|
||||
gestures with Realtek plus opportunistic ASIO music evidence, and
|
||||
|
|
|
|||
Loading…
Reference in a new issue