diff --git a/artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png b/artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png new file mode 100644 index 0000000..8f0b5b5 Binary files /dev/null and b/artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png differ diff --git a/config/mimir-runtime.perfect-machine.local.json b/config/mimir-runtime.perfect-machine.local.json new file mode 100644 index 0000000..de03cb8 --- /dev/null +++ b/config/mimir-runtime.perfect-machine.local.json @@ -0,0 +1,85 @@ +{ + "bufferSeconds": 5.0, + "audioSync": { + "mode": "passive", + "referenceSourceId": "loopback-scarlett-speakers", + "calibrationGain": 0.0, + "watermarkGain": 0.0 + }, + "streams": [ + { + "sourceId": "leap-stereo-ir", + "displayName": "LeapUVC stereo IR packed R/G", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "ks-camera", + "command": "../native/camera_capture/build/Release/mimir_camera_capture.dll", + "pathNeedle": "vid_f182&pid_0003&mi_00", + "width": 640, + "height": 240, + "inputFormat": "YUY2", + "pixelFormat": "LeapStereoIr", + "minimumFramesPerSecond": 100.0, + "queueDepth": 8 + }, + { + "sourceId": "ps3-eye-0", + "displayName": "PS3 Eye 0 high-rate tracking", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "ps3eye", + "command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll", + "cameraIndex": 0, + "width": 320, + "height": 240, + "framesPerSecond": 187 + }, + { + "sourceId": "ps3-eye-1", + "displayName": "PS3 Eye 1 high-rate tracking", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "ps3eye", + "command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll", + "cameraIndex": 1, + "width": 320, + "height": 240, + "framesPerSecond": 187 + }, + { + "sourceId": "kiyo-pro-rgb", + "displayName": "Kiyo Pro RGB context", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "ks-camera", + "command": "../native/camera_capture/build/Release/mimir_camera_capture.dll", + "pathNeedle": "vid_1532&pid_0e05&mi_00", + "width": 640, + "height": 480, + "inputFormat": "YUY2", + "pixelFormat": "YUY2", + "minimumFramesPerSecond": 25.0, + "queueDepth": 8 + }, + { + "sourceId": "kiyo-basic-rgb", + "displayName": "Kiyo RGB context", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "ks-camera", + "command": "../native/camera_capture/build/Release/mimir_camera_capture.dll", + "pathNeedle": "vid_1532&pid_0e03&mi_00", + "width": 640, + "height": 480, + "inputFormat": "YUY2", + "pixelFormat": "YUY2", + "minimumFramesPerSecond": 25.0, + "queueDepth": 8 + } + ] +} diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 1763b8d..e6ec3df 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -343,6 +343,15 @@ a named invariant that the native runtime cannot protect yet. Current direct-driver smokes prove real-frame upload for LeapUVC 640x240 YUY2, both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular Kiyo 640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open. + `config/mimir-runtime.perfect-machine.local.json` is the current local + all-sensor blast profile: Leap packed stereo IR, both PS3 Eyes at + 320x240@187, Kiyo Pro RGB, and regular Kiyo RGB. `Mimir.BufferSmoke + --perfect-machine-live-smoke` opens all five configured video sources, + attaches Fensalir texture leases, and passes only when at least five live + camera textures, a Leap stereo-depth lowering, a disparity SurfacePage, and a + derived point-cloud Mesh are present. It prints deferred camera texture + Feature requests explicitly; those are not capture failures, they are the + future Fensalir D3D12 feature-extraction cut. `MimirMediaFoundationGpuVideoCaptureDriver` is the first compressed camera path: `native/camera_capture/mimir_mf_gpu_capture.dll` uses Media Foundation SourceReader with a D3D11 device manager, selects MJPG/H264 camera modes, @@ -426,6 +435,14 @@ a named invariant that the native runtime cannot protect yet. stable tracks at 320x240@187: with sensitive wide tracking defaults, each Eye returned 160 live tracks from each Eye in a one-second pass, with 160 stable tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1`. + A full Fensalir headless proof using that all-sensor profile now runs the + actual D3D12 host path. The 2026-05-31 run started all five video sources, + declared eight field resources, resolved six Texture2D resources, dispatched + one packed-Leap stereo-depth kernel, generated one point-cloud/surface stream, + and reported 19,200 visible reservoir splats. The captured proof is + `artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png`. Generic + camera Feature claims still defer; live PS3/Kiyo feature extraction remains a + Fensalir compute shader owner, not the CPU tracker. Phase 6 program-output receipts are also structurally complete: Fensalir publishes the final program surface through named shared D3D12 texture/fence resources, the OBS plugin consumes that surface with an explicit D3D12-to- diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index 598bb19..7dd2830 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -40,6 +40,14 @@ flowchart TD now: it opens direct sensor feeds, applies KS/UVC manual exposure/gain where supported, scores actual luma frames, and reports the best setting per source. PS3 Eye and Leap-specific controls still need native control cuts. +- The current all-sensor GPU proof is + `config/mimir-runtime.perfect-machine.local.json` plus + `Mimir.BufferSmoke --perfect-machine-live-smoke`: Leap, both PS3 Eyes, Kiyo + Pro, and regular Kiyo all enter the runtime as Fensalir texture leases. The + smoke requires five live camera Texture2D resources, one Leap stereo-depth + lowering, one disparity SurfacePage, and one derived point-cloud Mesh. + Deferred generic camera Feature requests are printed as the selected future + shader work, not hidden as capture success. - PS3 Eyes own high-rate sparse tracking evidence before they own depth: `MimirSparseFeatureTracker` turns live Bayer/luma frames into stable sensor-local feature tracks, and `MimirFeatureTrackFieldCandidate` lowers diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 6714a13..dfe2a8a 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -546,6 +546,15 @@ WinUSB/libusb capture, emitting Bayer8 frames through the common upload lane. Driver smokes have pulled and uploaded real frames for LeapUVC 640x240 YUY2, both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular Kiyo 640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open. +`config/mimir-runtime.perfect-machine.local.json` is the current local +all-sensor runtime profile: Leap packed stereo IR, both PS3 Eyes at +320x240@187, Kiyo Pro RGB, and regular Kiyo RGB. `Mimir.BufferSmoke +--perfect-machine-live-smoke` opens all five video sources through the runtime, +attaches Fensalir texture leases, and requires five camera Texture2D resources, +one Leap stereo-depth lowering, one disparity SurfacePage, and one derived +point-cloud Mesh. It prints the remaining deferred generic camera Feature +claims so the gap stays visible: PS3/Kiyo/Leap texture feature extraction is +still a Fensalir D3D12 compute owner, not a CPU tracker hidden in the hot path. `MimirMediaFoundationGpuVideoCaptureDriver` plus `native/camera_capture/mimir_mf_gpu_capture.dll` is the compressed camera path: Media Foundation SourceReader runs with a D3D11 device manager, decodes MJPG or @@ -645,7 +654,13 @@ camera Feature claims through Fensalir field evidence. A one-second live pass at 320x240@187 with sensitive defaults returned 160 live tracks from each Eye: 160 stable tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1`. This is a CPU diagnostic owner; the hot production path remains Fensalir D3D12 -feature extraction. Once the frustum fit is coherent, detected +feature extraction. A full 2026-05-31 Fensalir headless run using the +all-sensor profile started all five video sources, declared eight field +resources, resolved six Texture2D resources, dispatched one packed-Leap +stereo-depth kernel, generated one point-cloud/surface stream, reported 19,200 +visible reservoir splats, and captured +`artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png`. Once the +frustum fit is coherent, detected surface candidates can be sampled across every synchronized view before Fensalir resolves them into 4D Gaussian/surface splat claims. diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 98e9714..fef2f6a 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -120,6 +120,11 @@ if (args.Any(arg => string.Equals(arg, "--perfect-machine-profile-smoke", String return RunPerfectMachineProfileSmoke(); } +if (args.Any(arg => string.Equals(arg, "--perfect-machine-live-smoke", StringComparison.OrdinalIgnoreCase))) +{ + return RunPerfectMachineLiveSmoke(args); +} + if (args.Any(arg => string.Equals(arg, "--eve-program-output-contract-smoke", StringComparison.OrdinalIgnoreCase))) { return RunEveProgramOutputContractSmoke(); @@ -3695,6 +3700,79 @@ static int RunPerfectMachineProfileSmoke() : 1; } +static int RunPerfectMachineLiveSmoke(string[] args) +{ + var configPath = Path.GetFullPath(ParseStringOption( + args, + "--config", + "config/mimir-runtime.perfect-machine.local.json")); + var seconds = Math.Max(0.25, ParseDoubleOption(args, "--seconds", 1.5)); + var previousConfig = Environment.GetEnvironmentVariable("MIMIR_RUNTIME_CONFIG"); + Environment.SetEnvironmentVariable("MIMIR_RUNTIME_CONFIG", configPath); + try + { + var configuration = MimirRuntimeConfiguration.Load(); + var broker = new FakeFieldResourceBroker(); + using var runtime = new MimirRuntime( + new AquariumRuntimeOptions(Headless: true, CultCachePath: null), + configuration); + runtime.AttachServices(new AquariumRuntimeServices(broker)); + runtime.OnSceneReady(); + var input = new InputState(); + var stopwatch = Stopwatch.StartNew(); + var updates = 0; + while (stopwatch.Elapsed.TotalSeconds < seconds) + { + runtime.Update(1.0f / 60.0f, input); + updates++; + Thread.Sleep(1); + } + + var field = runtime.Frame.Scene.FieldEvidenceFrame; + var validation = AquariumFieldEvidenceValidator.Validate(field); + var plan = AquariumFieldLoweringPlanner.Plan(field); + var videoResources = field.Resources + .Where(static resource => + resource.Kind == AquariumFieldResourceKind.Texture2D && + resource.Residency is AquariumFieldResourceResidency.SharedGpu or AquariumFieldResourceResidency.GpuResident) + .ToArray(); + var cameraTextures = videoResources + .Where(static resource => resource.ResourceKey.StartsWith("mimir:resource:camera:", StringComparison.Ordinal)) + .ToArray(); + var surfacePages = field.Resources.Count(static resource => resource.Kind == AquariumFieldResourceKind.SurfacePage); + var meshResources = field.Resources.Count(static resource => resource.Kind == AquariumFieldResourceKind.Mesh); + var stereoLowerings = field.StereoDepthLowerings.Count; + var resourceClaims = field.Claims.Count(static claim => + claim.Encoding is AquariumFieldEncoding.Radiance or AquariumFieldEncoding.Feature or AquariumFieldEncoding.Height or AquariumFieldEncoding.Mesh); + Console.WriteLine( + $"perfect-machine-live-smoke config={configPath} updates={updates} resources={field.Resources.Count} gpuTextures={videoResources.Length} cameraTextures={cameraTextures.Length} claims={field.Claims.Count} resourceClaims={resourceClaims} stereoLowerings={stereoLowerings} meshResources={meshResources} surfacePages={surfacePages} planned={plan.Packets.Count} deferred={plan.DeferredRequests.Count} uploaded={broker.UploadedResourceKey} committed={broker.CommittedResourceKey} errors={validation.HasErrors}"); + foreach (var resource in videoResources.OrderBy(static resource => resource.ResourceKey, StringComparer.Ordinal)) + { + Console.WriteLine( + $"perfect-machine-resource key={resource.ResourceKey} {resource.Width}x{resource.Height} format={resource.Format} residency={resource.Residency} handle={resource.NativeHandle} kind={resource.NativeHandleKind} version={resource.Version}"); + } + + foreach (var deferred in plan.DeferredRequests.OrderBy(static request => request.RequestKey, StringComparer.Ordinal)) + { + Console.WriteLine( + $"perfect-machine-deferred request={deferred.RequestKey} claim={deferred.ClaimKey} domain={deferred.DomainKey} encoding={deferred.Encoding} proposal={deferred.ProposalKind} payload={deferred.PayloadHandle}"); + } + + return cameraTextures.Length >= 5 && + stereoLowerings >= 1 && + meshResources >= 1 && + surfacePages >= 1 && + plan.Packets.Count >= 2 && + !validation.HasErrors + ? 0 + : 1; + } + finally + { + Environment.SetEnvironmentVariable("MIMIR_RUNTIME_CONFIG", previousConfig); + } +} + static MimirAudioSynchronizationReport? EstimateBioacousticDelay(float[] reference, float[] candidate, int sampleRate) { const string referenceSourceId = "actuator-reference"; diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 865ca96..ef0dc04 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -258,3 +258,4 @@ {"ts": "2026-05-31T03:05:00+01:00", "type": "verification", "status": "ok", "note": "Live LED spline sweep now runs on actual direct sensor feeds. `dotnet run --project src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj -- --led-spline-live-score --setting-seconds 0.25 --settle-ms 100 --expected-leds 38 --minimum-luma 0.55` selected exposure -9/gain 8 for `kiyo-pro-rgb` (score 0.666, 41 detections, usable) and `kiyo-basic-rgb` (score 0.650, 38 detections, usable). LeapUVC rejected generic KS exposure/gain writes and only recovered two LED components; both PS3 Eye feeds scored 0 at current fixed controls because their native wrapper does not yet expose gain/exposure."} {"ts": "2026-05-31T02:45:05+01:00", "type": "verification", "status": "ok", "note": "Eve dashboard now renders voidbot.swarm as a native VoidBot-style cockpit with CTB avatar URLs, selected Face/status pane, state tree, and detail panel; device build/install passed and broker smoke reported 8 CTB avatar URLs."} {"ts": "2026-05-31T03:22:00+01:00", "type": "verification", "status": "ok", "note": "PS3 Eye sparse tracking bootstrap is live. `--ps3eye-tracking-smoke` produced 13 synthetic tracks, 12 stable tracks, and one Feature claim. The final default `--ps3eye-tracking-live --seconds 1.0` captured both Eyes at 320x240@187 and returned 160 tracks / 160 stable tracks from `ps3-eye-0` and 160 tracks / 156 stable tracks from `ps3-eye-1`. This is CPU diagnostic tracking; production feature extraction remains a Fensalir D3D12 owner."} +{"ts": "2026-05-31T03:31:08+01:00", "type": "verification", "status": "ok", "note": "2026-05-31 all-sensor GPU proof: config/mimir-runtime.perfect-machine.local.json opened Leap, both PS3 Eyes, Kiyo Pro, and regular Kiyo; BufferSmoke --perfect-machine-live-smoke reported 5 camera Texture2D leases, Leap stereo lowering, SurfacePage, Mesh, 7 planned packets, and 5 deferred generic camera Feature requests. Mimir.App headless D3D12 dispatched packed-Leap stereo depth, generated one point-cloud/surface stream, reported 19,200 visible reservoir splats, and captured artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png."} diff --git a/state/map.yaml b/state/map.yaml index 67008ed..92faa37 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -99,6 +99,21 @@ invariants: `AQUARIUM_DISABLE_RESERVOIR_HISTORY`: Fensalir now compiles a dedicated lean `D3D12ReservoirHistoryUpdate.hlsl` for current-sample history/resolved output instead of compiling the monolithic post shader as a compute entry. + The 2026-05-31 all-sensor proof uses + `config/mimir-runtime.perfect-machine.local.json`: Leap packed stereo IR, + both PS3 Eyes at 320x240@187, Kiyo Pro RGB, and regular Kiyo RGB all open + through `MimirRuntime` and enter as Fensalir texture leases. + `Mimir.BufferSmoke --perfect-machine-live-smoke` now requires five live + camera Texture2D resources, one Leap stereo-depth lowering, one disparity + SurfacePage, and one derived point-cloud Mesh. The Fensalir headless run + started all five sources, declared eight field resources, resolved six + Texture2D resources, dispatched the packed-Leap stereo-depth kernel, + generated one point-cloud/surface stream, reported 19,200 visible reservoir + splats, and captured + `artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png`. + Generic camera Feature claims still defer because PS3/Kiyo/Leap texture + feature extraction belongs to a future Fensalir D3D12 compute lowering, not + the CPU diagnostic tracker. Calibrated projection constants, full SGM, temporal/spatial reservoir reuse in a compilable shader, and global residual/calibration ownership remain open. @@ -563,9 +578,13 @@ accepted_design: or GPU bridge-copy work. `Mimir.BufferSmoke --fensalir-camera-observation-smoke` verifies the split, and `--fensalir-texture-lease-smoke` verifies the - engine-owned lease path. Camera image claims stay debug-view-only for now; - production visual fusion/feature extraction lowerings are deferred until - the renderer and audio work are fixed. Fensalir now owns the first in-process + engine-owned lease path. `config/mimir-runtime.perfect-machine.local.json` + is the current local all-sensor runtime profile, and + `--perfect-machine-live-smoke` proves the five camera texture leases plus + Leap depth/page/mesh sockets are present in one live frame. Camera image + claims stay debug-view-only for now; production visual fusion/feature + extraction lowerings are deferred until Fensalir owns the D3D12 feature + shader. Fensalir now owns the first in-process D3D12 field resource registry: declared shared structured/curve buffers import/alias GPU-resident resources by handle, while Fensalir-owned resources allocate GPU slots only when Fensalir is the producer.