From 27c53f74a7cba14a2190fa7b81ed2eb264bbe4ba Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 31 May 2026 22:53:56 +0100 Subject: [PATCH] Generalize frustum solve to feature tracks --- docs/implementation-plan.md | 10 +- src/Mimir.BufferSmoke/Program.cs | 81 +++++++++++- .../MimirVisualCalibrationResidualSolver.cs | 118 ++++++++++++++---- state/evidence.jsonl | 1 + state/map.yaml | 10 +- 5 files changed, 191 insertions(+), 29 deletions(-) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 1f2f649..53321e8 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -500,9 +500,13 @@ a named invariant that the native runtime cannot protect yet. consumes stable LED/marker correspondences, scene-space marker points, and seed frustums, then emits per-source position, rotation, horizontal/vertical tangent half-FOV, correspondence count, clipspace reprojection residual, and - confidence. Synthetic smoke `--frustum-calibration-smoke` solves both Eyes, - Kiyo Pro, Leap stereo IR, and Eve camera, establishing the first fusion-ready - frustum state before dense surface fusion. + confidence. The same solver accepts `MimirFeatureTrackFieldCandidate` plus + scene-space `TrackId` anchors, so PS Move sphere histories, sparse Eye + tracks, and Eve-local marker tracks use the same Starfire frustum authority + as the LED strip. Synthetic smoke `--frustum-calibration-smoke` solves both + LED-spline and feature-track frustums for both Eyes, Kiyo Pro, Leap stereo + IR, and Eve camera, establishing the first fusion-ready frustum state before + dense surface fusion. Next useful cut is direct report decoding plus optical sphere observations from PS3 Eyes/Kiyos/Eve entering the same calibration-evidence path as LED spline constraints. diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 049b77e..69cd592 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -1128,15 +1128,52 @@ static int RunFrustumCalibrationSmoke() seeds, maximumDeltaMeters: 0.20, maximumRotationDeltaDegrees: 12.0); + var trackScenePoints = scenePoints + .Select(point => new MimirFeatureTrackScenePoint( + 10_000 + point.LedIndex, + point.PositionMeters, + point.Confidence)) + .ToArray(); + var featureCandidate = new MimirFeatureTrackFieldCandidate( + "frustum-solve:move-eve-markers:frame-1", + "room-frustum-calibration", + "mimir-local-witness-feature-tracks", + truth.Select((frustum, index) => + { + var size = sizes[frustum.SourceId]; + return ProjectFrustumFeatureTracks( + scenePoints, + frustum, + size.Width, + size.Height, + 3_000_100_000L + index); + }).ToArray(), + StableTrackCount: scenePoints.Length, + MeanTrackAgeFrames: 8.0, + MeanSpeedPixelsPerSecond: 0.0, + Confidence: 0.91, + ObservedTimeNs: 3_000_100_004L); + var featureFrame = new MimirCameraFrustumCalibrationSolver().FitFrustumsFromFeatureTracks( + featureCandidate, + trackScenePoints, + seeds, + markerSetId: "move-and-eve-feature-markers", + maximumDeltaMeters: 0.20, + maximumRotationDeltaDegrees: 12.0); Console.WriteLine( - $"frustum-calibration-smoke updates={frame.FrustumUpdates.Count} confidence={frame.Confidence:0.000} meanClipError={frame.MeanReprojectionErrorClip:0.000000} sources={string.Join(",", frame.FrustumUpdates.Select(static update => $"{update.SourceId}:{update.UsedPointCount}/{update.RotationDeltaDegrees:0.00}deg/{update.HorizontalTanHalfFov:0.000}x{update.VerticalTanHalfFov:0.000}"))}"); + $"frustum-calibration-smoke ledUpdates={frame.FrustumUpdates.Count} featureUpdates={featureFrame.FrustumUpdates.Count} confidence={frame.Confidence:0.000}/{featureFrame.Confidence:0.000} meanClipError={frame.MeanReprojectionErrorClip:0.000000}/{featureFrame.MeanReprojectionErrorClip:0.000000} sources={string.Join(",", frame.FrustumUpdates.Select(static update => $"{update.SourceId}:{update.UsedPointCount}/{update.RotationDeltaDegrees:0.00}deg/{update.HorizontalTanHalfFov:0.000}x{update.VerticalTanHalfFov:0.000}"))}"); return frame.HasFrustumUpdate && + featureFrame.HasFrustumUpdate && frame.FrustumUpdates.Count == truth.Length && + featureFrame.FrustumUpdates.Count == truth.Length && frame.FrustumUpdates.All(update => update.UsedPointCount == scenePoints.Length) && + featureFrame.FrustumUpdates.All(update => update.UsedPointCount == scenePoints.Length) && frame.FrustumUpdates.All(static update => update.MeanReprojectionErrorClip < 0.010) && - frame.Confidence >= 0.70 + featureFrame.FrustumUpdates.All(static update => update.MeanReprojectionErrorClip < 0.010) && + frame.Confidence >= 0.70 && + featureFrame.Confidence >= 0.70 ? 0 : 1; } @@ -1935,6 +1972,46 @@ static IReadOnlyList ProjectFrustumLedDetections( return output; } +static MimirFeatureTrackCameraObservation ProjectFrustumFeatureTracks( + IReadOnlyList scenePoints, + MimirCameraFrustumEstimate frustum, + int width, + int height, + long observedTimeNs) +{ + var worldToCamera = Quaternion.Inverse(frustum.CameraToWorldRotation); + var tracks = scenePoints.Select(point => + { + var cameraSpace = Vector3.Transform(point.PositionMeters - frustum.PositionMeters, worldToCamera); + var z = Math.Max(1.0e-6f, cameraSpace.Z); + var clipX = cameraSpace.X / (z * (float)Math.Max(1.0e-6, frustum.HorizontalTanHalfFov)); + var clipY = cameraSpace.Y / (z * (float)Math.Max(1.0e-6, frustum.VerticalTanHalfFov)); + var imageX = (clipX + 1.0f) * 0.5f * width; + var imageY = (1.0f - clipY) * 0.5f * height; + return new MimirFeatureTrackPoint( + 10_000 + point.LedIndex, + imageX, + imageY, + clipX, + clipY, + VelocityXPerSecond: 0.0, + VelocityYPerSecond: 0.0, + AgeFrames: 8, + Confidence: point.Confidence); + }).ToArray(); + return new MimirFeatureTrackCameraObservation( + $"{frustum.SourceId}:feature-frustum:{observedTimeNs}", + frustum.SourceId, + width, + height, + observedTimeNs, + tracks, + FrameCount: 1, + MeanTrackAgeFrames: 8.0, + MeanSpeedPixelsPerSecond: 0.0, + Confidence: tracks.Average(static track => track.Confidence)); +} + static IReadOnlyList SyntheticSweepDetections( string settingId, IReadOnlyList scenePoints, diff --git a/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs b/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs index 34a5de9..b666655 100644 --- a/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs +++ b/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs @@ -83,6 +83,11 @@ public readonly record struct MimirLedSplineScenePoint( Vector3 PositionMeters, double Confidence); +public readonly record struct MimirFeatureTrackScenePoint( + int TrackId, + Vector3 PositionMeters, + double Confidence); + public sealed record MimirCameraPoseEstimate( string SourceId, Vector3 PositionMeters, @@ -142,7 +147,7 @@ public sealed record MimirCameraFrustumSolveUpdate( public sealed record MimirCameraFrustumSolveFrame( string CalibrationId, string CandidateKey, - string SplineId, + string MarkerSetId, IReadOnlyList FrustumUpdates, double MeanReprojectionErrorClip, double Confidence, @@ -727,18 +732,10 @@ public sealed class MimirCameraFrustumCalibrationSolver .Where(static point => point.LedIndex >= 0 && point.Confidence > 0.0) .GroupBy(static point => point.LedIndex) .ToDictionary(static group => group.Key, static group => group.OrderByDescending(point => point.Confidence).First()); - var frustumBySource = currentFrustums.ToDictionary(static frustum => frustum.SourceId, StringComparer.Ordinal); - var updates = new List(); - - foreach (var observation in candidate.CameraObservations) - { - if (!frustumBySource.TryGetValue(observation.SourceId, out var seed) || - seed.Confidence <= 0.0) - { - continue; - } - - var correspondences = observation.Points + var observations = candidate.CameraObservations + .Select(observation => new FrustumObservation( + observation.SourceId, + observation.Points .Where(point => sceneByLed.ContainsKey(point.LedIndex) && point.Confidence > 0.0) .Select(point => { @@ -749,19 +746,96 @@ public sealed class MimirCameraFrustumCalibrationSolver point.ClipY, Math.Min(point.Confidence, scene.Confidence)); }) - .ToArray(); - if (correspondences.Length < 4) + .ToArray())) + .ToArray(); + return FitFrustums( + candidate.CalibrationId, + candidate.CandidateKey, + candidate.SplineId, + candidate.Confidence, + observations, + currentFrustums, + maximumDeltaMeters, + maximumRotationDeltaDegrees, + minimumTanHalfFov, + maximumTanHalfFov); + } + + public MimirCameraFrustumSolveFrame FitFrustumsFromFeatureTracks( + MimirFeatureTrackFieldCandidate candidate, + IEnumerable scenePoints, + IEnumerable currentFrustums, + string markerSetId = "feature-tracks", + double maximumDeltaMeters = 0.50, + double maximumRotationDeltaDegrees = 30.0, + double minimumTanHalfFov = 0.20, + double maximumTanHalfFov = 2.50) + { + var sceneByTrack = scenePoints + .Where(static point => point.TrackId >= 0 && point.Confidence > 0.0) + .GroupBy(static point => point.TrackId) + .ToDictionary(static group => group.Key, static group => group.OrderByDescending(point => point.Confidence).First()); + var observations = candidate.CameraObservations + .Select(observation => new FrustumObservation( + observation.SourceId, + observation.Tracks + .Where(track => sceneByTrack.ContainsKey(track.TrackId) && track.Confidence > 0.0) + .Select(track => + { + var scene = sceneByTrack[track.TrackId]; + return new FrustumCorrespondence( + scene.PositionMeters, + track.ClipX, + track.ClipY, + Math.Min(track.Confidence, scene.Confidence)); + }) + .ToArray())) + .ToArray(); + return FitFrustums( + candidate.CalibrationId, + candidate.CandidateKey, + markerSetId, + candidate.Confidence, + observations, + currentFrustums, + maximumDeltaMeters, + maximumRotationDeltaDegrees, + minimumTanHalfFov, + maximumTanHalfFov); + } + + private static MimirCameraFrustumSolveFrame FitFrustums( + string calibrationId, + string candidateKey, + string markerSetId, + double candidateConfidence, + IReadOnlyList observations, + IEnumerable currentFrustums, + double maximumDeltaMeters, + double maximumRotationDeltaDegrees, + double minimumTanHalfFov, + double maximumTanHalfFov) + { + var frustumBySource = currentFrustums.ToDictionary(static frustum => frustum.SourceId, StringComparer.Ordinal); + var updates = new List(); + + foreach (var observation in observations) + { + if (!frustumBySource.TryGetValue(observation.SourceId, out var seed) || + seed.Confidence <= 0.0 || + observation.Correspondences.Count < 4) { continue; } + var correspondences = observation.Correspondences; if (!TryFitFrustum( correspondences, seed, minimumTanHalfFov, maximumTanHalfFov, out var solved, - out var meanError)) + out _)) { continue; } @@ -785,7 +859,7 @@ public sealed class MimirCameraFrustumCalibrationSolver VerticalTanHalfFov = Clamp(solved.VerticalTanHalfFov, minimumTanHalfFov, maximumTanHalfFov), }; var finalError = MeanReprojectionError(correspondences, finalEstimate); - var confidence = Clamp01(candidate.Confidence) * + var confidence = Clamp01(candidateConfidence) * Clamp01(seed.Confidence) * correspondences.Average(static point => Clamp01(point.Confidence)) * (1.0 / (1.0 + finalError * 4.0)); @@ -797,7 +871,7 @@ public sealed class MimirCameraFrustumCalibrationSolver finalEstimate.VerticalTanHalfFov, clampedDelta, rotationDeltaDegrees, - correspondences.Length, + correspondences.Count, finalError, confidence)); } @@ -809,9 +883,9 @@ public sealed class MimirCameraFrustumCalibrationSolver ? 0.0 : updates.Average(static update => update.Confidence); return new MimirCameraFrustumSolveFrame( - candidate.CalibrationId, - candidate.CandidateKey, - candidate.SplineId, + calibrationId, + candidateKey, + markerSetId, updates, meanFrameError, frameConfidence, @@ -1110,5 +1184,7 @@ public sealed class MimirCameraFrustumCalibrationSolver private static double Clamp01(double value) => double.IsFinite(value) ? Math.Clamp(value, 0.0, 1.0) : 0.0; + private sealed record FrustumObservation(string SourceId, IReadOnlyList Correspondences); + private readonly record struct FrustumCorrespondence(Vector3 Scene, double ClipX, double ClipY, double Confidence); } diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 66221d6..b9b3d8d 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -289,3 +289,4 @@ {"ts": "2026-05-31T22:30:36+01:00", "type": "verification", "status": "partial", "note": "Added sync_sweep.py plus stdlib-only Nightwing witness helpers. Live receipt artifacts/runtime/sync-sweep-hints-live proved Nightwing owns Eye/Move localization locally: nw-eye-0 and nw-eye-1 each captured about 1874 frames at 320x240@187 with zero sequence gaps and emitted 8/8 compact mimir.move_controller_observation_state.v1 hints, while Starfire analyzed its own ASIO/DShow/Kiyo streams. Nightwing builtin mic capture works but the cheap local chirp hint is not yet reliable enough to score as a timing witness."} {"ts":"2026-05-31T22:45:00+01:00","type":"implementation","status":"ok","note":"Mimir editor authority was cut back to a 2D OBS-shaped program mixer. SceneEditorState no longer creates default SDF/model content, preview items are video-feed-only, runtime editor camera/SDF/spline output is silent, and the Eve/CultUI surface is Video Sources with source-adjacent layer actions left, chrome-free program canvas center, and Audio Mix right. Numeric transform inspector controls were removed from the live surface. `--scene-editor-smoke` now verifies only camera+video feed nodes, video-only preview items, and normalized program placement.","files":["src/Mimir.Runtime/MimirRuntime.cs","src/Mimir.Runtime/Synchronization/MimirSceneEditorState.cs","src/Mimir.BufferSmoke/Program.cs"],"source":"dotnet build src/Mimir.Runtime/Mimir.Runtime.csproj; dotnet build src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj; dotnet run --no-build --project src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj -- --scene-editor-smoke; Mimir.App --headless --headless-width 1280 --headless-height 720 --capture-frame artifacts/runtime/mimir-editor-flat-2d-program-mixer-headless.png"} {"ts": "2026-05-31T22:46:40+01:00", "type": "implementation", "status": "ok", "note": "Added the first Starfire-owned camera frustum solve. MimirCameraFrustumCalibrationSolver consumes stable LED/marker correspondences, scene-space marker points, and seed frustums, then emits position, rotation, horizontal/vertical tangent half-FOV, correspondence count, clipspace reprojection residual, and confidence. BufferSmoke --frustum-calibration-smoke solves five synthetic frustums for ps3-eye-0, ps3-eye-1, kiyo-pro-rgb, leap-stereo-ir, and eve-camera with mean clip error 0.000000; room calibration lock still passes. Eve is now modeled like Nightwing as eve-local-sensor-witness/eve-local-sensors: it owns local camera/mic flash and chirp solves and ships typed hints, not raw placement authority."} +{"ts": "2026-05-31T22:52:47+01:00", "type": "implementation", "status": "ok", "note": "Extended MimirCameraFrustumCalibrationSolver beyond LED strips: it now also consumes MimirFeatureTrackFieldCandidate plus scene-space TrackId anchors, so PS Move sphere histories, sparse Eye tracks, and Eve-local marker tracks enter the same Starfire-owned frustum authority. BufferSmoke --frustum-calibration-smoke now proves five LED-spline frustums and five feature-track frustums for ps3-eye-0, ps3-eye-1, kiyo-pro-rgb, leap-stereo-ir, and eve-camera with mean clip error 0.000000."} diff --git a/state/map.yaml b/state/map.yaml index 594643f..3753bc9 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -232,9 +232,13 @@ invariants: reprojection residual, and confidence. This supersedes translation-only rig updates for global fusion setup; the older `MimirCameraRigCalibrationSolver` remains a narrow bounded translation - residual helper. `Mimir.BufferSmoke --frustum-calibration-smoke` proves - five solved frustums for both Eyes, Kiyo Pro, Leap stereo IR, and Eve - camera. + residual helper. The same solver also consumes + `MimirFeatureTrackFieldCandidate` plus scene-space `TrackId` anchors, so + PS Move sphere histories, Eye feature tracks, and Eve-local marker tracks + can enter the frustum solve without becoming a separate authority. + `Mimir.BufferSmoke --frustum-calibration-smoke` proves five LED-spline + frustums and five feature-track frustums for both Eyes, Kiyo Pro, Leap + stereo IR, and Eve camera. The Eyes have physically moved to Nightwing and Linux sees them as stock `ov534`/`gspca` V4L2 cameras on `/dev/video2` and `/dev/video3`. A repo-owned `tools/nightwing_v4l2_cadence_probe.py` copied to Nightwing