From 571f9cdeecde12e7dd88bf280d8382bff93f297d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 31 May 2026 01:58:50 +0100 Subject: [PATCH] Add LED spline clipspace residuals --- docs/implementation-plan.md | 15 +- docs/sensor-fusion-architecture.md | 12 +- notes/current-system-map.md | 14 +- src/Mimir.BufferSmoke/Program.cs | 28 +-- .../MimirFensalirBridgeContracts.cs | 2 + .../MimirVisualCalibrationResidualSolver.cs | 172 ++++++++++++++++++ state/map.yaml | 7 +- 7 files changed, 223 insertions(+), 27 deletions(-) create mode 100644 src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index cb8f11f..1a6fb2b 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -376,14 +376,19 @@ a named invariant that the native runtime cannot protect yet. FieldEvidence authority: deterministic marker features plan as `DebugOverlay` packets while ambiguous raw camera features still defer. LED strip calibration now has the same typed evidence path: - `MimirLedSplineFieldCandidate` stores ordered per-camera bright-curve samples, - temporal-code/index stability flags, and shared calibration/spline identity. + `MimirLedSplineFieldCandidate` stores ordered per-camera bright-curve samples + in image pixels plus local clipspace/frustum coordinates, temporal-code/index + stability flags, and shared calibration/spline identity. `BuildLedSplineCandidateFrame` lowers that curve as deterministic camera-sensor Feature evidence, and `--led-spline-calibration-smoke` proves Fensalir plans it as one `DebugOverlay` packet with no deferred request. - This is calibration evidence only. Exact correspondence requires coded or - otherwise stable LED identity; uncoded repeated lights remain curve - constraints until the global residual solver resolves them. + `MimirVisualCalibrationResidualSolver` is the first higher-level owner shape: + it compares coded/indexed observations across camera frustums and emits + residual reports, but it does not write camera pose until the calibration + state contract exists. This is calibration evidence only. Exact + correspondence requires coded or otherwise stable LED identity; uncoded + repeated lights remain curve constraints until the global residual solver + resolves them. Phase 6 program-output receipts are also structurally complete: Fensalir publishes the final program surface through named shared D3D12 texture/fence resources, the OBS plugin consumes that surface with an explicit D3D12-to- diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index 94c9351..3478594 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -10,7 +10,7 @@ flowchart TD C["Leap stereo IR driver"] --> B B --> D["native reservoir handles"] D --> E["Fensalir GPU feature extraction"] - E --> J["LED spline observations"] + E --> J["LED spline observations in local clipspace"] E --> F["cross-view matching + flow"] J --> K["global residual calibration owner"] K --> F @@ -26,12 +26,16 @@ flowchart TD - Process capture is a bridge edge, not the local six-camera foundation. - Fensalir owns GPU extraction, fusion, material fitting, and render budgeting. - Runtime buffers own retention and stream health, not scene reconstruction. -- LED strips are active calibration evidence. Cameras may publish ordered - bright-curve observations, but only the global residual calibration owner may - update camera intrinsics/extrinsics. +- LED strips are active calibration evidence. Each camera publishes ordered + bright-curve observations in its own local frustum/clipspace; only the global + residual calibration owner may fit those frustums into scene-space camera + intrinsics/extrinsics. - Stable LED index identity requires temporal/color/address coding or another correspondence source; uncoded identical lights are spline constraints, not metric depth authority. +- Once camera frustums are coherent, every detected surface candidate can be + resampled across the synchronized view set before Fensalir resolves it into + spatiotemporal surface/splat claims. ## Next Cut diff --git a/notes/current-system-map.md b/notes/current-system-map.md index deb2029..df08859 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -604,17 +604,21 @@ world-space support, and Fensalir plans it as `DebugOverlay` evidence while ambiguous raw camera features still defer. LED strips now have a typed calibration-evidence socket instead of being folded into fake depth. `MimirLedSplineFieldCandidate` carries ordered per-camera -bright-curve samples, whether the LEDs were temporally coded, and whether the -sample indices are stable. `BuildLedSplineCandidateFrame` lowers that shared -curve as deterministic camera-sensor Feature evidence; Fensalir plans it as a +bright-curve samples in that sensor's image plane and local clipspace/frustum, +whether the LEDs were temporally coded, and whether the sample indices are +stable. `BuildLedSplineCandidateFrame` lowers that shared curve as +deterministic camera-sensor Feature evidence; Fensalir plans it as a `DebugOverlay` packet through the same FieldEvidence authority. Addressable or blink-coded strips can provide exact cross-camera LED identity. Plain identical white LEDs still provide a strong spline/epipolar constraint, but their point correspondence is ambiguous under occlusion, saturation, repeated spacing, and motion blur until temporal continuity, Leap/body anchors, or a future residual solver disambiguates it. Individual cameras do not write calibration state; the -future global residual owner consumes these claims and is the only authority -allowed to update camera intrinsics/extrinsics. +future global residual owner fits local frustums into a coherent scene and is +the only authority allowed to update camera intrinsics/extrinsics. Once that +frustum fit is coherent, detected surface candidates can be sampled across +every synchronized view before Fensalir resolves them into 4D Gaussian/surface +splat claims. The current teardown/migration map is `docs/fensalir-rendering-rebuild-migration.md`, paired with Fensalir's diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 56f1af7..8d6b0b6 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -813,11 +813,11 @@ static int RunLedSplineCalibrationSmoke() ObservedTimeNs: 1_000_000_000L, Points: [ - new MimirLedSplineObservationPoint(0, 0.00, 110.0, 82.0, 2.5, 0.91), - new MimirLedSplineObservationPoint(1, 0.25, 132.0, 98.0, 2.2, 0.94), - new MimirLedSplineObservationPoint(2, 0.50, 148.0, 126.0, 2.4, 0.92), - new MimirLedSplineObservationPoint(3, 0.75, 171.0, 152.0, 2.6, 0.90), - new MimirLedSplineObservationPoint(4, 1.00, 204.0, 184.0, 2.7, 0.89), + new MimirLedSplineObservationPoint(0, 0.00, 110.0, 82.0, -0.3125, 0.3167, 2.5, 0.91), + new MimirLedSplineObservationPoint(1, 0.25, 132.0, 98.0, -0.1750, 0.1833, 2.2, 0.94), + new MimirLedSplineObservationPoint(2, 0.50, 148.0, 126.0, -0.0750, -0.0500, 2.4, 0.92), + new MimirLedSplineObservationPoint(3, 0.75, 171.0, 152.0, 0.0688, -0.2667, 2.6, 0.90), + new MimirLedSplineObservationPoint(4, 1.00, 204.0, 184.0, 0.2750, -0.5333, 2.7, 0.89), ], Confidence: 0.91), new MimirLedSplineCameraObservation( @@ -828,11 +828,11 @@ static int RunLedSplineCalibrationSmoke() ObservedTimeNs: 1_000_000_004L, Points: [ - new MimirLedSplineObservationPoint(0, 0.00, 763.0, 356.0, 4.4, 0.88), - new MimirLedSplineObservationPoint(1, 0.25, 840.0, 432.0, 4.1, 0.93), - new MimirLedSplineObservationPoint(2, 0.50, 914.0, 552.0, 4.3, 0.94), - new MimirLedSplineObservationPoint(3, 0.75, 1018.0, 673.0, 4.6, 0.90), - new MimirLedSplineObservationPoint(4, 1.00, 1188.0, 816.0, 4.9, 0.86), + new MimirLedSplineObservationPoint(0, 0.00, 763.0, 356.0, -0.2052, 0.3407, 4.4, 0.88), + new MimirLedSplineObservationPoint(1, 0.25, 840.0, 432.0, -0.1250, 0.2000, 4.1, 0.93), + new MimirLedSplineObservationPoint(2, 0.50, 914.0, 552.0, -0.0479, -0.0222, 4.3, 0.94), + new MimirLedSplineObservationPoint(3, 0.75, 1018.0, 673.0, 0.0604, -0.2463, 4.6, 0.90), + new MimirLedSplineObservationPoint(4, 1.00, 1188.0, 816.0, 0.2375, -0.5111, 4.9, 0.86), ], Confidence: 0.90), ], @@ -847,10 +847,11 @@ static int RunLedSplineCalibrationSmoke() var frame = lowerer.BuildLedSplineCandidateFrame([candidate]); var validation = AquariumFieldEvidenceValidator.Validate(frame); var plan = AquariumFieldLoweringPlanner.Plan(frame); + var residual = new MimirVisualCalibrationResidualSolver().EvaluateLedSpline(candidate); var claim = frame.Claims.SingleOrDefault(); Console.WriteLine( - $"led-spline-calibration-smoke domains={frame.Domains.Count} claims={frame.Claims.Count} planned={plan.Packets.Count} deferred={plan.DeferredRequests.Count} points={candidate.CameraObservations.Sum(static observation => observation.Points.Count)} errors={validation.HasErrors}"); + $"led-spline-calibration-smoke domains={frame.Domains.Count} claims={frame.Claims.Count} planned={plan.Packets.Count} deferred={plan.DeferredRequests.Count} pairs={residual.PairResiduals.Count} clipResidual={residual.PairResiduals.FirstOrDefault()?.MeanClipspaceDistance ?? double.NaN:0.000000} points={candidate.CameraObservations.Sum(static observation => observation.Points.Count)} errors={validation.HasErrors}"); if (validation.HasErrors) { @@ -869,6 +870,11 @@ static int RunLedSplineCalibrationSmoke() claim.Proposal.Kind == AquariumFieldProposalKind.DeterministicStructural && claim.Proposal.RepresentedCandidateCount == 10 && claim.PayloadHandle.Contains("indexed:coded", StringComparison.Ordinal) && + residual.Status == MimirVisualCalibrationResidualStatus.ResidualOnly && + !residual.HasPoseUpdate && + residual.PairResiduals.Count == 1 && + residual.PairResiduals[0].SharedLedCount == 5 && + residual.PairResiduals[0].MeanClipspaceDistance < 0.12 && plan.Packets.Count == 1 && plan.Packets[0].Backend == AquariumFieldBackendKind.DebugOverlay && plan.DeferredRequests.Count == 0 diff --git a/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs b/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs index a2c411b..c9d379a 100644 --- a/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs +++ b/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs @@ -166,6 +166,8 @@ public readonly record struct MimirLedSplineObservationPoint( double CurveT, double ImageX, double ImageY, + double ClipX, + double ClipY, double RadiusPixels, double Confidence); diff --git a/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs b/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs new file mode 100644 index 0000000..e608ff1 --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs @@ -0,0 +1,172 @@ +namespace Mimir.Runtime.Synchronization; + +public enum MimirVisualCalibrationResidualStatus +{ + Empty, + InsufficientCorrespondence, + ResidualOnly, +} + +public sealed record MimirLedSplineCameraResidual( + string SourceId, + string ObservationKey, + int PointCount, + double MeanRadiusPixels, + double Confidence); + +public sealed record MimirLedSplinePairResidual( + string LeftSourceId, + string RightSourceId, + int SharedLedCount, + double MeanCurveParameterResidual, + double MeanClipspaceDistance, + double MeanRadiusRatio, + long MeanTimeOffsetNs, + double Confidence); + +public sealed record MimirVisualCalibrationResidualFrame( + string CalibrationId, + string CandidateKey, + string SplineId, + MimirVisualCalibrationResidualStatus Status, + IReadOnlyList CameraResiduals, + IReadOnlyList PairResiduals, + double MeanCurveParameterResidual, + double Confidence, + bool HasPoseUpdate); + +public sealed class MimirVisualCalibrationResidualSolver +{ + public MimirVisualCalibrationResidualFrame EvaluateLedSpline(MimirLedSplineFieldCandidate candidate) + { + if (candidate.CameraObservations.Count == 0) + { + return Empty(candidate); + } + + var cameraResiduals = candidate.CameraObservations + .Select(static observation => new MimirLedSplineCameraResidual( + observation.SourceId, + observation.ObservationKey, + observation.Points.Count, + observation.Points.Count == 0 + ? 0.0 + : observation.Points.Average(static point => Math.Max(0.0, point.RadiusPixels)), + Clamp01(observation.Confidence))) + .ToArray(); + + if (!candidate.HasStableLedIndices || candidate.CameraObservations.Count < 2) + { + return new MimirVisualCalibrationResidualFrame( + candidate.CalibrationId, + candidate.CandidateKey, + candidate.SplineId, + MimirVisualCalibrationResidualStatus.InsufficientCorrespondence, + cameraResiduals, + [], + double.NaN, + 0.0, + HasPoseUpdate: false); + } + + var pairResiduals = new List(); + for (var leftIndex = 0; leftIndex < candidate.CameraObservations.Count; leftIndex++) + { + for (var rightIndex = leftIndex + 1; rightIndex < candidate.CameraObservations.Count; rightIndex++) + { + var residual = PairResidual(candidate.CameraObservations[leftIndex], candidate.CameraObservations[rightIndex]); + if (residual is not null) + { + pairResiduals.Add(residual); + } + } + } + + if (pairResiduals.Count == 0) + { + return new MimirVisualCalibrationResidualFrame( + candidate.CalibrationId, + candidate.CandidateKey, + candidate.SplineId, + MimirVisualCalibrationResidualStatus.InsufficientCorrespondence, + cameraResiduals, + [], + double.NaN, + 0.0, + HasPoseUpdate: false); + } + + var meanResidual = pairResiduals.Average(static residual => residual.MeanCurveParameterResidual); + var confidence = Clamp01(candidate.Confidence) * + pairResiduals.Average(static residual => residual.Confidence) * + (candidate.HasTemporalCode ? 1.0 : 0.5); + return new MimirVisualCalibrationResidualFrame( + candidate.CalibrationId, + candidate.CandidateKey, + candidate.SplineId, + MimirVisualCalibrationResidualStatus.ResidualOnly, + cameraResiduals, + pairResiduals, + meanResidual, + confidence, + HasPoseUpdate: false); + } + + private static MimirVisualCalibrationResidualFrame Empty(MimirLedSplineFieldCandidate candidate) => + new( + candidate.CalibrationId, + candidate.CandidateKey, + candidate.SplineId, + MimirVisualCalibrationResidualStatus.Empty, + [], + [], + double.NaN, + 0.0, + HasPoseUpdate: false); + + private static MimirLedSplinePairResidual? PairResidual( + MimirLedSplineCameraObservation left, + MimirLedSplineCameraObservation right) + { + var leftByIndex = left.Points + .Where(static point => point.LedIndex >= 0) + .GroupBy(static point => point.LedIndex) + .ToDictionary(static group => group.Key, static group => group.OrderByDescending(point => point.Confidence).First()); + var rightByIndex = right.Points + .Where(static point => point.LedIndex >= 0) + .GroupBy(static point => point.LedIndex) + .ToDictionary(static group => group.Key, static group => group.OrderByDescending(point => point.Confidence).First()); + var shared = leftByIndex.Keys.Intersect(rightByIndex.Keys).Order().ToArray(); + if (shared.Length == 0) + { + return null; + } + + var curveResidual = shared.Average(index => Math.Abs(leftByIndex[index].CurveT - rightByIndex[index].CurveT)); + var clipDistance = shared.Average(index => + { + var dx = leftByIndex[index].ClipX - rightByIndex[index].ClipX; + var dy = leftByIndex[index].ClipY - rightByIndex[index].ClipY; + return Math.Sqrt(dx * dx + dy * dy); + }); + var radiusRatio = shared.Average(index => + { + var leftRadius = Math.Max(1.0e-6, leftByIndex[index].RadiusPixels); + var rightRadius = Math.Max(1.0e-6, rightByIndex[index].RadiusPixels); + return Math.Max(leftRadius, rightRadius) / Math.Min(leftRadius, rightRadius); + }); + var confidence = shared.Average(index => Clamp01(leftByIndex[index].Confidence) * Clamp01(rightByIndex[index].Confidence)); + return new MimirLedSplinePairResidual( + left.SourceId, + right.SourceId, + shared.Length, + curveResidual, + clipDistance, + radiusRatio, + right.ObservedTimeNs - left.ObservedTimeNs, + confidence); + } + + private static double Clamp01(double value) => + double.IsFinite(value) ? Math.Clamp(value, 0.0, 1.0) : 0.0; +} diff --git a/state/map.yaml b/state/map.yaml index f56382b..b9ac486 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -104,13 +104,16 @@ invariants: open. - LED strips on performers are active visual calibration evidence, not fake depth. `MimirLedSplineFieldCandidate` carries per-camera ordered bright - curve observations with optional temporal code / stable LED indices, and + curve observations in image pixels and local clipspace/frustum coordinates + with optional temporal code / stable LED indices, and `MimirFensalirFieldLowering.BuildLedSplineCandidateFrame` lowers them as deterministic camera-sensor Feature claims that Fensalir plans as `DebugOverlay` calibration evidence. Exact cross-camera correspondence requires addressable/blink-coded LEDs or another identity source; uncoded identical LEDs remain useful curve constraints but cannot own camera pose. - The future global residual/calibration owner consumes these claims and is + `MimirVisualCalibrationResidualSolver` now emits residual-only reports over + coded/indexed LED observations; a future global residual/calibration owner + must fit local frustums into coherent scene-space camera placement and be the only writer of camera intrinsics/extrinsics. - Fensalir packet payload handles are not authority. Live payload references must resolve through declared typed GPU/native resources with shape,