From 2f1e5f856edb5f6072f8f2a0ec6497afb60ef8d5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 31 May 2026 02:12:17 +0100 Subject: [PATCH] Score exposure sweeps by LED spline quality --- docs/implementation-plan.md | 6 + docs/sensor-fusion-architecture.md | 3 + notes/current-system-map.md | 12 +- src/Mimir.BufferSmoke/Program.cs | 117 +++++++++++++ .../MimirVisualCalibrationResidualSolver.cs | 157 +++++++++++++++++- state/evidence.jsonl | 1 + state/map.yaml | 6 +- 7 files changed, 296 insertions(+), 6 deletions(-) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 96b21ed..d1f7452 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -393,6 +393,12 @@ a named invariant that the native runtime cannot protect yet. correspondence requires coded or otherwise stable LED identity; uncoded repeated lights remain curve constraints until the global residual solver resolves them. + Exposure/gain sweeps must optimize recovered spline quality, not generic + contrast or brightness. `MimirLedSplineQualityScorer` scores each setting by + detected LED count, curve coverage, spacing coherence, smoothness, exposure + fitness, and saturation penalty. `--led-spline-quality-sweep-smoke` proves + the selector chooses the balanced setting over both underexposed missing-LED + detections and overexposed saturated halos. Phase 6 program-output receipts are also structurally complete: Fensalir publishes the final program surface through named shared D3D12 texture/fence resources, the OBS plugin consumes that surface with an explicit D3D12-to- diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index 41fec30..f50bf64 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -33,6 +33,9 @@ flowchart TD - Stable LED index identity requires temporal/color/address coding or another correspondence source; uncoded identical lights are spline constraints, not metric depth authority. +- Exposure/gain sweeps optimize recovered spline quality: detected LED count, + curve coverage, spacing coherence, smoothness, exposure fitness, and + saturation penalty. Bright clipped blobs are not good evidence. - The first camera-rig solver fits bounded camera translation updates from a scene-anchored LED curve by minimizing ray-to-curve distance. Rotation, intrinsics, and distortion are future owners. diff --git a/notes/current-system-map.md b/notes/current-system-map.md index b723672..9fc5b63 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -619,10 +619,14 @@ samples. `MimirCameraRigCalibrationSolver` is the first higher-level frustum fit owner: given stable LED indices and a scene-space LED curve anchor, it minimizes ray-to-curve distance and emits bounded camera translation updates. Individual cameras do not write calibration state. Rotation, focal length, and -distortion solving are still future cuts, not hidden behavior. Once the frustum -fit is coherent, detected surface candidates can be sampled across every -synchronized view before Fensalir resolves them into 4D Gaussian/surface splat -claims. +distortion solving are still future cuts, not hidden behavior. Exposure/gain +sweeps are scored by the recovered spline, not raw brightness: +`MimirLedSplineQualityScorer` combines detected LED count, curve coverage, +spacing coherence, smoothness, exposure fitness, and saturation penalty, and +`MimirLedSplineSweepSelector` picks the best calibration-usable setting. Once +the frustum fit is coherent, detected surface candidates can be sampled across +every synchronized view before Fensalir resolves them into 4D Gaussian/surface +splat claims. The current teardown/migration map is `docs/fensalir-rendering-rebuild-migration.md`, paired with Fensalir's diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 8581d56..2e4f9f7 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -145,6 +145,11 @@ if (args.Any(arg => string.Equals(arg, "--led-spline-calibration-smoke", StringC return RunLedSplineCalibrationSmoke(); } +if (args.Any(arg => string.Equals(arg, "--led-spline-quality-sweep-smoke", StringComparison.OrdinalIgnoreCase))) +{ + return RunLedSplineQualitySweepSmoke(); +} + if (args.Any(arg => string.Equals(arg, "--fensalir-texture-lease-smoke", StringComparison.OrdinalIgnoreCase))) { return RunFensalirTextureLeaseSmoke(); @@ -891,6 +896,72 @@ static int RunLedSplineCalibrationSmoke() : 1; } +static int RunLedSplineQualitySweepSmoke() +{ + var scenePoints = new[] + { + new MimirLedSplineScenePoint(0, new Vector3(-0.20f, 0.30f, 2.20f), 0.95), + new MimirLedSplineScenePoint(1, new Vector3(-0.08f, 0.18f, 2.05f), 0.96), + new MimirLedSplineScenePoint(2, new Vector3(0.04f, 0.02f, 1.95f), 0.97), + new MimirLedSplineScenePoint(3, new Vector3(0.18f, -0.14f, 1.88f), 0.95), + new MimirLedSplineScenePoint(4, new Vector3(0.35f, -0.28f, 1.82f), 0.93), + }; + var pose = new MimirCameraPoseEstimate("kiyo-basic", new Vector3(0.24f, 0.02f, -0.03f), Quaternion.Identity, 1.0); + var settings = new[] + { + new MimirCameraExposureGainSetting("underexposed", -10, 0, "LEDs barely clear detector threshold."), + new MimirCameraExposureGainSetting("balanced", -8, 8, "LED centroids are bright but not clipped."), + new MimirCameraExposureGainSetting("blown-out", -5, 32, "Halos merge and saturated centroids lose precision."), + }; + var curveSolver = new MimirLedSplineCurveSolver(); + var scorer = new MimirLedSplineQualityScorer(); + var results = new List(settings.Length); + foreach (var setting in settings) + { + var detections = SyntheticSweepDetections(setting.SettingId, scenePoints, pose, 1920, 1080); + var curve = curveSolver.SolveCameraCurve( + pose.SourceId, + $"{pose.SourceId}:led-spline:{setting.SettingId}", + 1920, + 1080, + 1_000_000_000L, + detections); + var detectionArray = detections.ToArray(); + var saturated = detectionArray.Length == 0 + ? 0.0 + : detectionArray.Count(static detection => detection.IsSaturated) / (double)detectionArray.Length; + var exposureFitness = detectionArray.Length == 0 + ? 0.0 + : detectionArray.Average(static detection => + { + var peak = Math.Clamp(detection.PeakLuma, 0.0, 1.0); + return peak < 0.18 ? peak / 0.18 : peak > 0.92 ? Math.Max(0.0, 1.0 - (peak - 0.92) / 0.08) : 1.0; + }); + results.Add(new MimirLedSplineSweepResult( + pose.SourceId, + setting, + curve, + scorer.Score(curve, scenePoints.Length, saturated, exposureFitness))); + } + + var best = new MimirLedSplineSweepSelector().SelectBest(results); + foreach (var result in results) + { + Console.WriteLine( + $"led-spline-quality source={result.SourceId} setting={result.Setting.SettingId} detected={result.Quality.DetectedLedCount} score={result.Quality.Score:0.000} coverage={result.Quality.CurveCoverage:0.000} spacing={result.Quality.SpacingCoherence:0.000} smooth={result.Quality.Smoothness:0.000} exposure={result.Quality.ExposureFitness:0.000} saturated={result.Quality.SaturatedFraction:0.000} usable={result.Quality.UsableForCalibration}"); + } + + Console.WriteLine($"led-spline-quality-best setting={best?.Setting.SettingId ?? "none"} score={best?.Quality.Score ?? 0.0:0.000}"); + + return best?.Setting.SettingId == "balanced" && + best.Quality.UsableForCalibration && + results.Single(static result => result.Setting.SettingId == "underexposed").Quality.Score < best.Quality.Score && + results.Single(static result => result.Setting.SettingId == "blown-out").Quality.SaturatedFraction > 0.0 && + results.Single(static result => result.Setting.SettingId == "blown-out").Quality.Score < best.Quality.Score + ? 0 + : 1; +} + static IReadOnlyList ProjectLedDetections( IReadOnlyList scenePoints, MimirCameraPoseEstimate pose, @@ -919,6 +990,52 @@ static IReadOnlyList ProjectLedDetections( return output; } +static IReadOnlyList SyntheticSweepDetections( + string settingId, + IReadOnlyList scenePoints, + MimirCameraPoseEstimate pose, + int width, + int height) +{ + var projected = ProjectLedDetections(scenePoints, pose, width, height, radiusPixels: 4.5).ToArray(); + return settingId switch + { + "underexposed" => projected + .Take(3) + .Select((detection, index) => detection with + { + ImageX = detection.ImageX + (index - 1) * 2.5, + ImageY = detection.ImageY + (index - 1) * -1.5, + Confidence = 0.32 + index * 0.04, + PeakLuma = 0.16 + index * 0.01, + LocalContrast = 0.22 + index * 0.03, + }) + .ToArray(), + "blown-out" => projected + .Select((detection, index) => detection with + { + ImageX = detection.ImageX + (index % 2 == 0 ? 14.0 : -11.0), + ImageY = detection.ImageY + (index % 2 == 0 ? -9.0 : 13.0), + RadiusPixels = 13.0 + index, + Confidence = 0.56, + PeakLuma = 1.0, + LocalContrast = 0.50, + IsSaturated = true, + }) + .ToArray(), + _ => projected + .Select((detection, index) => detection with + { + ImageX = detection.ImageX + (index - 2) * 0.35, + ImageY = detection.ImageY + (2 - index) * 0.20, + Confidence = 0.90 + index * 0.01, + PeakLuma = 0.74 + index * 0.02, + LocalContrast = 0.88, + }) + .ToArray(), + }; +} + static int RunFensalirTextureLeaseSmoke() { var broker = new FakeFieldResourceBroker(); diff --git a/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs b/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs index f345d23..9467106 100644 --- a/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs +++ b/src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs @@ -7,7 +7,10 @@ public readonly record struct MimirLedPixelDetection( double ImageX, double ImageY, double RadiusPixels, - double Confidence); + double Confidence, + double PeakLuma = 1.0, + double LocalContrast = 1.0, + bool IsSaturated = false); public sealed record MimirLedSplineCurveFit( string SourceId, @@ -16,6 +19,30 @@ public sealed record MimirLedSplineCurveFit( double CurveLengthPixels, double Confidence); +public sealed record MimirLedSplineQualityReport( + string SourceId, + string ObservationKey, + int DetectedLedCount, + double CurveCoverage, + double SpacingCoherence, + double Smoothness, + double ExposureFitness, + double SaturatedFraction, + double Score, + bool UsableForCalibration); + +public sealed record MimirCameraExposureGainSetting( + string SettingId, + int Exposure, + int Gain, + string Notes); + +public sealed record MimirLedSplineSweepResult( + string SourceId, + MimirCameraExposureGainSetting Setting, + MimirLedSplineCurveFit Curve, + MimirLedSplineQualityReport Quality); + public enum MimirVisualCalibrationResidualStatus { Empty, @@ -199,6 +226,134 @@ public sealed class MimirLedSplineCurveSolver double.IsFinite(value) ? Math.Clamp(value, 0.0, 1.0) : 0.0; } +public sealed class MimirLedSplineQualityScorer +{ + public MimirLedSplineQualityReport Score( + MimirLedSplineCurveFit curve, + int expectedLedCount, + double saturatedFraction = 0.0, + double exposureFitness = 1.0) + { + var points = curve.Points + .OrderBy(static point => point.CurveT) + .ToArray(); + if (points.Length == 0 || expectedLedCount <= 0) + { + return new MimirLedSplineQualityReport( + curve.SourceId, + curve.ObservationKey, + points.Length, + 0.0, + 0.0, + 0.0, + 0.0, + Clamp01(saturatedFraction), + 0.0, + UsableForCalibration: false); + } + + var coverage = Clamp01(points.Length / (double)Math.Max(1, expectedLedCount)); + var spacing = SpacingCoherence(points); + var smoothness = Smoothness(points); + var pointConfidence = points.Average(static point => Clamp01(point.Confidence)); + var saturationPenalty = 1.0 - Clamp01(saturatedFraction); + var exposure = Clamp01(exposureFitness); + var score = coverage * + (0.28 * spacing + 0.22 * smoothness + 0.25 * pointConfidence + 0.25 * exposure) * + saturationPenalty; + return new MimirLedSplineQualityReport( + curve.SourceId, + curve.ObservationKey, + points.Length, + coverage, + spacing, + smoothness, + exposure, + Clamp01(saturatedFraction), + Clamp01(score), + UsableForCalibration: points.Length >= 3 && coverage >= 0.60 && score >= 0.55); + } + + private static double SpacingCoherence(IReadOnlyList points) + { + if (points.Count < 3) + { + return points.Count >= 2 ? 0.75 : 0.0; + } + + var distances = new double[points.Count - 1]; + for (var index = 1; index < points.Count; index++) + { + var dx = points[index].ImageX - points[index - 1].ImageX; + var dy = points[index].ImageY - points[index - 1].ImageY; + distances[index - 1] = Math.Sqrt(dx * dx + dy * dy); + } + + var mean = distances.Average(); + if (mean <= 1.0e-9) + { + return 0.0; + } + + var variance = distances.Average(distance => + { + var delta = distance - mean; + return delta * delta; + }); + var coefficient = Math.Sqrt(variance) / mean; + return 1.0 / (1.0 + coefficient * 3.0); + } + + private static double Smoothness(IReadOnlyList points) + { + if (points.Count < 3) + { + return points.Count >= 2 ? 0.75 : 0.0; + } + + var turnSum = 0.0; + var turns = 0; + for (var index = 1; index < points.Count - 1; index++) + { + var ax = points[index].ImageX - points[index - 1].ImageX; + var ay = points[index].ImageY - points[index - 1].ImageY; + var bx = points[index + 1].ImageX - points[index].ImageX; + var by = points[index + 1].ImageY - points[index].ImageY; + var aLength = Math.Sqrt(ax * ax + ay * ay); + var bLength = Math.Sqrt(bx * bx + by * by); + if (aLength <= 1.0e-9 || bLength <= 1.0e-9) + { + continue; + } + + var dot = Math.Clamp((ax * bx + ay * by) / (aLength * bLength), -1.0, 1.0); + turnSum += Math.Acos(dot); + turns++; + } + + if (turns == 0) + { + return 0.0; + } + + var meanTurn = turnSum / turns; + return 1.0 / (1.0 + meanTurn); + } + + private static double Clamp01(double value) => + double.IsFinite(value) ? Math.Clamp(value, 0.0, 1.0) : 0.0; +} + +public sealed class MimirLedSplineSweepSelector +{ + public MimirLedSplineSweepResult? SelectBest(IEnumerable results) => + results + .Where(static result => result.Quality.UsableForCalibration) + .OrderByDescending(static result => result.Quality.Score) + .ThenBy(static result => result.Quality.SaturatedFraction) + .FirstOrDefault(); +} + public sealed class MimirVisualCalibrationResidualSolver { public MimirVisualCalibrationResidualFrame EvaluateLedSpline(MimirLedSplineFieldCandidate candidate) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index fbfae6d..671ba01 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -253,3 +253,4 @@ {"ts": "2026-05-31T01:22:17+01:00", "type": "verification", "status": "ok", "note": "Eve dashboard broker cut: /eve/deck now routes provider-owned retained dashboard trees, /eve/deck/providers exposes switchboard/Mimir/StreamPixels manifests, and EveCanvas connects to /eve/deck with provider-card open-provider commands. Added typed CultMesh-compatible document shapes mimir.eve_dashboard_manifest.v1 and mimir.eve_dashboard_state.v1 while keeping live iPad transport WebSocket/TCP for LAN and SSH tunnel use. Smoke verified /health, provider catalog, WebSocket switchboard state, and open-provider transition to mimir.stream.layout."} {"ts": "2026-05-31T01:34:32+01:00", "type": "verification", "status": "ok", "note": "Eve dashboard broker gained native VoidBot provider voidbot.swarm. It reads VoidBot's existing .voidbot/status/swarm-state.json projection, exposes a provider manifest with CTB/agent-status/state-tree capabilities, renders upcoming turns as CTB cards, agent status cards, and selected Face state/detail nodes. Smoke verified /eve/deck/providers includes voidbot.swarm and open-provider switched to a VoidBot snapshot with paused swarm state, CTB cards, and agent detail."} {"ts": "2026-05-31T02:05:12+01:00", "type": "implementation", "status": "ok", "note": "Added the first LED-strip calibration machine: MimirLedSplineCurveSolver orders bright detections into per-sensor image/clipspace curves, and MimirCameraRigCalibrationSolver consumes stable LED indices plus a scene-space LED curve anchor to emit bounded camera translation updates by minimizing ray-to-curve distance. Rotation, focal length, and distortion remain explicit future calibration owners."} +{"ts": "2026-05-31T02:12:01+01:00", "type": "decision", "status": "ok", "note": "LED exposure/gain sweeps must optimize recovered spline quality, not raw brightness/contrast. MimirLedSplineQualityScorer now scores detected LED count, curve coverage, spacing coherence, smoothness, exposure fitness, and saturation penalty; overexposed halos can see every LED and still lose because centroid/curve quality is bad."} diff --git a/state/map.yaml b/state/map.yaml index 6391a38..581a4e0 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -115,7 +115,11 @@ invariants: curves from bright detections. `MimirCameraRigCalibrationSolver` consumes stable LED indices plus a scene-space LED curve anchor and emits bounded camera translation updates by minimizing ray-to-curve distance. Rotation, - focal length, and distortion are still future calibration cuts. + focal length, and distortion are still future calibration cuts. Exposure + and gain sweeps must optimize `MimirLedSplineQualityScorer` output: + recovered LED count, curve coverage, spacing coherence, smoothness, + exposure fitness, and saturation penalty decide whether a setting is + usable for calibration. - Fensalir packet payload handles are not authority. Live payload references must resolve through declared typed GPU/native resources with shape, residency, shader access, validity, version, and handle metadata before