GameCult/Mimir
Watch
9
0
Fork
You've already forked Mimir
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Generalize frustum solve to feature tracks

Browse source
This commit is contained in:
Meta Vens 2026-05-31 22:53:56 +01:00
commit 27c53f74a7
5 changed files with 191 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

10
docs/implementation-plan.md
View file

@ -500,9 +500,13 @@ a named invariant that the native runtime cannot protect yet.
consumes stable LED/marker correspondences, scene-space marker points, and
seed frustums, then emits per-source position, rotation, horizontal/vertical
tangent half-FOV, correspondence count, clipspace reprojection residual, and
confidence. Synthetic smoke `--frustum-calibration-smoke` solves both Eyes,
Kiyo Pro, Leap stereo IR, and Eve camera, establishing the first fusion-ready
frustum state before dense surface fusion.
confidence. The same solver accepts `MimirFeatureTrackFieldCandidate` plus
scene-space `TrackId` anchors, so PS Move sphere histories, sparse Eye
tracks, and Eve-local marker tracks use the same Starfire frustum authority
as the LED strip. Synthetic smoke `--frustum-calibration-smoke` solves both
LED-spline and feature-track frustums for both Eyes, Kiyo Pro, Leap stereo
IR, and Eve camera, establishing the first fusion-ready frustum state before
dense surface fusion.
Next useful cut is direct report decoding plus optical sphere observations
from PS3 Eyes/Kiyos/Eve entering the same calibration-evidence path as LED
spline constraints.

81
src/Mimir.BufferSmoke/Program.cs
View file

@ -1128,15 +1128,52 @@ static int RunFrustumCalibrationSmoke()
seeds,
maximumDeltaMeters: 0.20,
maximumRotationDeltaDegrees: 12.0);
var trackScenePoints = scenePoints
.Select(point => new MimirFeatureTrackScenePoint(
10_000 + point.LedIndex,
point.PositionMeters,
point.Confidence))
.ToArray();
var featureCandidate = new MimirFeatureTrackFieldCandidate(
"frustum-solve:move-eve-markers:frame-1",
"room-frustum-calibration",
"mimir-local-witness-feature-tracks",
truth.Select((frustum, index) =>
{
var size = sizes[frustum.SourceId];
return ProjectFrustumFeatureTracks(
scenePoints,
frustum,
size.Width,
size.Height,
3_000_100_000L + index);
}).ToArray(),
StableTrackCount: scenePoints.Length,
MeanTrackAgeFrames: 8.0,
MeanSpeedPixelsPerSecond: 0.0,
Confidence: 0.91,
ObservedTimeNs: 3_000_100_004L);
var featureFrame = new MimirCameraFrustumCalibrationSolver().FitFrustumsFromFeatureTracks(
featureCandidate,
trackScenePoints,
seeds,
markerSetId: "move-and-eve-feature-markers",
maximumDeltaMeters: 0.20,
maximumRotationDeltaDegrees: 12.0);
Console.WriteLine(
$"frustum-calibration-smoke updates={frame.FrustumUpdates.Count} confidence={frame.Confidence:0.000} meanClipError={frame.MeanReprojectionErrorClip:0.000000} sources={string.Join(",", frame.FrustumUpdates.Select(static update => $"{update.SourceId}:{update.UsedPointCount}/{update.RotationDeltaDegrees:0.00}deg/{update.HorizontalTanHalfFov:0.000}x{update.VerticalTanHalfFov:0.000}"))}");
$"frustum-calibration-smoke ledUpdates={frame.FrustumUpdates.Count} featureUpdates={featureFrame.FrustumUpdates.Count} confidence={frame.Confidence:0.000}/{featureFrame.Confidence:0.000} meanClipError={frame.MeanReprojectionErrorClip:0.000000}/{featureFrame.MeanReprojectionErrorClip:0.000000} sources={string.Join(",", frame.FrustumUpdates.Select(static update => $"{update.SourceId}:{update.UsedPointCount}/{update.RotationDeltaDegrees:0.00}deg/{update.HorizontalTanHalfFov:0.000}x{update.VerticalTanHalfFov:0.000}"))}");
return frame.HasFrustumUpdate &&
featureFrame.HasFrustumUpdate &&
frame.FrustumUpdates.Count == truth.Length &&
featureFrame.FrustumUpdates.Count == truth.Length &&
frame.FrustumUpdates.All(update => update.UsedPointCount == scenePoints.Length) &&
featureFrame.FrustumUpdates.All(update => update.UsedPointCount == scenePoints.Length) &&
frame.FrustumUpdates.All(static update => update.MeanReprojectionErrorClip < 0.010) &&
frame.Confidence >= 0.70
featureFrame.FrustumUpdates.All(static update => update.MeanReprojectionErrorClip < 0.010) &&
frame.Confidence >= 0.70 &&
featureFrame.Confidence >= 0.70
? 0
: 1;
}
@ -1935,6 +1972,46 @@ static IReadOnlyList<MimirLedPixelDetection> ProjectFrustumLedDetections(
return output;
}
static MimirFeatureTrackCameraObservation ProjectFrustumFeatureTracks(
IReadOnlyList<MimirLedSplineScenePoint> scenePoints,
MimirCameraFrustumEstimate frustum,
int width,
int height,
long observedTimeNs)
{
var worldToCamera = Quaternion.Inverse(frustum.CameraToWorldRotation);
var tracks = scenePoints.Select(point =>
{
var cameraSpace = Vector3.Transform(point.PositionMeters - frustum.PositionMeters, worldToCamera);
var z = Math.Max(1.0e-6f, cameraSpace.Z);
var clipX = cameraSpace.X / (z * (float)Math.Max(1.0e-6, frustum.HorizontalTanHalfFov));
var clipY = cameraSpace.Y / (z * (float)Math.Max(1.0e-6, frustum.VerticalTanHalfFov));
var imageX = (clipX + 1.0f) * 0.5f * width;
var imageY = (1.0f - clipY) * 0.5f * height;
return new MimirFeatureTrackPoint(
10_000 + point.LedIndex,
imageX,
imageY,
clipX,
clipY,
VelocityXPerSecond: 0.0,
VelocityYPerSecond: 0.0,
AgeFrames: 8,
Confidence: point.Confidence);
}).ToArray();
return new MimirFeatureTrackCameraObservation(
$"{frustum.SourceId}:feature-frustum:{observedTimeNs}",
frustum.SourceId,
width,
height,
observedTimeNs,
tracks,
FrameCount: 1,
MeanTrackAgeFrames: 8.0,
MeanSpeedPixelsPerSecond: 0.0,
Confidence: tracks.Average(static track => track.Confidence));
}
static IReadOnlyList<MimirLedPixelDetection> SyntheticSweepDetections(
string settingId,
IReadOnlyList<MimirLedSplineScenePoint> scenePoints,

118
src/Mimir.Runtime/Synchronization/MimirVisualCalibrationResidualSolver.cs
View file

@ -83,6 +83,11 @@ public readonly record struct MimirLedSplineScenePoint(
Vector3 PositionMeters,
double Confidence);
public readonly record struct MimirFeatureTrackScenePoint(
int TrackId,
Vector3 PositionMeters,
double Confidence);
public sealed record MimirCameraPoseEstimate(
string SourceId,
Vector3 PositionMeters,
@ -142,7 +147,7 @@ public sealed record MimirCameraFrustumSolveUpdate(
public sealed record MimirCameraFrustumSolveFrame(
string CalibrationId,
string CandidateKey,
string SplineId,
string MarkerSetId,
IReadOnlyList<MimirCameraFrustumSolveUpdate> FrustumUpdates,
double MeanReprojectionErrorClip,
double Confidence,
@ -727,18 +732,10 @@ public sealed class MimirCameraFrustumCalibrationSolver
.Where(static point => point.LedIndex >= 0 && point.Confidence > 0.0)
.GroupBy(static point => point.LedIndex)
.ToDictionary(static group => group.Key, static group => group.OrderByDescending(point => point.Confidence).First());
var frustumBySource = currentFrustums.ToDictionary(static frustum => frustum.SourceId, StringComparer.Ordinal);
var updates = new List<MimirCameraFrustumSolveUpdate>();
foreach (var observation in candidate.CameraObservations)
{
if (!frustumBySource.TryGetValue(observation.SourceId, out var seed) ||
seed.Confidence <= 0.0)
{
continue;
}
var correspondences = observation.Points
var observations = candidate.CameraObservations
.Select(observation => new FrustumObservation(
observation.SourceId,
observation.Points
.Where(point => sceneByLed.ContainsKey(point.LedIndex) && point.Confidence > 0.0)
.Select(point =>
{
@ -749,19 +746,96 @@ public sealed class MimirCameraFrustumCalibrationSolver
point.ClipY,
Math.Min(point.Confidence, scene.Confidence));
})
.ToArray();
if (correspondences.Length < 4)
.ToArray()))
.ToArray();
return FitFrustums(
candidate.CalibrationId,
candidate.CandidateKey,
candidate.SplineId,
candidate.Confidence,
observations,
currentFrustums,
maximumDeltaMeters,
maximumRotationDeltaDegrees,
minimumTanHalfFov,
maximumTanHalfFov);
}
public MimirCameraFrustumSolveFrame FitFrustumsFromFeatureTracks(
MimirFeatureTrackFieldCandidate candidate,
IEnumerable<MimirFeatureTrackScenePoint> scenePoints,
IEnumerable<MimirCameraFrustumEstimate> currentFrustums,
string markerSetId = "feature-tracks",
double maximumDeltaMeters = 0.50,
double maximumRotationDeltaDegrees = 30.0,
double minimumTanHalfFov = 0.20,
double maximumTanHalfFov = 2.50)
{
var sceneByTrack = scenePoints
.Where(static point => point.TrackId >= 0 && point.Confidence > 0.0)
.GroupBy(static point => point.TrackId)
.ToDictionary(static group => group.Key, static group => group.OrderByDescending(point => point.Confidence).First());
var observations = candidate.CameraObservations
.Select(observation => new FrustumObservation(
observation.SourceId,
observation.Tracks
.Where(track => sceneByTrack.ContainsKey(track.TrackId) && track.Confidence > 0.0)
.Select(track =>
{
var scene = sceneByTrack[track.TrackId];
return new FrustumCorrespondence(
scene.PositionMeters,
track.ClipX,
track.ClipY,
Math.Min(track.Confidence, scene.Confidence));
})
.ToArray()))
.ToArray();
return FitFrustums(
candidate.CalibrationId,
candidate.CandidateKey,
markerSetId,
candidate.Confidence,
observations,
currentFrustums,
maximumDeltaMeters,
maximumRotationDeltaDegrees,
minimumTanHalfFov,
maximumTanHalfFov);
}
private static MimirCameraFrustumSolveFrame FitFrustums(
string calibrationId,
string candidateKey,
string markerSetId,
double candidateConfidence,
IReadOnlyList<FrustumObservation> observations,
IEnumerable<MimirCameraFrustumEstimate> currentFrustums,
double maximumDeltaMeters,
double maximumRotationDeltaDegrees,
double minimumTanHalfFov,
double maximumTanHalfFov)
{
var frustumBySource = currentFrustums.ToDictionary(static frustum => frustum.SourceId, StringComparer.Ordinal);
var updates = new List<MimirCameraFrustumSolveUpdate>();
foreach (var observation in observations)
{
if (!frustumBySource.TryGetValue(observation.SourceId, out var seed) ||
seed.Confidence <= 0.0 ||
observation.Correspondences.Count < 4)
{
continue;
}
var correspondences = observation.Correspondences;
if (!TryFitFrustum(
correspondences,
seed,
minimumTanHalfFov,
maximumTanHalfFov,
out var solved,
out var meanError))
out _))
{
continue;
}
@ -785,7 +859,7 @@ public sealed class MimirCameraFrustumCalibrationSolver
VerticalTanHalfFov = Clamp(solved.VerticalTanHalfFov, minimumTanHalfFov, maximumTanHalfFov),
};
var finalError = MeanReprojectionError(correspondences, finalEstimate);
var confidence = Clamp01(candidate.Confidence) *
var confidence = Clamp01(candidateConfidence) *
Clamp01(seed.Confidence) *
correspondences.Average(static point => Clamp01(point.Confidence)) *
(1.0 / (1.0 + finalError * 4.0));
@ -797,7 +871,7 @@ public sealed class MimirCameraFrustumCalibrationSolver
finalEstimate.VerticalTanHalfFov,
clampedDelta,
rotationDeltaDegrees,
correspondences.Length,
correspondences.Count,
finalError,
confidence));
}
@ -809,9 +883,9 @@ public sealed class MimirCameraFrustumCalibrationSolver
? 0.0
: updates.Average(static update => update.Confidence);
return new MimirCameraFrustumSolveFrame(
candidate.CalibrationId,
candidate.CandidateKey,
candidate.SplineId,
calibrationId,
candidateKey,
markerSetId,
updates,
meanFrameError,
frameConfidence,
@ -1110,5 +1184,7 @@ public sealed class MimirCameraFrustumCalibrationSolver
private static double Clamp01(double value) =>
double.IsFinite(value) ? Math.Clamp(value, 0.0, 1.0) : 0.0;
private sealed record FrustumObservation(string SourceId, IReadOnlyList<FrustumCorrespondence> Correspondences);
private readonly record struct FrustumCorrespondence(Vector3 Scene, double ClipX, double ClipY, double Confidence);
}

1
state/evidence.jsonl
View file

@ -289,3 +289,4 @@
{"ts": "2026-05-31T22:30:36+01:00", "type": "verification", "status": "partial", "note": "Added sync_sweep.py plus stdlib-only Nightwing witness helpers. Live receipt artifacts/runtime/sync-sweep-hints-live proved Nightwing owns Eye/Move localization locally: nw-eye-0 and nw-eye-1 each captured about 1874 frames at 320x240@187 with zero sequence gaps and emitted 8/8 compact mimir.move_controller_observation_state.v1 hints, while Starfire analyzed its own ASIO/DShow/Kiyo streams. Nightwing builtin mic capture works but the cheap local chirp hint is not yet reliable enough to score as a timing witness."}
{"ts":"2026-05-31T22:45:00+01:00","type":"implementation","status":"ok","note":"Mimir editor authority was cut back to a 2D OBS-shaped program mixer. SceneEditorState no longer creates default SDF/model content, preview items are video-feed-only, runtime editor camera/SDF/spline output is silent, and the Eve/CultUI surface is Video Sources with source-adjacent layer actions left, chrome-free program canvas center, and Audio Mix right. Numeric transform inspector controls were removed from the live surface. `--scene-editor-smoke` now verifies only camera+video feed nodes, video-only preview items, and normalized program placement.","files":["src/Mimir.Runtime/MimirRuntime.cs","src/Mimir.Runtime/Synchronization/MimirSceneEditorState.cs","src/Mimir.BufferSmoke/Program.cs"],"source":"dotnet build src/Mimir.Runtime/Mimir.Runtime.csproj; dotnet build src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj; dotnet run --no-build --project src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj -- --scene-editor-smoke; Mimir.App --headless --headless-width 1280 --headless-height 720 --capture-frame artifacts/runtime/mimir-editor-flat-2d-program-mixer-headless.png"}
{"ts": "2026-05-31T22:46:40+01:00", "type": "implementation", "status": "ok", "note": "Added the first Starfire-owned camera frustum solve. MimirCameraFrustumCalibrationSolver consumes stable LED/marker correspondences, scene-space marker points, and seed frustums, then emits position, rotation, horizontal/vertical tangent half-FOV, correspondence count, clipspace reprojection residual, and confidence. BufferSmoke --frustum-calibration-smoke solves five synthetic frustums for ps3-eye-0, ps3-eye-1, kiyo-pro-rgb, leap-stereo-ir, and eve-camera with mean clip error 0.000000; room calibration lock still passes. Eve is now modeled like Nightwing as eve-local-sensor-witness/eve-local-sensors: it owns local camera/mic flash and chirp solves and ships typed hints, not raw placement authority."}
{"ts": "2026-05-31T22:52:47+01:00", "type": "implementation", "status": "ok", "note": "Extended MimirCameraFrustumCalibrationSolver beyond LED strips: it now also consumes MimirFeatureTrackFieldCandidate plus scene-space TrackId anchors, so PS Move sphere histories, sparse Eye tracks, and Eve-local marker tracks enter the same Starfire-owned frustum authority. BufferSmoke --frustum-calibration-smoke now proves five LED-spline frustums and five feature-track frustums for ps3-eye-0, ps3-eye-1, kiyo-pro-rgb, leap-stereo-ir, and eve-camera with mean clip error 0.000000."}

10
state/map.yaml
View file

@ -232,9 +232,13 @@ invariants:
reprojection residual, and confidence. This supersedes translation-only
rig updates for global fusion setup; the older
`MimirCameraRigCalibrationSolver` remains a narrow bounded translation
residual helper. `Mimir.BufferSmoke --frustum-calibration-smoke` proves
five solved frustums for both Eyes, Kiyo Pro, Leap stereo IR, and Eve
camera.
residual helper. The same solver also consumes
`MimirFeatureTrackFieldCandidate` plus scene-space `TrackId` anchors, so
PS Move sphere histories, Eye feature tracks, and Eve-local marker tracks
can enter the frustum solve without becoming a separate authority.
`Mimir.BufferSmoke --frustum-calibration-smoke` proves five LED-spline
frustums and five feature-track frustums for both Eyes, Kiyo Pro, Leap
stereo IR, and Eve camera.
The Eyes have physically moved to Nightwing and Linux sees them as stock
`ov534`/`gspca` V4L2 cameras on `/dev/video2` and `/dev/video3`. A
repo-owned `tools/nightwing_v4l2_cadence_probe.py` copied to Nightwing