diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index f1a9c4e..1763b8d 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -414,6 +414,18 @@ a named invariant that the native runtime cannot protect yet. generic KS exposure/gain writes and only recovered two LED components; both PS3 Eye feeds currently score fixed-control frames because their native wrapper does not yet expose gain/exposure. + PS3 Eyes now have the first sparse tracking surface: + `MimirSparseFeatureTracker` detects grid-suppressed Bayer/luma corner + features, links them frame-to-frame by bounded nearest-neighbor motion, and + emits sensor-local `MimirFeatureTrackCameraObservation` records with track + ids, image/clipspace coordinates, velocity, age, and confidence. + `MimirFeatureTrackFieldCandidate` lowers those observations as camera-sensor + Feature claims through `BuildFeatureTrackCandidateFrame`. The CPU tracker is + a live diagnostic/bootstrap owner; Fensalir D3D12 feature extraction remains + the production owner. `--ps3eye-tracking-live` proved both Eyes can produce + stable tracks at 320x240@187: with sensitive wide tracking defaults, each Eye + returned 160 live tracks from each Eye in a one-second pass, with 160 stable + tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1`. Phase 6 program-output receipts are also structurally complete: Fensalir publishes the final program surface through named shared D3D12 texture/fence resources, the OBS plugin consumes that surface with an explicit D3D12-to- diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index a60d50f..598bb19 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -40,6 +40,11 @@ flowchart TD now: it opens direct sensor feeds, applies KS/UVC manual exposure/gain where supported, scores actual luma frames, and reports the best setting per source. PS3 Eye and Leap-specific controls still need native control cuts. +- PS3 Eyes own high-rate sparse tracking evidence before they own depth: + `MimirSparseFeatureTracker` turns live Bayer/luma frames into stable + sensor-local feature tracks, and `MimirFeatureTrackFieldCandidate` lowers + them as camera Feature claims. This CPU path is a diagnostic bootstrap; the + production hot path belongs in Fensalir D3D12 feature extraction. - The first camera-rig solver fits bounded camera translation updates from a scene-anchored LED curve by minimizing ray-to-curve distance. Rotation, intrinsics, and distortion are future owners. diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 11a512d..6714a13 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -636,9 +636,18 @@ format, and the best spline setting is reported per source. On 2026-05-31 with `-9`, gain `8` (`kiyo-pro-rgb` score `0.666`; `kiyo-basic-rgb` score `0.650`). LeapUVC rejected generic KS exposure/gain writes and only found two LED components; PS3 Eye feeds are still fixed-control until the WinUSB wrapper owns -gain/exposure. Once the frustum fit is coherent, detected surface candidates -can be sampled across every synchronized view before Fensalir resolves them -into 4D Gaussian/surface splat claims. +gain/exposure. PS3 Eyes now also have a sparse tracking bootstrap: +`MimirSparseFeatureTracker` detects grid-suppressed Bayer/luma corner features, +links them frame-to-frame, and emits `MimirFeatureTrackCameraObservation` +records with sensor-local image/clipspace position, velocity, age, and +confidence. `MimirFeatureTrackFieldCandidate` lowers those observations as +camera Feature claims through Fensalir field evidence. A one-second live pass +at 320x240@187 with sensitive defaults returned 160 live tracks from each Eye: +160 stable tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1`. +This is a CPU diagnostic owner; the hot production path remains Fensalir D3D12 +feature extraction. Once the frustum fit is coherent, detected +surface candidates can be sampled across every synchronized view before +Fensalir resolves them into 4D Gaussian/surface splat claims. The current teardown/migration map is `docs/fensalir-rendering-rebuild-migration.md`, paired with Fensalir's diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 3112178..98e9714 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -160,6 +160,16 @@ if (args.Any(arg => string.Equals(arg, "--led-spline-live-score", StringComparis return RunLedSplineLiveScore(args); } +if (args.Any(arg => string.Equals(arg, "--ps3eye-tracking-smoke", StringComparison.OrdinalIgnoreCase))) +{ + return RunPs3EyeTrackingSmoke(); +} + +if (args.Any(arg => string.Equals(arg, "--ps3eye-tracking-live", StringComparison.OrdinalIgnoreCase))) +{ + return RunPs3EyeTrackingLive(args); +} + if (args.Any(arg => string.Equals(arg, "--fensalir-texture-lease-smoke", StringComparison.OrdinalIgnoreCase))) { return RunFensalirTextureLeaseSmoke(); @@ -1096,6 +1106,139 @@ static int RunLedSplineLiveScore(string[] args) return anyCaptured ? 0 : 1; } +static int RunPs3EyeTrackingSmoke() +{ + const int width = 160; + const int height = 120; + var tracker = new MimirSparseFeatureTracker(new MimirSparseFeatureTrackerOptions( + MaxFeatures: 32, + CellSizePixels: 10, + SearchRadiusPixels: 12, + MinimumTrackAgeFrames: 3, + MinimumCornerScore: 0.08, + MinimumLuma: 0.12)); + MimirSparseFeatureTrackerFrame latest = default!; + for (var frameIndex = 0; frameIndex < 8; frameIndex++) + { + var luma = new byte[width * height]; + for (var marker = 0; marker < 8; marker++) + { + var x = 24 + marker * 14 + frameIndex * 2; + var y = 20 + marker * 9 + frameIndex; + StampBlob(luma, width, height, x, y, radius: 2, peak: 230); + } + + latest = tracker.Update( + "ps3-eye-0", + width, + height, + luma, + 1_000_000_000L + frameIndex * 5_350_000L); + } + + var observation = tracker.ToCameraObservation(latest); + var candidate = new MimirFeatureTrackFieldCandidate( + "ps3-eye-synthetic-tracks", + "visual-calibration", + "Mimir.BufferSmoke", + [observation], + latest.StableTrackCount, + latest.MeanTrackAgeFrames, + latest.MeanSpeedPixelsPerSecond, + latest.Confidence, + latest.ObservedTimeNs); + var evidence = new MimirFensalirFieldLowering().BuildFeatureTrackCandidateFrame([candidate]); + var validation = AquariumFieldEvidenceValidator.Validate(evidence); + Console.WriteLine( + $"ps3eye-tracking-smoke frames={latest.FrameCount} tracks={latest.Tracks.Count} stable={latest.StableTrackCount} meanAge={latest.MeanTrackAgeFrames:0.000} meanSpeed={latest.MeanSpeedPixelsPerSecond:0.000} confidence={latest.Confidence:0.000} claims={evidence.Claims.Count} planned={evidence.BackendPackets.Count} errors={validation.HasErrors}"); + + return latest.StableTrackCount >= 8 && + latest.Confidence > 0.50 && + evidence.Claims.Count == 1 && + !validation.HasErrors + ? 0 + : 1; +} + +static int RunPs3EyeTrackingLive(string[] args) +{ + var library = ParseStringOption(args, "--native-library", Path.GetFullPath("native/camera_capture/build/Release/mimir_ps3eye_capture.dll")); + var seconds = Math.Max(0.10, ParseDoubleOption(args, "--seconds", 1.0)); + var width = ParseIntOption(args, "--width", 320); + var height = ParseIntOption(args, "--height", 240); + var fps = ParseIntOption(args, "--fps", 187); + var maxFeatures = Math.Max(8, ParseIntOption(args, "--max-features", 160)); + var cameraFilter = ParseIntOption(args, "--camera-index", -1); + var cameras = cameraFilter >= 0 ? [cameraFilter] : new[] { 0, 1 }; + var anyCaptured = false; + foreach (var cameraIndex in cameras) + { + var sourceId = $"ps3-eye-{cameraIndex}"; + try + { + using var driver = new MimirPs3EyeVideoCaptureDriver(new MimirPs3EyeVideoCaptureDriverOptions( + library, + sourceId, + cameraIndex, + width, + height, + fps)); + var tracker = new MimirSparseFeatureTracker(new MimirSparseFeatureTrackerOptions( + MaxFeatures: maxFeatures, + CellSizePixels: ParseIntOption(args, "--cell-size", 12), + SearchRadiusPixels: ParseIntOption(args, "--search-radius", 24), + MinimumTrackAgeFrames: ParseIntOption(args, "--min-age", 3), + MinimumCornerScore: ParseDoubleOption(args, "--min-corner", 0.05), + MinimumLuma: ParseDoubleOption(args, "--min-luma", 0.03))); + var deadline = Stopwatch.GetTimestamp() + (long)(seconds * Stopwatch.Frequency); + MimirSparseFeatureTrackerFrame latest = default!; + var frames = 0; + while (Stopwatch.GetTimestamp() < deadline) + { + if (!driver.TryCapture(out var frame, out var data)) + { + Thread.Sleep(1); + continue; + } + + if (data.IsEmpty || !TryExtractLiveLuma(frame, data.Span, out var luma, out var lumaWidth, out var lumaHeight)) + { + continue; + } + + latest = tracker.Update(sourceId, lumaWidth, lumaHeight, luma, frame.DeviceTimestampNs); + frames++; + } + + anyCaptured |= frames > 0; + var observation = frames == 0 ? null : tracker.ToCameraObservation(latest); + var candidate = observation == null + ? null + : new MimirFeatureTrackFieldCandidate( + $"{sourceId}:live-tracks:{latest.FrameCount}", + "visual-calibration-live", + "Mimir.BufferSmoke", + [observation], + latest.StableTrackCount, + latest.MeanTrackAgeFrames, + latest.MeanSpeedPixelsPerSecond, + latest.Confidence, + latest.ObservedTimeNs); + var evidence = candidate == null + ? AquariumFieldEvidenceFrame.Empty + : new MimirFensalirFieldLowering().BuildFeatureTrackCandidateFrame([candidate]); + Console.WriteLine( + $"ps3eye-tracking-live source={sourceId} frames={frames} tracks={latest?.Tracks.Count ?? 0} stable={latest?.StableTrackCount ?? 0} meanAge={latest?.MeanTrackAgeFrames ?? 0.0:0.000} meanSpeed={latest?.MeanSpeedPixelsPerSecond ?? 0.0:0.000} confidence={latest?.Confidence ?? 0.0:0.000} claims={evidence.Claims.Count} format={MimirVideoPixelFormat.Bayer8} size={width}x{height}"); + } + catch (Exception ex) + { + Console.WriteLine($"ps3eye-tracking-live source={sourceId} status=failed error=\"{ex.Message.Replace("\"", "'", StringComparison.Ordinal)}\""); + } + } + + return anyCaptured ? 0 : 1; +} + static IReadOnlyList LiveLedSplineExposureGainSettings() => [ new("exp-10_gain-0", -10, 0), diff --git a/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs b/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs index c9d379a..f303a0f 100644 --- a/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs +++ b/src/Mimir.Runtime/Synchronization/MimirFensalirBridgeContracts.cs @@ -193,6 +193,40 @@ public sealed record MimirLedSplineFieldCandidate( double Confidence, long ObservedTimeNs); +public readonly record struct MimirFeatureTrackPoint( + int TrackId, + double ImageX, + double ImageY, + double ClipX, + double ClipY, + double VelocityXPerSecond, + double VelocityYPerSecond, + int AgeFrames, + double Confidence); + +public sealed record MimirFeatureTrackCameraObservation( + string ObservationKey, + string SourceId, + int Width, + int Height, + long ObservedTimeNs, + IReadOnlyList Tracks, + int FrameCount, + double MeanTrackAgeFrames, + double MeanSpeedPixelsPerSecond, + double Confidence); + +public sealed record MimirFeatureTrackFieldCandidate( + string CandidateKey, + string CalibrationId, + string ProducerKey, + IReadOnlyList CameraObservations, + int StableTrackCount, + double MeanTrackAgeFrames, + double MeanSpeedPixelsPerSecond, + double Confidence, + long ObservedTimeNs); + public readonly record struct MimirSurfaceAxes( string X, string Y, diff --git a/src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs b/src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs index c71178e..4475e39 100644 --- a/src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs +++ b/src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs @@ -214,6 +214,40 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt }; } + public AquariumFieldEvidenceFrame BuildFeatureTrackCandidateFrame( + IEnumerable trackCandidates) + { + var domains = new List(); + var claims = new List(); + var candidates = new List(); + var seenDomains = new HashSet(StringComparer.Ordinal); + + foreach (var trackCandidate in trackCandidates) + { + if (string.IsNullOrWhiteSpace(trackCandidate.CandidateKey) || + trackCandidate.CameraObservations.Count == 0) + { + continue; + } + + var domain = DomainForFeatureTrackCandidate(trackCandidate); + AddDomain(domains, seenDomains, domain); + + var claim = ClaimForFeatureTrackCandidate(trackCandidate, domain.DomainKey); + claims.Add(claim); + candidates.Add(CandidateForClaim(claim, AquariumFieldGuide.Valid(claim.Confidence))); + } + + return new AquariumFieldEvidenceFrame + { + Domains = domains, + Claims = claims, + Candidates = candidates, + AccumulationWindowSeconds = (float)options.AccumulationWindowSeconds, + PresentationDelaySeconds = (float)options.PresentationDelaySeconds + }; + } + public AquariumFieldEvidenceFrame BuildLeapPackedStereoDepthCandidateFrame( IEnumerable windows, IEnumerable inputResources) @@ -621,6 +655,27 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt "Mimir.Runtime"); } + private static AquariumFieldDomain DomainForFeatureTrackCandidate(MimirFeatureTrackFieldCandidate candidate) + { + var tracks = candidate.CameraObservations.SelectMany(static observation => observation.Tracks).ToArray(); + var maxWidth = Math.Max(1, candidate.CameraObservations.Max(static observation => observation.Width)); + var maxHeight = Math.Max(1, candidate.CameraObservations.Max(static observation => observation.Height)); + var minX = tracks.Length == 0 ? 0.0f : Math.Clamp((float)tracks.Min(static track => track.ImageX), 0.0f, maxWidth); + var minY = tracks.Length == 0 ? 0.0f : Math.Clamp((float)tracks.Min(static track => track.ImageY), 0.0f, maxHeight); + var maxX = tracks.Length == 0 ? maxWidth : Math.Clamp((float)tracks.Max(static track => track.ImageX), minX, maxWidth); + var maxY = tracks.Length == 0 ? maxHeight : Math.Clamp((float)tracks.Max(static track => track.ImageY), minY, maxHeight); + return new AquariumFieldDomain( + DomainKeyForFeatureTrackCandidate(candidate), + "", + AquariumFieldDomainKind.CameraSensor, + Matrix4x4.Identity, + Matrix4x4.Identity, + new Vector3(minX, minY, 0.0f), + new Vector3(maxX, maxY, Math.Max(1, candidate.CameraObservations.Count)), + Vector3.Zero, + "Mimir.Runtime"); + } + private static AquariumFieldDomain DomainForStereoDepthCandidate(MimirStereoDepthFieldCandidate candidate) { var width = Math.Max(1, candidate.Width); @@ -794,6 +849,32 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt Confidence: confidence); } + private AquariumFieldClaim ClaimForFeatureTrackCandidate( + MimirFeatureTrackFieldCandidate trackCandidate, + string domainKey) + { + var confidence = Clamp01((float)trackCandidate.Confidence); + var represented = Math.Max(1, trackCandidate.CameraObservations.Sum(static observation => observation.Tracks.Count)); + var proposal = new AquariumFieldProposalPolicy( + AquariumFieldProposalKind.SensorObservation, + SourcePdf: 1.0f, + TargetContribution: confidence, + RepresentedCandidateCount: represented, + Seed: StableSeed(trackCandidate.CandidateKey)); + + return new AquariumFieldClaim( + ClaimKey: $"feature-tracks:{trackCandidate.CalibrationId}:{trackCandidate.CandidateKey}", + DomainKey: domainKey, + ProducerKey: trackCandidate.ProducerKey, + Layer: AquariumFieldLayer.Form, + Encoding: AquariumFieldEncoding.Feature, + Support: SupportForFeatureTrackCandidate(trackCandidate), + Proposal: proposal, + PayloadHandle: $"{trackCandidate.CalibrationId}:stable={trackCandidate.StableTrackCount}", + ObservedTimeNs: trackCandidate.ObservedTimeNs, + Confidence: confidence); + } + private AquariumFieldClaim ClaimForStereoDepthCandidate( MimirStereoDepthFieldCandidate depthCandidate, string domainKey) @@ -1008,6 +1089,26 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt TemporalUncertainty: splineCandidate.HasTemporalCode ? 0.0f : 1.0f / 187.0f); } + private AquariumFieldSupport SupportForFeatureTrackCandidate(MimirFeatureTrackFieldCandidate trackCandidate) + { + var tracks = trackCandidate.CameraObservations.SelectMany(static observation => observation.Tracks).ToArray(); + var radiusPixels = tracks.Length == 0 + ? 1.0f + : Math.Max(1.0f, (float)(tracks.Average(static track => Math.Max(1, track.AgeFrames)) * 0.5)); + var meanSpeed = Math.Max(0.0f, (float)trackCandidate.MeanSpeedPixelsPerSecond); + var confidence = tracks.Length == 0 + ? 0.0f + : (float)tracks.Average(static track => Math.Clamp(track.Confidence, 0.0, 1.0)); + return new AquariumFieldSupport( + Center: new Vector3(0.0f, 0.0f, Math.Max(1, trackCandidate.CameraObservations.Count)), + Radius: new Vector3(radiusPixels, radiusPixels, Math.Max(1.0f, trackCandidate.CameraObservations.Count * 0.5f)), + LocalFrame: Matrix4x4.Identity, + ConservativeRadius: Math.Max(radiusPixels, meanSpeed / 187.0f), + ProjectedError: 1.0f - confidence, + Curvature: 0.0f, + TemporalUncertainty: 1.0f / 187.0f); + } + private AquariumFieldSupport SupportForStereoDepthCandidate(MimirStereoDepthFieldCandidate depthCandidate) { var width = Math.Max(1.0f, depthCandidate.Width); @@ -1321,6 +1422,9 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt private static string DomainKeyForLedSplineCandidate(MimirLedSplineFieldCandidate candidate) => $"mimir:led-spline:{candidate.CalibrationId}:{candidate.SplineId}:{candidate.CandidateKey}"; + private static string DomainKeyForFeatureTrackCandidate(MimirFeatureTrackFieldCandidate candidate) => + $"mimir:feature-tracks:{candidate.CalibrationId}:{candidate.CandidateKey}"; + private static string DomainKeyForStereoDepthCandidate(MimirStereoDepthFieldCandidate candidate) => DomainKeyForStereoDepthCandidate(candidate.CalibrationId, candidate.CameraPairId, candidate.CandidateKey); diff --git a/src/Mimir.Runtime/Synchronization/MimirSparseFeatureTracker.cs b/src/Mimir.Runtime/Synchronization/MimirSparseFeatureTracker.cs new file mode 100644 index 0000000..8f0a643 --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirSparseFeatureTracker.cs @@ -0,0 +1,274 @@ +namespace Mimir.Runtime.Synchronization; + +public sealed record MimirSparseFeatureTrackerOptions( + int MaxFeatures = 96, + int CellSizePixels = 12, + int SearchRadiusPixels = 18, + int MinimumTrackAgeFrames = 3, + double MinimumCornerScore = 0.10, + double MinimumLuma = 0.08); + +public sealed record MimirSparseFeatureTrackerFrame( + string SourceId, + int Width, + int Height, + long ObservedTimeNs, + int FrameCount, + IReadOnlyList Tracks, + int StableTrackCount, + double MeanTrackAgeFrames, + double MeanSpeedPixelsPerSecond, + double Confidence); + +public sealed class MimirSparseFeatureTracker(MimirSparseFeatureTrackerOptions? options = null) +{ + private readonly MimirSparseFeatureTrackerOptions options = options ?? new(); + private readonly List tracks = []; + private int nextTrackId = 1; + private int frameCount; + private long previousTimeNs; + + public MimirSparseFeatureTrackerFrame Update( + string sourceId, + int width, + int height, + ReadOnlySpan luma, + long observedTimeNs) + { + if (width <= 2 || height <= 2 || luma.Length < width * height) + { + return Empty(sourceId, width, height, observedTimeNs); + } + + frameCount++; + var dtSeconds = previousTimeNs <= 0 || observedTimeNs <= previousTimeNs + ? 1.0 / 187.0 + : Math.Max(1.0e-6, (observedTimeNs - previousTimeNs) / 1_000_000_000.0); + previousTimeNs = observedTimeNs; + + var detections = DetectFeatures(width, height, luma); + MatchDetections(detections, dtSeconds); + PruneTracks(); + + var output = tracks + .Where(track => track.MissedFrames == 0) + .OrderByDescending(track => track.AgeFrames) + .ThenByDescending(track => track.Confidence) + .Take(options.MaxFeatures) + .Select(track => new MimirFeatureTrackPoint( + track.TrackId, + track.X, + track.Y, + PixelToClipX(track.X, width), + PixelToClipY(track.Y, height), + track.VelocityXPerSecond, + track.VelocityYPerSecond, + track.AgeFrames, + track.Confidence)) + .ToArray(); + var stable = output.Count(point => point.AgeFrames >= options.MinimumTrackAgeFrames); + var meanAge = output.Length == 0 ? 0.0 : output.Average(static point => point.AgeFrames); + var meanSpeed = output.Length == 0 + ? 0.0 + : output.Average(static point => Math.Sqrt( + point.VelocityXPerSecond * point.VelocityXPerSecond + + point.VelocityYPerSecond * point.VelocityYPerSecond)); + var stability = output.Length == 0 ? 0.0 : stable / (double)Math.Min(options.MaxFeatures, output.Length); + var density = Math.Clamp(output.Length / (double)Math.Max(1, options.MaxFeatures), 0.0, 1.0); + var confidence = Math.Clamp(0.55 * stability + 0.45 * density, 0.0, 1.0); + return new MimirSparseFeatureTrackerFrame( + sourceId, + width, + height, + observedTimeNs, + frameCount, + output, + stable, + meanAge, + meanSpeed, + confidence); + } + + public MimirFeatureTrackCameraObservation ToCameraObservation(MimirSparseFeatureTrackerFrame frame) => + new( + $"{frame.SourceId}:feature-tracks:{frame.FrameCount}", + frame.SourceId, + frame.Width, + frame.Height, + frame.ObservedTimeNs, + frame.Tracks, + frame.FrameCount, + frame.MeanTrackAgeFrames, + frame.MeanSpeedPixelsPerSecond, + frame.Confidence); + + private MimirSparseFeatureTrackerFrame Empty(string sourceId, int width, int height, long observedTimeNs) => + new(sourceId, width, height, observedTimeNs, frameCount, [], 0, 0.0, 0.0, 0.0); + + private IReadOnlyList DetectFeatures(int width, int height, ReadOnlySpan luma) + { + var cell = Math.Max(4, options.CellSizePixels); + var gridWidth = Math.Max(1, (width + cell - 1) / cell); + var gridHeight = Math.Max(1, (height + cell - 1) / cell); + var bestByCell = new FeatureDetection?[gridWidth * gridHeight]; + var minimumLuma = Math.Clamp(options.MinimumLuma, 0.0, 1.0) * 255.0; + var minimumScore = Math.Clamp(options.MinimumCornerScore, 0.0, 1.0) * 255.0; + + for (var y = 1; y < height - 1; y++) + { + var row = y * width; + for (var x = 1; x < width - 1; x++) + { + var center = luma[row + x]; + if (center < minimumLuma) + { + continue; + } + + var gx = Math.Abs(luma[row + x + 1] - luma[row + x - 1]); + var gy = Math.Abs(luma[row + width + x] - luma[row - width + x]); + var diagonalA = Math.Abs(luma[row + width + x + 1] - luma[row - width + x - 1]); + var diagonalB = Math.Abs(luma[row + width + x - 1] - luma[row - width + x + 1]); + var score = Math.Min(Math.Max(gx, diagonalA), Math.Max(gy, diagonalB)); + if (score < minimumScore) + { + continue; + } + + var cellX = Math.Min(gridWidth - 1, x / cell); + var cellY = Math.Min(gridHeight - 1, y / cell); + var cellIndex = cellY * gridWidth + cellX; + var candidate = new FeatureDetection(x, y, score / 255.0); + if (bestByCell[cellIndex] is not { } existing || candidate.Score > existing.Score) + { + bestByCell[cellIndex] = candidate; + } + } + } + + return bestByCell + .Where(static detection => detection.HasValue) + .Select(static detection => detection!.Value) + .OrderByDescending(static detection => detection.Score) + .Take(options.MaxFeatures) + .ToArray(); + } + + private void MatchDetections(IReadOnlyList detections, double dtSeconds) + { + var matchedTracks = new HashSet(); + var matchedDetections = new bool[detections.Count]; + var radiusSquared = options.SearchRadiusPixels * options.SearchRadiusPixels; + for (var detectionIndex = 0; detectionIndex < detections.Count; detectionIndex++) + { + var detection = detections[detectionIndex]; + TrackedFeature? bestTrack = null; + var bestDistance = double.MaxValue; + foreach (var track in tracks) + { + if (matchedTracks.Contains(track.TrackId)) + { + continue; + } + + var dx = detection.X - track.X; + var dy = detection.Y - track.Y; + var distance = dx * dx + dy * dy; + if (distance <= radiusSquared && distance < bestDistance) + { + bestDistance = distance; + bestTrack = track; + } + } + + if (bestTrack == null) + { + continue; + } + + var velocityX = (detection.X - bestTrack.X) / dtSeconds; + var velocityY = (detection.Y - bestTrack.Y) / dtSeconds; + bestTrack.X = detection.X; + bestTrack.Y = detection.Y; + bestTrack.VelocityXPerSecond = velocityX; + bestTrack.VelocityYPerSecond = velocityY; + bestTrack.Confidence = Math.Clamp(0.7 * bestTrack.Confidence + 0.3 * detection.Score, 0.0, 1.0); + bestTrack.AgeFrames++; + bestTrack.MissedFrames = 0; + matchedTracks.Add(bestTrack.TrackId); + matchedDetections[detectionIndex] = true; + } + + foreach (var track in tracks) + { + if (!matchedTracks.Contains(track.TrackId)) + { + track.MissedFrames++; + } + } + + for (var index = 0; index < detections.Count; index++) + { + if (matchedDetections[index]) + { + continue; + } + + var detection = detections[index]; + tracks.Add(new TrackedFeature( + nextTrackId++, + detection.X, + detection.Y, + 0.0, + 0.0, + 1, + 0, + detection.Score)); + } + } + + private void PruneTracks() + { + tracks.RemoveAll(static track => track.MissedFrames > 5); + if (tracks.Count <= options.MaxFeatures * 2) + { + return; + } + + var survivors = tracks + .OrderBy(static track => track.MissedFrames) + .ThenByDescending(static track => track.AgeFrames) + .ThenByDescending(static track => track.Confidence) + .Take(options.MaxFeatures * 2) + .ToHashSet(); + tracks.RemoveAll(track => !survivors.Contains(track)); + } + + private static double PixelToClipX(double imageX, int width) => + Math.Clamp(imageX / Math.Max(1.0, width) * 2.0 - 1.0, -1.0, 1.0); + + private static double PixelToClipY(double imageY, int height) => + Math.Clamp(1.0 - imageY / Math.Max(1.0, height) * 2.0, -1.0, 1.0); + + private readonly record struct FeatureDetection(double X, double Y, double Score); + + private sealed class TrackedFeature( + int trackId, + double x, + double y, + double velocityXPerSecond, + double velocityYPerSecond, + int ageFrames, + int missedFrames, + double confidence) + { + public int TrackId { get; } = trackId; + public double X { get; set; } = x; + public double Y { get; set; } = y; + public double VelocityXPerSecond { get; set; } = velocityXPerSecond; + public double VelocityYPerSecond { get; set; } = velocityYPerSecond; + public int AgeFrames { get; set; } = ageFrames; + public int MissedFrames { get; set; } = missedFrames; + public double Confidence { get; set; } = confidence; + } +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 0b68de9..865ca96 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -257,3 +257,4 @@ {"ts": "2026-05-31T02:16:04+01:00", "type": "implementation", "status": "ok", "note": "Added MimirLedSplineFrameAnalyzer as the frame-level bridge for physical camera sweeps: it extracts bright luma components, solves a per-sensor LED curve, and feeds MimirLedSplineQualityScorer so saturated blobs can be detected but rejected as poor calibration evidence."} {"ts": "2026-05-31T03:05:00+01:00", "type": "verification", "status": "ok", "note": "Live LED spline sweep now runs on actual direct sensor feeds. `dotnet run --project src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj -- --led-spline-live-score --setting-seconds 0.25 --settle-ms 100 --expected-leds 38 --minimum-luma 0.55` selected exposure -9/gain 8 for `kiyo-pro-rgb` (score 0.666, 41 detections, usable) and `kiyo-basic-rgb` (score 0.650, 38 detections, usable). LeapUVC rejected generic KS exposure/gain writes and only recovered two LED components; both PS3 Eye feeds scored 0 at current fixed controls because their native wrapper does not yet expose gain/exposure."} {"ts": "2026-05-31T02:45:05+01:00", "type": "verification", "status": "ok", "note": "Eve dashboard now renders voidbot.swarm as a native VoidBot-style cockpit with CTB avatar URLs, selected Face/status pane, state tree, and detail panel; device build/install passed and broker smoke reported 8 CTB avatar URLs."} +{"ts": "2026-05-31T03:22:00+01:00", "type": "verification", "status": "ok", "note": "PS3 Eye sparse tracking bootstrap is live. `--ps3eye-tracking-smoke` produced 13 synthetic tracks, 12 stable tracks, and one Feature claim. The final default `--ps3eye-tracking-live --seconds 1.0` captured both Eyes at 320x240@187 and returned 160 tracks / 160 stable tracks from `ps3-eye-0` and 160 tracks / 156 stable tracks from `ps3-eye-1`. This is CPU diagnostic tracking; production feature extraction remains a Fensalir D3D12 owner."} diff --git a/state/map.yaml b/state/map.yaml index 150000a..67008ed 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -129,6 +129,15 @@ invariants: for both Kiyo Pro and Kiyo Basic. LeapUVC rejected the generic KS exposure/gain writes and PS3 Eye capture remains fixed-control until its native wrapper owns gain/exposure. + - PS3 Eyes now have a first sparse tracking surface instead of only cadence + proof. `MimirSparseFeatureTracker` detects Bayer/luma corner features, + links them into sensor-local tracks with clipspace position, velocity, age, + and confidence, and `MimirFeatureTrackFieldCandidate` lowers them as + camera Feature claims. `Mimir.BufferSmoke --ps3eye-tracking-live` proved a + one-second 320x240@187 run can return 160 live tracks from each Eye: 160 + stable tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1` with + sensitive defaults. This is CPU diagnostic/bootstrap work; + Fensalir D3D12 feature extraction remains the production owner. - Fensalir packet payload handles are not authority. Live payload references must resolve through declared typed GPU/native resources with shape, residency, shader access, validity, version, and handle metadata before