From c2e5d037642d465b7c1906b6a0d3911849b463b7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 10 Jun 2026 20:35:35 +0100 Subject: [PATCH] Add Rust Move sphere candidate tracker --- docs/implementation-plan.md | 6 + native/move_tracker/Cargo.lock | 7 + native/move_tracker/Cargo.toml | 10 + native/move_tracker/README.md | 23 ++ .../shaders/MoveSphereCandidate.comp.hlsl | 104 ++++++ native/move_tracker/src/lib.rs | 308 ++++++++++++++++++ notes/current-system-map.md | 8 + state/evidence.jsonl | 1 + state/map.yaml | 7 + 9 files changed, 474 insertions(+) create mode 100644 native/move_tracker/Cargo.lock create mode 100644 native/move_tracker/Cargo.toml create mode 100644 native/move_tracker/README.md create mode 100644 native/move_tracker/shaders/MoveSphereCandidate.comp.hlsl create mode 100644 native/move_tracker/src/lib.rs diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index db936a5..8c5b816 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -50,6 +50,12 @@ a named invariant that the native runtime cannot protect yet. owner. It launches `Mimir.PsMoveProbe` in steady hold mode so the locally attached `VID_054C/PID_03D5` Move sphere stays lit for calibration/tracking receipts. +- `native/move_tracker` is the first native/Rust Move optical candidate + extractor. Rust owns dispatch planning, FFI, validation, and a deterministic + CPU mirror; `shaders/MoveSphereCandidate.comp.hlsl` owns one 16px tile per + thread group and emits weighted bright-sphere candidates. This is marker + extraction only, not final pose, stereo triangulation, calibration, + association, IMU fusion, or prediction. - `MimirProcessStreamSource` for bridge/network command edges. - `MimirFrameEventProcessStreamSource` for temporary JSON-line frame metadata from native probes into the same rolling buffers Fensalir inspects. One probe diff --git a/native/move_tracker/Cargo.lock b/native/move_tracker/Cargo.lock new file mode 100644 index 0000000..8843462 --- /dev/null +++ b/native/move_tracker/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mimir-move-tracker" +version = "0.1.0" diff --git a/native/move_tracker/Cargo.toml b/native/move_tracker/Cargo.toml new file mode 100644 index 0000000..ff0060e --- /dev/null +++ b/native/move_tracker/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mimir-move-tracker" +version = "0.1.0" +edition = "2024" +description = "Rust orchestration and CPU mirror for PS Move sphere tracking compute" + +[lib] +crate-type = ["rlib", "cdylib", "staticlib"] + +[dependencies] diff --git a/native/move_tracker/README.md b/native/move_tracker/README.md new file mode 100644 index 0000000..cc9e954 --- /dev/null +++ b/native/move_tracker/README.md @@ -0,0 +1,23 @@ +# Mimir Move Tracker + +Rust owns orchestration for PS Move sphere tracking. The compute shader owns the +parallel image reduction stage. This crate deliberately stops at per-frame +marker candidates: final pose, stereo triangulation, IMU fusion, prediction, +and calibration remain separate owners. + +## Current Contract + +- Input: one luma frame (`Y8`) with width, height, stride, source hash, and frame + sequence. +- GPU stage: one 16px tile per thread group, emitting at most one candidate per + tile. +- Output: `MoveMarkerCandidate` records containing weighted centroid, radius, + area, peak/mean luma, and score. +- Rust mirror: deterministic CPU implementation and FFI wrappers for tests and + non-GPU smoke paths. +- Shader IDs are carried as `uint2` low/high pairs to avoid making 64-bit HLSL + integer support a dependency of the first extraction pass. + +This is the first stage required for PlayStation-class wand tracking. The +remaining stack needs calibrated camera rays, controlled sphere colors/exposure, +multi-camera association, triangulation, IMU fusion, and latency prediction. diff --git a/native/move_tracker/shaders/MoveSphereCandidate.comp.hlsl b/native/move_tracker/shaders/MoveSphereCandidate.comp.hlsl new file mode 100644 index 0000000..d00414f --- /dev/null +++ b/native/move_tracker/shaders/MoveSphereCandidate.comp.hlsl @@ -0,0 +1,104 @@ +// Mimir PS Move sphere candidate extraction. +// One thread group owns one image tile. The shader emits at most one bright +// marker candidate per 16px tile; Rust owns dispatch sizing and downstream ranking. + +struct MoveSphereCandidate +{ + uint2 sourceIdHash; + uint2 frameSequence; + uint tileX; + uint tileY; + float centerXPx; + float centerYPx; + float radiusPx; + uint areaPx; + float meanLuma; + uint peakLuma; + float score; +}; + +cbuffer MoveTrackerConfig : register(b0) +{ + uint width; + uint height; + uint strideBytes; + uint tileSize; + uint thresholdMin; + uint minAreaPx; + uint maxCandidates; + uint _pad0; + uint2 sourceIdHash; + uint2 frameSequence; +}; + +ByteAddressBuffer frameY8 : register(t0); +RWStructuredBuffer candidates : register(u0); +RWByteAddressBuffer candidateCounter : register(u1); + +groupshared uint gArea; +groupshared uint gSumX; +groupshared uint gSumY; +groupshared uint gSumW; +groupshared uint gPeak; + +[numthreads(16, 16, 1)] +void main(uint3 groupId : SV_GroupID, uint3 groupThreadId : SV_GroupThreadID) +{ + if (groupThreadId.x == 0 && groupThreadId.y == 0) + { + gArea = 0; + gSumX = 0; + gSumY = 0; + gSumW = 0; + gPeak = 0; + } + GroupMemoryBarrierWithGroupSync(); + + uint x = groupId.x * tileSize + groupThreadId.x; + uint y = groupId.y * tileSize + groupThreadId.y; + if (groupThreadId.x < tileSize && groupThreadId.y < tileSize && x < width && y < height) + { + uint byteOffset = y * strideBytes + x; + uint packed = frameY8.Load(byteOffset & ~3u); + uint shift = (byteOffset & 3u) * 8u; + uint luma = (packed >> shift) & 0xffu; + if (luma >= thresholdMin) + { + uint weight = luma - thresholdMin + 1u; + InterlockedAdd(gArea, 1u); + InterlockedAdd(gSumX, x * weight); + InterlockedAdd(gSumY, y * weight); + InterlockedAdd(gSumW, weight); + InterlockedMax(gPeak, luma); + } + } + + GroupMemoryBarrierWithGroupSync(); + + if (groupThreadId.x == 0 && groupThreadId.y == 0 && gArea >= minAreaPx && gSumW > 0) + { + uint index; + candidateCounter.InterlockedAdd(0, 1, index); + if (index < maxCandidates) + { + float area = (float)gArea; + float tilePixels = (float)(tileSize * tileSize); + float areaScore = min(1.0, area / tilePixels); + float brightnessScore = (float)gPeak / 255.0; + + MoveSphereCandidate candidate; + candidate.sourceIdHash = sourceIdHash; + candidate.frameSequence = frameSequence; + candidate.tileX = groupId.x; + candidate.tileY = groupId.y; + candidate.centerXPx = (float)gSumX / (float)gSumW; + candidate.centerYPx = (float)gSumY / (float)gSumW; + candidate.radiusPx = sqrt(area / 3.14159265358979323846); + candidate.areaPx = gArea; + candidate.meanLuma = (float)thresholdMin + (((float)gSumW / area) - 1.0); + candidate.peakLuma = gPeak; + candidate.score = brightnessScore * (0.65 + areaScore * 0.35); + candidates[index] = candidate; + } + } +} diff --git a/native/move_tracker/src/lib.rs b/native/move_tracker/src/lib.rs new file mode 100644 index 0000000..cf59ba2 --- /dev/null +++ b/native/move_tracker/src/lib.rs @@ -0,0 +1,308 @@ +#![deny(unsafe_op_in_unsafe_fn)] + +use core::ffi::c_int; + +pub const MIMIR_MOVE_TRACKER_OK: c_int = 0; +pub const MIMIR_MOVE_TRACKER_NULL: c_int = -1; +pub const MIMIR_MOVE_TRACKER_INVALID: c_int = -2; + +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MoveTrackerConfig { + pub width: u32, + pub height: u32, + pub stride_bytes: u32, + pub tile_size: u32, + pub threshold_min: u8, + pub min_area_px: u32, + pub max_candidates: u32, + pub source_id_hash: u64, + pub frame_sequence: u64, +} + +impl Default for MoveTrackerConfig { + fn default() -> Self { + Self { + width: 320, + height: 240, + stride_bytes: 320, + tile_size: 16, + threshold_min: 180, + min_area_px: 4, + max_candidates: 64, + source_id_hash: 0, + frame_sequence: 0, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct MoveTrackerDispatchPlan { + pub group_size_x: u32, + pub group_size_y: u32, + pub groups_x: u32, + pub groups_y: u32, + pub max_candidates: u32, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct MoveMarkerCandidate { + pub source_id_hash: u64, + pub frame_sequence: u64, + pub tile_x: u32, + pub tile_y: u32, + pub center_x_px: f32, + pub center_y_px: f32, + pub radius_px: f32, + pub area_px: u32, + pub mean_luma: f32, + pub peak_luma: u32, + pub score: f32, +} + +pub fn dispatch_plan(config: MoveTrackerConfig) -> Option { + if !config_is_valid(config) { + return None; + } + + let tile = config.tile_size; + Some(MoveTrackerDispatchPlan { + group_size_x: tile, + group_size_y: tile, + groups_x: config.width.div_ceil(tile), + groups_y: config.height.div_ceil(tile), + max_candidates: config.max_candidates, + }) +} + +pub fn extract_luma_candidates( + frame: &[u8], + config: MoveTrackerConfig, +) -> Option> { + if !config_is_valid(config) { + return None; + } + + let required = required_frame_bytes(config)?; + if frame.len() < required { + return None; + } + + let mut candidates = Vec::new(); + let tile = config.tile_size; + let groups_x = config.width.div_ceil(tile); + let groups_y = config.height.div_ceil(tile); + + for tile_y in 0..groups_y { + for tile_x in 0..groups_x { + if candidates.len() >= config.max_candidates as usize { + return Some(candidates); + } + + if let Some(candidate) = extract_tile(frame, config, tile_x, tile_y) { + candidates.push(candidate); + } + } + } + + candidates.sort_by(|left, right| { + right + .score + .partial_cmp(&left.score) + .unwrap_or(core::cmp::Ordering::Equal) + }); + Some(candidates) +} + +fn extract_tile( + frame: &[u8], + config: MoveTrackerConfig, + tile_x: u32, + tile_y: u32, +) -> Option { + let x0 = tile_x * config.tile_size; + let y0 = tile_y * config.tile_size; + let x1 = (x0 + config.tile_size).min(config.width); + let y1 = (y0 + config.tile_size).min(config.height); + let threshold = u32::from(config.threshold_min); + let mut area = 0u32; + let mut sum_x = 0u64; + let mut sum_y = 0u64; + let mut sum_w = 0u64; + let mut peak = 0u32; + + for y in y0..y1 { + let row = y as usize * config.stride_bytes as usize; + for x in x0..x1 { + let luma = u32::from(frame[row + x as usize]); + if luma < threshold { + continue; + } + + area += 1; + peak = peak.max(luma); + let weight = u64::from(luma - threshold + 1); + sum_x += u64::from(x) * weight; + sum_y += u64::from(y) * weight; + sum_w += weight; + } + } + + if area < config.min_area_px || sum_w == 0 { + return None; + } + + let center_x = sum_x as f32 / sum_w as f32; + let center_y = sum_y as f32 / sum_w as f32; + let radius = (area as f32 / core::f32::consts::PI).sqrt(); + let mean_luma = threshold as f32 + ((sum_w as f32 / area as f32) - 1.0); + let area_score = (area as f32 / (config.tile_size * config.tile_size) as f32).min(1.0); + let brightness_score = peak as f32 / 255.0; + + Some(MoveMarkerCandidate { + source_id_hash: config.source_id_hash, + frame_sequence: config.frame_sequence, + tile_x, + tile_y, + center_x_px: center_x, + center_y_px: center_y, + radius_px: radius, + area_px: area, + mean_luma, + peak_luma: peak, + score: brightness_score * (0.65 + area_score * 0.35), + }) +} + +fn config_is_valid(config: MoveTrackerConfig) -> bool { + config.width > 0 + && config.height > 0 + && config.stride_bytes >= config.width + && config.tile_size == 16 + && config.min_area_px > 0 + && config.max_candidates > 0 +} + +fn required_frame_bytes(config: MoveTrackerConfig) -> Option { + let rows = usize::try_from(config.height).ok()?; + let stride = usize::try_from(config.stride_bytes).ok()?; + rows.checked_mul(stride) +} + +#[unsafe(no_mangle)] +pub extern "C" fn mimir_move_tracker_dispatch_plan( + config: MoveTrackerConfig, + out_plan: *mut MoveTrackerDispatchPlan, +) -> c_int { + if out_plan.is_null() { + return MIMIR_MOVE_TRACKER_NULL; + } + + let Some(plan) = dispatch_plan(config) else { + return MIMIR_MOVE_TRACKER_INVALID; + }; + + unsafe { + *out_plan = plan; + } + MIMIR_MOVE_TRACKER_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn mimir_move_tracker_extract_luma_cpu( + config: MoveTrackerConfig, + frame: *const u8, + frame_len: usize, + out_candidates: *mut MoveMarkerCandidate, + out_capacity: usize, + out_count: *mut usize, +) -> c_int { + if frame.is_null() || out_candidates.is_null() || out_count.is_null() { + return MIMIR_MOVE_TRACKER_NULL; + } + + let frame = unsafe { core::slice::from_raw_parts(frame, frame_len) }; + let Some(candidates) = extract_luma_candidates(frame, config) else { + return MIMIR_MOVE_TRACKER_INVALID; + }; + + let written = candidates.len().min(out_capacity); + unsafe { + core::ptr::copy_nonoverlapping(candidates.as_ptr(), out_candidates, written); + *out_count = written; + } + MIMIR_MOVE_TRACKER_OK +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dispatch_plan_covers_partial_tiles() { + let config = MoveTrackerConfig { + width: 33, + height: 17, + stride_bytes: 40, + tile_size: 16, + ..MoveTrackerConfig::default() + }; + + let plan = dispatch_plan(config).expect("valid plan"); + assert_eq!(3, plan.groups_x); + assert_eq!(2, plan.groups_y); + assert_eq!(16, plan.group_size_x); + assert_eq!(16, plan.group_size_y); + } + + #[test] + fn extracts_weighted_marker_candidate() { + let config = MoveTrackerConfig { + width: 32, + height: 24, + stride_bytes: 32, + tile_size: 16, + threshold_min: 180, + min_area_px: 3, + max_candidates: 8, + source_id_hash: 42, + frame_sequence: 7, + }; + let mut frame = vec![0u8; (config.stride_bytes * config.height) as usize]; + for (x, y, luma) in [ + (10u32, 8u32, 230u8), + (11, 8, 250), + (10, 9, 240), + (11, 9, 255), + ] { + frame[(y * config.stride_bytes + x) as usize] = luma; + } + + let candidates = extract_luma_candidates(&frame, config).expect("candidates"); + assert_eq!(1, candidates.len()); + let candidate = candidates[0]; + assert_eq!(42, candidate.source_id_hash); + assert_eq!(7, candidate.frame_sequence); + assert_eq!(0, candidate.tile_x); + assert_eq!(0, candidate.tile_y); + assert_eq!(4, candidate.area_px); + assert_eq!(255, candidate.peak_luma); + assert!(candidate.center_x_px > 10.4 && candidate.center_x_px < 10.7); + assert!(candidate.center_y_px > 8.4 && candidate.center_y_px < 8.7); + assert!(candidate.score > 0.65); + } + + #[test] + fn rejects_short_frames_and_empty_config() { + assert!(extract_luma_candidates(&[], MoveTrackerConfig::default()).is_none()); + assert!( + dispatch_plan(MoveTrackerConfig { + width: 0, + ..MoveTrackerConfig::default() + }) + .is_none() + ); + } +} diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 22b0d7a..afc0c7e 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -136,6 +136,14 @@ That launcher runs `Mimir.PsMoveProbe` against the local Windows HID col01 output collection and refreshes the PS Move LED report so the USB-attached Move stays visible as a tracking/calibration marker. +`native/move_tracker` owns the first native/GPU-shaped Move optical extraction +stage. Rust owns dispatch planning, FFI, configuration validation, and a +deterministic CPU mirror; `MoveSphereCandidate.comp.hlsl` owns the parallel +16px-tile luma reduction that emits ranked sphere candidates. This stage owns +per-frame marker candidates only. Stereo triangulation, camera calibration, +controller association, IMU fusion, prediction, and final 6DoF pose remain +separate later owners. + ## Program Composition [[docs/mimir-program-composition|Mimir Program Composition]] is the live diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 1eb8cac..35b69a7 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -126,3 +126,4 @@ {"ts": "2026-06-10T16:41:42+01:00", "type": "implementation", "status": "ok", "note": "Added typed PS Move tracking observations as Mimir runtime streams: MimirStreamKind.Tracking, mimir.move_tracking_observation.v1, local Starfire and remote Nightwing Move capture profiles, Muninn producer identity, Odin discovery provenance, runtime buffer readout, and BufferSmoke proofs for Starfire/Nightwing USB-attached Moves entering Mimir buffers."} {"ts": "2026-06-10T17:38:22+01:00", "type": "verification", "status": "ok", "note": "Restored and launched the narrow Nightwing Move tracking witness path. scripts/start-nightwing-move-tracking.ps1 started Mimir.EveSensorReceiver on Starfire port 8796, staged Nightwing Eye/Move Python tools, launched Nightwing worker PID 1131578 with --move-light move-usb=/dev/hidraw1:#35ff6c, and VerseRecorder C:\\Users\\Meta\\Videos\\Mimir\\VerseCaptures\\20260610-173549 recorded mimir.psmove_light_state.v1 HID success plus mimir.move_controller_observation_state.v1 blobs from both ps3-eye-0 and ps3-eye-1."} {"ts": "2026-06-10T18:11:18+01:00", "type": "verification", "status": "ok", "note": "Started the Starfire-local USB Move light owner. Windows sees Sony VID_054C/PID_03D5 with col01 output reports; scripts/start-starfire-move-light.ps1 launched Mimir.PsMoveProbe PID 27344 with --rgb #35ff6c --hold-seconds 0 --refresh-ms 250, and artifacts/runtime/starfire-move-light-20260610-180943/move-light-events.jsonl shows repeated mimir.psmove_usb_pulse_event.v1 hold writes succeeding with ok rgb=#35FF6C bytes=49."} +{"ts": "2026-06-10T20:33:54+01:00", "type": "implementation", "status": "ok", "note": "Added native/move_tracker as Rust-orchestrated PS Move optical candidate extraction: Rust owns dispatch/FFI/CPU mirror; HLSL owns 16px-tile luma reduction. Scope is marker candidates only, before calibration/triangulation/IMU fusion/prediction."} diff --git a/state/map.yaml b/state/map.yaml index 23e3099..0508890 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -91,6 +91,13 @@ accepted_design: blob extraction, and the stream currently emits `mimir.psmove_light_state.v1` plus `mimir.move_controller_observation_state.v1` optical witness records before later fusion into full tracking poses. + - `native/move_tracker` owns the first Rust/native Move optical candidate + stage: Rust validates config, plans dispatch, exposes FFI, and mirrors the + compute path on CPU; `MoveSphereCandidate.comp.hlsl` reduces one 16px luma + tile per thread group into weighted sphere candidates. This stage emits + marker candidates only. Calibration, multi-camera association, + triangulation, IMU fusion, prediction, and final 6DoF pose are still + separate owners. - `MimirFrameEventProcessStreamSource` is a temporary diagnostic adapter for JSON-line video frame and audio block metadata from native probes into runtime rolling buffers; it does not carry pixels/audio payloads or own the