From a4b44734996e1043d26069c50643f908ade2c6d8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 18 May 2026 22:55:13 +0100 Subject: [PATCH] Prioritize Leap evidence in LOD cache --- docs/implementation-plan.md | 8 +- localcast/sensor_fusion/__init__.py | 8 ++ localcast/sensor_fusion/stochastic_mapping.py | 92 +++++++++++++++++++ notes/current-system-map.md | 6 +- scripts/live_sensor_fusion.py | 64 +++++++++---- state/map.yaml | 14 ++- tests/test_stochastic_mapping.py | 15 +++ 7 files changed, 183 insertions(+), 24 deletions(-) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 923ad35..32c6fe8 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -28,6 +28,7 @@ - `localcast.sensor_fusion.cultcache_docs` stores live visual state as typed CultCache MessagePack documents. - `localcast.sensor_fusion.spout_output` renders render-frame packets into a GPU texture and publishes it as a named Spout sender for OBS. - `scripts/live_sensor_fusion.py` writes fused render frames into `calibration/runs/visual-state.msgpack`. + - The live visual producer writes a multi-LOD scene cache with source kind and priority. Real Leap frames are promoted as the highest-priority visual timing/spatial evidence; Leap fallback frames remain lower-priority diagnostics. - `scripts/stream_spout.py` runs the deadline Spout sender loop from the typed cache with typed and JSON heartbeat status. - `docs/obs-spout-streaming.md` documents OBS setup and the Aquarium replacement boundary. - `docs/typed-visual-state.md` documents the CultCache/CultNet visual boundary. @@ -70,6 +71,7 @@ 11. Move the render-frame consumer into Aquarium Engine so dense brush/splat rendering replaces the deadline OpenGL point sink behind the same OBS Spout boundary. 12. Capture fixed-board ChArUco observations for every camera, solve `config/sensor-fusion.json`, and verify cross-view feature triangulation with `scripts/triangulate_surface_features.py`. 13. Replace synthetic live-fusion observations with PS3 Eye detector observations and calibrated surface-feature tracks. -14. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present. -15. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video. -16. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests. +14. Fix the live Leap capture backend so actual Leap frames, not fallback frames, populate `leap-ground-truth` cells in the multi-LOD cache. +15. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present. +16. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video. +17. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests. diff --git a/localcast/sensor_fusion/__init__.py b/localcast/sensor_fusion/__init__.py index e36564c..6538da4 100644 --- a/localcast/sensor_fusion/__init__.py +++ b/localcast/sensor_fusion/__init__.py @@ -40,11 +40,15 @@ from .surface_features import ( from .stochastic_mapping import ( ImageRayBiasMap, LODCell, + LODEvidencePoint, MultiLODSceneCache, RayBiasSample, SceneRay, TransientMatchHypothesis, + evidence_from_fusion_items, + evidence_from_render_points, multilod_cache_from_points, + multilod_cache_from_evidence, pixel_scene_ray, stochastic_transient_matches, ) @@ -103,6 +107,7 @@ __all__ = [ "SurfaceFeatureTrack", "ImageRayBiasMap", "LODCell", + "LODEvidencePoint", "MultiLODSceneCache", "RayBiasSample", "SceneRay", @@ -134,7 +139,10 @@ __all__ = [ "match_surface_features", "measure_frame_quality", "orb_surface_observations", + "evidence_from_fusion_items", + "evidence_from_render_points", "multilod_cache_from_points", + "multilod_cache_from_evidence", "overlay_audio_events", "pixel_scene_ray", "stochastic_transient_matches", diff --git a/localcast/sensor_fusion/stochastic_mapping.py b/localcast/sensor_fusion/stochastic_mapping.py index 754d8d3..8c2b3ba 100644 --- a/localcast/sensor_fusion/stochastic_mapping.py +++ b/localcast/sensor_fusion/stochastic_mapping.py @@ -95,6 +95,8 @@ class LODCell: count: int source_time_min_ns: int source_time_max_ns: int + source_priority: float = 1.0 + source_kind: str = "generic" @dataclass(frozen=True) @@ -119,6 +121,8 @@ class MultiLODSceneCache: "count": cell.count, "sourceTimeMinNs": cell.source_time_min_ns, "sourceTimeMaxNs": cell.source_time_max_ns, + "sourcePriority": cell.source_priority, + "sourceKind": cell.source_kind, } for cell in self.cells ], @@ -304,6 +308,94 @@ def multilod_cache_from_points( ) +@dataclass(frozen=True) +class LODEvidencePoint: + stable_key: str + xyz: np.ndarray + confidence: float + timestamp_ns: int + source_priority: float + source_kind: str + + +def multilod_cache_from_evidence( + evidence: Iterable[LODEvidencePoint], + *, + levels: tuple[float, ...] = (0.02, 0.08, 0.32), + created_monotonic_ns: int = 0, +) -> MultiLODSceneCache: + items = list(evidence) + cells: list[LODCell] = [] + for level_index, cell_size in enumerate(levels): + buckets: dict[tuple[int, int, int], list[LODEvidencePoint]] = {} + for item in items: + key = tuple(int(np.floor(float(value) / float(cell_size))) for value in np.asarray(item.xyz, dtype=np.float64)) + buckets.setdefault(key, []).append(item) + for key, bucket in buckets.items(): + weights = np.asarray([max(1.0e-6, item.confidence * item.source_priority) for item in bucket], dtype=np.float64) + positions = np.asarray([np.asarray(item.xyz, dtype=np.float64) for item in bucket], dtype=np.float64) + center = np.average(positions, axis=0, weights=weights) + priority = float(max(item.source_priority for item in bucket)) + source_kind = max(bucket, key=lambda item: item.source_priority).source_kind + cells.append( + LODCell( + level=level_index, + cell=key, + center=tuple(float(value) for value in center), + radius_m=float(cell_size) * 0.5, + confidence=float(min(1.0, np.mean(weights))), + count=len(bucket), + source_time_min_ns=min(item.timestamp_ns for item in bucket), + source_time_max_ns=max(item.timestamp_ns for item in bucket), + source_priority=priority, + source_kind=source_kind, + ) + ) + return MultiLODSceneCache( + schema="localcast.sensor_fusion.multilod_scene_cache.v1", + created_monotonic_ns=int(created_monotonic_ns), + levels=tuple(float(level) for level in levels), + cells=tuple(cells), + ) + + +def evidence_from_render_points(points: Iterable, *, source_kind: str, source_priority: float) -> tuple[LODEvidencePoint, ...]: + rows: list[LODEvidencePoint] = [] + for point in points: + rows.append( + LODEvidencePoint( + stable_key=str(point.stable_key), + xyz=np.asarray(point.xyz, dtype=np.float64), + confidence=float(point.confidence), + timestamp_ns=int(point.source_timestamp_ns), + source_priority=float(source_priority), + source_kind=str(source_kind), + ) + ) + return tuple(rows) + + +def evidence_from_fusion_items( + items: Iterable[TriangulatedPoint | TransientMatchHypothesis], + *, + source_kind: str, + source_priority: float, +) -> tuple[LODEvidencePoint, ...]: + rows: list[LODEvidencePoint] = [] + for item in items: + rows.append( + LODEvidencePoint( + stable_key=str(getattr(item, "marker_id", getattr(item, "stable_key", "scene-claim"))), + xyz=np.asarray(item.xyz, dtype=np.float64), + confidence=float(item.confidence), + timestamp_ns=item_timestamp_ns(item), + source_priority=float(source_priority), + source_kind=str(source_kind), + ) + ) + return tuple(rows) + + def item_timestamp_ns(item: TriangulatedPoint | TransientMatchHypothesis) -> int: if hasattr(item, "timestamp_ns"): return int(getattr(item, "timestamp_ns")) diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 5fbdca9..a34ac52 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -91,16 +91,20 @@ flowchart TD E --> F["ScreenBrushPacket lowering"] F --> G["Spout sender"] G --> H["OBS Spout2 Capture"] + I["Leap motion evidence"] --> J["multi-LOD scene cache"] + K["RGB/stochastic/synthetic evidence"] --> J + J --> L["Aquarium compute reconciliation"] ``` Ownership: - `localcast.sensor_fusion.cultcache_docs` owns typed visual state documents. - `scripts/live_sensor_fusion.py` currently owns the live producer and writes `localcast.visual.render_frame` into CultCache. +- The live producer also writes a multi-LOD scene cache with source kind and priority. Real Leap frames are the highest-priority timing/spatial evidence; Leap fallback frames are tagged as fallback and do not become ground truth. - `scripts/stream_spout.py` consumes typed CultCache state and publishes Spout. - CultNet document replication is the intended API boundary for other producers/consumers. -Invariant: JSON is not the visual-state authority. The current renderer can still write a JSON heartbeat for human inspection, but live visual state lives in typed CultCache MessagePack documents. +Invariant: JSON is not the visual-state authority. The current renderer can still write a JSON heartbeat for human inspection, but live visual state lives in typed CultCache MessagePack documents. Leap outranks camera surfaces only when the frame source is actual Leap capture; diagnostics must stay labeled as diagnostics. ## OBS Program Surface diff --git a/scripts/live_sensor_fusion.py b/scripts/live_sensor_fusion.py index 9de860f..36e0a9e 100644 --- a/scripts/live_sensor_fusion.py +++ b/scripts/live_sensor_fusion.py @@ -21,8 +21,10 @@ from localcast.sensor_fusion import ( SensorRig, SurfaceFeatureObservation, dense_stereo_points, + evidence_from_fusion_items, + evidence_from_render_points, measure_frame_quality, - multilod_cache_from_points, + multilod_cache_from_evidence, put_live_render_frame, lower_points_to_render_frame, stochastic_transient_matches, @@ -599,11 +601,17 @@ class LeapPackedMotionSampler: self._capture = None self._previous_channels: dict[str, np.ndarray] = {} self._last_timestamp_ns: int | None = None + self._last_frame_kind: str | None = None + self._fallback_index = 0 @property def last_timestamp_ns(self) -> int | None: return self._last_timestamp_ns + @property + def last_frame_kind(self) -> str | None: + return self._last_frame_kind + def close(self) -> None: if self._capture is not None: try: @@ -646,23 +654,36 @@ class LeapPackedMotionSampler: return self._fallback_frame() ok, frame = self._capture.read() if ok and frame is not None: + self._last_frame_kind = "capture" return frame return self._fallback_frame() def _fallback_frame(self) -> np.ndarray | None: import cv2 - for path in ( - self.fallback_dir / f"rgb-probe-{self.api}-{self.index}.png", - self.fallback_dir / f"rgb-probe-dshow-{self.index}.png", - self.fallback_dir / f"rgb-probe-msmf-{self.index}.png", - self.fallback_dir / "leap-probe.png", - ): - if path.exists(): - frame = cv2.imread(str(path)) - if frame is not None: - return frame - return None + path = self.fallback_dir / "leap-probe.png" + if path.exists(): + frame = cv2.imread(str(path)) + if frame is not None: + self._last_frame_kind = "fallback-file" + return frame + return self._synthetic_motion_frame() + + def _synthetic_motion_frame(self) -> np.ndarray: + self._last_frame_kind = "fallback-synthetic" + frame = np.zeros((self.height, self.width, 3), dtype=np.uint8) + t = self._fallback_index + self._fallback_index += 1 + cx = int((self.width * 0.30) + (self.width * 0.40) * ((np.sin(t * 0.37) + 1.0) * 0.5)) + cy = int((self.height * 0.30) + (self.height * 0.34) * ((np.cos(t * 0.41) + 1.0) * 0.5)) + y0 = max(0, cy - 4) + y1 = min(self.height, cy + 5) + x0 = max(0, cx - 4) + x1 = min(self.width, cx + 5) + frame[y0:y1, x0:x1, 1] = 220 + frame[max(0, y0 - 8) : min(self.height, y1 + 8), max(0, x0 - 8) : min(self.width, x1 + 8), 2] = 80 + frame[max(0, y0 - 12) : min(self.height, y1 + 12), max(0, x0 - 12) : min(self.width, x1 + 12), 0] = 55 + return frame def unpack_leap_packed_channels(frame_bgr: np.ndarray) -> dict[str, np.ndarray]: @@ -804,11 +825,6 @@ def main() -> None: samples_per_observation=2, seed=frame_id, ) - multilod_cache_from_points( - tuple(result.points) + tuple(transient_matches), - levels=(0.02, 0.08, 0.32), - created_monotonic_ns=time.monotonic_ns(), - ).write_json(lod_cache_path) frame = lower_points_to_render_frame( result.points, render_config, @@ -820,8 +836,22 @@ def main() -> None: leap_points = () if leap_sampler is None else leap_sampler.motion_splats(timestamp_ns) if leap_sampler is not None and leap_sampler.last_timestamp_ns is not None: timestamp_ns = max(timestamp_ns, leap_sampler.last_timestamp_ns) + leap_source_kind = ( + "leap-ground-truth" + if leap_sampler is not None and leap_sampler.last_frame_kind == "capture" + else "leap-fallback" + ) + leap_source_priority = 3.0 if leap_source_kind == "leap-ground-truth" else 1.1 support_points = dim_fusion_points(tuple(frame.points)) if not rgb_points else () stochastic_points = transient_match_render_points(transient_matches, timestamp_ns) + multilod_cache_from_evidence( + evidence_from_render_points(leap_points, source_kind=leap_source_kind, source_priority=leap_source_priority) + + evidence_from_render_points(rgb_points, source_kind="rgb-surface", source_priority=1.4) + + evidence_from_fusion_items(transient_matches, source_kind="stochastic-transient", source_priority=1.2) + + evidence_from_fusion_items(result.points, source_kind="synthetic-fusion", source_priority=0.8), + levels=(0.01, 0.04, 0.16, 0.64), + created_monotonic_ns=time.monotonic_ns(), + ).write_json(lod_cache_path) frame = RenderFramePacket( schema=frame.schema, frame_id=frame.frame_id, diff --git a/state/map.yaml b/state/map.yaml index 6dd5b2c..d13d1b0 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -158,7 +158,10 @@ research: cache rule: semantic claims first, cached backend lowerings second. Stochastic transient matching now lowers descriptor-consistent image observations into scene-ray hypotheses and a multi-LOD world-space cache - for Aquarium compute-shader reconciliation. + for Aquarium compute-shader reconciliation. Leap evidence is the + highest-priority temporal/spatial source when live Leap frames are present; + fallback Leap frames stay tagged as fallback evidence and must not be + treated as ground truth. typed_visual_state: summary: docs/typed-visual-state.md key_decision: > @@ -267,7 +270,10 @@ current_status: observations can be descriptor-matched and triangulated into world-space tracks. Unit tests inject synthetic cameras and frames so the core does not depend on live camera - drivers. + drivers. The multi-LOD visual cache now carries source kind and source + priority so Aquarium can prefer real Leap evidence over RGB surfaces, + stochastic transient hypotheses, and synthetic fusion support while still + seeing lower-priority fallback diagnostics when the Leap capture path stalls. next_action: > Capture real ChArUco intrinsics for the PS3 Eyes and Kiyos, capture fixed-board observations from every camera, solve a local @@ -280,7 +286,9 @@ current_status: Then replace the synthetic live producer with a detector adapter that turns FFmpeg PS3 Eye frames into marker `Observation2D` records for the fusion core and publishes the same typed CultCache/CultNet render-frame - document. In parallel, move the render-frame consumer into Aquarium Engine so + document. Also fix the live Leap capture backend so the cache emits + `leap-ground-truth` rather than `leap-fallback`; current MSMF probing can + still stall on frame reads. In parallel, move the render-frame consumer into Aquarium Engine so Aquarium renders cached points/brushes into its GPU render target and publishes or hands off the same Spout sender `LocalCastBridge Point Cloud` for OBS, replacing the deadline OpenGL point sink without changing packet diff --git a/tests/test_stochastic_mapping.py b/tests/test_stochastic_mapping.py index 193ece4..0615291 100644 --- a/tests/test_stochastic_mapping.py +++ b/tests/test_stochastic_mapping.py @@ -7,7 +7,9 @@ import numpy as np from localcast.sensor_fusion.core import CameraModel, SensorRig, Observation2D from localcast.sensor_fusion.stochastic_mapping import ( ImageRayBiasMap, + LODEvidencePoint, RayBiasSample, + multilod_cache_from_evidence, multilod_cache_from_points, pixel_scene_ray, stochastic_transient_matches, @@ -95,6 +97,19 @@ class StochasticMappingTests(unittest.TestCase): cache.write_json(path) self.assertIn("multilod_scene_cache", path.read_text(encoding="utf-8")) + def test_multilod_evidence_prefers_leap_priority(self): + evidence = [ + LODEvidencePoint("rgb", np.array([0.01, 0.0, 1.0]), 0.7, 100, 1.0, "rgb"), + LODEvidencePoint("leap", np.array([0.03, 0.0, 1.0]), 0.8, 120, 3.0, "leap-ground-truth"), + ] + + cache = multilod_cache_from_evidence(evidence, levels=(0.10,), created_monotonic_ns=5) + cell = cache.cells[0] + + self.assertEqual("leap-ground-truth", cell.source_kind) + self.assertEqual(3.0, cell.source_priority) + self.assertGreater(cell.center[0], 0.02) + if __name__ == "__main__": unittest.main()