diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index b1160b4..d3268b5 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -29,6 +29,7 @@ - `localcast.sensor_fusion.spout_output` renders render-frame packets into a GPU texture and publishes it as a named Spout sender for OBS. - `scripts/live_sensor_fusion.py` writes fused render frames into `calibration/runs/visual-state.msgpack`. - The live visual producer writes a multi-LOD scene cache with source kind and priority. Real Leap frames are promoted as the highest-priority visual timing/spatial evidence; Leap fallback frames remain lower-priority diagnostics. + - Live clap calibration is wired into the visual producer. It keeps a rolling frame window from the Kiyo pair plus Leap, reads the live spatial audio frame for transient candidates, publishes `clap-events.msgpack`, injects clap calibration markers into the render frame, and writes clap evidence into the LOD cache. Kiyo stereo owns the current rough 3D solve; Leap owns the best visual timing witness until its geometric model is calibrated. - `scripts/stream_spout.py` runs the deadline Spout sender loop from the typed cache with typed and JSON heartbeat status. - The deadline Spout renderer has a named `kiyo-mid-deru` virtual camera preset: eye at the midpoint between the two Kiyo-class cameras and target on the co-streamer body volume. It also applies a renderer-owned point budget with stable high-confidence anchors plus frame-varying remainder samples so downstream TAA/supersampling can accumulate detail without forcing every source claim into every OBS frame. - `docs/obs-spout-streaming.md` documents OBS setup and the Aquarium replacement boundary. @@ -72,7 +73,8 @@ 11. Move the render-frame consumer into Aquarium Engine so dense brush/splat rendering replaces the deadline OpenGL point sink behind the same OBS Spout boundary. 12. Capture fixed-board ChArUco observations for every camera, solve `config/sensor-fusion.json`, and verify cross-view feature triangulation with `scripts/triangulate_surface_features.py`. 13. Replace synthetic live-fusion observations with PS3 Eye detector observations and calibrated surface-feature tracks. -14. Fix the live Leap capture backend so actual Leap frames, not fallback frames, populate `leap-ground-truth` cells in the multi-LOD cache. -15. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present. -16. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video. -17. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests. +14. Calibrate Leap as a geometric camera in the shared rig so the timing authority can also contribute direct 3D rays instead of only visual timing peaks. +15. Fix the live Leap capture backend so actual Leap frames, not fallback frames, populate `leap-ground-truth` cells in the multi-LOD cache. +16. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present. +17. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video. +18. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests. diff --git a/notes/current-system-map.md b/notes/current-system-map.md index eb9eb7b..7ebaaf8 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -93,6 +93,10 @@ flowchart TD G --> H["OBS Spout2 Capture"] I["Leap motion evidence"] --> J["multi-LOD scene cache"] K["RGB/stochastic/synthetic evidence"] --> J + M["live clap audio transient"] --> N["Leap-timed clap calibration"] + O["Kiyo stereo motion window"] --> N + N --> C + N --> J J --> L["Aquarium compute reconciliation"] ``` @@ -100,6 +104,7 @@ Ownership: - `localcast.sensor_fusion.cultcache_docs` owns typed visual state documents. - `scripts/live_sensor_fusion.py` currently owns the live producer and writes `localcast.visual.render_frame` into CultCache. +- The live producer now keeps a rolling RGB/Leap frame window for clap calibration. Kiyo stereo motion gives the first rough 3D clap position; Leap participates as the high-precision visual timing witness until a calibrated Leap geometric model joins the rig. - The live producer also writes a multi-LOD scene cache with source kind and priority. Real Leap frames are the highest-priority timing/spatial evidence; Leap fallback frames are tagged as fallback and do not become ground truth. - `scripts/stream_spout.py` consumes typed CultCache state and publishes Spout. Its presentation camera defaults to `kiyo-mid-deru`: the virtual eye sits halfway between the two Kiyo-class cameras and aims at the co-streamer body target. Its render budget is temporal: high-confidence anchors stay stable while the remaining samples rotate by frame id for TAA/supersampling reconciliation. - CultNet document replication is the intended API boundary for other producers/consumers. diff --git a/scripts/live_sensor_fusion.py b/scripts/live_sensor_fusion.py index 36e0a9e..1e39784 100644 --- a/scripts/live_sensor_fusion.py +++ b/scripts/live_sensor_fusion.py @@ -20,16 +20,22 @@ from localcast.sensor_fusion import ( RenderPointPacket, SensorRig, SurfaceFeatureObservation, + TimestampedFrame, + ClapDetectorConfig, + detect_clap_events, dense_stereo_points, evidence_from_fusion_items, evidence_from_render_points, + make_clap_events_frame, measure_frame_quality, multilod_cache_from_evidence, + put_live_clap_events, put_live_render_frame, lower_points_to_render_frame, stochastic_transient_matches, ) from localcast.sensor_fusion.camera_control import AdaptiveCameraController, OpenCvCameraSettingPort +from audio_field.cultcache_audio import frame_to_numpy, get_live_spatial_audio_frame def acquire_runtime_lock(path: Path) -> BinaryIO | None: @@ -276,6 +282,7 @@ class RgbSplatSampler: self.fallback_dir = fallback_dir self._captures: dict[int, object] = {} self._frames: dict[int, np.ndarray] = {} + self._timestamped_frames: dict[str, TimestampedFrame] = {} self._controllers: dict[int, AdaptiveCameraController] = {} self._setting_ports: dict[int, OpenCvCameraSettingPort] = {} @@ -303,6 +310,9 @@ class RgbSplatSampler: points.extend(rgb_body_splats("deru-rgb", secondary, np.array([0.48, 0.28, 0.0]), now_s + 0.6, timestamp_ns, side=1.0, step=self.sample_step)) return tuple(points) + def latest_timestamped_frames(self) -> dict[str, TimestampedFrame]: + return dict(self._timestamped_frames) + def _read_frame(self, index: int) -> np.ndarray | None: import cv2 @@ -328,6 +338,8 @@ class RgbSplatSampler: for _ in range(2): ok, frame = capture.read() if ok and frame is not None: + sensor_id = self._sensor_id(index) + self._timestamped_frames[sensor_id] = TimestampedFrame(sensor_id, time.monotonic_ns(), frame.copy()) self._tune_capture(index, frame) self._frames[index] = frame return frame @@ -357,9 +369,18 @@ class RgbSplatSampler: if path.exists(): frame = cv2.imread(str(path)) if frame is not None: + sensor_id = self._sensor_id(index) + self._timestamped_frames[sensor_id] = TimestampedFrame(sensor_id, time.monotonic_ns(), frame.copy()) return frame return None + def _sensor_id(self, index: int) -> str: + if index == self.primary_index: + return "kiyo-primary" + if index == self.secondary_index: + return "kiyo-secondary" + return f"rgb-{index}" + def cv2_api(name: str) -> int: import cv2 @@ -424,6 +445,132 @@ def rgb_dense_camera(sensor_id: str, x: float, width: int, height: int) -> Camer ) +class LiveClapCalibrator: + def __init__( + self, + *, + audio_cache: Path, + clap_cache: Path, + camera_width: int, + camera_height: int, + max_frames_per_sensor: int = 24, + ) -> None: + self.audio_cache = audio_cache + self.clap_cache = clap_cache + self.max_frames_per_sensor = max(2, int(max_frames_per_sensor)) + self.frames_by_sensor: dict[str, list[TimestampedFrame]] = {} + self.last_audio_frame_id: int | None = None + self.last_event_key: str | None = None + left = rgb_dense_camera("kiyo-primary", -0.18, camera_width, camera_height) + right = rgb_dense_camera("kiyo-secondary", 0.18, camera_width, camera_height) + self.rig = SensorRig( + cameras={left.sensor_id: left, right.sensor_id: right}, + config=FusionConfig(max_pair_dt_ns=80_000_000, max_reprojection_error_px=80.0, cache_ttl_ns=1_000_000_000), + ) + self.config = ClapDetectorConfig( + audio_window_samples=192, + audio_hop_samples=48, + audio_onset_ratio=4.0, + min_audio_energy=1.0e-6, + min_camera_motion_score=0.025, + camera_window_ns=90_000_000, + min_camera_count=2, + min_event_spacing_ns=220_000_000, + ) + self.events = () + + def observe_frames(self, frames: dict[str, TimestampedFrame]) -> None: + for sensor_id, frame in frames.items(): + bucket = self.frames_by_sensor.setdefault(sensor_id, []) + if bucket and bucket[-1].timestamp_ns == frame.timestamp_ns: + continue + bucket.append(frame) + del bucket[:-self.max_frames_per_sensor] + + def update(self, frame_id: int) -> tuple[RenderPointPacket, ...]: + audio = self._read_audio() + if audio is None or audio.frame_id == self.last_audio_frame_id: + return self._event_points(frame_id) + self.last_audio_frame_id = audio.frame_id + block = frame_to_numpy(audio) + events = detect_clap_events( + block, + audio.sample_rate, + self._frames_for_detection(), + audio_time_ns=audio.audio_time_ns, + start_sample=audio.start_sample, + rig=self.rig, + config=self.config, + ) + if events: + self.events = tuple(events) + self.last_event_key = events[-1].stable_key + put_live_clap_events(self.clap_cache, make_clap_events_frame(frame_id=frame_id, events=events)) + elif not self.events: + put_live_clap_events(self.clap_cache, make_clap_events_frame(frame_id=frame_id, events=())) + return self._event_points(frame_id) + + def _read_audio(self): + if not self.audio_cache.exists(): + return None + return get_live_spatial_audio_frame(self.audio_cache) + + def _frames_for_detection(self) -> dict[str, list[TimestampedFrame]]: + return {sensor_id: list(frames) for sensor_id, frames in self.frames_by_sensor.items()} + + def _event_points(self, timestamp_ns: int) -> tuple[RenderPointPacket, ...]: + points: list[RenderPointPacket] = [] + for event in self.events[-4:]: + confidence = max(0.0, min(1.0, event.visual_confidence * event.acoustic_confidence)) + points.append( + render_point( + f"clap-calibration:{event.stable_key}", + np.asarray(event.position_m, dtype=np.float64), + 0.075, + (0.15, 0.95, 1.0, 0.72), + confidence, + timestamp_ns, + ) + ) + for peak in event.camera_peaks: + camera = self.rig.cameras.get(peak.sensor_id) + if peak.sensor_id == "leap": + points.append( + render_point( + f"clap-timing:{event.stable_key}:leap", + np.asarray(event.position_m, dtype=np.float64) + np.array([0.0, -0.035, 0.12], dtype=np.float64), + 0.04, + (0.45, 1.0, 0.72, 0.82), + max(0.0, min(1.0, peak.score)), + int(peak.timestamp_ns), + ) + ) + continue + if camera is None: + continue + ray = pixel_ray_point(camera, peak.uv, distance_m=1.2) + points.append( + render_point( + f"clap-ray:{event.stable_key}:{peak.sensor_id}", + ray, + 0.026, + (0.2, 0.85, 1.0, 0.42), + max(0.0, min(1.0, peak.score)), + int(peak.timestamp_ns), + ) + ) + return tuple(points) + + +def pixel_ray_point(camera: CameraModel, uv: np.ndarray, *, distance_m: float) -> np.ndarray: + point = np.asarray([float(uv[0]), float(uv[1]), 1.0], dtype=np.float64) + direction_sensor = np.linalg.inv(camera.camera_matrix) @ point + direction_sensor = direction_sensor / max(1.0e-12, np.linalg.norm(direction_sensor)) + direction_world = camera.world_from_sensor[:3, :3] @ direction_sensor + direction_world = direction_world / max(1.0e-12, np.linalg.norm(direction_world)) + return camera.position_world + direction_world * float(distance_m) + + def rgb_body_splats( prefix: str, frame_bgr: np.ndarray, @@ -602,6 +749,7 @@ class LeapPackedMotionSampler: self._previous_channels: dict[str, np.ndarray] = {} self._last_timestamp_ns: int | None = None self._last_frame_kind: str | None = None + self._latest_timestamped_frame: TimestampedFrame | None = None self._fallback_index = 0 @property @@ -612,6 +760,11 @@ class LeapPackedMotionSampler: def last_frame_kind(self) -> str | None: return self._last_frame_kind + def latest_timestamped_frames(self) -> dict[str, TimestampedFrame]: + if self._latest_timestamped_frame is None: + return {} + return {self._latest_timestamped_frame.sensor_id: self._latest_timestamped_frame} + def close(self) -> None: if self._capture is not None: try: @@ -625,6 +778,7 @@ class LeapPackedMotionSampler: if frame is None: return () self._last_timestamp_ns = time.monotonic_ns() + self._latest_timestamped_frame = TimestampedFrame("leap", self._last_timestamp_ns, frame.copy()) channels = unpack_leap_packed_channels(frame) points: list[RenderPointPacket] = [] for channel_name, channel in channels.items(): @@ -770,6 +924,9 @@ def main() -> None: parser.add_argument("--leap-fps", type=float, default=120.0) parser.add_argument("--leap-step", type=int, default=12) parser.add_argument("--lod-cache", default=str(ROOT / "calibration" / "runs" / "visual-lod-cache.json")) + parser.add_argument("--audio-cache", default=str(ROOT / "calibration" / "runs" / "audio-state.msgpack")) + parser.add_argument("--clap-cache", default=str(ROOT / "calibration" / "runs" / "clap-events.msgpack")) + parser.add_argument("--no-clap-calibration", action="store_true") parser.add_argument("--no-stochastic-transients", action="store_true") args = parser.parse_args() @@ -810,6 +967,12 @@ def main() -> None: step=args.leap_step, fallback_dir=ROOT / "calibration" / "runs", ) + clap_calibrator = None if args.no_clap_calibration else LiveClapCalibrator( + audio_cache=Path(args.audio_cache), + clap_cache=Path(args.clap_cache), + camera_width=args.rgb_width, + camera_height=args.rgb_height, + ) try: while True: now = time.monotonic() @@ -834,6 +997,11 @@ def main() -> None: timestamp_ns = frame.source_time_max_ns rgb_points = () if rgb_sampler is None else rgb_sampler.splats(now, timestamp_ns) leap_points = () if leap_sampler is None else leap_sampler.motion_splats(timestamp_ns) + if clap_calibrator is not None: + if rgb_sampler is not None: + clap_calibrator.observe_frames(rgb_sampler.latest_timestamped_frames()) + if leap_sampler is not None: + clap_calibrator.observe_frames(leap_sampler.latest_timestamped_frames()) if leap_sampler is not None and leap_sampler.last_timestamp_ns is not None: timestamp_ns = max(timestamp_ns, leap_sampler.last_timestamp_ns) leap_source_kind = ( @@ -844,9 +1012,11 @@ def main() -> None: leap_source_priority = 3.0 if leap_source_kind == "leap-ground-truth" else 1.1 support_points = dim_fusion_points(tuple(frame.points)) if not rgb_points else () stochastic_points = transient_match_render_points(transient_matches, timestamp_ns) + clap_points = () if clap_calibrator is None else clap_calibrator.update(timestamp_ns) multilod_cache_from_evidence( evidence_from_render_points(leap_points, source_kind=leap_source_kind, source_priority=leap_source_priority) + evidence_from_render_points(rgb_points, source_kind="rgb-surface", source_priority=1.4) + + evidence_from_render_points(clap_points, source_kind="clap-calibration", source_priority=2.4) + evidence_from_fusion_items(transient_matches, source_kind="stochastic-transient", source_priority=1.2) + evidence_from_fusion_items(result.points, source_kind="synthetic-fusion", source_priority=0.8), levels=(0.01, 0.04, 0.16, 0.64), @@ -863,7 +1033,7 @@ def main() -> None: spout_sender_name=frame.spout_sender_name, target_width=frame.target_width, target_height=frame.target_height, - points=leap_points + rgb_points + stochastic_points + support_points, + points=clap_points + leap_points + rgb_points + stochastic_points + support_points, ) put_live_render_frame(cache_path, frame) frame_id += 1 diff --git a/scripts/live_status.py b/scripts/live_status.py index dba6fc7..6a479e8 100644 --- a/scripts/live_status.py +++ b/scripts/live_status.py @@ -100,6 +100,23 @@ def msgpack_head(path: Path): ) elif summary["type"] in ("localcast.visual.stream_status", "localcast.audio.stream_status"): summary["payloadHead"] = head + elif summary["type"] == "localcast.calibration.clap_events": + summary.update( + { + "frameId": payload[1], + "eventCount": len(payload[3]) if len(payload) > 3 else 0, + "latestEvent": None + if len(payload) <= 3 or not payload[3] + else { + "stableKey": payload[3][-1][0], + "positionM": payload[3][-1][1], + "acousticOracleNs": payload[3][-1][2], + "visualObservedNs": payload[3][-1][3], + "timingUncertaintyUs": payload[3][-1][4], + "cameraCount": len(payload[3][-1][7]) if len(payload[3][-1]) > 7 else 0, + }, + } + ) return summary @@ -123,6 +140,7 @@ def main() -> None: "audio-stream-status.msgpack", "audio-phase-field.msgpack", "audio-mic-field.msgpack", + "clap-events.msgpack", "active-probes/active-probes.jsonl", "visual-lod-cache.json", "stream-spout-status.json", @@ -139,6 +157,7 @@ def main() -> None: "eventsHead": msgpack_head(runs / "audio-events.msgpack"), "phaseFieldHead": msgpack_head(runs / "audio-phase-field.msgpack"), "micFieldHead": msgpack_head(runs / "audio-mic-field.msgpack"), + "clapHead": msgpack_head(runs / "clap-events.msgpack"), "lastActiveProbe": read_last_jsonl(runs / "active-probes" / "active-probes.jsonl"), } print(json.dumps(payload, indent=2, default=str)) diff --git a/state/map.yaml b/state/map.yaml index 92c6634..d773c64 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -165,7 +165,10 @@ research: for Aquarium compute-shader reconciliation. Leap evidence is the highest-priority temporal/spatial source when live Leap frames are present; fallback Leap frames stay tagged as fallback evidence and must not be - treated as ground truth. + treated as ground truth. Live clap calibration now uses the Kiyo stereo + pair for the first rough 3D impact solve and includes Leap motion as the + high-precision visual timing witness; Leap should become a geometric + camera authority after shared calibration. typed_visual_state: summary: docs/typed-visual-state.md key_decision: > @@ -290,7 +293,8 @@ current_status: Then replace the synthetic live producer with a detector adapter that turns FFmpeg PS3 Eye frames into marker `Observation2D` records for the fusion core and publishes the same typed CultCache/CultNet render-frame - document. Also fix the live Leap capture backend so the cache emits + document. Calibrate Leap as a geometric camera model so clap timing peaks + can become direct 3D rays instead of timing-only evidence. Also fix the live Leap capture backend so the cache emits `leap-ground-truth` rather than `leap-fallback`; current MSMF probing can still stall on frame reads. In parallel, move the render-frame consumer into Aquarium Engine so Aquarium renders cached points/brushes into its GPU render target and diff --git a/tests/test_sensor_fusion.py b/tests/test_sensor_fusion.py index 79940c7..6b063ca 100644 --- a/tests/test_sensor_fusion.py +++ b/tests/test_sensor_fusion.py @@ -45,13 +45,15 @@ from localcast.sensor_fusion import ( ) from localcast.sensor_fusion.spout_output import rasterize_frame_rgba from scripts.live_sensor_fusion import ( + LiveClapCalibrator, leap_channel_motion_points, + rgb_dense_camera, rgb_dense_stereo_splats, rgb_room_splats, unpack_leap_packed_channels, ) from localcast.sensor_fusion.active_illumination import ActiveIlluminationController, IlluminationPulsePlan -from audio_field.cultcache_audio import make_audio_source_events, make_spatial_audio_frame +from audio_field.cultcache_audio import make_audio_source_events, make_spatial_audio_frame, put_live_spatial_audio_frame from localcast.sensor_fusion.adapters import read_raw_bgr_frames @@ -457,6 +459,57 @@ class SensorFusionTests(unittest.TestCase): self.assertEqual(1, len(loaded.events)) self.assertEqual(events[0].acoustic_oracle_ns, loaded.events[0].acoustic_oracle_ns) + def test_live_clap_calibrator_uses_audio_and_kiyo_motion(self): + width, height = 320, 240 + target = np.array([0.0, -0.5, 2.0]) + sample_rate = 48_000 + audio_time_ns = 4_000_000_000 + clap_sample = 960 + clap_time_ns = audio_time_ns + round(clap_sample * 1_000_000_000 / sample_rate) + block = np.zeros((2048, 4), dtype=np.float32) + block[clap_sample] = np.array([1.0, -0.8, 0.6, -0.4], dtype=np.float32) + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + audio_cache = tmp_path / "audio-state.msgpack" + clap_cache = tmp_path / "clap-events.msgpack" + put_live_spatial_audio_frame( + audio_cache, + make_spatial_audio_frame( + block, + frame_id=1, + sample_rate=sample_rate, + start_sample=0, + audio_time_ns=audio_time_ns, + ), + ) + calibrator = LiveClapCalibrator( + audio_cache=audio_cache, + clap_cache=clap_cache, + camera_width=width, + camera_height=height, + ) + frames = {} + for sensor_id, x in (("kiyo-primary", -0.18), ("kiyo-secondary", 0.18)): + cam = rgb_dense_camera(sensor_id, x, width, height) + uv = cam.project_world(target) + previous = np.zeros((height, width, 3), dtype=np.uint8) + current = previous.copy() + col, row = int(round(uv[0])), int(round(uv[1])) + current[row - 3 : row + 4, col - 3 : col + 4, :] = 255 + calibrator.observe_frames( + { + sensor_id: TimestampedFrame(sensor_id, clap_time_ns - 8_000_000, previous), + } + ) + frames[sensor_id] = TimestampedFrame(sensor_id, clap_time_ns + 8_000_000, current) + calibrator.observe_frames(frames) + + points = calibrator.update(frame_id=12) + + self.assertTrue(clap_cache.exists()) + self.assertTrue(any(point.stable_key.startswith("clap-calibration:") for point in points)) + def test_audio_source_events_overlay_against_audio_alignment_time(self): frame = RenderFramePacket( schema="localcast.sensor_fusion.render_frame.v1",