From bd28491b42d82d511b1983c5d52a770f0e2fc2ba Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 18 May 2026 20:12:00 +0100 Subject: [PATCH] Add camera quality control and debug dense stereo --- docs/sensor-fusion-architecture.md | 48 ++++--- docs/typed-visual-state.md | 17 ++- localcast/sensor_fusion/__init__.py | 17 +++ localcast/sensor_fusion/camera_control.py | 154 ++++++++++++++++++++++ localcast/sensor_fusion/dense_stereo.py | 126 ++++++++++++++++++ scripts/live_sensor_fusion.py | 91 ++++++++++++- state/map.yaml | 37 ++++-- tests/test_sensor_fusion.py | 70 ++++++++++ 8 files changed, 523 insertions(+), 37 deletions(-) create mode 100644 localcast/sensor_fusion/camera_control.py create mode 100644 localcast/sensor_fusion/dense_stereo.py diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index 39c20c7..7adacf0 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -13,7 +13,9 @@ The rig currently has usable capture surfaces: - LeapUVC raw IR through OpenCV for near-field evidence. - ChArUco tooling for intrinsics capture, but no shared fusion state yet. -That is capture. Capture is not a world model. +That is capture. Capture is not a world model. The previous live producer also +painted camera colors onto guessed body and room surfaces. That was useful +deadline scaffolding, but it is not enough for detailed reconstruction. ## Invariants @@ -25,27 +27,41 @@ That is capture. Capture is not a world model. ## Intended Change +The live pipeline now has two measurement-quality paths: sparse marker fusion +for calibrated tracked objects and GPU-owned dense RGB stereo/flow for surface +detail. A CPU dense stereo matcher exists only as a debug reference and is +opt-in through `--cpu-dense-stereo`; production dense fusion may not route +through Python loops. The provisional Kiyo geometry remains a lie detector, not +truth, until `config/sensor-fusion.json` carries measured extrinsics. + The first pipeline slice adds these boundaries: ```mermaid flowchart TD A["Capture adapters"] --> B["FramePacket ring"] - B --> C["Detection adapter"] - C --> D["Observation2D"] - E["SensorRig calibration"] --> F["Fusion core"] - D --> F - F --> G["TrackCache"] - G --> H["PointCloud PLY"] - G --> I["CultCache RenderFrame document"] - I --> J["Spout sink / Aquarium GPU renderer"] - J --> K["Spout2 sender texture"] - K --> L["OBS Spout2 source"] + B --> C["Camera quality controller"] + C --> D["manual exposure/gain/focus commands"] + B --> E["Detection adapter"] + E --> F["Observation2D"] + B --> G["Dense stereo matcher"] + H["SensorRig calibration"] --> I["Fusion core"] + F --> I + I --> J["TrackCache"] + G --> K["RGB surface claims"] + J --> L["PointCloud PLY"] + J --> M["CultCache RenderFrame document"] + K --> M + M --> N["Spout sink / Aquarium GPU renderer"] + N --> O["Spout2 sender texture"] + O --> P["OBS Spout2 source"] ``` ## Ownership - `localcast.sensor_fusion.core` owns calibrated camera models, 2D observations, DLT triangulation, reprojection gating, confidence scoring, point-cloud export, and track cache expiry. - `localcast.sensor_fusion.adapters` owns driver-facing frame ingress. Its first concrete adapter is an FFmpeg raw BGR reader for the PS3 Eye DirectShow path. +- `localcast.sensor_fusion.camera_control` owns measurement-quality policy: luminance, clipping, contrast, and sharpness in; normalized exposure/gain/focus commands out. Driver adapters own translating those commands to OpenCV, DirectShow, vendor tools, or no-op mocks. +- `localcast.sensor_fusion.dense_stereo` owns a debug CPU reference for dense calibrated RGB surface claims. It is useful for tests and shader parity, but the live million-splat path belongs to Aquarium/GPU compute. - `localcast.sensor_fusion.render_bridge` owns the in-process render-frame ABI: cached point claims plus target dimensions, Spout sender name, source timestamp range, intended visual presentation time, and ambisonic/audio alignment time. - `localcast.sensor_fusion.cultcache_docs` owns the typed CultCache document boundary for live visual state. - `localcast.sensor_fusion.spout_output` owns the deadline OBS publication sink: render-frame packet in, GPU texture plus Spout sender heartbeat out. @@ -80,6 +96,8 @@ The core can be tested without cameras: - timestamp windows reject stale pairs - track cache expiry is deterministic - raw-frame adapter can be tested from an in-memory byte stream +- adaptive camera control responds to overexposure without live hardware +- dense stereo turns shifted textured camera pairs into calibrated RGB surface claims Those tests matter because camera drivers are not unit-test dependencies. They are weather. @@ -127,10 +145,10 @@ That lets the final stream compositor buffer point-cloud visuals until they alig ## Next Cut -1. Add detector adapters that turn PS3 Eye frames into `Observation2D` marker detections. -2. Load real ChArUco intrinsics/extrinsics into `config/sensor-fusion.json`. -3. Record an observation run from the two PS3 Eyes into typed CultCache docs. -4. Generate a live sparse PLY stream or debug preview. +1. Load real ChArUco intrinsics/extrinsics into `config/sensor-fusion.json`; the current dense Kiyo pair geometry is provisional and should not be mistaken for truth. +2. Move dense matching from CPU block search to GPU-resident stereo/flow so the million-sample target is not murdered by Python loops. +3. Add detector adapters that turn PS3 Eye frames into `Observation2D` marker detections. +4. Record an observation run from the two PS3 Eyes into typed CultCache docs. 5. Feed `RenderFramePacket` into Aquarium Engine. 6. Add Aquarium-side GPU point/brush rendering to a D3D render target. 7. Publish the render target as a Spout2 sender and receive it in OBS. diff --git a/docs/typed-visual-state.md b/docs/typed-visual-state.md index 04ce8f7..cf5df94 100644 --- a/docs/typed-visual-state.md +++ b/docs/typed-visual-state.md @@ -58,13 +58,16 @@ CultNetDocumentPut(localcast.visual.render_frame) For the deadline rig, both producer and renderer share the local CultCache file. The boundary is still the document contract: a future camera process, Aquarium process, or remote sensor node should publish the same document through CultNet `document_put` messages rather than inventing another transport shape. -The live visual frame may contain multiple claim families. `room-rgb:*` claims -are camera-resolved room/background surface samples, `host-rgb:*` and -`deru-rgb:*` claims are RGB body/object surface samples, and `leap-motion:*` -claims are high-rate LeapUVC packed-channel motion samples. Leap packed maps are -split into explicit green, magenta, red, and blue channels before publication so -the downstream accumulator can use Leap as the strongest timing/motion witness -without pretending the packed image is one ordinary RGB camera. +The live visual frame may contain multiple claim families. `dense-rgb:*` claims +are calibrated two-camera RGB surface samples from the debug CPU stereo +reference or, in the intended production path, Aquarium/GPU dense stereo and +flow. `room-rgb:*` claims are camera-resolved room/background surface samples; +`host-rgb:*` and `deru-rgb:*` claims are fallback RGB body/object surface +samples; and `leap-motion:*` claims are high-rate LeapUVC packed-channel motion +samples. Leap packed maps are split into explicit green, magenta, red, and blue +channels before publication so the downstream accumulator can use Leap as the +strongest timing/motion witness without pretending the packed image is one +ordinary RGB camera. LocalCastBridge does not own the final splat budget. It publishes dense typed claims with stable keys and timestamps; Aquarium owns temporal accumulation, diff --git a/localcast/sensor_fusion/__init__.py b/localcast/sensor_fusion/__init__.py index 5d8bc10..9570f2f 100644 --- a/localcast/sensor_fusion/__init__.py +++ b/localcast/sensor_fusion/__init__.py @@ -12,6 +12,15 @@ from .core import ( load_fusion_config, ) from .render_bridge import RenderBridgeConfig, RenderFramePacket, RenderPointPacket, lower_points_to_render_frame +from .camera_control import ( + AdaptiveCameraController, + CameraControlCommand, + CameraControlState, + CameraQualityTarget, + FrameQuality, + measure_frame_quality, +) +from .dense_stereo import DenseStereoConfig, dense_stereo_points from .audio_overlay import AudioVisualSyncStatus, overlay_audio_events from .media_artifacts import RemoteVideoArtifact, remote_video_artifact_for_present_time from .cultcache_docs import ( @@ -37,6 +46,12 @@ __all__ = [ "RenderBridgeConfig", "RenderFramePacket", "RenderPointPacket", + "AdaptiveCameraController", + "CameraControlCommand", + "CameraControlState", + "CameraQualityTarget", + "DenseStereoConfig", + "FrameQuality", "AudioVisualSyncStatus", "RemoteVideoArtifact", "LIVE_RENDER_FRAME_KEY", @@ -49,6 +64,8 @@ __all__ = [ "get_live_render_frame", "lower_frame_to_screen_brushes", "lower_points_to_render_frame", + "dense_stereo_points", + "measure_frame_quality", "overlay_audio_events", "remote_video_artifact_for_present_time", "put_live_render_frame", diff --git a/localcast/sensor_fusion/camera_control.py b/localcast/sensor_fusion/camera_control.py new file mode 100644 index 0000000..2f26e1a --- /dev/null +++ b/localcast/sensor_fusion/camera_control.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +import numpy as np + + +@dataclass(frozen=True) +class FrameQuality: + mean_luma: float + dark_clip: float + bright_clip: float + contrast: float + sharpness: float + + +@dataclass(frozen=True) +class CameraControlState: + exposure: float = 0.5 + gain: float = 0.25 + focus: float = 0.5 + + +@dataclass(frozen=True) +class CameraControlCommand: + exposure: float | None = None + gain: float | None = None + focus: float | None = None + + @property + def changed(self) -> bool: + return self.exposure is not None or self.gain is not None or self.focus is not None + + +@dataclass(frozen=True) +class CameraQualityTarget: + mean_luma: float = 0.52 + luma_deadband: float = 0.045 + max_bright_clip: float = 0.015 + max_dark_clip: float = 0.035 + min_sharpness: float = 0.035 + exposure_step: float = 0.035 + gain_step: float = 0.025 + focus_step: float = 0.020 + + +class CameraSettingPort(Protocol): + def apply(self, command: CameraControlCommand) -> None: + ... + + +def measure_frame_quality(frame_bgr: np.ndarray) -> FrameQuality: + if frame_bgr.ndim != 3 or frame_bgr.shape[2] != 3: + raise ValueError("frame_bgr must have shape HxWx3") + frame = frame_bgr.astype(np.float32) / 255.0 + luma = 0.0722 * frame[:, :, 0] + 0.7152 * frame[:, :, 1] + 0.2126 * frame[:, :, 2] + gx = np.diff(luma, axis=1) + gy = np.diff(luma, axis=0) + sharpness = float(np.mean(np.abs(gx)) + np.mean(np.abs(gy))) + return FrameQuality( + mean_luma=float(np.mean(luma)), + dark_clip=float(np.mean(luma < 0.025)), + bright_clip=float(np.mean(luma > 0.975)), + contrast=float(np.percentile(luma, 95) - np.percentile(luma, 5)), + sharpness=sharpness, + ) + + +class AdaptiveCameraController: + """Small bounded controller for manual capture settings. + + The controller owns measurement quality policy. It deliberately emits + normalized commands; the driver adapter owns mapping those commands to + OpenCV, DirectShow, vendor tools, or a no-op mock in tests. + """ + + def __init__( + self, + state: CameraControlState | None = None, + target: CameraQualityTarget | None = None, + ) -> None: + self.state = state or CameraControlState() + self.target = target or CameraQualityTarget() + self._focus_direction = 1.0 + + def update(self, quality: FrameQuality) -> CameraControlCommand: + exposure = self.state.exposure + gain = self.state.gain + focus = self.state.focus + + too_bright = quality.mean_luma > self.target.mean_luma + self.target.luma_deadband + too_dark = quality.mean_luma < self.target.mean_luma - self.target.luma_deadband + + if too_bright or quality.bright_clip > self.target.max_bright_clip: + exposure -= self.target.exposure_step + if quality.bright_clip > self.target.max_bright_clip * 2.0: + gain -= self.target.gain_step + elif too_dark or quality.dark_clip > self.target.max_dark_clip: + exposure += self.target.exposure_step + if exposure >= 0.92: + gain += self.target.gain_step + + if quality.sharpness < self.target.min_sharpness: + focus += self._focus_direction * self.target.focus_step + if focus <= 0.05 or focus >= 0.95: + self._focus_direction *= -1.0 + + exposure = _clamp01(exposure) + gain = _clamp01(gain) + focus = _clamp01(focus) + + command = CameraControlCommand( + exposure=None if abs(exposure - self.state.exposure) < 1e-9 else exposure, + gain=None if abs(gain - self.state.gain) < 1e-9 else gain, + focus=None if abs(focus - self.state.focus) < 1e-9 else focus, + ) + self.state = CameraControlState(exposure=exposure, gain=gain, focus=focus) + return command + + +class OpenCvCameraSettingPort: + def __init__(self, capture: object) -> None: + self.capture = capture + + def apply(self, command: CameraControlCommand) -> None: + if not command.changed: + return + import cv2 + + # Disable auto controls where OpenCV exposes the knobs, then apply a + # normalized manual target. Driver backends differ; failed sets are + # non-fatal because image quality feedback will continue next frame. + try: + self.capture.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.25) + except Exception: + pass + if command.exposure is not None: + _try_set(self.capture, cv2.CAP_PROP_EXPOSURE, -13.0 + command.exposure * 12.0) + if command.gain is not None: + _try_set(self.capture, cv2.CAP_PROP_GAIN, command.gain * 255.0) + if command.focus is not None and hasattr(cv2, "CAP_PROP_FOCUS"): + _try_set(self.capture, cv2.CAP_PROP_FOCUS, command.focus * 255.0) + + +def _try_set(capture: object, prop: int, value: float) -> None: + try: + capture.set(prop, float(value)) + except Exception: + pass + + +def _clamp01(value: float) -> float: + return max(0.0, min(1.0, float(value))) diff --git a/localcast/sensor_fusion/dense_stereo.py b/localcast/sensor_fusion/dense_stereo.py new file mode 100644 index 0000000..79a3ee9 --- /dev/null +++ b/localcast/sensor_fusion/dense_stereo.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +from .core import CameraModel, triangulate_dlt +from .render_bridge import RenderPointPacket + + +@dataclass(frozen=True) +class DenseStereoConfig: + sample_step: int = 4 + block_radius: int = 3 + max_disparity_px: int = 96 + min_texture: float = 0.020 + min_ncc: float = 0.72 + max_reprojection_error_px: float = 6.0 + radius_m: float = 0.006 + + +def dense_stereo_points( + *, + prefix: str, + left_frame_bgr: np.ndarray, + right_frame_bgr: np.ndarray, + left_camera: CameraModel, + right_camera: CameraModel, + timestamp_ns: int, + config: DenseStereoConfig | None = None, +) -> tuple[RenderPointPacket, ...]: + """Match textured pixels in a calibrated pair and emit RGB surface claims.""" + + cfg = config or DenseStereoConfig() + left = _luma(left_frame_bgr) + right = _luma(right_frame_bgr) + height = min(left.shape[0], right.shape[0], left_camera.height, right_camera.height) + width = min(left.shape[1], right.shape[1], left_camera.width, right_camera.width) + radius = max(1, int(cfg.block_radius)) + step = max(1, int(cfg.sample_step)) + points: list[RenderPointPacket] = [] + + for row in range(radius, height - radius, step): + for col in range(radius + 1, width - radius, step): + patch = _patch(left, row, col, radius) + texture = float(np.std(patch)) + if texture < cfg.min_texture: + continue + best_col, score = _best_horizontal_match( + patch, + right, + row, + col, + radius, + max_disparity_px=cfg.max_disparity_px, + ) + if best_col is None or score < cfg.min_ncc: + continue + uv_left = np.array([float(col), float(row)], dtype=np.float64) + uv_right = np.array([float(best_col), float(row)], dtype=np.float64) + try: + xyz = triangulate_dlt(left_camera.projection_matrix, uv_left, right_camera.projection_matrix, uv_right) + except ValueError: + continue + if not np.all(np.isfinite(xyz)): + continue + error = 0.5 * ( + left_camera.reprojection_error(xyz, uv_left) + right_camera.reprojection_error(xyz, uv_right) + ) + if error > cfg.max_reprojection_error_px: + continue + b, g, r = (left_frame_bgr[row, col].astype(np.float32) / 255.0).clip(0.0, 1.0) + confidence = max(0.0, min(1.0, (score - cfg.min_ncc) / max(1e-6, 1.0 - cfg.min_ncc))) + confidence = max(0.20, min(1.0, 0.35 + confidence * 0.55 + min(0.25, texture))) + points.append( + RenderPointPacket( + stable_key=f"{prefix}:{row}:{col}:{best_col}", + xyz=xyz.astype(np.float64), + radius_m=cfg.radius_m, + color_rgba=(float(r), float(g), float(b), 0.38 + 0.48 * confidence), + confidence=confidence, + source_timestamp_ns=timestamp_ns, + ) + ) + return tuple(points) + + +def _best_horizontal_match( + left_patch: np.ndarray, + right: np.ndarray, + row: int, + col: int, + radius: int, + *, + max_disparity_px: int, +) -> tuple[int | None, float]: + best_col: int | None = None + best_score = -1.0 + min_col = max(radius, col - max_disparity_px) + max_col = min(right.shape[1] - radius - 1, col + max_disparity_px // 8) + for candidate_col in range(min_col, max_col + 1): + score = _ncc(left_patch, _patch(right, row, candidate_col, radius)) + if score > best_score: + best_col = candidate_col + best_score = score + return best_col, best_score + + +def _luma(frame_bgr: np.ndarray) -> np.ndarray: + if frame_bgr.ndim != 3 or frame_bgr.shape[2] != 3: + raise ValueError("frame_bgr must have shape HxWx3") + frame = frame_bgr.astype(np.float32) / 255.0 + return 0.0722 * frame[:, :, 0] + 0.7152 * frame[:, :, 1] + 0.2126 * frame[:, :, 2] + + +def _patch(image: np.ndarray, row: int, col: int, radius: int) -> np.ndarray: + return image[row - radius : row + radius + 1, col - radius : col + radius + 1] + + +def _ncc(a: np.ndarray, b: np.ndarray) -> float: + aa = a.astype(np.float32) - float(np.mean(a)) + bb = b.astype(np.float32) - float(np.mean(b)) + denom = float(np.linalg.norm(aa) * np.linalg.norm(bb)) + if denom < 1e-9: + return -1.0 + return float(np.sum(aa * bb) / denom) diff --git a/scripts/live_sensor_fusion.py b/scripts/live_sensor_fusion.py index 0528a76..5ac9dd9 100644 --- a/scripts/live_sensor_fusion.py +++ b/scripts/live_sensor_fusion.py @@ -12,15 +12,19 @@ if str(ROOT) not in sys.path: from localcast.sensor_fusion import ( CameraModel, + DenseStereoConfig, FusionConfig, Observation2D, RenderBridgeConfig, RenderFramePacket, RenderPointPacket, SensorRig, + dense_stereo_points, + measure_frame_quality, put_live_render_frame, lower_points_to_render_frame, ) +from localcast.sensor_fusion.camera_control import AdaptiveCameraController, OpenCvCameraSettingPort def acquire_runtime_lock(path: Path) -> BinaryIO | None: @@ -214,6 +218,9 @@ class RgbSplatSampler: height: int, sample_step: int, room_step: int, + dense_step: int, + enable_cpu_dense_stereo: bool, + adaptive_controls: bool, fallback_dir: Path, ) -> None: self.api = api @@ -223,9 +230,14 @@ class RgbSplatSampler: self.height = height self.sample_step = max(4, int(sample_step)) self.room_step = max(self.sample_step, int(room_step)) + self.dense_step = max(1, int(dense_step)) + self.enable_cpu_dense_stereo = enable_cpu_dense_stereo + self.adaptive_controls = adaptive_controls self.fallback_dir = fallback_dir self._captures: dict[int, object] = {} self._frames: dict[int, np.ndarray] = {} + self._controllers: dict[int, AdaptiveCameraController] = {} + self._setting_ports: dict[int, OpenCvCameraSettingPort] = {} def close(self) -> None: for capture in self._captures.values(): @@ -239,12 +251,16 @@ class RgbSplatSampler: primary = self._read_frame(self.primary_index) secondary = self._read_frame(self.secondary_index) points: list[RenderPointPacket] = [] + if self.enable_cpu_dense_stereo and primary is not None and secondary is not None: + points.extend(rgb_dense_stereo_splats(primary, secondary, timestamp_ns, step=self.dense_step)) if primary is not None: points.extend(rgb_room_splats("room-rgb:primary", primary, np.array([-0.42, 0.05, 0.0]), timestamp_ns, step=self.room_step)) - points.extend(rgb_body_splats("host-rgb", primary, np.array([-0.42, 0.05, 0.0]), now_s, timestamp_ns, side=-1.0, step=self.sample_step)) + if not points: + points.extend(rgb_body_splats("host-rgb", primary, np.array([-0.42, 0.05, 0.0]), now_s, timestamp_ns, side=-1.0, step=self.sample_step)) if secondary is not None: points.extend(rgb_room_splats("room-rgb:secondary", secondary, np.array([0.48, 0.28, 0.0]), timestamp_ns, step=self.room_step)) - points.extend(rgb_body_splats("deru-rgb", secondary, np.array([0.48, 0.28, 0.0]), now_s + 0.6, timestamp_ns, side=1.0, step=self.sample_step)) + if not points: + points.extend(rgb_body_splats("deru-rgb", secondary, np.array([0.48, 0.28, 0.0]), now_s + 0.6, timestamp_ns, side=1.0, step=self.sample_step)) return tuple(points) def _read_frame(self, index: int) -> np.ndarray | None: @@ -258,6 +274,9 @@ class RgbSplatSampler: capture.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height) capture.set(cv2.CAP_PROP_BUFFERSIZE, 1) self._captures[index] = capture + if self.adaptive_controls: + self._controllers[index] = AdaptiveCameraController() + self._setting_ports[index] = OpenCvCameraSettingPort(capture) else: try: capture.release() @@ -269,6 +288,7 @@ class RgbSplatSampler: for _ in range(2): ok, frame = capture.read() if ok and frame is not None: + self._tune_capture(index, frame) self._frames[index] = frame return frame cached = self._frames.get(index) @@ -276,6 +296,14 @@ class RgbSplatSampler: return cached return self._fallback_frame(index) + def _tune_capture(self, index: int, frame: np.ndarray) -> None: + controller = self._controllers.get(index) + port = self._setting_ports.get(index) + if controller is None or port is None: + return + command = controller.update(measure_frame_quality(frame)) + port.apply(command) + def _fallback_frame(self, index: int) -> np.ndarray | None: import cv2 @@ -303,6 +331,59 @@ def cv2_api(name: str) -> int: }[name] +def rgb_dense_stereo_splats( + primary_bgr: np.ndarray, + secondary_bgr: np.ndarray, + timestamp_ns: int, + *, + step: int, +) -> tuple[RenderPointPacket, ...]: + height, width = primary_bgr.shape[:2] + secondary_height, secondary_width = secondary_bgr.shape[:2] + width = min(width, secondary_width) + height = min(height, secondary_height) + left = rgb_dense_camera("kiyo-primary", -0.18, width, height) + right = rgb_dense_camera("kiyo-secondary", 0.18, width, height) + return dense_stereo_points( + prefix="dense-rgb", + left_frame_bgr=primary_bgr[:height, :width], + right_frame_bgr=secondary_bgr[:height, :width], + left_camera=left, + right_camera=right, + timestamp_ns=timestamp_ns, + config=DenseStereoConfig(sample_step=step, block_radius=3, max_disparity_px=max(24, width // 3), radius_m=0.0045), + ) + + +def rgb_dense_camera(sensor_id: str, x: float, width: int, height: int) -> CameraModel: + focal = 0.82 * float(width) + return CameraModel( + sensor_id=sensor_id, + camera_matrix=np.array( + [ + [focal, 0.0, width * 0.5], + [0.0, focal, height * 0.5], + [0.0, 0.0, 1.0], + ], + dtype=np.float64, + ), + dist_coeffs=np.zeros(5), + world_from_sensor=np.array( + [ + [1.0, 0.0, 0.0, x], + [0.0, 1.0, 0.0, -0.72], + [0.0, 0.0, 1.0, 1.28], + [0.0, 0.0, 0.0, 1.0], + ], + dtype=np.float64, + ), + width=width, + height=height, + role="rgb_dense_stereo", + latency_ms=45.0, + ) + + def rgb_body_splats( prefix: str, frame_bgr: np.ndarray, @@ -619,6 +700,9 @@ def main() -> None: parser.add_argument("--rgb-height", type=int, default=480) parser.add_argument("--rgb-sample-step", type=int, default=10) parser.add_argument("--rgb-room-step", type=int, default=28) + parser.add_argument("--cpu-dense-stereo", action="store_true", help="Debug-only CPU stereo matcher. Production dense fusion belongs on the GPU.") + parser.add_argument("--rgb-dense-step", type=int, default=16) + parser.add_argument("--no-adaptive-camera-controls", action="store_true") parser.add_argument("--no-leap", action="store_true") parser.add_argument("--leap-api", choices=["any", "dshow", "msmf"], default="msmf") parser.add_argument("--leap-index", type=int, default=0) @@ -650,6 +734,9 @@ def main() -> None: height=args.rgb_height, sample_step=args.rgb_sample_step, room_step=args.rgb_room_step, + dense_step=args.rgb_dense_step, + enable_cpu_dense_stereo=args.cpu_dense_stereo, + adaptive_controls=not args.no_adaptive_camera_controls, fallback_dir=ROOT / "calibration" / "runs", ) leap_sampler = None if args.no_leap else LeapPackedMotionSampler( diff --git a/state/map.yaml b/state/map.yaml index d19f97e..2dfc80b 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -215,21 +215,32 @@ current_status: for the stem packer. The neighbor desktop launcher no longer uses Voicemeeter; it restores the Focusrite speakers as the primary playback device and the Focusrite analogue input as the default capture device. A - direct Core Audio WASAPI loopback shim exists, but the current Focusrite - render endpoint rejects loopback initialization with 0x88890008. + direct Core Audio WASAPI loopback shim now works when launched in the + neighbor's interactive console session; SSH-only launches can see the + endpoint but receive no render packets. Missing synchronized neighbor feeds are explicit silent placeholders, not raw unsynchronized substitutes. The live producer currently uses synthetic calibrated observations through the real `SensorRig.fuse` path until camera detector - observations replace them. Unit tests inject synthetic cameras and frames so - the core does not depend on live camera drivers. + observations replace them. The RGB path now has a debug-only CPU dense + stereo reference: the two Kiyo-class frames can be block-matched as a + provisional calibrated pair and triangulated into `dense-rgb:*` surface + claims, but production dense reconstruction must move to Aquarium/GPU + compute rather than Python loops. A separate + adaptive camera-control policy measures luminance, clipping, contrast, and + sharpness, then emits normalized exposure/gain/focus commands through the + OpenCV capture adapter when the driver accepts them. Unit tests inject + synthetic cameras and frames so the core does not depend on live camera + drivers. next_action: > Capture real ChArUco intrinsics/extrinsics for the PS3 Eyes and Kiyos, replace `config/sensor-fusion.example.json` placeholder transforms with a - local `config/sensor-fusion.json`, then replace the synthetic live producer - with a detector adapter that turns FFmpeg PS3 Eye frames into marker - `Observation2D` records for the fusion core and publishes the same typed - CultCache/CultNet render-frame document. In parallel, move the render-frame - consumer into Aquarium Engine so + local `config/sensor-fusion.json`, then move the dense RGB stereo matcher + to a GPU-resident stereo/flow pass so high-detail surface claims can scale + toward the million-splat temporal accumulator without Python owning each + point. Then replace the synthetic live producer with a detector adapter that + turns FFmpeg PS3 Eye frames into marker `Observation2D` records for the + fusion core and publishes the same typed CultCache/CultNet render-frame + document. In parallel, move the render-frame consumer into Aquarium Engine so Aquarium renders cached points/brushes into its GPU render target and publishes or hands off the same Spout sender `LocalCastBridge Point Cloud` for OBS, replacing the deadline OpenGL point sink without changing packet @@ -239,7 +250,7 @@ current_status: shotgun capture/transport path, replace placeholder mic/speaker geometry with measured world coordinates, then build the delay/SRO alignment stage that feeds the bounded field cache and emits aligned six-channel blocks for - FOA encoding. Fix neighbor loopback by using a render endpoint/driver that - accepts WASAPI loopback or a sender FFmpeg build with native WASAPI input; - do not reintroduce a virtual mixer unless direct loopback fails a concrete - invariant that the mixer actually satisfies. + FOA encoding. Keep neighbor loopback on the scheduled interactive WASAPI + capture path and use it as the timing witness once actual neighbor app + audio is present; do not reintroduce a virtual mixer unless direct loopback + fails a concrete invariant that the mixer actually satisfies. diff --git a/tests/test_sensor_fusion.py b/tests/test_sensor_fusion.py index 1ef925a..83cc038 100644 --- a/tests/test_sensor_fusion.py +++ b/tests/test_sensor_fusion.py @@ -6,7 +6,10 @@ import unittest import numpy as np from localcast.sensor_fusion import ( + AdaptiveCameraController, CameraModel, + CameraQualityTarget, + DenseStereoConfig, FusionConfig, Observation2D, PointCloud, @@ -17,6 +20,7 @@ from localcast.sensor_fusion import ( SpoutOutputConfig, TrackCache, frame_to_vertex_array, + dense_stereo_points, get_live_render_frame, lower_frame_to_screen_brushes, lower_points_to_render_frame, @@ -27,6 +31,7 @@ from localcast.sensor_fusion import ( from localcast.sensor_fusion.spout_output import rasterize_frame_rgba from scripts.live_sensor_fusion import ( leap_channel_motion_points, + rgb_dense_stereo_splats, rgb_room_splats, unpack_leap_packed_channels, ) @@ -327,6 +332,71 @@ class SensorFusionTests(unittest.TestCase): self.assertTrue(all(point.stable_key.startswith("leap-motion:green") for point in points)) self.assertTrue(all(point.source_timestamp_ns == 456 for point in points)) + def test_adaptive_camera_controller_reduces_overexposure(self): + controller = AdaptiveCameraController(target=CameraQualityTarget(mean_luma=0.5, luma_deadband=0.02)) + + command = controller.update( + quality=type( + "Quality", + (), + { + "mean_luma": 0.82, + "dark_clip": 0.0, + "bright_clip": 0.12, + "contrast": 0.2, + "sharpness": 0.2, + }, + )() + ) + + self.assertTrue(command.changed) + self.assertIsNotNone(command.exposure) + self.assertLess(command.exposure, 0.5) + self.assertIsNotNone(command.gain) + self.assertLess(command.gain, 0.25) + + def test_dense_stereo_emits_calibrated_rgb_surface_claims(self): + height, width = 48, 72 + rng = np.random.default_rng(42) + left_frame = rng.integers(0, 255, size=(height, width, 3), dtype=np.uint8) + right_frame = np.roll(left_frame, -8, axis=1) + left = camera("left", -0.25) + right = camera("right", 0.25) + left = CameraModel("left", left.camera_matrix, left.dist_coeffs, left.world_from_sensor, width, height) + right = CameraModel("right", right.camera_matrix, right.dist_coeffs, right.world_from_sensor, width, height) + + points = dense_stereo_points( + prefix="dense-test", + left_frame_bgr=left_frame, + right_frame_bgr=right_frame, + left_camera=left, + right_camera=right, + timestamp_ns=789, + config=DenseStereoConfig( + sample_step=8, + block_radius=2, + max_disparity_px=16, + min_ncc=0.55, + max_reprojection_error_px=200.0, + ), + ) + + self.assertGreater(len(points), 0) + self.assertTrue(all(point.stable_key.startswith("dense-test") for point in points)) + self.assertTrue(all(point.source_timestamp_ns == 789 for point in points)) + self.assertTrue(any(point.radius_m < 0.01 for point in points)) + + def test_live_rgb_sampler_dense_path_uses_camera_pair_before_fallback_surfaces(self): + height, width = 48, 72 + rng = np.random.default_rng(7) + primary = rng.integers(0, 255, size=(height, width, 3), dtype=np.uint8) + secondary = np.roll(primary, -7, axis=1) + + points = rgb_dense_stereo_splats(primary, secondary, 321, step=8) + + self.assertGreater(len(points), 0) + self.assertTrue(all(point.stable_key.startswith("dense-rgb") for point in points)) + if __name__ == "__main__": unittest.main()