From aa632d98bbd3c85293cb6a85df6ebb01505e4cbe Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 19 May 2026 10:06:53 +0100 Subject: [PATCH] Split diagnostic render math from Spout --- docs/implementation-plan.md | 6 +- docs/sensor-fusion-architecture.md | 7 +- localcast/diagnostics/render_math.py | 474 ++++++++++++++++++++++++++ localcast/diagnostics/spout_output.py | 467 +------------------------ notes/current-system-map.md | 3 + state/evidence.jsonl | 1 + tests/test_sensor_fusion.py | 2 +- 7 files changed, 501 insertions(+), 459 deletions(-) create mode 100644 localcast/diagnostics/render_math.py diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 6dba551..63aa750 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -53,7 +53,11 @@ shape removes independent per-kind storage from the live foundation. - `localcast.sensor_fusion.render_bridge` emits render-frame packets with visual/audio timing metadata and Spout sender identity. - `localcast.diagnostics.visual_cache` stores diagnostic visual state as typed CultCache MessagePack documents. It is not the production live boundary. - - `localcast.diagnostics.spout_output` renders render-frame packets into a GPU texture and publishes it as a named Spout sender for OBS. This is diagnostic/migration code, not production runtime. + - `localcast.diagnostics.render_math` owns pure diagnostic render budgeting, + camera projection, brush lowering, and CPU rasterization tests without an + OpenGL or OpenCV requirement. + - `localcast.diagnostics.spout_output` is now only the diagnostic Spout/OpenGL + publisher shell around that math. It is not production runtime. - `scripts/diagnostic_live_sensor_fusion.py` writes fused render frames into `calibration/runs/visual-state.msgpack`. This is diagnostic/migration code, not production runtime. - The live visual producer writes a multi-LOD scene cache with source kind and priority. Real Leap frames are promoted as the highest-priority visual timing/spatial evidence; Leap fallback frames remain lower-priority diagnostics. - Live clap calibration is wired into the visual producer. It keeps a rolling frame window from the Kiyo pair plus Leap, reads the live spatial audio frame for transient candidates, publishes `clap-events.msgpack`, injects clap calibration markers into the render frame, and writes clap evidence into the LOD cache. Kiyo stereo owns the current rough 3D solve; Leap owns the best visual timing witness until its geometric model is calibrated. diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index 8cdf8b6..011123e 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -75,7 +75,12 @@ flowchart TD - `localcast.diagnostics.visual_cache` owns the typed CultCache document shape used by diagnostic visual-state files. Production visual state should cross the native/CultNet boundary directly, not through this Python file adapter. -- `localcast.diagnostics.spout_output` owns the deadline OBS publication sink: render-frame packet in, GPU texture plus Spout sender heartbeat out. It is diagnostic/migration code, not the production visual authority. +- `localcast.diagnostics.render_math` owns pure diagnostic render budgeting, + camera projection, brush lowering, and CPU rasterization. It has unit tests + and does not require a GPU context. +- `localcast.diagnostics.spout_output` owns only the deadline OBS publication + sink: render-frame packet in, GPU texture plus Spout sender heartbeat out. It + is diagnostic/migration code, not the production visual authority. - `scripts/sensor_fusion.py` owns CLI composition for offline observation files. - `scripts/diagnostic_stream_spout.py` owns the diagnostic Spout sender loop for OBS. - `config/sensor-fusion.example.json` owns the declarative rig shape: capture hints, intrinsics, extrinsics, latency, and fusion thresholds. diff --git a/localcast/diagnostics/render_math.py b/localcast/diagnostics/render_math.py new file mode 100644 index 0000000..d32fe2c --- /dev/null +++ b/localcast/diagnostics/render_math.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import math +import time + +import numpy as np + +from localcast.sensor_fusion.render_bridge import RenderFramePacket, RenderPointPacket + + +@dataclass(frozen=True) +class SpoutOutputConfig: + sender_name: str = "LocalCastBridge Point Cloud" + width: int = 1920 + height: int = 1080 + fps: float = 60.0 + point_scale: float = 1.0 + demo_point_count: int = 4096 + camera_preset: str = "kiyo-mid-deru" + max_render_points: int = 5000 + + +@dataclass(frozen=True) +class VirtualCamera: + eye: tuple[float, float, float] + target: tuple[float, float, float] + up: tuple[float, float, float] = (0.0, 0.0, 1.0) + fov_degrees: float = 62.0 + near_m: float = 0.05 + far_m: float = 50.0 + + @property + def eye_np(self) -> np.ndarray: + return np.asarray(self.eye, dtype=np.float64) + + @property + def target_np(self) -> np.ndarray: + return np.asarray(self.target, dtype=np.float64) + + @property + def up_np(self) -> np.ndarray: + return np.asarray(self.up, dtype=np.float64) + + +@dataclass(frozen=True) +class ScreenBrushPacket: + center_px: tuple[int, int] + radii_px: tuple[int, int] + rotation_deg: float + color_rgba: tuple[int, int, int, int] + depth: float = 0.0 + + +def demo_render_frame(now_s: float, config: SpoutOutputConfig) -> RenderFramePacket: + count = max(1, int(config.demo_point_count)) + points: list[RenderPointPacket] = [] + for i in range(count): + u = i / count + ring = 0.15 + 1.15 * math.sqrt(u) + theta = i * 2.399963229728653 + now_s * 0.45 + z = 1.2 + 0.55 * math.sin(i * 0.031 + now_s * 0.9) + confidence = 0.35 + 0.65 * ((math.sin(i * 0.017 + now_s) + 1.0) * 0.5) + color = ( + 0.15 + 0.85 * confidence, + 0.35 + 0.35 * math.sin(theta + 1.7), + 0.70 + 0.25 * math.cos(theta), + 0.85, + ) + points.append( + RenderPointPacket( + stable_key=f"demo-{i}", + xyz=np.array([ring * math.cos(theta), ring * math.sin(theta), z], dtype=np.float64), + radius_m=0.018 + 0.035 * confidence, + color_rgba=color, + confidence=confidence, + source_timestamp_ns=time.monotonic_ns(), + ) + ) + now_ns = time.monotonic_ns() + return RenderFramePacket( + schema="localcast.sensor_fusion.render_frame.v1", + frame_id=int(now_s * config.fps), + created_monotonic_ns=now_ns, + source_time_min_ns=now_ns, + source_time_max_ns=now_ns, + present_time_ns=now_ns, + audio_alignment_time_ns=now_ns, + spout_sender_name=config.sender_name, + target_width=config.width, + target_height=config.height, + points=tuple(points), + ) + + +def frame_to_vertex_array(frame: RenderFramePacket, point_scale: float = 1.0) -> np.ndarray: + vertices = np.zeros((len(frame.points), 8), dtype=np.float32) + for index, point in enumerate(frame.points): + vertices[index, 0:3] = point.xyz.astype(np.float32) + rgba = np.asarray(point.color_rgba, dtype=np.float32) + confidence = max(0.0, min(1.0, float(point.confidence))) + vertices[index, 3:7] = np.clip(rgba, 0.0, 1.0) + vertices[index, 6] *= confidence + vertices[index, 7] = max(0.002, float(point.radius_m) * point_scale) + return vertices + + +def frame_with_point_budget( + frame: RenderFramePacket, + max_points: int, + camera_preset_name: str = "kiyo-mid-deru", +) -> RenderFramePacket: + if max_points <= 0 or len(frame.points) <= max_points: + return frame + pinned = sorted( + (point for point in frame.points if is_pinned_render_point(point)), + key=lambda point: (-render_priority(point, camera_preset_name), point.stable_key), + )[:max_points] + pinned_keys = {point.stable_key for point in pinned} + remaining_budget = max(0, max_points - len(pinned)) + if remaining_budget <= 0: + points = pinned + return _frame_with_points(frame, points) + priority_count = max(1, int(remaining_budget * 0.55)) + temporal_count = max(0, remaining_budget - priority_count) + priority = sorted( + (point for point in frame.points if point.stable_key not in pinned_keys), + key=lambda point: (-render_priority(point, camera_preset_name), stable_hash(point.stable_key, 0)), + )[:priority_count] + selected_keys = pinned_keys | {point.stable_key for point in priority} + remainder = [point for point in frame.points if point.stable_key not in selected_keys] + temporal = sorted( + remainder, + key=lambda point: stable_hash(point.stable_key, frame.frame_id) / max(0.05, render_priority(point, camera_preset_name)), + )[:temporal_count] + points = pinned + priority + temporal + points = sorted(points, key=lambda point: point.stable_key) + return _frame_with_points(frame, points) + + +def _frame_with_points(frame: RenderFramePacket, points: list[RenderPointPacket]) -> RenderFramePacket: + return RenderFramePacket( + schema=frame.schema, + frame_id=frame.frame_id, + created_monotonic_ns=frame.created_monotonic_ns, + source_time_min_ns=frame.source_time_min_ns, + source_time_max_ns=frame.source_time_max_ns, + present_time_ns=frame.present_time_ns, + audio_alignment_time_ns=frame.audio_alignment_time_ns, + spout_sender_name=frame.spout_sender_name, + target_width=frame.target_width, + target_height=frame.target_height, + points=tuple(points), + ) + + +def is_pinned_render_point(point: RenderPointPacket) -> bool: + return point.stable_key.startswith( + ( + "camera-chirp:", + "camera-pose-correction:", + "camera-sync:", + "clap-calibration:", + "clap-ray:", + "clap-timing:", + "audio-event-", + ) + ) + + +def point_prefix_counts(points: tuple[RenderPointPacket, ...] | list[RenderPointPacket]) -> dict[str, int]: + counts: dict[str, int] = {} + for point in points: + prefix = point_prefix(point.stable_key) + counts[prefix] = counts.get(prefix, 0) + 1 + return dict(sorted(counts.items())) + + +def point_prefix(stable_key: str) -> str: + if ":" in stable_key: + return stable_key.split(":", 1)[0] + if stable_key.startswith("audio-event-"): + return "audio-event" + return stable_key + + +def stable_hash(key: str, salt: int) -> int: + digest = hashlib.blake2s(f"{salt}:{key}".encode("utf-8"), digest_size=8).digest() + return int.from_bytes(digest, "little") + + +def render_priority(point: RenderPointPacket, camera_preset_name: str) -> float: + key = point.stable_key + confidence = max(0.0, min(1.0, float(point.confidence))) + semantic = 1.0 + if key.startswith("clap-calibration:"): + semantic = 3.2 + elif key.startswith(("camera-sync:", "camera-chirp:", "camera-pose-correction:")): + semantic = 2.6 + elif key.startswith(("clap-ray:", "clap-timing:")): + semantic = 2.2 + elif key.startswith("dense-rgb:"): + semantic = 2.0 + elif key.startswith("leap-motion:"): + semantic = 1.45 + elif key.startswith("room-rgb:"): + semantic = 1.15 + elif key.startswith("stochastic:"): + semantic = 0.36 + elif key.startswith("audio-event-"): + semantic = 0.24 + elif key.startswith(("sensor:", "frustum:")): + semantic = 0.12 + focus = render_focus_weight(point, camera_preset_name) + return max(0.001, confidence * semantic * focus) + + +def render_focus_weight(point: RenderPointPacket, camera_preset_name: str) -> float: + if camera_preset_name != "kiyo-mid-deru": + return 1.0 + xyz = np.asarray(point.xyz, dtype=np.float64) + deru = np.array([0.48, 0.28, 1.24], dtype=np.float64) + distance = float(np.linalg.norm((xyz - deru) * np.array([1.0, 1.0, 0.75], dtype=np.float64))) + return 0.22 + 1.78 / (1.0 + distance * 1.85) + + +def lower_frame_to_screen_brushes( + frame: RenderFramePacket, + width: int, + height: int, + point_scale: float = 1.0, + camera: VirtualCamera | None = None, +) -> tuple[ScreenBrushPacket, ...]: + brushes: list[ScreenBrushPacket] = [] + view_proj = camera_matrix(0.0, width / max(1, height), camera) + for point in frame.points: + projected = project_world_to_screen(point.xyz, view_proj, width, height) + if projected is None: + continue + px, py, depth, clip_w = projected + radius_scale = semantic_radius_scale(point.stable_key) + radius = max(2, int(round(float(point.radius_m) * point_scale * height * 1.45 * radius_scale / max(0.42, clip_w)))) + if px < -radius * 3 or px >= width + radius * 3 or py < -radius * 3 or py >= height + radius * 3: + continue + stable = hashlib.blake2s(point.stable_key.encode("utf-8"), digest_size=4).digest() + angle = int.from_bytes(stable[:2], "little") / 65535.0 * 180.0 + semantic_stretch = 2.8 if point.stable_key.startswith(("frustum:", "ray:", "floor:", "wall:")) else 1.0 + stretch = semantic_stretch * (1.0 + (stable[2] / 255.0) * 1.45) + radii = (max(2, int(round(radius * stretch))), radius) + rgba = np.clip(np.asarray(point.color_rgba, dtype=np.float32), 0.0, 1.0) + rgba[3] *= max(0.0, min(1.0, float(point.confidence))) + rgba[3] *= semantic_alpha_scale(point.stable_key) + color = tuple(int(v) for v in (rgba * 255.0)[:4]) + brushes.append(ScreenBrushPacket((px, py), radii, angle, color, depth)) + return tuple(sorted(brushes, key=lambda brush: brush.depth, reverse=True)) + + +def semantic_radius_scale(stable_key: str) -> float: + if stable_key.startswith("audio-event-"): + return 0.42 + if stable_key.startswith("stochastic:"): + return 0.50 + return 1.0 + + +def semantic_alpha_scale(stable_key: str) -> float: + if stable_key.startswith("audio-event-"): + return 0.36 + if stable_key.startswith("stochastic:"): + return 0.46 + return 1.0 + + +def rasterize_frame_rgba( + frame: RenderFramePacket, + width: int, + height: int, + point_scale: float = 1.0, + camera: VirtualCamera | None = None, +) -> np.ndarray: + image = np.zeros((height, width, 4), dtype=np.uint8) + image[:, :, 0] = 7 + image[:, :, 1] = 10 + image[:, :, 2] = 15 + image[:, :, 3] = 255 + draw_reconstruction_guides(image, camera) + draw_floor_shadows(image, frame, point_scale, camera) + canvas = image.astype(np.float32) / 255.0 + for brush in lower_frame_to_screen_brushes(frame, width, height, point_scale, camera): + composite_gaussian_brush(canvas, brush) + return np.clip(canvas * 255.0, 0.0, 255.0).astype(np.uint8) + + +def composite_gaussian_brush(canvas: np.ndarray, brush: ScreenBrushPacket) -> None: + height, width = canvas.shape[:2] + cx, cy = brush.center_px + rx, ry = max(1, brush.radii_px[0]), max(1, brush.radii_px[1]) + bound = int(math.ceil(max(rx, ry) * 1.35)) + x0 = max(0, cx - bound) + x1 = min(width, cx + bound + 1) + y0 = max(0, cy - bound) + y1 = min(height, cy + bound + 1) + if x0 >= x1 or y0 >= y1: + return + + yy, xx = np.mgrid[y0:y1, x0:x1].astype(np.float32) + dx = xx - float(cx) + dy = yy - float(cy) + angle = math.radians(float(brush.rotation_deg)) + ca = math.cos(angle) + sa = math.sin(angle) + local_x = dx * ca + dy * sa + local_y = -dx * sa + dy * ca + q = (local_x / float(rx)) ** 2 + (local_y / float(ry)) ** 2 + mask = q < 1.0 + if not np.any(mask): + return + + # Compact Gaussian envelope: smooth center, finite edge, no solid paint bucket. + falloff = 4.4 + edge = math.exp(-falloff) + gaussian = np.exp(-falloff * q) + envelope = np.clip((gaussian - edge) / max(1.0 - edge, 1e-6), 0.0, 1.0) ** 1.35 + envelope *= mask + + src = np.asarray(brush.color_rgba, dtype=np.float32) / 255.0 + alpha = np.clip(envelope * src[3], 0.0, 0.96) + if float(alpha.max()) <= 0.0: + return + target = canvas[y0:y1, x0:x1, :] + target[:, :, :3] = src[:3] * alpha[:, :, None] + target[:, :, :3] * (1.0 - alpha[:, :, None]) + target[:, :, 3] = np.maximum(target[:, :, 3], alpha) + + +def project_world_to_screen( + xyz: np.ndarray, + view_proj: np.ndarray, + width: int, + height: int, +) -> tuple[int, int, float, float] | None: + point = np.ones(4, dtype=np.float64) + point[:3] = np.asarray(xyz, dtype=np.float64) + clip = view_proj @ point + if clip[3] <= 0.001: + return None + ndc = clip[:3] / clip[3] + if ndc[0] < -1.35 or ndc[0] > 1.35 or ndc[1] < -1.35 or ndc[1] > 1.35: + return None + px = int(round((ndc[0] * 0.5 + 0.5) * width)) + py = int(round((0.5 - ndc[1] * 0.5) * height)) + return px, py, float(ndc[2]), float(clip[3]) + + +def draw_reconstruction_guides(image: np.ndarray, camera: VirtualCamera | None = None) -> None: + try: + import cv2 + except ModuleNotFoundError: + return + + height, width = image.shape[:2] + view_proj = camera_matrix(0.0, width / max(1, height), camera) + grid_color = (36, 50, 62, 255) + wall_color = (24, 36, 50, 255) + for x in np.linspace(-1.8, 1.8, 13): + draw_world_line(image, view_proj, np.array([x, -1.0, 0.0]), np.array([x, 1.15, 0.0]), grid_color, 1) + for y in np.linspace(-1.0, 1.15, 9): + draw_world_line(image, view_proj, np.array([-1.8, y, 0.0]), np.array([1.8, y, 0.0]), grid_color, 1) + for x in np.linspace(-1.8, 1.8, 9): + draw_world_line(image, view_proj, np.array([x, 1.15, 0.0]), np.array([x, 1.15, 2.2]), wall_color, 1) + for z in np.linspace(0.35, 2.1, 6): + draw_world_line(image, view_proj, np.array([-1.8, 1.15, z]), np.array([1.8, 1.15, z]), wall_color, 1) + +def draw_floor_shadows( + image: np.ndarray, + frame: RenderFramePacket, + point_scale: float, + camera: VirtualCamera | None = None, +) -> None: + try: + import cv2 + except ModuleNotFoundError: + return + + height, width = image.shape[:2] + view_proj = camera_matrix(0.0, width / max(1, height), camera) + for point in frame.points: + xyz = np.asarray(point.xyz, dtype=np.float64) + if xyz[2] <= 0.10 or point.stable_key.startswith(("sensor:", "frustum:")): + continue + shadow = xyz.copy() + shadow[2] = 0.012 + projected = project_world_to_screen(shadow, view_proj, width, height) + if projected is None: + continue + px, py, _, clip_w = projected + radius = max(2, int(round(float(point.radius_m) * point_scale * height * 0.9 / max(0.42, clip_w)))) + shade = int(18 + 18 * max(0.0, min(1.0, float(point.confidence)))) + cv2.ellipse(image, (px, py), (radius * 2, max(1, radius // 2)), 0.0, 0.0, 360.0, (shade // 3, shade // 2, shade, 255), -1, cv2.LINE_AA) + + +def draw_world_line( + image: np.ndarray, + view_proj: np.ndarray, + start: np.ndarray, + end: np.ndarray, + color: tuple[int, int, int, int], + thickness: int, +) -> None: + try: + import cv2 + except ModuleNotFoundError: + return + + height, width = image.shape[:2] + a = project_world_to_screen(start, view_proj, width, height) + b = project_world_to_screen(end, view_proj, width, height) + if a is None or b is None: + return + cv2.line(image, a[:2], b[:2], color, thickness=thickness, lineType=cv2.LINE_AA) + + + +def camera_preset(name: str, now_s: float = 0.0) -> VirtualCamera: + if name == "orbit": + return VirtualCamera( + eye=(2.4 * math.sin(now_s * 0.08), -3.4, 1.85 + 0.2 * math.cos(now_s * 0.11)), + target=(0.0, 0.0, 1.15), + fov_degrees=58.0, + ) + if name == "kiyo-mid-deru": + return VirtualCamera( + eye=(0.05, -1.035, 1.64), + target=(0.48, 0.28, 1.24), + fov_degrees=64.0, + ) + raise ValueError(f"unknown camera preset: {name}") + + +def camera_matrix(now_s: float, aspect: float, camera: VirtualCamera | None = None) -> np.ndarray: + view = camera_preset("orbit", now_s) if camera is None else camera + return perspective(math.radians(view.fov_degrees), aspect, view.near_m, view.far_m) @ look_at( + view.eye_np, + view.target_np, + view.up_np, + ) + + +def look_at(eye: np.ndarray, target: np.ndarray, up: np.ndarray) -> np.ndarray: + forward = target - eye + forward = forward / np.linalg.norm(forward) + side = np.cross(forward, up) + side = side / np.linalg.norm(side) + true_up = np.cross(side, forward) + matrix = np.eye(4, dtype=np.float64) + matrix[0, :3] = side + matrix[1, :3] = true_up + matrix[2, :3] = -forward + translate = np.eye(4, dtype=np.float64) + translate[:3, 3] = -eye + return matrix @ translate + + +def perspective(fov_y: float, aspect: float, near: float, far: float) -> np.ndarray: + f = 1.0 / math.tan(fov_y * 0.5) + matrix = np.zeros((4, 4), dtype=np.float64) + matrix[0, 0] = f / aspect + matrix[1, 1] = f + matrix[2, 2] = (far + near) / (near - far) + matrix[2, 3] = (2.0 * far * near) / (near - far) + matrix[3, 2] = -1.0 + return matrix + + diff --git a/localcast/diagnostics/spout_output.py b/localcast/diagnostics/spout_output.py index d243ae6..a2d0e9f 100644 --- a/localcast/diagnostics/spout_output.py +++ b/localcast/diagnostics/spout_output.py @@ -1,10 +1,7 @@ -from __future__ import annotations +from __future__ import annotations -from dataclasses import dataclass import ctypes -import hashlib import json -import math from pathlib import Path import time @@ -14,52 +11,19 @@ from localcast.sensor_fusion.audio_overlay import AudioVisualSyncStatus, overlay from localcast.sensor_fusion.media_artifacts import remote_video_artifact_for_present_time from localcast.sensor_fusion.render_bridge import RenderFramePacket, RenderPointPacket from localcast.diagnostics.render_frame_json import load_render_frame_json +from localcast.diagnostics.render_math import ( + SpoutOutputConfig, + camera_matrix, + camera_preset, + demo_render_frame, + frame_to_vertex_array, + frame_with_point_budget, + point_prefix_counts, + rasterize_frame_rgba, +) from localcast.diagnostics.visual_cache import CultStreamStatus, get_live_render_frame, put_stream_status -@dataclass(frozen=True) -class SpoutOutputConfig: - sender_name: str = "LocalCastBridge Point Cloud" - width: int = 1920 - height: int = 1080 - fps: float = 60.0 - point_scale: float = 1.0 - demo_point_count: int = 4096 - camera_preset: str = "kiyo-mid-deru" - max_render_points: int = 5000 - - -@dataclass(frozen=True) -class VirtualCamera: - eye: tuple[float, float, float] - target: tuple[float, float, float] - up: tuple[float, float, float] = (0.0, 0.0, 1.0) - fov_degrees: float = 62.0 - near_m: float = 0.05 - far_m: float = 50.0 - - @property - def eye_np(self) -> np.ndarray: - return np.asarray(self.eye, dtype=np.float64) - - @property - def target_np(self) -> np.ndarray: - return np.asarray(self.target, dtype=np.float64) - - @property - def up_np(self) -> np.ndarray: - return np.asarray(self.up, dtype=np.float64) - - -@dataclass(frozen=True) -class ScreenBrushPacket: - center_px: tuple[int, int] - radii_px: tuple[int, int] - rotation_deg: float - color_rgba: tuple[int, int, int, int] - depth: float = 0.0 - - class RenderFrameFileSource: def __init__(self, path: Path, demo_if_missing: bool = False, demo_point_count: int = 4096) -> None: self.path = path @@ -173,364 +137,6 @@ class CultCacheRenderFrameSource: return self._events -def demo_render_frame(now_s: float, config: SpoutOutputConfig) -> RenderFramePacket: - count = max(1, int(config.demo_point_count)) - points: list[RenderPointPacket] = [] - for i in range(count): - u = i / count - ring = 0.15 + 1.15 * math.sqrt(u) - theta = i * 2.399963229728653 + now_s * 0.45 - z = 1.2 + 0.55 * math.sin(i * 0.031 + now_s * 0.9) - confidence = 0.35 + 0.65 * ((math.sin(i * 0.017 + now_s) + 1.0) * 0.5) - color = ( - 0.15 + 0.85 * confidence, - 0.35 + 0.35 * math.sin(theta + 1.7), - 0.70 + 0.25 * math.cos(theta), - 0.85, - ) - points.append( - RenderPointPacket( - stable_key=f"demo-{i}", - xyz=np.array([ring * math.cos(theta), ring * math.sin(theta), z], dtype=np.float64), - radius_m=0.018 + 0.035 * confidence, - color_rgba=color, - confidence=confidence, - source_timestamp_ns=time.monotonic_ns(), - ) - ) - now_ns = time.monotonic_ns() - return RenderFramePacket( - schema="localcast.sensor_fusion.render_frame.v1", - frame_id=int(now_s * config.fps), - created_monotonic_ns=now_ns, - source_time_min_ns=now_ns, - source_time_max_ns=now_ns, - present_time_ns=now_ns, - audio_alignment_time_ns=now_ns, - spout_sender_name=config.sender_name, - target_width=config.width, - target_height=config.height, - points=tuple(points), - ) - - -def frame_to_vertex_array(frame: RenderFramePacket, point_scale: float = 1.0) -> np.ndarray: - vertices = np.zeros((len(frame.points), 8), dtype=np.float32) - for index, point in enumerate(frame.points): - vertices[index, 0:3] = point.xyz.astype(np.float32) - rgba = np.asarray(point.color_rgba, dtype=np.float32) - confidence = max(0.0, min(1.0, float(point.confidence))) - vertices[index, 3:7] = np.clip(rgba, 0.0, 1.0) - vertices[index, 6] *= confidence - vertices[index, 7] = max(0.002, float(point.radius_m) * point_scale) - return vertices - - -def frame_with_point_budget( - frame: RenderFramePacket, - max_points: int, - camera_preset_name: str = "kiyo-mid-deru", -) -> RenderFramePacket: - if max_points <= 0 or len(frame.points) <= max_points: - return frame - pinned = sorted( - (point for point in frame.points if is_pinned_render_point(point)), - key=lambda point: (-render_priority(point, camera_preset_name), point.stable_key), - )[:max_points] - pinned_keys = {point.stable_key for point in pinned} - remaining_budget = max(0, max_points - len(pinned)) - if remaining_budget <= 0: - points = pinned - return _frame_with_points(frame, points) - priority_count = max(1, int(remaining_budget * 0.55)) - temporal_count = max(0, remaining_budget - priority_count) - priority = sorted( - (point for point in frame.points if point.stable_key not in pinned_keys), - key=lambda point: (-render_priority(point, camera_preset_name), stable_hash(point.stable_key, 0)), - )[:priority_count] - selected_keys = pinned_keys | {point.stable_key for point in priority} - remainder = [point for point in frame.points if point.stable_key not in selected_keys] - temporal = sorted( - remainder, - key=lambda point: stable_hash(point.stable_key, frame.frame_id) / max(0.05, render_priority(point, camera_preset_name)), - )[:temporal_count] - points = pinned + priority + temporal - points = sorted(points, key=lambda point: point.stable_key) - return _frame_with_points(frame, points) - - -def _frame_with_points(frame: RenderFramePacket, points: list[RenderPointPacket]) -> RenderFramePacket: - return RenderFramePacket( - schema=frame.schema, - frame_id=frame.frame_id, - created_monotonic_ns=frame.created_monotonic_ns, - source_time_min_ns=frame.source_time_min_ns, - source_time_max_ns=frame.source_time_max_ns, - present_time_ns=frame.present_time_ns, - audio_alignment_time_ns=frame.audio_alignment_time_ns, - spout_sender_name=frame.spout_sender_name, - target_width=frame.target_width, - target_height=frame.target_height, - points=tuple(points), - ) - - -def is_pinned_render_point(point: RenderPointPacket) -> bool: - return point.stable_key.startswith( - ( - "camera-chirp:", - "camera-pose-correction:", - "camera-sync:", - "clap-calibration:", - "clap-ray:", - "clap-timing:", - "audio-event-", - ) - ) - - -def point_prefix_counts(points: tuple[RenderPointPacket, ...] | list[RenderPointPacket]) -> dict[str, int]: - counts: dict[str, int] = {} - for point in points: - prefix = point_prefix(point.stable_key) - counts[prefix] = counts.get(prefix, 0) + 1 - return dict(sorted(counts.items())) - - -def point_prefix(stable_key: str) -> str: - if ":" in stable_key: - return stable_key.split(":", 1)[0] - if stable_key.startswith("audio-event-"): - return "audio-event" - return stable_key - - -def stable_hash(key: str, salt: int) -> int: - digest = hashlib.blake2s(f"{salt}:{key}".encode("utf-8"), digest_size=8).digest() - return int.from_bytes(digest, "little") - - -def render_priority(point: RenderPointPacket, camera_preset_name: str) -> float: - key = point.stable_key - confidence = max(0.0, min(1.0, float(point.confidence))) - semantic = 1.0 - if key.startswith("clap-calibration:"): - semantic = 3.2 - elif key.startswith(("camera-sync:", "camera-chirp:", "camera-pose-correction:")): - semantic = 2.6 - elif key.startswith(("clap-ray:", "clap-timing:")): - semantic = 2.2 - elif key.startswith("dense-rgb:"): - semantic = 2.0 - elif key.startswith("leap-motion:"): - semantic = 1.45 - elif key.startswith("room-rgb:"): - semantic = 1.15 - elif key.startswith("stochastic:"): - semantic = 0.36 - elif key.startswith("audio-event-"): - semantic = 0.24 - elif key.startswith(("sensor:", "frustum:")): - semantic = 0.12 - focus = render_focus_weight(point, camera_preset_name) - return max(0.001, confidence * semantic * focus) - - -def render_focus_weight(point: RenderPointPacket, camera_preset_name: str) -> float: - if camera_preset_name != "kiyo-mid-deru": - return 1.0 - xyz = np.asarray(point.xyz, dtype=np.float64) - deru = np.array([0.48, 0.28, 1.24], dtype=np.float64) - distance = float(np.linalg.norm((xyz - deru) * np.array([1.0, 1.0, 0.75], dtype=np.float64))) - return 0.22 + 1.78 / (1.0 + distance * 1.85) - - -def lower_frame_to_screen_brushes( - frame: RenderFramePacket, - width: int, - height: int, - point_scale: float = 1.0, - camera: VirtualCamera | None = None, -) -> tuple[ScreenBrushPacket, ...]: - brushes: list[ScreenBrushPacket] = [] - view_proj = camera_matrix(0.0, width / max(1, height), camera) - for point in frame.points: - projected = project_world_to_screen(point.xyz, view_proj, width, height) - if projected is None: - continue - px, py, depth, clip_w = projected - radius_scale = semantic_radius_scale(point.stable_key) - radius = max(2, int(round(float(point.radius_m) * point_scale * height * 1.45 * radius_scale / max(0.42, clip_w)))) - if px < -radius * 3 or px >= width + radius * 3 or py < -radius * 3 or py >= height + radius * 3: - continue - stable = hashlib.blake2s(point.stable_key.encode("utf-8"), digest_size=4).digest() - angle = int.from_bytes(stable[:2], "little") / 65535.0 * 180.0 - semantic_stretch = 2.8 if point.stable_key.startswith(("frustum:", "ray:", "floor:", "wall:")) else 1.0 - stretch = semantic_stretch * (1.0 + (stable[2] / 255.0) * 1.45) - radii = (max(2, int(round(radius * stretch))), radius) - rgba = np.clip(np.asarray(point.color_rgba, dtype=np.float32), 0.0, 1.0) - rgba[3] *= max(0.0, min(1.0, float(point.confidence))) - rgba[3] *= semantic_alpha_scale(point.stable_key) - color = tuple(int(v) for v in (rgba * 255.0)[:4]) - brushes.append(ScreenBrushPacket((px, py), radii, angle, color, depth)) - return tuple(sorted(brushes, key=lambda brush: brush.depth, reverse=True)) - - -def semantic_radius_scale(stable_key: str) -> float: - if stable_key.startswith("audio-event-"): - return 0.42 - if stable_key.startswith("stochastic:"): - return 0.50 - return 1.0 - - -def semantic_alpha_scale(stable_key: str) -> float: - if stable_key.startswith("audio-event-"): - return 0.36 - if stable_key.startswith("stochastic:"): - return 0.46 - return 1.0 - - -def rasterize_frame_rgba( - frame: RenderFramePacket, - width: int, - height: int, - point_scale: float = 1.0, - camera: VirtualCamera | None = None, -) -> np.ndarray: - image = np.zeros((height, width, 4), dtype=np.uint8) - image[:, :, 0] = 7 - image[:, :, 1] = 10 - image[:, :, 2] = 15 - image[:, :, 3] = 255 - draw_reconstruction_guides(image, camera) - draw_floor_shadows(image, frame, point_scale, camera) - canvas = image.astype(np.float32) / 255.0 - for brush in lower_frame_to_screen_brushes(frame, width, height, point_scale, camera): - composite_gaussian_brush(canvas, brush) - return np.clip(canvas * 255.0, 0.0, 255.0).astype(np.uint8) - - -def composite_gaussian_brush(canvas: np.ndarray, brush: ScreenBrushPacket) -> None: - height, width = canvas.shape[:2] - cx, cy = brush.center_px - rx, ry = max(1, brush.radii_px[0]), max(1, brush.radii_px[1]) - bound = int(math.ceil(max(rx, ry) * 1.35)) - x0 = max(0, cx - bound) - x1 = min(width, cx + bound + 1) - y0 = max(0, cy - bound) - y1 = min(height, cy + bound + 1) - if x0 >= x1 or y0 >= y1: - return - - yy, xx = np.mgrid[y0:y1, x0:x1].astype(np.float32) - dx = xx - float(cx) - dy = yy - float(cy) - angle = math.radians(float(brush.rotation_deg)) - ca = math.cos(angle) - sa = math.sin(angle) - local_x = dx * ca + dy * sa - local_y = -dx * sa + dy * ca - q = (local_x / float(rx)) ** 2 + (local_y / float(ry)) ** 2 - mask = q < 1.0 - if not np.any(mask): - return - - # Compact Gaussian envelope: smooth center, finite edge, no solid paint bucket. - falloff = 4.4 - edge = math.exp(-falloff) - gaussian = np.exp(-falloff * q) - envelope = np.clip((gaussian - edge) / max(1.0 - edge, 1e-6), 0.0, 1.0) ** 1.35 - envelope *= mask - - src = np.asarray(brush.color_rgba, dtype=np.float32) / 255.0 - alpha = np.clip(envelope * src[3], 0.0, 0.96) - if float(alpha.max()) <= 0.0: - return - target = canvas[y0:y1, x0:x1, :] - target[:, :, :3] = src[:3] * alpha[:, :, None] + target[:, :, :3] * (1.0 - alpha[:, :, None]) - target[:, :, 3] = np.maximum(target[:, :, 3], alpha) - - -def project_world_to_screen( - xyz: np.ndarray, - view_proj: np.ndarray, - width: int, - height: int, -) -> tuple[int, int, float, float] | None: - point = np.ones(4, dtype=np.float64) - point[:3] = np.asarray(xyz, dtype=np.float64) - clip = view_proj @ point - if clip[3] <= 0.001: - return None - ndc = clip[:3] / clip[3] - if ndc[0] < -1.35 or ndc[0] > 1.35 or ndc[1] < -1.35 or ndc[1] > 1.35: - return None - px = int(round((ndc[0] * 0.5 + 0.5) * width)) - py = int(round((0.5 - ndc[1] * 0.5) * height)) - return px, py, float(ndc[2]), float(clip[3]) - - -def draw_reconstruction_guides(image: np.ndarray, camera: VirtualCamera | None = None) -> None: - import cv2 - - height, width = image.shape[:2] - view_proj = camera_matrix(0.0, width / max(1, height), camera) - grid_color = (36, 50, 62, 255) - wall_color = (24, 36, 50, 255) - for x in np.linspace(-1.8, 1.8, 13): - draw_world_line(image, view_proj, np.array([x, -1.0, 0.0]), np.array([x, 1.15, 0.0]), grid_color, 1) - for y in np.linspace(-1.0, 1.15, 9): - draw_world_line(image, view_proj, np.array([-1.8, y, 0.0]), np.array([1.8, y, 0.0]), grid_color, 1) - for x in np.linspace(-1.8, 1.8, 9): - draw_world_line(image, view_proj, np.array([x, 1.15, 0.0]), np.array([x, 1.15, 2.2]), wall_color, 1) - for z in np.linspace(0.35, 2.1, 6): - draw_world_line(image, view_proj, np.array([-1.8, 1.15, z]), np.array([1.8, 1.15, z]), wall_color, 1) - -def draw_floor_shadows( - image: np.ndarray, - frame: RenderFramePacket, - point_scale: float, - camera: VirtualCamera | None = None, -) -> None: - import cv2 - - height, width = image.shape[:2] - view_proj = camera_matrix(0.0, width / max(1, height), camera) - for point in frame.points: - xyz = np.asarray(point.xyz, dtype=np.float64) - if xyz[2] <= 0.10 or point.stable_key.startswith(("sensor:", "frustum:")): - continue - shadow = xyz.copy() - shadow[2] = 0.012 - projected = project_world_to_screen(shadow, view_proj, width, height) - if projected is None: - continue - px, py, _, clip_w = projected - radius = max(2, int(round(float(point.radius_m) * point_scale * height * 0.9 / max(0.42, clip_w)))) - shade = int(18 + 18 * max(0.0, min(1.0, float(point.confidence)))) - cv2.ellipse(image, (px, py), (radius * 2, max(1, radius // 2)), 0.0, 0.0, 360.0, (shade // 3, shade // 2, shade, 255), -1, cv2.LINE_AA) - - -def draw_world_line( - image: np.ndarray, - view_proj: np.ndarray, - start: np.ndarray, - end: np.ndarray, - color: tuple[int, int, int, int], - thickness: int, -) -> None: - import cv2 - - height, width = image.shape[:2] - a = project_world_to_screen(start, view_proj, width, height) - b = project_world_to_screen(end, view_proj, width, height) - if a is None or b is None: - return - cv2.line(image, a[:2], b[:2], color, thickness=thickness, lineType=cv2.LINE_AA) - - def write_status( path: Path, *, @@ -787,57 +393,6 @@ class OpenGLSpoutPointRenderer: self.sender = None -def camera_preset(name: str, now_s: float = 0.0) -> VirtualCamera: - if name == "orbit": - return VirtualCamera( - eye=(2.4 * math.sin(now_s * 0.08), -3.4, 1.85 + 0.2 * math.cos(now_s * 0.11)), - target=(0.0, 0.0, 1.15), - fov_degrees=58.0, - ) - if name == "kiyo-mid-deru": - return VirtualCamera( - eye=(0.05, -1.035, 1.64), - target=(0.48, 0.28, 1.24), - fov_degrees=64.0, - ) - raise ValueError(f"unknown camera preset: {name}") - - -def camera_matrix(now_s: float, aspect: float, camera: VirtualCamera | None = None) -> np.ndarray: - view = camera_preset("orbit", now_s) if camera is None else camera - return perspective(math.radians(view.fov_degrees), aspect, view.near_m, view.far_m) @ look_at( - view.eye_np, - view.target_np, - view.up_np, - ) - - -def look_at(eye: np.ndarray, target: np.ndarray, up: np.ndarray) -> np.ndarray: - forward = target - eye - forward = forward / np.linalg.norm(forward) - side = np.cross(forward, up) - side = side / np.linalg.norm(side) - true_up = np.cross(side, forward) - matrix = np.eye(4, dtype=np.float64) - matrix[0, :3] = side - matrix[1, :3] = true_up - matrix[2, :3] = -forward - translate = np.eye(4, dtype=np.float64) - translate[:3, 3] = -eye - return matrix @ translate - - -def perspective(fov_y: float, aspect: float, near: float, far: float) -> np.ndarray: - f = 1.0 / math.tan(fov_y * 0.5) - matrix = np.zeros((4, 4), dtype=np.float64) - matrix[0, 0] = f / aspect - matrix[1, 1] = f - matrix[2, 2] = (far + near) / (near - far) - matrix[2, 3] = (2.0 * far * near) / (near - far) - matrix[3, 2] = -1.0 - return matrix - - VERTEX_SHADER = """ #version 330 core layout(location = 0) in vec3 in_pos; diff --git a/notes/current-system-map.md b/notes/current-system-map.md index ec7ce1a..0796370 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -151,6 +151,9 @@ Ownership: - `localcast.diagnostics.render_frame_json` and `localcast.diagnostics.lod_json` are explicit compatibility adapters; model modules do not own file serialization. +- `localcast.diagnostics.render_math` is the pure, testable diagnostic renderer + math split away from the OpenGL publisher. `spout_output` is now only the + disposable Spout/OpenGL shell. Invariant: fallback/demo evidence cannot enter ground-truth reservoir kinds. Render packets, LOD cells, brush packets, and overlays are derived from reservoir diff --git a/state/evidence.jsonl b/state/evidence.jsonl index a5eb342..90ed54e 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -62,3 +62,4 @@ {"ts": "2026-05-19T09:36:00+01:00", "type": "implementation", "status": "ok", "note": "RenderFramePacket no longer owns JSON file serialization. Compatibility JSON load/write moved to localcast.diagnostics.render_frame_json, diagnostic Spout and sparse-fusion scripts import it explicitly, and tests/import guards prove the production packet module is typed data only."} {"ts": "2026-05-19T09:45:00+01:00", "type": "implementation", "status": "ok", "note": "MultiLODSceneCache no longer owns JSON file serialization. Diagnostic LOD writes moved to localcast.diagnostics.lod_json, diagnostic_live_sensor_fusion imports that adapter explicitly, and tests/import guards prove stochastic_mapping is model/algorithm code only."} {"ts": "2026-05-19T10:02:00+01:00", "type": "implementation", "status": "ok", "note": "Native reservoir tightened sample provenance: live samples must have flags == 0, so diagnostic and unknown future flags are rejected at both raw reservoir and LocalcastRuntime typed producer boundaries. Unknown flags no longer slip through as speculative metadata."} +{"ts": "2026-05-19T10:20:00+01:00", "type": "implementation", "status": "ok", "note": "Diagnostic render math split away from the OpenGL Spout publisher: localcast.diagnostics.render_math owns pure budget/projection/brush/rasterization code and tests import it directly, while spout_output is reduced to frame sources, status writers, and the disposable OpenGL/Spout shell."} diff --git a/tests/test_sensor_fusion.py b/tests/test_sensor_fusion.py index 4c468ef..54d2ad5 100644 --- a/tests/test_sensor_fusion.py +++ b/tests/test_sensor_fusion.py @@ -54,7 +54,7 @@ from localcast.diagnostics.visual_cache import ( put_live_render_frame, ) from localcast.sensor_fusion.audio_overlay import clamp_frame_to_audio_reservoir -from localcast.diagnostics.spout_output import ( +from localcast.diagnostics.render_math import ( SpoutOutputConfig, VirtualCamera, frame_to_vertex_array,