From b3e41fa2bdc5ea75e761e587fab170ce91afcca7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 19 May 2026 09:20:32 +0100 Subject: [PATCH] Quarantine visual cache adapter --- docs/implementation-plan.md | 6 +++-- docs/obs-spout-streaming.md | 4 ++- docs/sensor-fusion-architecture.md | 9 +++++-- docs/typed-visual-state.md | 20 +++++++++----- localcast/diagnostics/spout_output.py | 2 +- .../visual_cache.py} | 2 +- localcast/sensor_fusion/__init__.py | 26 ------------------- scripts/diagnostic_live_sensor_fusion.py | 10 ++++--- state/evidence.jsonl | 1 + state/map.yaml | 23 +++++++++------- tests/test_sensor_fusion.py | 12 +++++---- 11 files changed, 57 insertions(+), 58 deletions(-) rename localcast/{sensor_fusion/cultcache_docs.py => diagnostics/visual_cache.py} (99%) diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 5839cd3..2c4bbae 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -51,7 +51,8 @@ shape removes independent per-kind storage from the live foundation. - `localcast.sensor_fusion.calibration_space` solves fixed-board ChArUco observations into camera poses in one common world frame. - `localcast.sensor_fusion.surface_features` matches cross-view features and triangulates calibrated surface tracks. - `localcast.sensor_fusion.render_bridge` emits render-frame packets with visual/audio timing metadata and Spout sender identity. - - `localcast.sensor_fusion.cultcache_docs` stores live visual state as typed CultCache MessagePack documents. + - `localcast.diagnostics.visual_cache` stores diagnostic visual state as typed + CultCache MessagePack documents. It is not the production live boundary. - `localcast.diagnostics.spout_output` renders render-frame packets into a GPU texture and publishes it as a named Spout sender for OBS. This is diagnostic/migration code, not production runtime. - `scripts/diagnostic_live_sensor_fusion.py` writes fused render frames into `calibration/runs/visual-state.msgpack`. This is diagnostic/migration code, not production runtime. - The live visual producer writes a multi-LOD scene cache with source kind and priority. Real Leap frames are promoted as the highest-priority visual timing/spatial evidence; Leap fallback frames remain lower-priority diagnostics. @@ -66,7 +67,8 @@ shape removes independent per-kind storage from the live foundation. - `scripts/diagnostic_stream_spout.py` runs the deadline Spout sender loop from the typed cache with typed and JSON heartbeat status. This is diagnostic/migration code, not production runtime. - The deadline Spout renderer has a named `kiyo-mid-deru` virtual camera preset: eye at the midpoint between the two Kiyo-class cameras and target on the co-streamer body volume. It also applies a renderer-owned point budget with pinned calibration/cross-modal constraints, stable high-confidence anchors, and frame-varying remainder samples so downstream TAA/supersampling can accumulate detail without forcing every source claim into every OBS frame. Its JSON heartbeat reports the prefix counts that survived the render budget. - `docs/obs-spout-streaming.md` documents OBS setup and the Aquarium replacement boundary. - - `docs/typed-visual-state.md` documents the CultCache/CultNet visual boundary. + - `docs/typed-visual-state.md` documents the diagnostic CultCache file shape + and the CultNet visual schema target. - OBS synchronized program surface: - `scripts/setup_obs_synced_program.py` derives OBS-controllable stems from an aligned program audio timeline: host voice, co-streamer voice, ambient, transients, co-streamer loopback, and local loopback. - `scripts/capture_co_streamer_surfaces.py` captures neighbor Focusrite and neighbor loopback with local loopback ground truth, estimates the late remote-family offset, and writes aligned co-streamer surfaces for the stem packer. diff --git a/docs/obs-spout-streaming.md b/docs/obs-spout-streaming.md index 1529e7e..0889518 100644 --- a/docs/obs-spout-streaming.md +++ b/docs/obs-spout-streaming.md @@ -90,7 +90,9 @@ The sync heartbeat is written to `calibration/runs/av-sync-status.json`. It incl - OBS receives a named Spout texture. - The Spout sink owns presentation only; it does not own camera calibration, fusion, or scene truth. -- CultCache MessagePack documents are the live local state authority. CultNet document replication is the process/network boundary. +- CultCache MessagePack documents are diagnostic local state. Production visual + state crosses native/CultNet typed document boundaries; the Python file + adapter is not live authority. - JSON render-frame polling is compatibility scaffolding only. - Latency is explicit in the packet timestamps. The renderer may buffer, but it may not silently erase the visual/audio alignment fields. - The neighbor SRT video feed is a timed artifact. OBS must not be left to eyeball-sync it against the Spout texture and AmbiX bed. diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index bd97e25..d5e6c44 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -72,7 +72,9 @@ flowchart TD - `localcast.sensor_fusion.active_illumination` owns deliberate light pulses as calibration telemetry. A Tuya bulb is an optional local-network actuator, not a rendering authority; its pulse timestamps are evidence for camera exposure/depth response. - `localcast.sensor_fusion.clap_calibration` owns deliberate clap detection: chirplet-synced audio transients nominate oracle timestamps, camera frame windows around that oracle look for motion derivatives, calibrated camera peaks triangulate the impact point, and only audio-plus-visual agreement becomes a clap calibration event. - `localcast.sensor_fusion.render_bridge` owns the in-process render-frame ABI: cached point claims plus target dimensions, Spout sender name, source timestamp range, intended visual presentation time, and ambisonic/audio alignment time. -- `localcast.sensor_fusion.cultcache_docs` owns the typed CultCache document boundary for live visual state. +- `localcast.diagnostics.visual_cache` owns the typed CultCache document shape + used by diagnostic visual-state files. Production visual state should cross + the native/CultNet boundary directly, not through this Python file adapter. - `localcast.diagnostics.spout_output` owns the deadline OBS publication sink: render-frame packet in, GPU texture plus Spout sender heartbeat out. It is diagnostic/migration code, not the production visual authority. - `scripts/sensor_fusion.py` owns CLI composition for offline observation files. - `scripts/diagnostic_stream_spout.py` owns the diagnostic Spout sender loop for OBS. @@ -133,7 +135,10 @@ LocalCastBridge TrackCache -> OBS Spout2 Capture source ``` -`RenderFramePacket` is the in-process shape. The live boundary is now a typed CultCache MessagePack document, with CultNet `document_put` as the intended process/network API boundary. JSON render-frame files are compatibility scaffolding only. +`RenderFramePacket` is the diagnostic Python in-process shape. The diagnostic +file boundary is a typed CultCache MessagePack document. The production live +boundary is native typed state plus CultNet `document_put`; JSON render-frame +files are compatibility scaffolding only. Clap calibration events use the same typed-state rule. A clap is not every sharp audio transient. The detector first finds a sharp broadband audio onset in the diff --git a/docs/typed-visual-state.md b/docs/typed-visual-state.md index 0855b7d..119a8c1 100644 --- a/docs/typed-visual-state.md +++ b/docs/typed-visual-state.md @@ -1,10 +1,14 @@ -# Typed Visual State +# Diagnostic Typed Visual State ## Objective -Keep visual fusion state as typed CultCache documents and expose the boundary as CultNet document messages. +Keep the diagnostic visual file shape as typed CultCache documents and expose +the production boundary as native CultNet document messages. -JSON is no longer the authority for live visual state. The renderer consumes `calibration/runs/visual-state.msgpack`, which stores a `localcast.visual.render_frame` CultCache document. +JSON is no longer the authority for visual state. The diagnostic renderer +consumes `calibration/runs/visual-state.msgpack`, which stores a +`localcast.visual.render_frame` CultCache document. Production visual state must +not be owned by this Python file adapter. ## Current Mechanism @@ -23,9 +27,9 @@ flowchart TD H --> I["OBS Spout2 Capture"] ``` -Live files: +Diagnostic files: -- `calibration/runs/visual-state.msgpack`: typed live visual frame state. +- `calibration/runs/visual-state.msgpack`: typed diagnostic visual frame state. - `calibration/runs/visual-stream-status.msgpack`: typed stream status state. - `calibration/runs/stream-spout-status.json`: compatibility heartbeat for quick terminal checks. - `calibration/runs/av-sync-status.json`: visual/audio sync heartbeat from the Spout/Aquarium publisher. @@ -64,7 +68,11 @@ CultNetDocumentPut(localcast.visual.render_frame) -> renderer frame source ``` -For the deadline rig, both producer and renderer share the local CultCache file. The boundary is still the document contract: a future camera process, Aquarium process, or remote sensor node should publish the same document through CultNet `document_put` messages rather than inventing another transport shape. +For the deadline rig, both producer and renderer share the local CultCache file. +That arrangement is diagnostic scaffolding. Aquarium, native capture, and remote +sensor nodes should publish the same typed document through CultNet +`document_put` messages rather than treating a Python-polled file as live +authority. The live visual frame may contain multiple claim families. `dense-rgb:*` claims are calibrated two-camera RGB surface samples from the debug CPU stereo diff --git a/localcast/diagnostics/spout_output.py b/localcast/diagnostics/spout_output.py index 7d20727..5d542d7 100644 --- a/localcast/diagnostics/spout_output.py +++ b/localcast/diagnostics/spout_output.py @@ -13,7 +13,7 @@ import numpy as np from localcast.sensor_fusion.audio_overlay import AudioVisualSyncStatus, overlay_audio_events, with_remote_video_status from localcast.sensor_fusion.media_artifacts import remote_video_artifact_for_present_time from localcast.sensor_fusion.render_bridge import RenderFramePacket, RenderPointPacket, load_render_frame -from localcast.sensor_fusion.cultcache_docs import CultStreamStatus, get_live_render_frame, put_stream_status +from localcast.diagnostics.visual_cache import CultStreamStatus, get_live_render_frame, put_stream_status @dataclass(frozen=True) diff --git a/localcast/sensor_fusion/cultcache_docs.py b/localcast/diagnostics/visual_cache.py similarity index 99% rename from localcast/sensor_fusion/cultcache_docs.py rename to localcast/diagnostics/visual_cache.py index c29a07d..c4bd72b 100644 --- a/localcast/sensor_fusion/cultcache_docs.py +++ b/localcast/diagnostics/visual_cache.py @@ -8,7 +8,7 @@ from typing import Any import numpy as np -from .render_bridge import RenderFramePacket, RenderPointPacket +from localcast.sensor_fusion.render_bridge import RenderFramePacket, RenderPointPacket ROOT = Path(__file__).resolve().parents[2] diff --git a/localcast/sensor_fusion/__init__.py b/localcast/sensor_fusion/__init__.py index 679cf15..9d5cd50 100644 --- a/localcast/sensor_fusion/__init__.py +++ b/localcast/sensor_fusion/__init__.py @@ -81,20 +81,6 @@ from .chirp_pose import ( phase_source_id_for_microphone, speaker_geometry_from_audio_profile, ) -from .cultcache_docs import ( - CLAP_EVENTS_KEY, - LIVE_RENDER_FRAME_KEY, - CultClapEvent, - CultClapEventsFrame, - CultRenderFrame, - CultRenderPoint, - CultStreamStatus, - get_live_clap_events, - get_live_render_frame, - make_clap_events_frame, - put_live_clap_events, - put_live_render_frame, -) __all__ = [ "CameraModel", "CameraIntrinsics", @@ -143,16 +129,6 @@ __all__ = [ "TimestampedFrame", "TimestampedFrameHistory", "RemoteVideoArtifact", - "CLAP_EVENTS_KEY", - "LIVE_RENDER_FRAME_KEY", - "CultClapEvent", - "CultClapEventsFrame", - "CultRenderFrame", - "CultRenderPoint", - "CultStreamStatus", - "get_live_render_frame", - "get_live_clap_events", - "make_clap_events_frame", "lower_points_to_render_frame", "dense_stereo_points", "camera_models_to_config", @@ -176,8 +152,6 @@ __all__ = [ "phase_source_id_for_microphone", "speaker_geometry_from_audio_profile", "remote_video_artifact_for_present_time", - "put_live_clap_events", - "put_live_render_frame", "solve_camera_pose_from_fixed_board", "solve_common_space_from_fixed_board", "triangulate_surface_tracks", diff --git a/scripts/diagnostic_live_sensor_fusion.py b/scripts/diagnostic_live_sensor_fusion.py index 1e02d0d..5e4f76c 100644 --- a/scripts/diagnostic_live_sensor_fusion.py +++ b/scripts/diagnostic_live_sensor_fusion.py @@ -34,16 +34,18 @@ from localcast.sensor_fusion import ( estimate_pose_corrections, evidence_from_fusion_items, evidence_from_render_points, - get_live_clap_events, - make_clap_events_frame, measure_frame_quality, multilod_cache_from_evidence, - put_live_clap_events, - put_live_render_frame, lower_points_to_render_frame, stochastic_transient_matches, speaker_geometry_from_audio_profile, ) +from localcast.diagnostics.visual_cache import ( + get_live_clap_events, + make_clap_events_frame, + put_live_clap_events, + put_live_render_frame, +) from localcast.sensor_fusion.camera_control import AdaptiveCameraController, OpenCvCameraSettingPort from localcast.sensor_fusion.adapters import LatestFramePump, OpenCvCaptureConfig, OpenCvFrameSource from audio_field.cultcache_audio import frame_to_numpy, get_live_audio_phase_field, get_live_spatial_audio_frame diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 4a22e82..6995637 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -58,3 +58,4 @@ {"ts": "2026-05-19T08:03:24+00:00", "type": "implementation", "status": "ok", "note": "Diagnostic Spout launcher renamed: scripts/stream_spout.py is now scripts/diagnostic_stream_spout.py, keeping the OpenGL bridge visibly outside the production path while Aquarium Spout publication remains the target."} {"ts": "2026-05-19T08:07:19+00:00", "type": "implementation", "status": "ok", "note": "Native reservoir ABI now exposes indexed reads over the total rolling buffer and typed views via localcast_reservoir_sample_at and localcast_reservoir_view_sample_at, allowing Aquarium/Faust consumers to walk the live window without owning private histories."} {"ts": "2026-05-19T09:11:35+01:00", "type": "implementation", "status": "ok", "note": "Native reservoir provenance guard added: LocalcastSampleHandle now carries flags/reserved fields, diagnostic samples are rejected by localcast_reservoir_push and LocalcastRuntime producer calls, and tests prove fallback/diagnostic handles cannot enter the live rolling buffer."} +{"ts": "2026-05-19T09:25:00+01:00", "type": "implementation", "status": "partial", "note": "Python visual CultCache file adapter quarantined under localcast.diagnostics.visual_cache and removed from localcast.sensor_fusion exports; diagnostic producer/Spout/tests now import it explicitly. Import guard passed, but bundled Python cannot run visual-cache round-trip tests because msgpack is not installed."} diff --git a/state/map.yaml b/state/map.yaml index 08de588..0f4bc02 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -236,13 +236,15 @@ research: typed_visual_state: summary: docs/typed-visual-state.md key_decision: > - Live visual state now lives in typed CultCache MessagePack documents: - `localcast.visual.render_frame` in `calibration/runs/visual-state.msgpack` - and `localcast.visual.stream_status` in - `calibration/runs/visual-stream-status.msgpack`. CultNet document_put is - the intended process/network API boundary. Edge JSON may define schemas - or export diagnostics, but runtime data should remain typed documents - because every networked subsystem is under our control. + Diagnostic visual state can still be exported as typed CultCache + MessagePack documents: `localcast.visual.render_frame` in + `calibration/runs/visual-state.msgpack` and + `localcast.visual.stream_status` in + `calibration/runs/visual-stream-status.msgpack`. Production visual state + crosses native/CultNet typed document boundaries; the Python file adapter + is not live authority. Edge JSON may define schemas or export + diagnostics, but runtime data should remain typed documents because every + networked subsystem is under our control. current_status: summary: > @@ -298,9 +300,10 @@ current_status: source time ranges, intended visual presentation time, audio-alignment time, target render size, Spout sender name, and point draw metadata for Aquarium/Spout. A deadline OBS Spout sender is now live-shaped: - `scripts/diagnostic_live_sensor_fusion.py` now writes fused render frames into typed - CultCache state at `calibration/runs/visual-state.msgpack`. `scripts/diagnostic_stream_spout.py` - consumes that CultCache document, lowers points into compact anisotropic + `scripts/diagnostic_live_sensor_fusion.py` writes fused render frames into + diagnostic typed CultCache state at `calibration/runs/visual-state.msgpack`. + `scripts/diagnostic_stream_spout.py` consumes that diagnostic document, + lowers points into compact anisotropic screen brush packets, uploads an OpenGL texture, publishes Spout sender `LocalCastBridge Point Cloud`, and writes both typed status `calibration/runs/visual-stream-status.msgpack` and compatibility JSON diff --git a/tests/test_sensor_fusion.py b/tests/test_sensor_fusion.py index 7db6706..4c468ef 100644 --- a/tests/test_sensor_fusion.py +++ b/tests/test_sensor_fusion.py @@ -36,14 +36,9 @@ from localcast.sensor_fusion import ( detect_clap_events, estimate_pose_corrections, dense_stereo_points, - get_live_clap_events, - get_live_render_frame, lower_points_to_render_frame, match_surface_features, overlay_audio_events, - make_clap_events_frame, - put_live_clap_events, - put_live_render_frame, remote_video_artifact_for_present_time, phase_source_id_for_microphone, solve_common_space_from_fixed_board, @@ -51,6 +46,13 @@ from localcast.sensor_fusion import ( SurfaceFeatureObservation, triangulate_surface_tracks, ) +from localcast.diagnostics.visual_cache import ( + get_live_clap_events, + get_live_render_frame, + make_clap_events_frame, + put_live_clap_events, + put_live_render_frame, +) from localcast.sensor_fusion.audio_overlay import clamp_frame_to_audio_reservoir from localcast.diagnostics.spout_output import ( SpoutOutputConfig,