diff --git a/docs/aquarium-spatial-audio.md b/docs/aquarium-spatial-audio.md index 9f8a978..d8da422 100644 --- a/docs/aquarium-spatial-audio.md +++ b/docs/aquarium-spatial-audio.md @@ -17,6 +17,7 @@ flowchart TD I --> J["audio-events.msgpack / CultNet document_put"] E --> F["Aquarium client"] J --> F + K["visual-state.msgpack"] --> F F --> G["native Faust spatial DSP + volumetric renderer"] ``` @@ -25,6 +26,7 @@ Live files: - `calibration/runs/audio-state.msgpack`: latest typed AmbiX audio block. - `calibration/runs/audio-stream-status.msgpack`: audio publisher heartbeat. - `calibration/runs/audio-events.msgpack`: latest typed dialogue-focus and transient vector field. +- `calibration/runs/av-sync-status.json`: current Aquarium/Spout publisher view of visual frame id, audio frame id, audio delta, and synchronized event overlay count. Document type: @@ -54,10 +56,17 @@ Source-event analysis command: .\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-source-events --profile .\calibration\runs\audio-field-loud-profile.json --input .\calibration\runs\\field-cleaned.wav --cache .\calibration\runs\audio-events.msgpack ``` +Synchronized Aquarium/Spout publisher command: + +```powershell +.\.venv\Scripts\python.exe .\scripts\stream_spout.py --source cultcache --frame-cache .\calibration\runs\visual-state.msgpack --audio-cache .\calibration\runs\audio-state.msgpack --audio-events-cache .\calibration\runs\audio-events.msgpack --sync-status .\calibration\runs\av-sync-status.json +``` + ## Invariants - Aquarium owns rendering and Faust DSP, not microphone capture or clock correction. - LocalCastBridge publishes declared AmbiX blocks, not undecoded heterogeneous microphone channels. - LocalCastBridge publishes source-event geometry separately from AmbiX audio; render effects do not have to infer transient positions from the sound bed. +- Aquarium/Spout packaging selects audio source events against each visual frame's `audio_alignment_time_ns`; OBS should receive the already synchronized output rather than trying to align independent media sources. - The audio frame carries `start_sample` and `audio_time_ns` so visual packets can align against the same bounded-latency timeline. - OBS may receive a rendered monitor output later, but OBS is not the spatial bus authority. diff --git a/docs/typed-visual-state.md b/docs/typed-visual-state.md index 5b395cf..f33a1fc 100644 --- a/docs/typed-visual-state.md +++ b/docs/typed-visual-state.md @@ -13,7 +13,10 @@ flowchart TD A["live_sensor_fusion.py"] --> B["SensorRig.fuse"] B --> C["RenderFramePacket"] C --> D["CultRenderFrame"] - D --> E["CultCache MessagePack store"] + X["audio-state.msgpack"] --> Y["audio event overlay"] + Z["audio-events.msgpack"] --> Y + D --> Y + Y --> E["CultCache MessagePack store"] E --> F["stream_spout.py"] F --> G["ScreenBrushPacket lowering"] G --> H["Spout sender"] @@ -25,6 +28,7 @@ Live files: - `calibration/runs/visual-state.msgpack`: typed live visual frame state. - `calibration/runs/visual-stream-status.msgpack`: typed stream status state. - `calibration/runs/stream-spout-status.json`: compatibility heartbeat for quick terminal checks. +- `calibration/runs/av-sync-status.json`: visual/audio sync heartbeat from the Spout/Aquarium publisher. ## Document Types @@ -54,9 +58,16 @@ CultNetDocumentPut(localcast.visual.render_frame) For the deadline rig, both producer and renderer share the local CultCache file. The boundary is still the document contract: a future camera process, Aquarium process, or remote sensor node should publish the same document through CultNet `document_put` messages rather than inventing another transport shape. +## Audio Synchronization + +`stream_spout.py` can now read `calibration/runs/audio-state.msgpack` and `calibration/runs/audio-events.msgpack` while it renders the visual frame. It selects source events against `RenderFramePacket.audio_alignment_time_ns`, adds synchronized audio-event points to the render packet, and writes `calibration/runs/av-sync-status.json` with the active visual frame id, audio frame id, audio delta, event count, and overlay count. + +The audio bed still travels as `localcast.audio.spatial_frame`; the visual render packet only receives renderer-visible transient geometry. That keeps the machine legible: audio owns sound, source-event analysis owns acoustic facts, Aquarium owns the final pixel/audio package for OBS. + ## Cut Line - Keep: typed CultCache document contracts and MessagePack payloads. - Keep temporarily: JSON heartbeat for human shell checks. - Cut next: JSON render-frame polling once no script depends on it. - Do not add a second scene authority. Sensor fusion owns world claims; renderer owns pixels. +- Do not make OBS synchronize separate sources. Aquarium receives synchronized visual/audio documents and emits the OBS-facing package. diff --git a/localcast/sensor_fusion/__init__.py b/localcast/sensor_fusion/__init__.py index 163d742..8632333 100644 --- a/localcast/sensor_fusion/__init__.py +++ b/localcast/sensor_fusion/__init__.py @@ -12,6 +12,7 @@ from .core import ( load_fusion_config, ) from .render_bridge import RenderBridgeConfig, RenderFramePacket, RenderPointPacket, lower_points_to_render_frame +from .audio_overlay import AudioVisualSyncStatus, overlay_audio_events from .cultcache_docs import ( LIVE_RENDER_FRAME_KEY, CultRenderFrame, @@ -35,6 +36,7 @@ __all__ = [ "RenderBridgeConfig", "RenderFramePacket", "RenderPointPacket", + "AudioVisualSyncStatus", "LIVE_RENDER_FRAME_KEY", "CultRenderFrame", "CultRenderPoint", @@ -45,5 +47,6 @@ __all__ = [ "get_live_render_frame", "lower_frame_to_screen_brushes", "lower_points_to_render_frame", + "overlay_audio_events", "put_live_render_frame", ] diff --git a/localcast/sensor_fusion/audio_overlay.py b/localcast/sensor_fusion/audio_overlay.py new file mode 100644 index 0000000..8445e3e --- /dev/null +++ b/localcast/sensor_fusion/audio_overlay.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import sys + +import numpy as np + +from .render_bridge import RenderFramePacket, RenderPointPacket + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from audio_field.cultcache_audio import CultAudioSourceEvents, CultSpatialAudioFrame # noqa: E402 + + +@dataclass(frozen=True) +class AudioVisualSyncStatus: + visual_frame_id: int + visual_present_time_ns: int + audio_frame_id: int | None + audio_time_ns: int | None + audio_delta_ns: int | None + source_event_count: int + overlay_event_count: int + synchronized: bool + + +def overlay_audio_events( + frame: RenderFramePacket, + events: CultAudioSourceEvents | None, + audio_frame: CultSpatialAudioFrame | None, + *, + window_ns: int = 120_000_000, + radius_m: float = 0.045, + max_events: int = 128, +) -> tuple[RenderFramePacket, AudioVisualSyncStatus]: + if events is None: + return frame, sync_status(frame, None, 0, 0, False) + overlay = select_events_for_visual_frame(frame, events, audio_frame, int(window_ns), int(max_events)) + points = list(frame.points) + for event in overlay: + confidence = max(0.0, min(1.0, float(event.get("confidence", 0.5)))) + energy = max(0.0, float(event.get("energy", 0.0))) + alpha = max(0.25, min(1.0, 0.35 + confidence * 0.65)) + intensity = max(0.25, min(1.0, 0.45 + energy * 8000.0)) + points.append( + RenderPointPacket( + stable_key=f"audio-event-{event.get('eventId', len(points))}", + xyz=np.asarray(event.get("positionMeters", [0.0, 0.0, 1.2]), dtype=np.float64), + radius_m=radius_m * (0.75 + confidence), + color_rgba=(1.0, 0.35 + 0.45 * intensity, 0.15, alpha), + confidence=confidence, + source_timestamp_ns=event_time_ns(event, events, audio_frame), + ) + ) + return ( + RenderFramePacket( + schema=frame.schema, + frame_id=frame.frame_id, + created_monotonic_ns=frame.created_monotonic_ns, + source_time_min_ns=frame.source_time_min_ns, + source_time_max_ns=frame.source_time_max_ns, + present_time_ns=frame.present_time_ns, + audio_alignment_time_ns=frame.audio_alignment_time_ns, + spout_sender_name=frame.spout_sender_name, + target_width=frame.target_width, + target_height=frame.target_height, + points=tuple(points), + ), + sync_status(frame, audio_frame, len(events.events), len(overlay), True), + ) + + +def select_events_for_visual_frame( + frame: RenderFramePacket, + events: CultAudioSourceEvents, + audio_frame: CultSpatialAudioFrame | None, + window_ns: int, + max_events: int, +) -> list[dict]: + if audio_frame is None: + return list(events.events[:max_events]) + selected = [] + half_window = max(1, window_ns // 2) + for event in events.events: + when_ns = event_time_ns(event, events, audio_frame) + if abs(when_ns - frame.audio_alignment_time_ns) <= half_window: + selected.append((abs(when_ns - frame.audio_alignment_time_ns), dict(event))) + selected.sort(key=lambda item: item[0]) + return [event for _, event in selected[:max_events]] + + +def event_time_ns(event: dict, events: CultAudioSourceEvents, audio_frame: CultSpatialAudioFrame | None) -> int: + start_sample = int(event.get("startSample", 0)) + if audio_frame is None: + return events.audio_time_ns + int((start_sample - events.start_sample) * 1_000_000_000 / events.sample_rate) + event_span = max(1, int(events.frame_count)) + audio_mod = audio_frame.start_sample % event_span + event_mod = start_sample % event_span + delta_samples = event_mod - audio_mod + half_span = event_span // 2 + if delta_samples > half_span: + delta_samples -= event_span + elif delta_samples < -half_span: + delta_samples += event_span + return audio_frame.audio_time_ns + int(delta_samples * 1_000_000_000 / audio_frame.sample_rate) + + +def sync_status( + frame: RenderFramePacket, + audio_frame: CultSpatialAudioFrame | None, + source_event_count: int, + overlay_event_count: int, + synchronized: bool, +) -> AudioVisualSyncStatus: + audio_delta_ns = None if audio_frame is None else frame.audio_alignment_time_ns - audio_frame.audio_time_ns + return AudioVisualSyncStatus( + visual_frame_id=frame.frame_id, + visual_present_time_ns=frame.present_time_ns, + audio_frame_id=None if audio_frame is None else audio_frame.frame_id, + audio_time_ns=None if audio_frame is None else audio_frame.audio_time_ns, + audio_delta_ns=audio_delta_ns, + source_event_count=source_event_count, + overlay_event_count=overlay_event_count, + synchronized=bool(synchronized and audio_frame is not None), + ) + diff --git a/localcast/sensor_fusion/spout_output.py b/localcast/sensor_fusion/spout_output.py index e829161..2d3adfa 100644 --- a/localcast/sensor_fusion/spout_output.py +++ b/localcast/sensor_fusion/spout_output.py @@ -10,6 +10,7 @@ import time import numpy as np +from .audio_overlay import AudioVisualSyncStatus, overlay_audio_events from .render_bridge import RenderFramePacket, RenderPointPacket, load_render_frame from .cultcache_docs import CultStreamStatus, get_live_render_frame, put_stream_status @@ -53,12 +54,28 @@ class RenderFrameFileSource: class CultCacheRenderFrameSource: - def __init__(self, path: Path, demo_if_missing: bool = False, demo_point_count: int = 4096) -> None: + def __init__( + self, + path: Path, + demo_if_missing: bool = False, + demo_point_count: int = 4096, + audio_cache: Path | None = None, + audio_events_cache: Path | None = None, + audio_event_window_ns: int = 120_000_000, + ) -> None: self.path = path self.demo_if_missing = demo_if_missing self.demo_point_count = demo_point_count + self.audio_cache = audio_cache + self.audio_events_cache = audio_events_cache + self.audio_event_window_ns = int(audio_event_window_ns) self._mtime_ns: int | None = None self._frame: RenderFramePacket | None = None + self._audio_mtime_ns: int | None = None + self._audio_frame = None + self._events_mtime_ns: int | None = None + self._events = None + self.last_sync_status: AudioVisualSyncStatus | None = None def current_frame(self, now_s: float, config: SpoutOutputConfig) -> RenderFramePacket: if self.path.exists(): @@ -67,10 +84,47 @@ class CultCacheRenderFrameSource: self._frame = get_live_render_frame(self.path) self._mtime_ns = mtime_ns if self._frame is not None: - return self._frame + return self._with_audio_overlay(self._frame) if not self.demo_if_missing: raise FileNotFoundError(self.path) - return demo_render_frame(now_s, config) + return self._with_audio_overlay(demo_render_frame(now_s, config)) + + def _with_audio_overlay(self, frame: RenderFramePacket) -> RenderFramePacket: + if self.audio_cache is None and self.audio_events_cache is None: + self.last_sync_status = None + return frame + audio_frame = self._read_audio_frame() + events = self._read_audio_events() + augmented, status = overlay_audio_events( + frame, + events, + audio_frame, + window_ns=self.audio_event_window_ns, + ) + self.last_sync_status = status + return augmented + + def _read_audio_frame(self): + if self.audio_cache is None or not self.audio_cache.exists(): + return None + mtime_ns = self.audio_cache.stat().st_mtime_ns + if self._audio_frame is None or mtime_ns != self._audio_mtime_ns: + from audio_field.cultcache_audio import get_live_spatial_audio_frame + + self._audio_frame = get_live_spatial_audio_frame(self.audio_cache) + self._audio_mtime_ns = mtime_ns + return self._audio_frame + + def _read_audio_events(self): + if self.audio_events_cache is None or not self.audio_events_cache.exists(): + return None + mtime_ns = self.audio_events_cache.stat().st_mtime_ns + if self._events is None or mtime_ns != self._events_mtime_ns: + from audio_field.cultcache_audio import get_live_audio_source_events + + self._events = get_live_audio_source_events(self.audio_events_cache) + self._events_mtime_ns = mtime_ns + return self._events def demo_render_frame(now_s: float, config: SpoutOutputConfig) -> RenderFramePacket: @@ -217,6 +271,29 @@ def write_cult_status( ) +def write_sync_status(path: Path, status: AudioVisualSyncStatus | None) -> None: + if status is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "visual_frame_id": status.visual_frame_id, + "visual_present_time_ns": status.visual_present_time_ns, + "audio_frame_id": status.audio_frame_id, + "audio_time_ns": status.audio_time_ns, + "audio_delta_ns": status.audio_delta_ns, + "source_event_count": status.source_event_count, + "overlay_event_count": status.overlay_event_count, + "synchronized": status.synchronized, + "updated_monotonic_ns": time.monotonic_ns(), + }, + indent=2, + ), + encoding="utf-8", + ) + + class OpenGLSpoutPointRenderer: def __init__(self, config: SpoutOutputConfig) -> None: self.config = config diff --git a/scripts/stream_spatial_audio.py b/scripts/stream_spatial_audio.py index 6dd7c17..aee1d7b 100644 --- a/scripts/stream_spatial_audio.py +++ b/scripts/stream_spatial_audio.py @@ -74,6 +74,7 @@ def main() -> None: parser.add_argument("--loop", action="store_true") parser.add_argument("--synthetic", action="store_true") parser.add_argument("--smoke-readback", action="store_true") + parser.add_argument("--max-lag-ms", type=float, default=250.0) args = parser.parse_args() source = None @@ -92,6 +93,10 @@ def main() -> None: now = time.monotonic() if args.duration is not None and now - start_monotonic >= args.duration: break + realtime_start_sample = int((now - start_monotonic) * sample_rate) + max_lag_samples = int(float(args.max_lag_ms) * sample_rate / 1000.0) + if start_sample + args.chunk_frames < realtime_start_sample - max_lag_samples: + start_sample = max(0, realtime_start_sample - (realtime_start_sample % args.chunk_frames)) block = source_block(source, sample_rate, start_sample, args.chunk_frames, loop=args.loop) if block is None: break diff --git a/scripts/stream_spout.py b/scripts/stream_spout.py index 35181d5..572453e 100644 --- a/scripts/stream_spout.py +++ b/scripts/stream_spout.py @@ -13,6 +13,7 @@ from localcast.sensor_fusion.spout_output import ( RenderFrameFileSource, SpoutOutputConfig, write_cult_status, + write_sync_status, write_status, ) @@ -30,6 +31,11 @@ def main() -> None: parser.add_argument("--demo-point-count", type=int, default=4096) parser.add_argument("--status", default=str(ROOT / "calibration" / "runs" / "stream-spout-status.json")) parser.add_argument("--status-cache", default=str(ROOT / "calibration" / "runs" / "visual-stream-status.msgpack")) + parser.add_argument("--sync-status", default=str(ROOT / "calibration" / "runs" / "av-sync-status.json")) + parser.add_argument("--audio-cache", default=str(ROOT / "calibration" / "runs" / "audio-state.msgpack")) + parser.add_argument("--audio-events-cache", default=str(ROOT / "calibration" / "runs" / "audio-events.msgpack")) + parser.add_argument("--audio-event-window-ms", type=float, default=120.0) + parser.add_argument("--no-audio-overlay", action="store_true") parser.add_argument("--source", choices=["cultcache", "json"], default="cultcache") parser.add_argument("--duration", type=float, help="Optional run duration in seconds for smoke tests.") args = parser.parse_args() @@ -38,6 +44,7 @@ def main() -> None: frame_cache_path = Path(args.frame_cache) status_path = Path(args.status) status_cache_path = Path(args.status_cache) + sync_status_path = Path(args.sync_status) config = SpoutOutputConfig( sender_name=args.sender_name, width=args.width, @@ -47,7 +54,14 @@ def main() -> None: demo_point_count=args.demo_point_count, ) if args.source == "cultcache": - source = CultCacheRenderFrameSource(frame_cache_path, demo_if_missing=args.demo_if_missing, demo_point_count=args.demo_point_count) + source = CultCacheRenderFrameSource( + frame_cache_path, + demo_if_missing=args.demo_if_missing, + demo_point_count=args.demo_point_count, + audio_cache=None if args.no_audio_overlay else Path(args.audio_cache), + audio_events_cache=None if args.no_audio_overlay else Path(args.audio_events_cache), + audio_event_window_ns=int(float(args.audio_event_window_ms) * 1_000_000), + ) frame_source_name = str(frame_cache_path) else: source = RenderFrameFileSource(frame_path, demo_if_missing=args.demo_if_missing, demo_point_count=args.demo_point_count) @@ -80,6 +94,7 @@ def main() -> None: frame_source=frame_source_name, last_error=None if ok else "SpoutGL sendTexture returned false", ) + write_sync_status(sync_status_path, getattr(source, "last_sync_status", None)) last_status = now sleep_for = frame_interval - (time.monotonic() - now) if sleep_for > 0: diff --git a/tests/test_sensor_fusion.py b/tests/test_sensor_fusion.py index 29ac01f..c4484b3 100644 --- a/tests/test_sensor_fusion.py +++ b/tests/test_sensor_fusion.py @@ -19,8 +19,10 @@ from localcast.sensor_fusion import ( get_live_render_frame, lower_frame_to_screen_brushes, lower_points_to_render_frame, + overlay_audio_events, put_live_render_frame, ) +from audio_field.cultcache_audio import make_audio_source_events, make_spatial_audio_frame from localcast.sensor_fusion.adapters import read_raw_bgr_frames @@ -189,6 +191,47 @@ class SensorFusionTests(unittest.TestCase): self.assertEqual(1, len(loaded.points)) self.assertEqual("ball", loaded.points[0].stable_key) + def test_audio_source_events_overlay_against_audio_alignment_time(self): + frame = RenderFramePacket( + schema="localcast.sensor_fusion.render_frame.v1", + frame_id=9, + created_monotonic_ns=1_000, + source_time_min_ns=1_000, + source_time_max_ns=1_000, + present_time_ns=1_100, + audio_alignment_time_ns=1_000_000_000, + spout_sender_name="test", + target_width=128, + target_height=72, + points=(), + ) + audio = make_spatial_audio_frame( + np.zeros((1024, 4), dtype=np.float32), + frame_id=4, + sample_rate=48000, + start_sample=48000, + audio_time_ns=1_000_000_000, + ) + events = make_audio_source_events( + frame_id=5, + sample_rate=48000, + start_sample=0, + frame_count=96000, + audio_time_ns=0, + events=[ + {"eventId": 1, "startSample": 48000, "positionMeters": [0.2, 0.3, 1.4], "confidence": 0.9, "energy": 0.001}, + {"eventId": 2, "startSample": 70000, "positionMeters": [2.0, 0.0, 1.0], "confidence": 0.9, "energy": 0.001}, + ], + voice_focus=[], + ) + + augmented, status = overlay_audio_events(frame, events, audio, window_ns=40_000_000) + + self.assertEqual(1, len(augmented.points)) + self.assertEqual("audio-event-1", augmented.points[0].stable_key) + self.assertTrue(status.synchronized) + self.assertEqual(1, status.overlay_event_count) + if __name__ == "__main__": unittest.main()