Wire synchronized audio into Aquarium render path

This commit is contained in:
Meta Vens 2026-05-18 18:32:23 +01:00
commit aa8b5f926e
8 changed files with 298 additions and 5 deletions

View file

@ -17,6 +17,7 @@ flowchart TD
I --> J["audio-events.msgpack / CultNet document_put"]
E --> F["Aquarium client"]
J --> F
K["visual-state.msgpack"] --> F
F --> G["native Faust spatial DSP + volumetric renderer"]
```
@ -25,6 +26,7 @@ Live files:
- `calibration/runs/audio-state.msgpack`: latest typed AmbiX audio block.
- `calibration/runs/audio-stream-status.msgpack`: audio publisher heartbeat.
- `calibration/runs/audio-events.msgpack`: latest typed dialogue-focus and transient vector field.
- `calibration/runs/av-sync-status.json`: current Aquarium/Spout publisher view of visual frame id, audio frame id, audio delta, and synchronized event overlay count.
Document type:
@ -54,10 +56,17 @@ Source-event analysis command:
.\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-source-events --profile .\calibration\runs\audio-field-loud-profile.json --input .\calibration\runs\<run-folder>\field-cleaned.wav --cache .\calibration\runs\audio-events.msgpack
```
Synchronized Aquarium/Spout publisher command:
```powershell
.\.venv\Scripts\python.exe .\scripts\stream_spout.py --source cultcache --frame-cache .\calibration\runs\visual-state.msgpack --audio-cache .\calibration\runs\audio-state.msgpack --audio-events-cache .\calibration\runs\audio-events.msgpack --sync-status .\calibration\runs\av-sync-status.json
```
## Invariants
- Aquarium owns rendering and Faust DSP, not microphone capture or clock correction.
- LocalCastBridge publishes declared AmbiX blocks, not undecoded heterogeneous microphone channels.
- LocalCastBridge publishes source-event geometry separately from AmbiX audio; render effects do not have to infer transient positions from the sound bed.
- Aquarium/Spout packaging selects audio source events against each visual frame's `audio_alignment_time_ns`; OBS should receive the already synchronized output rather than trying to align independent media sources.
- The audio frame carries `start_sample` and `audio_time_ns` so visual packets can align against the same bounded-latency timeline.
- OBS may receive a rendered monitor output later, but OBS is not the spatial bus authority.

View file

@ -13,7 +13,10 @@ flowchart TD
A["live_sensor_fusion.py"] --> B["SensorRig.fuse"]
B --> C["RenderFramePacket"]
C --> D["CultRenderFrame"]
D --> E["CultCache MessagePack store"]
X["audio-state.msgpack"] --> Y["audio event overlay"]
Z["audio-events.msgpack"] --> Y
D --> Y
Y --> E["CultCache MessagePack store"]
E --> F["stream_spout.py"]
F --> G["ScreenBrushPacket lowering"]
G --> H["Spout sender"]
@ -25,6 +28,7 @@ Live files:
- `calibration/runs/visual-state.msgpack`: typed live visual frame state.
- `calibration/runs/visual-stream-status.msgpack`: typed stream status state.
- `calibration/runs/stream-spout-status.json`: compatibility heartbeat for quick terminal checks.
- `calibration/runs/av-sync-status.json`: visual/audio sync heartbeat from the Spout/Aquarium publisher.
## Document Types
@ -54,9 +58,16 @@ CultNetDocumentPut(localcast.visual.render_frame)
For the deadline rig, both producer and renderer share the local CultCache file. The boundary is still the document contract: a future camera process, Aquarium process, or remote sensor node should publish the same document through CultNet `document_put` messages rather than inventing another transport shape.
## Audio Synchronization
`stream_spout.py` can now read `calibration/runs/audio-state.msgpack` and `calibration/runs/audio-events.msgpack` while it renders the visual frame. It selects source events against `RenderFramePacket.audio_alignment_time_ns`, adds synchronized audio-event points to the render packet, and writes `calibration/runs/av-sync-status.json` with the active visual frame id, audio frame id, audio delta, event count, and overlay count.
The audio bed still travels as `localcast.audio.spatial_frame`; the visual render packet only receives renderer-visible transient geometry. That keeps the machine legible: audio owns sound, source-event analysis owns acoustic facts, Aquarium owns the final pixel/audio package for OBS.
## Cut Line
- Keep: typed CultCache document contracts and MessagePack payloads.
- Keep temporarily: JSON heartbeat for human shell checks.
- Cut next: JSON render-frame polling once no script depends on it.
- Do not add a second scene authority. Sensor fusion owns world claims; renderer owns pixels.
- Do not make OBS synchronize separate sources. Aquarium receives synchronized visual/audio documents and emits the OBS-facing package.

View file

@ -12,6 +12,7 @@ from .core import (
load_fusion_config,
)
from .render_bridge import RenderBridgeConfig, RenderFramePacket, RenderPointPacket, lower_points_to_render_frame
from .audio_overlay import AudioVisualSyncStatus, overlay_audio_events
from .cultcache_docs import (
LIVE_RENDER_FRAME_KEY,
CultRenderFrame,
@ -35,6 +36,7 @@ __all__ = [
"RenderBridgeConfig",
"RenderFramePacket",
"RenderPointPacket",
"AudioVisualSyncStatus",
"LIVE_RENDER_FRAME_KEY",
"CultRenderFrame",
"CultRenderPoint",
@ -45,5 +47,6 @@ __all__ = [
"get_live_render_frame",
"lower_frame_to_screen_brushes",
"lower_points_to_render_frame",
"overlay_audio_events",
"put_live_render_frame",
]

View file

@ -0,0 +1,130 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
import sys
import numpy as np
from .render_bridge import RenderFramePacket, RenderPointPacket
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from audio_field.cultcache_audio import CultAudioSourceEvents, CultSpatialAudioFrame # noqa: E402
@dataclass(frozen=True)
class AudioVisualSyncStatus:
visual_frame_id: int
visual_present_time_ns: int
audio_frame_id: int | None
audio_time_ns: int | None
audio_delta_ns: int | None
source_event_count: int
overlay_event_count: int
synchronized: bool
def overlay_audio_events(
frame: RenderFramePacket,
events: CultAudioSourceEvents | None,
audio_frame: CultSpatialAudioFrame | None,
*,
window_ns: int = 120_000_000,
radius_m: float = 0.045,
max_events: int = 128,
) -> tuple[RenderFramePacket, AudioVisualSyncStatus]:
if events is None:
return frame, sync_status(frame, None, 0, 0, False)
overlay = select_events_for_visual_frame(frame, events, audio_frame, int(window_ns), int(max_events))
points = list(frame.points)
for event in overlay:
confidence = max(0.0, min(1.0, float(event.get("confidence", 0.5))))
energy = max(0.0, float(event.get("energy", 0.0)))
alpha = max(0.25, min(1.0, 0.35 + confidence * 0.65))
intensity = max(0.25, min(1.0, 0.45 + energy * 8000.0))
points.append(
RenderPointPacket(
stable_key=f"audio-event-{event.get('eventId', len(points))}",
xyz=np.asarray(event.get("positionMeters", [0.0, 0.0, 1.2]), dtype=np.float64),
radius_m=radius_m * (0.75 + confidence),
color_rgba=(1.0, 0.35 + 0.45 * intensity, 0.15, alpha),
confidence=confidence,
source_timestamp_ns=event_time_ns(event, events, audio_frame),
)
)
return (
RenderFramePacket(
schema=frame.schema,
frame_id=frame.frame_id,
created_monotonic_ns=frame.created_monotonic_ns,
source_time_min_ns=frame.source_time_min_ns,
source_time_max_ns=frame.source_time_max_ns,
present_time_ns=frame.present_time_ns,
audio_alignment_time_ns=frame.audio_alignment_time_ns,
spout_sender_name=frame.spout_sender_name,
target_width=frame.target_width,
target_height=frame.target_height,
points=tuple(points),
),
sync_status(frame, audio_frame, len(events.events), len(overlay), True),
)
def select_events_for_visual_frame(
frame: RenderFramePacket,
events: CultAudioSourceEvents,
audio_frame: CultSpatialAudioFrame | None,
window_ns: int,
max_events: int,
) -> list[dict]:
if audio_frame is None:
return list(events.events[:max_events])
selected = []
half_window = max(1, window_ns // 2)
for event in events.events:
when_ns = event_time_ns(event, events, audio_frame)
if abs(when_ns - frame.audio_alignment_time_ns) <= half_window:
selected.append((abs(when_ns - frame.audio_alignment_time_ns), dict(event)))
selected.sort(key=lambda item: item[0])
return [event for _, event in selected[:max_events]]
def event_time_ns(event: dict, events: CultAudioSourceEvents, audio_frame: CultSpatialAudioFrame | None) -> int:
start_sample = int(event.get("startSample", 0))
if audio_frame is None:
return events.audio_time_ns + int((start_sample - events.start_sample) * 1_000_000_000 / events.sample_rate)
event_span = max(1, int(events.frame_count))
audio_mod = audio_frame.start_sample % event_span
event_mod = start_sample % event_span
delta_samples = event_mod - audio_mod
half_span = event_span // 2
if delta_samples > half_span:
delta_samples -= event_span
elif delta_samples < -half_span:
delta_samples += event_span
return audio_frame.audio_time_ns + int(delta_samples * 1_000_000_000 / audio_frame.sample_rate)
def sync_status(
frame: RenderFramePacket,
audio_frame: CultSpatialAudioFrame | None,
source_event_count: int,
overlay_event_count: int,
synchronized: bool,
) -> AudioVisualSyncStatus:
audio_delta_ns = None if audio_frame is None else frame.audio_alignment_time_ns - audio_frame.audio_time_ns
return AudioVisualSyncStatus(
visual_frame_id=frame.frame_id,
visual_present_time_ns=frame.present_time_ns,
audio_frame_id=None if audio_frame is None else audio_frame.frame_id,
audio_time_ns=None if audio_frame is None else audio_frame.audio_time_ns,
audio_delta_ns=audio_delta_ns,
source_event_count=source_event_count,
overlay_event_count=overlay_event_count,
synchronized=bool(synchronized and audio_frame is not None),
)

View file

@ -10,6 +10,7 @@ import time
import numpy as np
from .audio_overlay import AudioVisualSyncStatus, overlay_audio_events
from .render_bridge import RenderFramePacket, RenderPointPacket, load_render_frame
from .cultcache_docs import CultStreamStatus, get_live_render_frame, put_stream_status
@ -53,12 +54,28 @@ class RenderFrameFileSource:
class CultCacheRenderFrameSource:
def __init__(self, path: Path, demo_if_missing: bool = False, demo_point_count: int = 4096) -> None:
def __init__(
self,
path: Path,
demo_if_missing: bool = False,
demo_point_count: int = 4096,
audio_cache: Path | None = None,
audio_events_cache: Path | None = None,
audio_event_window_ns: int = 120_000_000,
) -> None:
self.path = path
self.demo_if_missing = demo_if_missing
self.demo_point_count = demo_point_count
self.audio_cache = audio_cache
self.audio_events_cache = audio_events_cache
self.audio_event_window_ns = int(audio_event_window_ns)
self._mtime_ns: int | None = None
self._frame: RenderFramePacket | None = None
self._audio_mtime_ns: int | None = None
self._audio_frame = None
self._events_mtime_ns: int | None = None
self._events = None
self.last_sync_status: AudioVisualSyncStatus | None = None
def current_frame(self, now_s: float, config: SpoutOutputConfig) -> RenderFramePacket:
if self.path.exists():
@ -67,10 +84,47 @@ class CultCacheRenderFrameSource:
self._frame = get_live_render_frame(self.path)
self._mtime_ns = mtime_ns
if self._frame is not None:
return self._frame
return self._with_audio_overlay(self._frame)
if not self.demo_if_missing:
raise FileNotFoundError(self.path)
return demo_render_frame(now_s, config)
return self._with_audio_overlay(demo_render_frame(now_s, config))
def _with_audio_overlay(self, frame: RenderFramePacket) -> RenderFramePacket:
if self.audio_cache is None and self.audio_events_cache is None:
self.last_sync_status = None
return frame
audio_frame = self._read_audio_frame()
events = self._read_audio_events()
augmented, status = overlay_audio_events(
frame,
events,
audio_frame,
window_ns=self.audio_event_window_ns,
)
self.last_sync_status = status
return augmented
def _read_audio_frame(self):
if self.audio_cache is None or not self.audio_cache.exists():
return None
mtime_ns = self.audio_cache.stat().st_mtime_ns
if self._audio_frame is None or mtime_ns != self._audio_mtime_ns:
from audio_field.cultcache_audio import get_live_spatial_audio_frame
self._audio_frame = get_live_spatial_audio_frame(self.audio_cache)
self._audio_mtime_ns = mtime_ns
return self._audio_frame
def _read_audio_events(self):
if self.audio_events_cache is None or not self.audio_events_cache.exists():
return None
mtime_ns = self.audio_events_cache.stat().st_mtime_ns
if self._events is None or mtime_ns != self._events_mtime_ns:
from audio_field.cultcache_audio import get_live_audio_source_events
self._events = get_live_audio_source_events(self.audio_events_cache)
self._events_mtime_ns = mtime_ns
return self._events
def demo_render_frame(now_s: float, config: SpoutOutputConfig) -> RenderFramePacket:
@ -217,6 +271,29 @@ def write_cult_status(
)
def write_sync_status(path: Path, status: AudioVisualSyncStatus | None) -> None:
if status is None:
return
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(
{
"visual_frame_id": status.visual_frame_id,
"visual_present_time_ns": status.visual_present_time_ns,
"audio_frame_id": status.audio_frame_id,
"audio_time_ns": status.audio_time_ns,
"audio_delta_ns": status.audio_delta_ns,
"source_event_count": status.source_event_count,
"overlay_event_count": status.overlay_event_count,
"synchronized": status.synchronized,
"updated_monotonic_ns": time.monotonic_ns(),
},
indent=2,
),
encoding="utf-8",
)
class OpenGLSpoutPointRenderer:
def __init__(self, config: SpoutOutputConfig) -> None:
self.config = config

View file

@ -74,6 +74,7 @@ def main() -> None:
parser.add_argument("--loop", action="store_true")
parser.add_argument("--synthetic", action="store_true")
parser.add_argument("--smoke-readback", action="store_true")
parser.add_argument("--max-lag-ms", type=float, default=250.0)
args = parser.parse_args()
source = None
@ -92,6 +93,10 @@ def main() -> None:
now = time.monotonic()
if args.duration is not None and now - start_monotonic >= args.duration:
break
realtime_start_sample = int((now - start_monotonic) * sample_rate)
max_lag_samples = int(float(args.max_lag_ms) * sample_rate / 1000.0)
if start_sample + args.chunk_frames < realtime_start_sample - max_lag_samples:
start_sample = max(0, realtime_start_sample - (realtime_start_sample % args.chunk_frames))
block = source_block(source, sample_rate, start_sample, args.chunk_frames, loop=args.loop)
if block is None:
break

View file

@ -13,6 +13,7 @@ from localcast.sensor_fusion.spout_output import (
RenderFrameFileSource,
SpoutOutputConfig,
write_cult_status,
write_sync_status,
write_status,
)
@ -30,6 +31,11 @@ def main() -> None:
parser.add_argument("--demo-point-count", type=int, default=4096)
parser.add_argument("--status", default=str(ROOT / "calibration" / "runs" / "stream-spout-status.json"))
parser.add_argument("--status-cache", default=str(ROOT / "calibration" / "runs" / "visual-stream-status.msgpack"))
parser.add_argument("--sync-status", default=str(ROOT / "calibration" / "runs" / "av-sync-status.json"))
parser.add_argument("--audio-cache", default=str(ROOT / "calibration" / "runs" / "audio-state.msgpack"))
parser.add_argument("--audio-events-cache", default=str(ROOT / "calibration" / "runs" / "audio-events.msgpack"))
parser.add_argument("--audio-event-window-ms", type=float, default=120.0)
parser.add_argument("--no-audio-overlay", action="store_true")
parser.add_argument("--source", choices=["cultcache", "json"], default="cultcache")
parser.add_argument("--duration", type=float, help="Optional run duration in seconds for smoke tests.")
args = parser.parse_args()
@ -38,6 +44,7 @@ def main() -> None:
frame_cache_path = Path(args.frame_cache)
status_path = Path(args.status)
status_cache_path = Path(args.status_cache)
sync_status_path = Path(args.sync_status)
config = SpoutOutputConfig(
sender_name=args.sender_name,
width=args.width,
@ -47,7 +54,14 @@ def main() -> None:
demo_point_count=args.demo_point_count,
)
if args.source == "cultcache":
source = CultCacheRenderFrameSource(frame_cache_path, demo_if_missing=args.demo_if_missing, demo_point_count=args.demo_point_count)
source = CultCacheRenderFrameSource(
frame_cache_path,
demo_if_missing=args.demo_if_missing,
demo_point_count=args.demo_point_count,
audio_cache=None if args.no_audio_overlay else Path(args.audio_cache),
audio_events_cache=None if args.no_audio_overlay else Path(args.audio_events_cache),
audio_event_window_ns=int(float(args.audio_event_window_ms) * 1_000_000),
)
frame_source_name = str(frame_cache_path)
else:
source = RenderFrameFileSource(frame_path, demo_if_missing=args.demo_if_missing, demo_point_count=args.demo_point_count)
@ -80,6 +94,7 @@ def main() -> None:
frame_source=frame_source_name,
last_error=None if ok else "SpoutGL sendTexture returned false",
)
write_sync_status(sync_status_path, getattr(source, "last_sync_status", None))
last_status = now
sleep_for = frame_interval - (time.monotonic() - now)
if sleep_for > 0:

View file

@ -19,8 +19,10 @@ from localcast.sensor_fusion import (
get_live_render_frame,
lower_frame_to_screen_brushes,
lower_points_to_render_frame,
overlay_audio_events,
put_live_render_frame,
)
from audio_field.cultcache_audio import make_audio_source_events, make_spatial_audio_frame
from localcast.sensor_fusion.adapters import read_raw_bgr_frames
@ -189,6 +191,47 @@ class SensorFusionTests(unittest.TestCase):
self.assertEqual(1, len(loaded.points))
self.assertEqual("ball", loaded.points[0].stable_key)
def test_audio_source_events_overlay_against_audio_alignment_time(self):
frame = RenderFramePacket(
schema="localcast.sensor_fusion.render_frame.v1",
frame_id=9,
created_monotonic_ns=1_000,
source_time_min_ns=1_000,
source_time_max_ns=1_000,
present_time_ns=1_100,
audio_alignment_time_ns=1_000_000_000,
spout_sender_name="test",
target_width=128,
target_height=72,
points=(),
)
audio = make_spatial_audio_frame(
np.zeros((1024, 4), dtype=np.float32),
frame_id=4,
sample_rate=48000,
start_sample=48000,
audio_time_ns=1_000_000_000,
)
events = make_audio_source_events(
frame_id=5,
sample_rate=48000,
start_sample=0,
frame_count=96000,
audio_time_ns=0,
events=[
{"eventId": 1, "startSample": 48000, "positionMeters": [0.2, 0.3, 1.4], "confidence": 0.9, "energy": 0.001},
{"eventId": 2, "startSample": 70000, "positionMeters": [2.0, 0.0, 1.0], "confidence": 0.9, "energy": 0.001},
],
voice_focus=[],
)
augmented, status = overlay_audio_events(frame, events, audio, window_ns=40_000_000)
self.assertEqual(1, len(augmented.points))
self.assertEqual("audio-event-1", augmented.points[0].stable_key)
self.assertTrue(status.synchronized)
self.assertEqual(1, status.overlay_event_count)
if __name__ == "__main__":
unittest.main()