Wire synchronized audio into Aquarium render path
This commit is contained in:
parent
0384df9f57
commit
aa8b5f926e
8 changed files with 298 additions and 5 deletions
|
|
@ -17,6 +17,7 @@ flowchart TD
|
|||
I --> J["audio-events.msgpack / CultNet document_put"]
|
||||
E --> F["Aquarium client"]
|
||||
J --> F
|
||||
K["visual-state.msgpack"] --> F
|
||||
F --> G["native Faust spatial DSP + volumetric renderer"]
|
||||
```
|
||||
|
||||
|
|
@ -25,6 +26,7 @@ Live files:
|
|||
- `calibration/runs/audio-state.msgpack`: latest typed AmbiX audio block.
|
||||
- `calibration/runs/audio-stream-status.msgpack`: audio publisher heartbeat.
|
||||
- `calibration/runs/audio-events.msgpack`: latest typed dialogue-focus and transient vector field.
|
||||
- `calibration/runs/av-sync-status.json`: current Aquarium/Spout publisher view of visual frame id, audio frame id, audio delta, and synchronized event overlay count.
|
||||
|
||||
Document type:
|
||||
|
||||
|
|
@ -54,10 +56,17 @@ Source-event analysis command:
|
|||
.\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-source-events --profile .\calibration\runs\audio-field-loud-profile.json --input .\calibration\runs\<run-folder>\field-cleaned.wav --cache .\calibration\runs\audio-events.msgpack
|
||||
```
|
||||
|
||||
Synchronized Aquarium/Spout publisher command:
|
||||
|
||||
```powershell
|
||||
.\.venv\Scripts\python.exe .\scripts\stream_spout.py --source cultcache --frame-cache .\calibration\runs\visual-state.msgpack --audio-cache .\calibration\runs\audio-state.msgpack --audio-events-cache .\calibration\runs\audio-events.msgpack --sync-status .\calibration\runs\av-sync-status.json
|
||||
```
|
||||
|
||||
## Invariants
|
||||
|
||||
- Aquarium owns rendering and Faust DSP, not microphone capture or clock correction.
|
||||
- LocalCastBridge publishes declared AmbiX blocks, not undecoded heterogeneous microphone channels.
|
||||
- LocalCastBridge publishes source-event geometry separately from AmbiX audio; render effects do not have to infer transient positions from the sound bed.
|
||||
- Aquarium/Spout packaging selects audio source events against each visual frame's `audio_alignment_time_ns`; OBS should receive the already synchronized output rather than trying to align independent media sources.
|
||||
- The audio frame carries `start_sample` and `audio_time_ns` so visual packets can align against the same bounded-latency timeline.
|
||||
- OBS may receive a rendered monitor output later, but OBS is not the spatial bus authority.
|
||||
|
|
|
|||
|
|
@ -13,7 +13,10 @@ flowchart TD
|
|||
A["live_sensor_fusion.py"] --> B["SensorRig.fuse"]
|
||||
B --> C["RenderFramePacket"]
|
||||
C --> D["CultRenderFrame"]
|
||||
D --> E["CultCache MessagePack store"]
|
||||
X["audio-state.msgpack"] --> Y["audio event overlay"]
|
||||
Z["audio-events.msgpack"] --> Y
|
||||
D --> Y
|
||||
Y --> E["CultCache MessagePack store"]
|
||||
E --> F["stream_spout.py"]
|
||||
F --> G["ScreenBrushPacket lowering"]
|
||||
G --> H["Spout sender"]
|
||||
|
|
@ -25,6 +28,7 @@ Live files:
|
|||
- `calibration/runs/visual-state.msgpack`: typed live visual frame state.
|
||||
- `calibration/runs/visual-stream-status.msgpack`: typed stream status state.
|
||||
- `calibration/runs/stream-spout-status.json`: compatibility heartbeat for quick terminal checks.
|
||||
- `calibration/runs/av-sync-status.json`: visual/audio sync heartbeat from the Spout/Aquarium publisher.
|
||||
|
||||
## Document Types
|
||||
|
||||
|
|
@ -54,9 +58,16 @@ CultNetDocumentPut(localcast.visual.render_frame)
|
|||
|
||||
For the deadline rig, both producer and renderer share the local CultCache file. The boundary is still the document contract: a future camera process, Aquarium process, or remote sensor node should publish the same document through CultNet `document_put` messages rather than inventing another transport shape.
|
||||
|
||||
## Audio Synchronization
|
||||
|
||||
`stream_spout.py` can now read `calibration/runs/audio-state.msgpack` and `calibration/runs/audio-events.msgpack` while it renders the visual frame. It selects source events against `RenderFramePacket.audio_alignment_time_ns`, adds synchronized audio-event points to the render packet, and writes `calibration/runs/av-sync-status.json` with the active visual frame id, audio frame id, audio delta, event count, and overlay count.
|
||||
|
||||
The audio bed still travels as `localcast.audio.spatial_frame`; the visual render packet only receives renderer-visible transient geometry. That keeps the machine legible: audio owns sound, source-event analysis owns acoustic facts, Aquarium owns the final pixel/audio package for OBS.
|
||||
|
||||
## Cut Line
|
||||
|
||||
- Keep: typed CultCache document contracts and MessagePack payloads.
|
||||
- Keep temporarily: JSON heartbeat for human shell checks.
|
||||
- Cut next: JSON render-frame polling once no script depends on it.
|
||||
- Do not add a second scene authority. Sensor fusion owns world claims; renderer owns pixels.
|
||||
- Do not make OBS synchronize separate sources. Aquarium receives synchronized visual/audio documents and emits the OBS-facing package.
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from .core import (
|
|||
load_fusion_config,
|
||||
)
|
||||
from .render_bridge import RenderBridgeConfig, RenderFramePacket, RenderPointPacket, lower_points_to_render_frame
|
||||
from .audio_overlay import AudioVisualSyncStatus, overlay_audio_events
|
||||
from .cultcache_docs import (
|
||||
LIVE_RENDER_FRAME_KEY,
|
||||
CultRenderFrame,
|
||||
|
|
@ -35,6 +36,7 @@ __all__ = [
|
|||
"RenderBridgeConfig",
|
||||
"RenderFramePacket",
|
||||
"RenderPointPacket",
|
||||
"AudioVisualSyncStatus",
|
||||
"LIVE_RENDER_FRAME_KEY",
|
||||
"CultRenderFrame",
|
||||
"CultRenderPoint",
|
||||
|
|
@ -45,5 +47,6 @@ __all__ = [
|
|||
"get_live_render_frame",
|
||||
"lower_frame_to_screen_brushes",
|
||||
"lower_points_to_render_frame",
|
||||
"overlay_audio_events",
|
||||
"put_live_render_frame",
|
||||
]
|
||||
|
|
|
|||
130
localcast/sensor_fusion/audio_overlay.py
Normal file
130
localcast/sensor_fusion/audio_overlay.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .render_bridge import RenderFramePacket, RenderPointPacket
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from audio_field.cultcache_audio import CultAudioSourceEvents, CultSpatialAudioFrame # noqa: E402
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AudioVisualSyncStatus:
|
||||
visual_frame_id: int
|
||||
visual_present_time_ns: int
|
||||
audio_frame_id: int | None
|
||||
audio_time_ns: int | None
|
||||
audio_delta_ns: int | None
|
||||
source_event_count: int
|
||||
overlay_event_count: int
|
||||
synchronized: bool
|
||||
|
||||
|
||||
def overlay_audio_events(
|
||||
frame: RenderFramePacket,
|
||||
events: CultAudioSourceEvents | None,
|
||||
audio_frame: CultSpatialAudioFrame | None,
|
||||
*,
|
||||
window_ns: int = 120_000_000,
|
||||
radius_m: float = 0.045,
|
||||
max_events: int = 128,
|
||||
) -> tuple[RenderFramePacket, AudioVisualSyncStatus]:
|
||||
if events is None:
|
||||
return frame, sync_status(frame, None, 0, 0, False)
|
||||
overlay = select_events_for_visual_frame(frame, events, audio_frame, int(window_ns), int(max_events))
|
||||
points = list(frame.points)
|
||||
for event in overlay:
|
||||
confidence = max(0.0, min(1.0, float(event.get("confidence", 0.5))))
|
||||
energy = max(0.0, float(event.get("energy", 0.0)))
|
||||
alpha = max(0.25, min(1.0, 0.35 + confidence * 0.65))
|
||||
intensity = max(0.25, min(1.0, 0.45 + energy * 8000.0))
|
||||
points.append(
|
||||
RenderPointPacket(
|
||||
stable_key=f"audio-event-{event.get('eventId', len(points))}",
|
||||
xyz=np.asarray(event.get("positionMeters", [0.0, 0.0, 1.2]), dtype=np.float64),
|
||||
radius_m=radius_m * (0.75 + confidence),
|
||||
color_rgba=(1.0, 0.35 + 0.45 * intensity, 0.15, alpha),
|
||||
confidence=confidence,
|
||||
source_timestamp_ns=event_time_ns(event, events, audio_frame),
|
||||
)
|
||||
)
|
||||
return (
|
||||
RenderFramePacket(
|
||||
schema=frame.schema,
|
||||
frame_id=frame.frame_id,
|
||||
created_monotonic_ns=frame.created_monotonic_ns,
|
||||
source_time_min_ns=frame.source_time_min_ns,
|
||||
source_time_max_ns=frame.source_time_max_ns,
|
||||
present_time_ns=frame.present_time_ns,
|
||||
audio_alignment_time_ns=frame.audio_alignment_time_ns,
|
||||
spout_sender_name=frame.spout_sender_name,
|
||||
target_width=frame.target_width,
|
||||
target_height=frame.target_height,
|
||||
points=tuple(points),
|
||||
),
|
||||
sync_status(frame, audio_frame, len(events.events), len(overlay), True),
|
||||
)
|
||||
|
||||
|
||||
def select_events_for_visual_frame(
|
||||
frame: RenderFramePacket,
|
||||
events: CultAudioSourceEvents,
|
||||
audio_frame: CultSpatialAudioFrame | None,
|
||||
window_ns: int,
|
||||
max_events: int,
|
||||
) -> list[dict]:
|
||||
if audio_frame is None:
|
||||
return list(events.events[:max_events])
|
||||
selected = []
|
||||
half_window = max(1, window_ns // 2)
|
||||
for event in events.events:
|
||||
when_ns = event_time_ns(event, events, audio_frame)
|
||||
if abs(when_ns - frame.audio_alignment_time_ns) <= half_window:
|
||||
selected.append((abs(when_ns - frame.audio_alignment_time_ns), dict(event)))
|
||||
selected.sort(key=lambda item: item[0])
|
||||
return [event for _, event in selected[:max_events]]
|
||||
|
||||
|
||||
def event_time_ns(event: dict, events: CultAudioSourceEvents, audio_frame: CultSpatialAudioFrame | None) -> int:
|
||||
start_sample = int(event.get("startSample", 0))
|
||||
if audio_frame is None:
|
||||
return events.audio_time_ns + int((start_sample - events.start_sample) * 1_000_000_000 / events.sample_rate)
|
||||
event_span = max(1, int(events.frame_count))
|
||||
audio_mod = audio_frame.start_sample % event_span
|
||||
event_mod = start_sample % event_span
|
||||
delta_samples = event_mod - audio_mod
|
||||
half_span = event_span // 2
|
||||
if delta_samples > half_span:
|
||||
delta_samples -= event_span
|
||||
elif delta_samples < -half_span:
|
||||
delta_samples += event_span
|
||||
return audio_frame.audio_time_ns + int(delta_samples * 1_000_000_000 / audio_frame.sample_rate)
|
||||
|
||||
|
||||
def sync_status(
|
||||
frame: RenderFramePacket,
|
||||
audio_frame: CultSpatialAudioFrame | None,
|
||||
source_event_count: int,
|
||||
overlay_event_count: int,
|
||||
synchronized: bool,
|
||||
) -> AudioVisualSyncStatus:
|
||||
audio_delta_ns = None if audio_frame is None else frame.audio_alignment_time_ns - audio_frame.audio_time_ns
|
||||
return AudioVisualSyncStatus(
|
||||
visual_frame_id=frame.frame_id,
|
||||
visual_present_time_ns=frame.present_time_ns,
|
||||
audio_frame_id=None if audio_frame is None else audio_frame.frame_id,
|
||||
audio_time_ns=None if audio_frame is None else audio_frame.audio_time_ns,
|
||||
audio_delta_ns=audio_delta_ns,
|
||||
source_event_count=source_event_count,
|
||||
overlay_event_count=overlay_event_count,
|
||||
synchronized=bool(synchronized and audio_frame is not None),
|
||||
)
|
||||
|
||||
|
|
@ -10,6 +10,7 @@ import time
|
|||
|
||||
import numpy as np
|
||||
|
||||
from .audio_overlay import AudioVisualSyncStatus, overlay_audio_events
|
||||
from .render_bridge import RenderFramePacket, RenderPointPacket, load_render_frame
|
||||
from .cultcache_docs import CultStreamStatus, get_live_render_frame, put_stream_status
|
||||
|
||||
|
|
@ -53,12 +54,28 @@ class RenderFrameFileSource:
|
|||
|
||||
|
||||
class CultCacheRenderFrameSource:
|
||||
def __init__(self, path: Path, demo_if_missing: bool = False, demo_point_count: int = 4096) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
path: Path,
|
||||
demo_if_missing: bool = False,
|
||||
demo_point_count: int = 4096,
|
||||
audio_cache: Path | None = None,
|
||||
audio_events_cache: Path | None = None,
|
||||
audio_event_window_ns: int = 120_000_000,
|
||||
) -> None:
|
||||
self.path = path
|
||||
self.demo_if_missing = demo_if_missing
|
||||
self.demo_point_count = demo_point_count
|
||||
self.audio_cache = audio_cache
|
||||
self.audio_events_cache = audio_events_cache
|
||||
self.audio_event_window_ns = int(audio_event_window_ns)
|
||||
self._mtime_ns: int | None = None
|
||||
self._frame: RenderFramePacket | None = None
|
||||
self._audio_mtime_ns: int | None = None
|
||||
self._audio_frame = None
|
||||
self._events_mtime_ns: int | None = None
|
||||
self._events = None
|
||||
self.last_sync_status: AudioVisualSyncStatus | None = None
|
||||
|
||||
def current_frame(self, now_s: float, config: SpoutOutputConfig) -> RenderFramePacket:
|
||||
if self.path.exists():
|
||||
|
|
@ -67,10 +84,47 @@ class CultCacheRenderFrameSource:
|
|||
self._frame = get_live_render_frame(self.path)
|
||||
self._mtime_ns = mtime_ns
|
||||
if self._frame is not None:
|
||||
return self._frame
|
||||
return self._with_audio_overlay(self._frame)
|
||||
if not self.demo_if_missing:
|
||||
raise FileNotFoundError(self.path)
|
||||
return demo_render_frame(now_s, config)
|
||||
return self._with_audio_overlay(demo_render_frame(now_s, config))
|
||||
|
||||
def _with_audio_overlay(self, frame: RenderFramePacket) -> RenderFramePacket:
|
||||
if self.audio_cache is None and self.audio_events_cache is None:
|
||||
self.last_sync_status = None
|
||||
return frame
|
||||
audio_frame = self._read_audio_frame()
|
||||
events = self._read_audio_events()
|
||||
augmented, status = overlay_audio_events(
|
||||
frame,
|
||||
events,
|
||||
audio_frame,
|
||||
window_ns=self.audio_event_window_ns,
|
||||
)
|
||||
self.last_sync_status = status
|
||||
return augmented
|
||||
|
||||
def _read_audio_frame(self):
|
||||
if self.audio_cache is None or not self.audio_cache.exists():
|
||||
return None
|
||||
mtime_ns = self.audio_cache.stat().st_mtime_ns
|
||||
if self._audio_frame is None or mtime_ns != self._audio_mtime_ns:
|
||||
from audio_field.cultcache_audio import get_live_spatial_audio_frame
|
||||
|
||||
self._audio_frame = get_live_spatial_audio_frame(self.audio_cache)
|
||||
self._audio_mtime_ns = mtime_ns
|
||||
return self._audio_frame
|
||||
|
||||
def _read_audio_events(self):
|
||||
if self.audio_events_cache is None or not self.audio_events_cache.exists():
|
||||
return None
|
||||
mtime_ns = self.audio_events_cache.stat().st_mtime_ns
|
||||
if self._events is None or mtime_ns != self._events_mtime_ns:
|
||||
from audio_field.cultcache_audio import get_live_audio_source_events
|
||||
|
||||
self._events = get_live_audio_source_events(self.audio_events_cache)
|
||||
self._events_mtime_ns = mtime_ns
|
||||
return self._events
|
||||
|
||||
|
||||
def demo_render_frame(now_s: float, config: SpoutOutputConfig) -> RenderFramePacket:
|
||||
|
|
@ -217,6 +271,29 @@ def write_cult_status(
|
|||
)
|
||||
|
||||
|
||||
def write_sync_status(path: Path, status: AudioVisualSyncStatus | None) -> None:
|
||||
if status is None:
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"visual_frame_id": status.visual_frame_id,
|
||||
"visual_present_time_ns": status.visual_present_time_ns,
|
||||
"audio_frame_id": status.audio_frame_id,
|
||||
"audio_time_ns": status.audio_time_ns,
|
||||
"audio_delta_ns": status.audio_delta_ns,
|
||||
"source_event_count": status.source_event_count,
|
||||
"overlay_event_count": status.overlay_event_count,
|
||||
"synchronized": status.synchronized,
|
||||
"updated_monotonic_ns": time.monotonic_ns(),
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
class OpenGLSpoutPointRenderer:
|
||||
def __init__(self, config: SpoutOutputConfig) -> None:
|
||||
self.config = config
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ def main() -> None:
|
|||
parser.add_argument("--loop", action="store_true")
|
||||
parser.add_argument("--synthetic", action="store_true")
|
||||
parser.add_argument("--smoke-readback", action="store_true")
|
||||
parser.add_argument("--max-lag-ms", type=float, default=250.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
source = None
|
||||
|
|
@ -92,6 +93,10 @@ def main() -> None:
|
|||
now = time.monotonic()
|
||||
if args.duration is not None and now - start_monotonic >= args.duration:
|
||||
break
|
||||
realtime_start_sample = int((now - start_monotonic) * sample_rate)
|
||||
max_lag_samples = int(float(args.max_lag_ms) * sample_rate / 1000.0)
|
||||
if start_sample + args.chunk_frames < realtime_start_sample - max_lag_samples:
|
||||
start_sample = max(0, realtime_start_sample - (realtime_start_sample % args.chunk_frames))
|
||||
block = source_block(source, sample_rate, start_sample, args.chunk_frames, loop=args.loop)
|
||||
if block is None:
|
||||
break
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from localcast.sensor_fusion.spout_output import (
|
|||
RenderFrameFileSource,
|
||||
SpoutOutputConfig,
|
||||
write_cult_status,
|
||||
write_sync_status,
|
||||
write_status,
|
||||
)
|
||||
|
||||
|
|
@ -30,6 +31,11 @@ def main() -> None:
|
|||
parser.add_argument("--demo-point-count", type=int, default=4096)
|
||||
parser.add_argument("--status", default=str(ROOT / "calibration" / "runs" / "stream-spout-status.json"))
|
||||
parser.add_argument("--status-cache", default=str(ROOT / "calibration" / "runs" / "visual-stream-status.msgpack"))
|
||||
parser.add_argument("--sync-status", default=str(ROOT / "calibration" / "runs" / "av-sync-status.json"))
|
||||
parser.add_argument("--audio-cache", default=str(ROOT / "calibration" / "runs" / "audio-state.msgpack"))
|
||||
parser.add_argument("--audio-events-cache", default=str(ROOT / "calibration" / "runs" / "audio-events.msgpack"))
|
||||
parser.add_argument("--audio-event-window-ms", type=float, default=120.0)
|
||||
parser.add_argument("--no-audio-overlay", action="store_true")
|
||||
parser.add_argument("--source", choices=["cultcache", "json"], default="cultcache")
|
||||
parser.add_argument("--duration", type=float, help="Optional run duration in seconds for smoke tests.")
|
||||
args = parser.parse_args()
|
||||
|
|
@ -38,6 +44,7 @@ def main() -> None:
|
|||
frame_cache_path = Path(args.frame_cache)
|
||||
status_path = Path(args.status)
|
||||
status_cache_path = Path(args.status_cache)
|
||||
sync_status_path = Path(args.sync_status)
|
||||
config = SpoutOutputConfig(
|
||||
sender_name=args.sender_name,
|
||||
width=args.width,
|
||||
|
|
@ -47,7 +54,14 @@ def main() -> None:
|
|||
demo_point_count=args.demo_point_count,
|
||||
)
|
||||
if args.source == "cultcache":
|
||||
source = CultCacheRenderFrameSource(frame_cache_path, demo_if_missing=args.demo_if_missing, demo_point_count=args.demo_point_count)
|
||||
source = CultCacheRenderFrameSource(
|
||||
frame_cache_path,
|
||||
demo_if_missing=args.demo_if_missing,
|
||||
demo_point_count=args.demo_point_count,
|
||||
audio_cache=None if args.no_audio_overlay else Path(args.audio_cache),
|
||||
audio_events_cache=None if args.no_audio_overlay else Path(args.audio_events_cache),
|
||||
audio_event_window_ns=int(float(args.audio_event_window_ms) * 1_000_000),
|
||||
)
|
||||
frame_source_name = str(frame_cache_path)
|
||||
else:
|
||||
source = RenderFrameFileSource(frame_path, demo_if_missing=args.demo_if_missing, demo_point_count=args.demo_point_count)
|
||||
|
|
@ -80,6 +94,7 @@ def main() -> None:
|
|||
frame_source=frame_source_name,
|
||||
last_error=None if ok else "SpoutGL sendTexture returned false",
|
||||
)
|
||||
write_sync_status(sync_status_path, getattr(source, "last_sync_status", None))
|
||||
last_status = now
|
||||
sleep_for = frame_interval - (time.monotonic() - now)
|
||||
if sleep_for > 0:
|
||||
|
|
|
|||
|
|
@ -19,8 +19,10 @@ from localcast.sensor_fusion import (
|
|||
get_live_render_frame,
|
||||
lower_frame_to_screen_brushes,
|
||||
lower_points_to_render_frame,
|
||||
overlay_audio_events,
|
||||
put_live_render_frame,
|
||||
)
|
||||
from audio_field.cultcache_audio import make_audio_source_events, make_spatial_audio_frame
|
||||
from localcast.sensor_fusion.adapters import read_raw_bgr_frames
|
||||
|
||||
|
||||
|
|
@ -189,6 +191,47 @@ class SensorFusionTests(unittest.TestCase):
|
|||
self.assertEqual(1, len(loaded.points))
|
||||
self.assertEqual("ball", loaded.points[0].stable_key)
|
||||
|
||||
def test_audio_source_events_overlay_against_audio_alignment_time(self):
|
||||
frame = RenderFramePacket(
|
||||
schema="localcast.sensor_fusion.render_frame.v1",
|
||||
frame_id=9,
|
||||
created_monotonic_ns=1_000,
|
||||
source_time_min_ns=1_000,
|
||||
source_time_max_ns=1_000,
|
||||
present_time_ns=1_100,
|
||||
audio_alignment_time_ns=1_000_000_000,
|
||||
spout_sender_name="test",
|
||||
target_width=128,
|
||||
target_height=72,
|
||||
points=(),
|
||||
)
|
||||
audio = make_spatial_audio_frame(
|
||||
np.zeros((1024, 4), dtype=np.float32),
|
||||
frame_id=4,
|
||||
sample_rate=48000,
|
||||
start_sample=48000,
|
||||
audio_time_ns=1_000_000_000,
|
||||
)
|
||||
events = make_audio_source_events(
|
||||
frame_id=5,
|
||||
sample_rate=48000,
|
||||
start_sample=0,
|
||||
frame_count=96000,
|
||||
audio_time_ns=0,
|
||||
events=[
|
||||
{"eventId": 1, "startSample": 48000, "positionMeters": [0.2, 0.3, 1.4], "confidence": 0.9, "energy": 0.001},
|
||||
{"eventId": 2, "startSample": 70000, "positionMeters": [2.0, 0.0, 1.0], "confidence": 0.9, "energy": 0.001},
|
||||
],
|
||||
voice_focus=[],
|
||||
)
|
||||
|
||||
augmented, status = overlay_audio_events(frame, events, audio, window_ns=40_000_000)
|
||||
|
||||
self.assertEqual(1, len(augmented.points))
|
||||
self.assertEqual("audio-event-1", augmented.points[0].stable_key)
|
||||
self.assertTrue(status.synchronized)
|
||||
self.assertEqual(1, status.overlay_event_count)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in a new issue