diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index 6bd1bbc..9a8646b 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -65,6 +65,7 @@ flowchart TD - `localcast.sensor_fusion.camera_control` owns measurement-quality policy: luminance, clipping, contrast, and sharpness in; normalized exposure/gain/focus commands out. Driver adapters own translating those commands to OpenCV, DirectShow, vendor tools, or no-op mocks. - `localcast.sensor_fusion.dense_stereo` owns a debug CPU reference for dense calibrated RGB surface claims. It is useful for tests and shader parity, but the live million-splat path belongs to Aquarium/GPU compute. - `localcast.sensor_fusion.active_illumination` owns deliberate light pulses as calibration telemetry. A Tuya bulb is an optional local-network actuator, not a rendering authority; its pulse timestamps are evidence for camera exposure/depth response. +- `localcast.sensor_fusion.clap_calibration` owns deliberate clap detection: chirplet-synced audio transients nominate oracle timestamps, camera frame windows around that oracle look for motion derivatives, calibrated camera peaks triangulate the impact point, and only audio-plus-visual agreement becomes a clap calibration event. - `localcast.sensor_fusion.render_bridge` owns the in-process render-frame ABI: cached point claims plus target dimensions, Spout sender name, source timestamp range, intended visual presentation time, and ambisonic/audio alignment time. - `localcast.sensor_fusion.cultcache_docs` owns the typed CultCache document boundary for live visual state. - `localcast.sensor_fusion.spout_output` owns the deadline OBS publication sink: render-frame packet in, GPU texture plus Spout sender heartbeat out. @@ -124,6 +125,17 @@ LocalCastBridge TrackCache `RenderFramePacket` is the in-process shape. The live boundary is now a typed CultCache MessagePack document, with CultNet `document_put` as the intended process/network API boundary. JSON render-frame files are compatibility scaffolding only. +Clap calibration events use the same typed-state rule. A clap is not every sharp +audio transient. The detector first finds a sharp broadband audio onset in the +audio oracle timeline, then opens a short camera window around that timestamp +and looks for high frame-derivative peaks per camera. At least the configured +minimum number of cameras must agree; when a calibrated rig is available, their +motion centroids are triangulated into a world-space impact point. The typed +`localcast.calibration.clap_events` document carries stable event id, acoustic +oracle time, visual observed time, timing uncertainty, confidence, world +position, and the contributing camera peaks. Aquarium consumes those events as +high-confidence calibration constraints beside ultrasonic chirplet room evidence. + The current streamable cut is: ```text diff --git a/docs/typed-visual-state.md b/docs/typed-visual-state.md index cf5df94..0ce10af 100644 --- a/docs/typed-visual-state.md +++ b/docs/typed-visual-state.md @@ -46,6 +46,14 @@ Live files: - payload: MessagePack array, decoded as `CultStreamStatus` - owns: sender heartbeat and renderer health +`localcast.calibration.clap_events` + +- schema id: `gamecult.localcast.calibration.clap_events.v1` +- key: `localcast.calibration.clap-events.live` +- payload: MessagePack array, decoded as `CultClapEventsFrame` +- owns: deliberate clap calibration events aligned against the audio timing oracle +- fields: stable key, world position, acoustic oracle timestamp, visual observed timestamp, timing uncertainty in microseconds, visual/acoustic confidence, and contributing camera motion peaks + ## CultNet Boundary The API boundary is CultNet document replication, not ad hoc file polling: diff --git a/localcast/sensor_fusion/__init__.py b/localcast/sensor_fusion/__init__.py index d108d2f..c375762 100644 --- a/localcast/sensor_fusion/__init__.py +++ b/localcast/sensor_fusion/__init__.py @@ -39,12 +39,28 @@ from .surface_features import ( ) from .audio_overlay import AudioVisualSyncStatus, overlay_audio_events from .media_artifacts import RemoteVideoArtifact, remote_video_artifact_for_present_time +from .clap_calibration import ( + AudioTransientCandidate, + CameraMotionPeak, + ClapCalibrationEvent, + ClapDetectorConfig, + TimestampedFrame, + detect_audio_transients, + detect_camera_motion_peaks, + detect_clap_events, +) from .cultcache_docs import ( + CLAP_EVENTS_KEY, LIVE_RENDER_FRAME_KEY, + CultClapEvent, + CultClapEventsFrame, CultRenderFrame, CultRenderPoint, CultStreamStatus, + get_live_clap_events, get_live_render_frame, + make_clap_events_frame, + put_live_clap_events, put_live_render_frame, ) from .spout_output import ScreenBrushPacket, SpoutOutputConfig, frame_to_vertex_array, lower_frame_to_screen_brushes @@ -75,8 +91,16 @@ __all__ = [ "SurfaceFeatureObservation", "SurfaceFeatureTrack", "AudioVisualSyncStatus", + "AudioTransientCandidate", + "CameraMotionPeak", + "ClapCalibrationEvent", + "ClapDetectorConfig", + "TimestampedFrame", "RemoteVideoArtifact", + "CLAP_EVENTS_KEY", "LIVE_RENDER_FRAME_KEY", + "CultClapEvent", + "CultClapEventsFrame", "CultRenderFrame", "CultRenderPoint", "CultStreamStatus", @@ -84,6 +108,8 @@ __all__ = [ "SpoutOutputConfig", "frame_to_vertex_array", "get_live_render_frame", + "get_live_clap_events", + "make_clap_events_frame", "lower_frame_to_screen_brushes", "lower_points_to_render_frame", "dense_stereo_points", @@ -92,7 +118,11 @@ __all__ = [ "measure_frame_quality", "orb_surface_observations", "overlay_audio_events", + "detect_audio_transients", + "detect_camera_motion_peaks", + "detect_clap_events", "remote_video_artifact_for_present_time", + "put_live_clap_events", "put_live_render_frame", "solve_camera_pose_from_fixed_board", "solve_common_space_from_fixed_board", diff --git a/localcast/sensor_fusion/clap_calibration.py b/localcast/sensor_fusion/clap_calibration.py new file mode 100644 index 0000000..65364c1 --- /dev/null +++ b/localcast/sensor_fusion/clap_calibration.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from dataclasses import dataclass +import math +from typing import Mapping, Sequence + +import numpy as np + +from .core import Observation2D, SensorRig + + +@dataclass(frozen=True) +class ClapDetectorConfig: + audio_window_samples: int = 256 + audio_hop_samples: int = 64 + audio_onset_ratio: float = 6.0 + min_audio_energy: float = 1.0e-5 + min_camera_motion_score: float = 0.035 + camera_window_ns: int = 35_000_000 + min_camera_count: int = 2 + min_event_spacing_ns: int = 180_000_000 + timing_uncertainty_us: float = 2.0 + + +@dataclass(frozen=True) +class AudioTransientCandidate: + oracle_time_ns: int + start_sample: int + energy: float + onset_ratio: float + confidence: float + + +@dataclass(frozen=True) +class TimestampedFrame: + sensor_id: str + timestamp_ns: int + image: np.ndarray + + +@dataclass(frozen=True) +class CameraMotionPeak: + sensor_id: str + timestamp_ns: int + uv: np.ndarray + score: float + + +@dataclass(frozen=True) +class ClapCalibrationEvent: + stable_key: str + position_m: tuple[float, float, float] + acoustic_oracle_ns: int + visual_observed_ns: int + timing_uncertainty_us: float + visual_confidence: float + acoustic_confidence: float + camera_peaks: tuple[CameraMotionPeak, ...] + + +def detect_audio_transients( + field: np.ndarray, + sample_rate: int, + *, + audio_time_ns: int, + start_sample: int = 0, + config: ClapDetectorConfig = ClapDetectorConfig(), +) -> list[AudioTransientCandidate]: + samples = np.asarray(field, dtype=np.float32) + if samples.ndim == 1: + samples = samples[:, None] + if samples.ndim != 2: + raise ValueError("audio field must be samples or samples-by-channels") + if sample_rate <= 0: + raise ValueError("sample_rate must be positive") + window = int(config.audio_window_samples) + hop = int(config.audio_hop_samples) + if window <= 0 or hop <= 0: + raise ValueError("audio windows must be positive") + + candidates: list[AudioTransientCandidate] = [] + previous_energy = 0.0 + for offset in range(0, max(0, samples.shape[0] - window + 1), hop): + block = samples[offset : offset + window] + diff = np.diff(block, axis=0) + energy = float(np.mean(block.astype(np.float64) ** 2)) + derivative_energy = float(np.mean(diff.astype(np.float64) ** 2)) if diff.size else 0.0 + onset = max(energy, derivative_energy) / max(previous_energy, 1.0e-12) + previous_energy = max(energy, 1.0e-12) + if energy < config.min_audio_energy or onset < config.audio_onset_ratio: + continue + sample = start_sample + offset + oracle_time_ns = audio_time_ns + round((sample - start_sample) * 1_000_000_000 / sample_rate) + confidence = max(0.0, min(1.0, 0.35 + math.log10(max(onset, 1.0)) * 0.28)) + candidates.append(AudioTransientCandidate(oracle_time_ns, sample, energy, onset, confidence)) + + return suppress_nearby_audio_candidates(candidates, config.min_event_spacing_ns) + + +def detect_camera_motion_peaks( + frames_by_sensor: Mapping[str, Sequence[TimestampedFrame]], + oracle_time_ns: int, + *, + config: ClapDetectorConfig = ClapDetectorConfig(), +) -> list[CameraMotionPeak]: + peaks: list[CameraMotionPeak] = [] + for sensor_id, frames in frames_by_sensor.items(): + ordered = sorted(frames, key=lambda frame: frame.timestamp_ns) + best: CameraMotionPeak | None = None + for previous, current in zip(ordered, ordered[1:]): + midpoint_ns = (previous.timestamp_ns + current.timestamp_ns) // 2 + if abs(midpoint_ns - oracle_time_ns) > config.camera_window_ns: + continue + score, uv = frame_motion_score(previous.image, current.image) + if score < config.min_camera_motion_score: + continue + peak = CameraMotionPeak(sensor_id, midpoint_ns, uv, score) + if best is None or peak.score > best.score: + best = peak + if best is not None: + peaks.append(best) + return sorted(peaks, key=lambda peak: peak.score, reverse=True) + + +def detect_clap_events( + field: np.ndarray, + sample_rate: int, + frames_by_sensor: Mapping[str, Sequence[TimestampedFrame]], + *, + audio_time_ns: int, + rig: SensorRig | None = None, + start_sample: int = 0, + config: ClapDetectorConfig = ClapDetectorConfig(), +) -> list[ClapCalibrationEvent]: + events: list[ClapCalibrationEvent] = [] + for index, candidate in enumerate( + detect_audio_transients(field, sample_rate, audio_time_ns=audio_time_ns, start_sample=start_sample, config=config) + ): + peaks = detect_camera_motion_peaks(frames_by_sensor, candidate.oracle_time_ns, config=config) + if len(peaks) < config.min_camera_count: + continue + position, visual_confidence = localize_clap(peaks, rig) + if visual_confidence <= 0.0: + continue + visual_time_ns = round(sum(peak.timestamp_ns * peak.score for peak in peaks) / max(1.0e-12, sum(peak.score for peak in peaks))) + timing_spread_us = max(abs(peak.timestamp_ns - candidate.oracle_time_ns) for peak in peaks) / 1000.0 + events.append( + ClapCalibrationEvent( + stable_key=f"clap:{candidate.oracle_time_ns}:{index}", + position_m=position, + acoustic_oracle_ns=candidate.oracle_time_ns, + visual_observed_ns=visual_time_ns, + timing_uncertainty_us=max(config.timing_uncertainty_us, timing_spread_us), + visual_confidence=visual_confidence, + acoustic_confidence=candidate.confidence, + camera_peaks=tuple(peaks), + ) + ) + return events + + +def frame_motion_score(previous: np.ndarray, current: np.ndarray) -> tuple[float, np.ndarray]: + a = to_luma(previous) + b = to_luma(current) + if a.shape != b.shape: + raise ValueError(f"frame shape mismatch: {a.shape} vs {b.shape}") + diff = np.abs(b - a) + threshold = max(0.02, float(np.percentile(diff, 92))) + mask = diff >= threshold + if not np.any(mask): + return 0.0, np.array([a.shape[1] * 0.5, a.shape[0] * 0.5], dtype=np.float64) + weights = diff * mask + yy, xx = np.indices(a.shape, dtype=np.float64) + total = float(np.sum(weights)) + uv = np.array([float(np.sum(xx * weights) / total), float(np.sum(yy * weights) / total)], dtype=np.float64) + score = float(np.mean(diff[mask])) + return score, uv + + +def localize_clap(peaks: Sequence[CameraMotionPeak], rig: SensorRig | None) -> tuple[tuple[float, float, float], float]: + if rig is None: + best = peaks[0] + return (float(best.uv[0]), float(best.uv[1]), 0.0), max(0.0, min(1.0, best.score)) + observations = [ + Observation2D(peak.sensor_id, "clap-impact", peak.timestamp_ns, peak.uv, max(0.0, min(1.0, peak.score))) + for peak in peaks + if peak.sensor_id in rig.cameras + ] + result = rig.fuse(observations) + if not result.points: + return (0.0, 0.0, 0.0), 0.0 + point = max(result.points, key=lambda item: item.confidence) + return tuple(float(value) for value in point.xyz), float(point.confidence) + + +def suppress_nearby_audio_candidates( + candidates: list[AudioTransientCandidate], + min_spacing_ns: int, +) -> list[AudioTransientCandidate]: + kept: list[AudioTransientCandidate] = [] + for candidate in sorted(candidates, key=lambda item: item.energy, reverse=True): + if any(abs(candidate.oracle_time_ns - existing.oracle_time_ns) < min_spacing_ns for existing in kept): + continue + kept.append(candidate) + return sorted(kept, key=lambda item: item.oracle_time_ns) + + +def to_luma(frame: np.ndarray) -> np.ndarray: + image = np.asarray(frame) + if image.ndim == 2: + return normalize_image(image) + if image.ndim != 3 or image.shape[2] < 3: + raise ValueError("camera frame must be grayscale or color image") + rgb = image[:, :, :3].astype(np.float32) + return normalize_image((0.2126 * rgb[:, :, 2]) + (0.7152 * rgb[:, :, 1]) + (0.0722 * rgb[:, :, 0])) + + +def normalize_image(image: np.ndarray) -> np.ndarray: + arr = np.asarray(image, dtype=np.float32) + if arr.size == 0: + raise ValueError("camera frame is empty") + max_value = 255.0 if np.max(arr) > 1.5 else 1.0 + return np.clip(arr / max_value, 0.0, 1.0) diff --git a/localcast/sensor_fusion/cultcache_docs.py b/localcast/sensor_fusion/cultcache_docs.py index 75d4449..c29a07d 100644 --- a/localcast/sensor_fusion/cultcache_docs.py +++ b/localcast/sensor_fusion/cultcache_docs.py @@ -23,6 +23,8 @@ LIVE_RENDER_FRAME_KEY = "localcast.visual.render-frame.live" RENDER_FRAME_SCHEMA_ID = "gamecult.localcast.visual.render_frame.v1" STREAM_STATUS_KEY = "localcast.visual.stream-status.live" STREAM_STATUS_SCHEMA_ID = "gamecult.localcast.visual.stream_status.v1" +CLAP_EVENTS_KEY = "localcast.calibration.clap-events.live" +CLAP_EVENTS_SCHEMA_ID = "gamecult.localcast.calibration.clap_events.v1" @dataclass(frozen=True) @@ -60,6 +62,26 @@ class CultStreamStatus: last_error: str +@dataclass(frozen=True) +class CultClapEvent: + stable_key: str + position_m: tuple[float, float, float] + acoustic_oracle_ns: int + visual_observed_ns: int + timing_uncertainty_us: float + visual_confidence: float + acoustic_confidence: float + camera_peaks: tuple[dict[str, Any], ...] + + +@dataclass(frozen=True) +class CultClapEventsFrame: + schema_version: str + frame_id: int + created_monotonic_ns: int + events: tuple[CultClapEvent, ...] + + def _pack(value: Any) -> bytes: import msgpack @@ -148,6 +170,50 @@ def _decode_status(raw: Any) -> CultStreamStatus: ) +def _encode_clap_event(event: CultClapEvent) -> list[Any]: + return [ + event.stable_key, + list(event.position_m), + event.acoustic_oracle_ns, + event.visual_observed_ns, + event.timing_uncertainty_us, + event.visual_confidence, + event.acoustic_confidence, + list(event.camera_peaks), + ] + + +def _decode_clap_event(raw: Any) -> CultClapEvent: + return CultClapEvent( + stable_key=str(raw[0]), + position_m=(float(raw[1][0]), float(raw[1][1]), float(raw[1][2])), + acoustic_oracle_ns=int(raw[2]), + visual_observed_ns=int(raw[3]), + timing_uncertainty_us=float(raw[4]), + visual_confidence=float(raw[5]), + acoustic_confidence=float(raw[6]), + camera_peaks=tuple(dict(item) for item in raw[7]), + ) + + +def _encode_clap_events_frame(frame: CultClapEventsFrame) -> list[Any]: + return [ + frame.schema_version, + frame.frame_id, + frame.created_monotonic_ns, + [_encode_clap_event(event) for event in frame.events], + ] + + +def _decode_clap_events_frame(raw: Any) -> CultClapEventsFrame: + return CultClapEventsFrame( + schema_version=str(raw[0]), + frame_id=int(raw[1]), + created_monotonic_ns=int(raw[2]), + events=tuple(_decode_clap_event(item) for item in raw[3]), + ) + + RENDER_FRAME_DOCUMENT = define_document_type( "localcast.visual.render_frame", encode=_encode_render_frame, @@ -166,6 +232,15 @@ STREAM_STATUS_DOCUMENT = define_document_type( payload_decoder=_unpack, ) +CLAP_EVENTS_DOCUMENT = define_document_type( + "localcast.calibration.clap_events", + encode=_encode_clap_events_frame, + decode=_decode_clap_events_frame, + name=lambda frame: CLAP_EVENTS_KEY if isinstance(frame, CultClapEventsFrame) else None, + payload_encoder=_pack, + payload_decoder=_unpack, +) + def _retry_cache_io(operation): last_error: Exception | None = None @@ -184,6 +259,7 @@ def _open_visual_cache_once(path: Path) -> CultCache: CultCache.builder() .register_document_type(RENDER_FRAME_DOCUMENT) .register_document_type(STREAM_STATUS_DOCUMENT) + .register_document_type(CLAP_EVENTS_DOCUMENT) .add_generic_store(SingleFileMessagePackBackingStore(path)) .build() ) @@ -270,3 +346,48 @@ def put_stream_status(path: Path, status: CultStreamStatus) -> None: cache.put(STREAM_STATUS_DOCUMENT, STREAM_STATUS_KEY, status) _retry_cache_io(operation) + + +def make_clap_events_frame(*, frame_id: int, events: list[Any] | tuple[Any, ...]) -> CultClapEventsFrame: + return CultClapEventsFrame( + schema_version=CLAP_EVENTS_SCHEMA_ID, + frame_id=int(frame_id), + created_monotonic_ns=time.monotonic_ns(), + events=tuple( + CultClapEvent( + stable_key=str(event.stable_key), + position_m=tuple(float(value) for value in event.position_m), + acoustic_oracle_ns=int(event.acoustic_oracle_ns), + visual_observed_ns=int(event.visual_observed_ns), + timing_uncertainty_us=float(event.timing_uncertainty_us), + visual_confidence=float(event.visual_confidence), + acoustic_confidence=float(event.acoustic_confidence), + camera_peaks=tuple( + { + "sensorId": peak.sensor_id, + "timestampNs": int(peak.timestamp_ns), + "uv": [float(peak.uv[0]), float(peak.uv[1])], + "score": float(peak.score), + } + for peak in event.camera_peaks + ), + ) + for event in events + ), + ) + + +def put_live_clap_events(path: Path, frame: CultClapEventsFrame) -> None: + def operation() -> None: + cache = open_visual_cache(path) + cache.put(CLAP_EVENTS_DOCUMENT, CLAP_EVENTS_KEY, frame) + + _retry_cache_io(operation) + + +def get_live_clap_events(path: Path) -> CultClapEventsFrame | None: + def operation() -> CultClapEventsFrame | None: + cache = open_visual_cache(path) + return cache.get(CLAP_EVENTS_DOCUMENT, CLAP_EVENTS_KEY) + + return _retry_cache_io(operation) diff --git a/tests/test_sensor_fusion.py b/tests/test_sensor_fusion.py index 415d67c..dc02f87 100644 --- a/tests/test_sensor_fusion.py +++ b/tests/test_sensor_fusion.py @@ -12,6 +12,7 @@ from localcast.sensor_fusion import ( CameraModel, CameraIntrinsics, CameraQualityTarget, + ClapDetectorConfig, DenseStereoConfig, FusionConfig, Observation2D, @@ -21,14 +22,19 @@ from localcast.sensor_fusion import ( RenderPointPacket, SensorRig, SpoutOutputConfig, + TimestampedFrame, TrackCache, + detect_clap_events, frame_to_vertex_array, dense_stereo_points, + get_live_clap_events, get_live_render_frame, lower_frame_to_screen_brushes, lower_points_to_render_frame, match_surface_features, overlay_audio_events, + make_clap_events_frame, + put_live_clap_events, put_live_render_frame, remote_video_artifact_for_present_time, solve_common_space_from_fixed_board, @@ -245,6 +251,98 @@ class SensorFusionTests(unittest.TestCase): self.assertEqual(1, len(loaded.points)) self.assertEqual("ball", loaded.points[0].stable_key) + def test_clap_detector_promotes_audio_transient_with_camera_motion(self): + left = camera("left", -0.25) + right = camera("right", 0.25) + rig = SensorRig( + cameras={"left": left, "right": right}, + config=FusionConfig(max_pair_dt_ns=40_000_000, max_reprojection_error_px=2.0), + ) + target = np.array([0.02, 0.01, 2.0]) + sample_rate = 48_000 + start_ns = 1_000_000_000 + clap_sample = 960 + oracle_ns = start_ns + round(clap_sample * 1_000_000_000 / sample_rate) + field = np.zeros((4096, 4), dtype=np.float32) + field[clap_sample] = np.array([0.85, -0.75, 0.6, -0.55], dtype=np.float32) + + frames = {} + for cam in (left, right): + uv = cam.project_world(target) + previous = np.zeros((cam.height, cam.width), dtype=np.float32) + current = previous.copy() + x, y = int(round(uv[0])), int(round(uv[1])) + current[y - 2 : y + 3, x - 2 : x + 3] = 1.0 + frames[cam.sensor_id] = ( + TimestampedFrame(cam.sensor_id, oracle_ns - 8_000_000, previous), + TimestampedFrame(cam.sensor_id, oracle_ns + 8_000_000, current), + ) + + events = detect_clap_events( + field, + sample_rate, + frames, + audio_time_ns=start_ns, + rig=rig, + config=ClapDetectorConfig( + audio_window_samples=128, + audio_hop_samples=32, + audio_onset_ratio=3.0, + min_audio_energy=1.0e-6, + min_camera_motion_score=0.2, + ), + ) + + self.assertEqual(1, len(events)) + event = events[0] + np.testing.assert_allclose(np.asarray(event.position_m), target, atol=0.04) + self.assertLessEqual(abs(oracle_ns - event.acoustic_oracle_ns), 3_000_000) + self.assertGreater(event.acoustic_confidence, 0.5) + self.assertGreater(event.visual_confidence, 0.5) + + def test_clap_events_round_trip_through_typed_cultcache_doc(self): + left = camera("left", -0.25) + sample_rate = 48_000 + start_ns = 2_000_000_000 + clap_sample = 960 + oracle_ns = start_ns + round(clap_sample * 1_000_000_000 / sample_rate) + field = np.zeros((2048, 1), dtype=np.float32) + field[clap_sample] = 1.0 + previous = np.zeros((left.height, left.width), dtype=np.float32) + current = previous.copy() + current[100:108, 120:128] = 1.0 + events = detect_clap_events( + field, + sample_rate, + { + "left": ( + TimestampedFrame("left", oracle_ns - 5_000_000, previous), + TimestampedFrame("left", oracle_ns + 5_000_000, current), + ) + }, + audio_time_ns=start_ns, + rig=None, + config=ClapDetectorConfig( + audio_window_samples=128, + audio_hop_samples=32, + audio_onset_ratio=3.0, + min_audio_energy=1.0e-6, + min_camera_count=1, + min_camera_motion_score=0.2, + ), + ) + frame = make_clap_events_frame(frame_id=12, events=events) + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "visual-state.msgpack" + put_live_clap_events(path, frame) + loaded = get_live_clap_events(path) + + self.assertIsNotNone(loaded) + self.assertEqual(12, loaded.frame_id) + self.assertEqual(1, len(loaded.events)) + self.assertEqual(events[0].acoustic_oracle_ns, loaded.events[0].acoustic_oracle_ns) + def test_audio_source_events_overlay_against_audio_alignment_time(self): frame = RenderFramePacket( schema="localcast.sensor_fusion.render_frame.v1",