diff --git a/audio_field/probe_optimizer.py b/audio_field/probe_optimizer.py new file mode 100644 index 0000000..a6d17e4 --- /dev/null +++ b/audio_field/probe_optimizer.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from audio_field.runtime_sync import SyncState + + +@dataclass(frozen=True) +class ProbePolicy: + target_confidence: float = 0.65 + trigger_confidence: float = 0.35 + min_interval_frames: int = 45 + max_probe_level_dbfs: float = -18.0 + prefer_masked_windows: bool = True + + +@dataclass(frozen=True) +class ProbeRequest: + frame_index: int + source_id: str + reason: str + level_dbfs: float + urgency: float + + +class ActiveProbeOptimizer: + """Schedules intentional chirplets when runtime sync confidence is weak.""" + + def __init__(self, policy: ProbePolicy): + self.policy = policy + self._last_probe_frame: int | None = None + + def choose_probe( + self, + frame_index: int, + states: dict[str, SyncState], + masked_window: bool = False, + ) -> ProbeRequest | None: + if self._last_probe_frame is not None: + if frame_index - self._last_probe_frame < self.policy.min_interval_frames: + return None + if self.policy.prefer_masked_windows and not masked_window: + return None + + weak = [ + (source_id, state) + for source_id, state in states.items() + if state.confidence < self.policy.trigger_confidence + ] + if not weak: + return None + + source_id, state = min(weak, key=lambda item: item[1].confidence) + urgency = max(0.0, self.policy.target_confidence - state.confidence) + self._last_probe_frame = frame_index + return ProbeRequest( + frame_index=frame_index, + source_id=source_id, + reason="sync-confidence-below-trigger", + level_dbfs=self.policy.max_probe_level_dbfs, + urgency=urgency, + ) diff --git a/audio_field/runtime_sync.py b/audio_field/runtime_sync.py new file mode 100644 index 0000000..7ecc47d --- /dev/null +++ b/audio_field/runtime_sync.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ChirpletObservation: + source_id: str + frame_index: int + delay_samples: float + phase_radians: float + score: float + rate_scale: float = 1.0 + + +@dataclass(frozen=True) +class SyncState: + delay_samples: float = 0.0 + phase_radians: float = 0.0 + rate_scale: float = 1.0 + confidence: float = 0.0 + last_frame_index: int = -1 + + +class RuntimeSyncEstimator: + """Maintains per-source delay/SRO/phase state from known speaker chirplets.""" + + def __init__(self, min_score: float = 0.18, smoothing: float = 0.2): + if not 0.0 <= smoothing <= 1.0: + raise ValueError("smoothing must be between 0 and 1") + self.min_score = min_score + self.smoothing = smoothing + self._states: dict[str, SyncState] = {} + + def state_for(self, source_id: str) -> SyncState: + return self._states.get(source_id, SyncState()) + + def update(self, observation: ChirpletObservation) -> SyncState: + previous = self.state_for(observation.source_id) + if observation.score < self.min_score: + frozen = SyncState( + delay_samples=previous.delay_samples, + phase_radians=previous.phase_radians, + rate_scale=previous.rate_scale, + confidence=observation.score, + last_frame_index=previous.last_frame_index, + ) + self._states[observation.source_id] = frozen + return frozen + + alpha = self.smoothing + updated = SyncState( + delay_samples=blend(previous.delay_samples, observation.delay_samples, alpha, previous.confidence), + phase_radians=blend(previous.phase_radians, observation.phase_radians, alpha, previous.confidence), + rate_scale=blend(previous.rate_scale, observation.rate_scale, alpha, previous.confidence), + confidence=observation.score, + last_frame_index=observation.frame_index, + ) + self._states[observation.source_id] = updated + return updated + + +def blend(previous: float, current: float, alpha: float, previous_confidence: float) -> float: + if previous_confidence <= 0.0: + return current + return (1.0 - alpha) * previous + alpha * current diff --git a/config/audio-field.example.json b/config/audio-field.example.json index 8f6b59f..76f7e0f 100644 --- a/config/audio-field.example.json +++ b/config/audio-field.example.json @@ -21,6 +21,31 @@ "maxLatencyMs": 2500, "convergesToRealtime": true, "dropOrStretchWhenBehind": "stretch-first-drop-only-on-overrun" + }, + "runtimeSync": { + "enabled": true, + "speakerChirpletsAsTelemetry": true, + "updateEveryBlock": true, + "phaseFieldUpdate": "per-frame", + "confidenceGate": { + "minChirpletScore": 0.18, + "freezeOnLowConfidence": true + }, + "estimates": [ + "delay", + "samplingRateOffset", + "polarity", + "phase" + ], + "activeProbeOptimization": { + "enabled": true, + "minIntervalMs": 1500, + "maxProbeLevelDbfs": -18, + "targetConfidence": 0.65, + "probeWhenAnyAnchorBelow": 0.35, + "preferMaskedWindows": true, + "objective": "maximize sync confidence gain per audible intrusion" + } } }, "ambisonicBus": { diff --git a/docs/audio-field.md b/docs/audio-field.md index cb4995d..857eb38 100644 --- a/docs/audio-field.md +++ b/docs/audio-field.md @@ -17,6 +17,10 @@ The default profile is `distributed-clocks`. The coherent field is not captured Latency is allowed. The live goal is bounded buffered real-time: hold enough audio to estimate delay and slow clock drift, emit blocks behind the live edge, and converge instead of chasing instant output with bad math. The profile owns that latency policy. +Runtime sync is not optional for the real show. Static sweep calibration gives the first estimate; the live system must keep updating delay, sampling-rate offset, polarity, and phase confidence as devices drift. Known speaker output is telemetry: every calibration chirplet or intentionally embedded probe gives the estimator a fresh phase-field observation. + +If passive telemetry is not enough, the system should emit extra chirplets automatically. That is an optimization loop: spend the smallest audible probe budget that raises sync confidence enough to protect the field. + ## Pipeline ```mermaid @@ -33,6 +37,12 @@ flowchart TD F --> K H --> K K --> L["bounded field cache"] + P["known speaker chirplets"] --> Q["runtime delay/SRO/phase estimator"] + R["confidence probe optimizer"] --> P + D --> Q + F --> Q + H --> Q + Q --> K L --> M["aligned six-channel blocks"] M --> N["FOA encoder"] N --> O["AmbiX ACN/SN3D output"] @@ -107,6 +117,14 @@ After dropping the neighbor Focusrite WAV into the run's `sources/` folder, esti .\.venv\Scripts\python.exe .\scripts\audio_field.py assemble-aligned --profile .\config\audio-field.json --run .\calibration\runs\ ``` +`analyze-distributed` now does a deadline-grade chirplet refinement by default: it uses the coarse sweep matched-filter peak as the initial time-of-arrival, then searches a small fractional-delay and chirp-rate neighborhood with chirplet atoms. Tune the local search when needed: + +```powershell +.\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-distributed --profile .\config\audio-field.json --run .\calibration\runs\ --chirplet-refine --search-samples 3 --fractional-steps 8 --rate-ppm 150 +``` + +This is not a full adaptive chirplet decomposition engine. It is the stream-day version: enough chirplet parameter search to get sub-sample timing refinement without turning calibration into a thesis defense in the middle of setup. + Encode an offline aligned six-channel WAV to FOA AmbiX: ```powershell @@ -124,6 +142,9 @@ The `play-record` and `record-field` commands remain for a `shared-input-device` - Playback sample rate is its own device fact. Probe it separately; do not assume the Scarlett input and output paths accept the same rates through PortAudio/WASAPI. - Distributed-clock microphones must be aligned and resampled into one reference timeline before FOA encoding. - Output may run behind live input by the configured latency budget, but it must converge toward real-time instead of letting cache depth grow without bound. +- Sweep arrival estimates should use chirplet refinement when calibration signal quality allows it; plain matched-filter peaks are a fallback, not the precision path. +- Runtime sync updates delay/SRO/phase estimates from known speaker chirplets every block/frame, gated by confidence so a bad observation freezes rather than poisons the field. +- Active chirplet probes are scheduled by an optimization loop, not by panic. It should prefer masked windows, respect minimum spacing and level caps, and maximize expected confidence gain per audible intrusion. - FOA output is AmbiX: ACN channel order, SN3D normalization, channels `W,Y,Z,X`. - Speaker calibration is a measurement path, not a generic monitor output. - Camera fusion may publish listener or source pose later, but it does not own audio channel timing. @@ -136,6 +157,8 @@ The audio code is split so the hard parts can be tested without the room, driver - `audio_field.latency` owns the bounded-delay real-time convergence policy. - `audio_field.ports` declares injectable capture, alignment, encoder, and sink protocols. - `audio_field.pipeline` wires those ports together without knowing whether the source is a real Focusrite, an SRT receiver, a WAV fixture, or a test double. +- The next runtime estimator module should own chirplet observations, delay/SRO/phase state, confidence gates, and phase-field updates. It should feed the aligner; it should not be hidden inside the FOA encoder. +- The active probe optimizer owns whether to emit extra chirplets. It reads sync confidence and probe budget, then decides when and where an intentional chirplet is worth the cost. - `scripts/audio_field.py` stays as the operator CLI and hardware probe surface. This is the portfolio-piece line: each module owns one invariant, and the tests use mocks/fakes instead of asking a driver stack to please be emotionally available. diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 9be52cc..373c61c 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -49,9 +49,12 @@ flowchart TD J --> D J --> F J --> H - D --> K["delay + SRO alignment"] - F --> K - H --> K + R["confidence probe optimizer"] --> P["known speaker chirplets"] + P --> Q["runtime delay/SRO/phase estimator"] + D --> Q + F --> Q + H --> Q + Q --> K["delay + SRO alignment"] K --> L["bounded field cache"] L --> M["aligned six-channel blocks"] M --> N["FOA encoder"] @@ -63,7 +66,9 @@ Ownership: - `config/audio-field.json` owns mic/speaker identity, machine/device mapping, clock domains, field channel order, geometry, gain, delay, polarity, role/quality priority, capture policy, and Ambisonic bus format. - `scripts/audio_field.py` owns profile validation, local device checks, calibration stimulus generation, clock-domain planning, shared-input capture helpers, and FOA encoding of already aligned six-channel WAVs. - `audio_field/` owns unit-testable buffering, bounded-latency convergence, injectable port protocols, and pipeline orchestration. +- Runtime sync owns per-block chirplet observations from known speaker output and updates delay/SRO/phase estimates with confidence gates before alignment. +- Active probe optimization owns extra chirplet emission when confidence drops, bounded by level/spacing/audibility budget. - The camera/sensor-fusion pipeline may publish world poses later; it does not own audio clocks or channel timing. - OBS may ingest rendered output later; it is not the authority for the Ambisonic field. -Invariant: distributed camera/Focusrite microphones must be aligned and resampled into one reference timeline before FOA encoding. Latency is allowed as bounded buffering, but cache depth must converge toward real-time. The local shielded cardioid and neighbor shotgun are the high-quality dialogue anchors; camera mics provide spatial/context evidence. +Invariant: distributed camera/Focusrite microphones must be aligned and resampled into one reference timeline before FOA encoding. Latency is allowed as bounded buffering, but cache depth must converge toward real-time. Speaker output chirplets are live telemetry; delay/SRO/phase state must update during runtime, not only during setup. Extra chirplets may be emitted automatically when confidence drops, but only under the active probe optimizer's budget. The local shielded cardioid and neighbor shotgun are the high-quality dialogue anchors; camera mics provide spatial/context evidence. diff --git a/scripts/audio_field.py b/scripts/audio_field.py index b3beb86..e1f761c 100644 --- a/scripts/audio_field.py +++ b/scripts/audio_field.py @@ -665,6 +665,16 @@ def cmd_analyze_distributed(args): recorded = as_float_matrix(recorded).reshape(-1) stim_for_rate = resample_if_needed(stimulus, int(stim_rate), int(rec_rate)) delay, peak, polarity = estimate_delay(recorded, stim_for_rate) + refined = None + if args.chirplet_refine: + refined = refine_chirplet_delay( + recorded, + stim_for_rate, + delay, + search_samples=args.search_samples, + fractional_steps=args.fractional_steps, + rate_ppm=args.rate_ppm, + ) rms = float(np.sqrt(np.mean(recorded * recorded))) if recorded.size else 0.0 item = { "micId": source["micId"], @@ -678,16 +688,23 @@ def cmd_analyze_distributed(args): "polarity": int(polarity), "rms": rms, } + if refined: + item["chirpletDelaySamples"] = refined["delaySamples"] + item["chirpletDelayMs"] = 1000.0 * refined["delaySamples"] / rec_rate + item["chirpletScore"] = refined["score"] + item["chirpletRateScale"] = refined["rateScale"] results["sources"].append(item) detections[source["micId"]] = item reference = detections.get(results["referenceMicId"]) if reference: - ref_time = reference["sweepStartSample"] / reference["sampleRate"] + ref_delay = reference.get("chirpletDelaySamples", reference["sweepStartSample"]) + ref_time = ref_delay / reference["sampleRate"] for item in results["sources"]: if item.get("status") != "ok": continue - item_time = item["sweepStartSample"] / item["sampleRate"] + item_delay = item.get("chirpletDelaySamples", item["sweepStartSample"]) + item_time = item_delay / item["sampleRate"] item["relativeDelaySeconds"] = item_time - ref_time item["relativeDelaySamplesAtFieldRate"] = int(round((item_time - ref_time) * int(profile["sampleRate"]))) @@ -778,6 +795,61 @@ def estimate_delay(recorded, sweep): return delay, peak, polarity +def refine_chirplet_delay(recorded, sweep, coarse_delay, search_samples=3, fractional_steps=8, rate_ppm=150): + recorded = np.asarray(recorded, dtype=np.float32).reshape(-1) + sweep = np.asarray(sweep, dtype=np.float32).reshape(-1) + if recorded.size == 0 or sweep.size == 0: + return None + + rate_scales = [1.0] + if rate_ppm: + delta = float(rate_ppm) / 1_000_000.0 + rate_scales = [1.0 - delta, 1.0, 1.0 + delta] + + best = None + start = int(round(coarse_delay)) - int(search_samples) + stop = int(round(coarse_delay)) + int(search_samples) + fractions = np.arange(max(1, int(fractional_steps)), dtype=np.float64) / max(1, int(fractional_steps)) + for rate_scale in rate_scales: + atom = chirplet_atom(sweep, rate_scale) + atom_norm = float(np.linalg.norm(atom)) + if atom_norm <= 0: + continue + for integer_delay in range(start, stop + 1): + for frac in fractions: + delay = float(integer_delay) + float(frac) + segment = sample_fractional_window(recorded, delay, len(atom)) + segment_norm = float(np.linalg.norm(segment)) + if segment_norm <= 0: + continue + score_signed = float(np.dot(segment, atom) / (segment_norm * atom_norm)) + score = abs(score_signed) + if best is None or score > best["score"]: + best = { + "delaySamples": delay, + "score": score, + "signedScore": score_signed, + "rateScale": rate_scale, + } + return best + + +def chirplet_atom(sweep, rate_scale): + if rate_scale == 1.0: + atom = sweep.astype(np.float32) + else: + x = np.arange(len(sweep), dtype=np.float64) * rate_scale + atom = np.interp(x, np.arange(len(sweep), dtype=np.float64), sweep, left=0.0, right=0.0).astype(np.float32) + atom = atom - float(np.mean(atom)) + norm = float(np.linalg.norm(atom)) + return atom / norm if norm > 0 else atom + + +def sample_fractional_window(samples, start, frame_count): + x = start + np.arange(frame_count, dtype=np.float64) + return np.interp(x, np.arange(len(samples), dtype=np.float64), samples, left=0.0, right=0.0).astype(np.float32) + + def cmd_record_field(args): profile = load_profile(args.profile) if profile.get("captureMode", "shared-input-device") == "distributed-clocks": @@ -980,6 +1052,10 @@ def main(): p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") p.add_argument("--run", type=Path, required=True) p.add_argument("--reference-mic") + p.add_argument("--chirplet-refine", action=argparse.BooleanOptionalAction, default=True) + p.add_argument("--search-samples", type=int, default=3) + p.add_argument("--fractional-steps", type=int, default=8) + p.add_argument("--rate-ppm", type=int, default=150) p.set_defaults(func=cmd_analyze_distributed) p = sub.add_parser("assemble-aligned", help="Assemble a distributed run into one aligned six-channel WAV.") diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 3efa5dc..f0c0327 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -34,3 +34,5 @@ {"ts": "2026-05-18T15:12:32+00:00", "type": "verification", "status": "partial", "note": "Deadline audio path now supports distributed run manifests, local concurrent mic capture, sweep-arrival analysis, aligned six-channel WAV assembly with missing channels zero-filled, and FOA encoding. Live smoke captured local Kiyo-left, both PS Eye mics, and local Focusrite at 48 kHz; Kiyo-right profile query is currently missing and neighbor Focusrite remains remote/manual import."} {"ts": "2026-05-18T15:19:47+00:00", "type": "decision", "status": "ok", "note": "Deadline OBS Spout path is now a live GPU sink: RenderFramePacket JSON feeds scripts/stream_spout.py, which renders to an OpenGL FBO and publishes Spout sender LocalCastBridge Point Cloud with heartbeat status. Aquarium remains the intended dense splat renderer behind the same packet boundary."} {"ts": "2026-05-18T15:21:20+00:00", "type": "decision", "status": "ok", "note": "Calibration delay analysis now uses local chirplet refinement: coarse matched-filter sweep detection initializes a small fractional-delay/chirp-rate atom search, following chirplet parameter-estimation guidance. Plugged-in left/right sweeps produced sub-sample chirplet delays for Kiyo-left, both PS Eye mics, and local Focusrite; Kiyo-right remains missing and neighbor Focusrite still needs import."} +{"ts": "2026-05-18T15:22:33+00:00", "type": "decision", "status": "ok", "note": "Runtime mic sync must treat known speaker output as chirplet telemetry every block/frame, updating delay, SRO/rate-scale, phase, and confidence fields dynamically. Static sweep offsets are only initial state; low-confidence runtime chirplet observations should freeze updates rather than corrupt alignment."} +{"ts": "2026-05-18T15:23:38+00:00", "type": "decision", "status": "ok", "note": "Active sync probes are now an explicit optimizer responsibility: when passive chirplet telemetry leaves any source below confidence threshold, the system may emit bounded extra chirplets in masked windows, maximizing confidence gain per audible intrusion while respecting spacing and level limits."} diff --git a/state/map.yaml b/state/map.yaml index f350e98..2c4b19c 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -171,7 +171,12 @@ current_status: buffering, latency convergence, injectable capture/alignment/encode/sink ports, and pipeline orchestration. Calibration analysis now has a local chirplet refinement pass: coarse sweep match followed by fractional-delay - and chirp-rate grid search around the detected arrival. The first visual + and chirp-rate grid search around the detected arrival. Runtime sync is now + modeled as per-block known-speaker chirplet telemetry updating delay, SRO, + phase, and confidence state; low-confidence observations freeze rather than + poisoning alignment. Active probe optimization owns extra chirplet emission + when sync confidence drops, bounded by spacing, level, and audibility budget. + The first visual sensor-fusion sidecar is in place: `localcast.sensor_fusion.core` owns calibrated camera models, timestamped 2D observations, DLT triangulation, reprojection/confidence diff --git a/tests/test_probe_optimizer.py b/tests/test_probe_optimizer.py new file mode 100644 index 0000000..77bbf39 --- /dev/null +++ b/tests/test_probe_optimizer.py @@ -0,0 +1,40 @@ +import unittest + +from audio_field.probe_optimizer import ActiveProbeOptimizer, ProbePolicy +from audio_field.runtime_sync import SyncState + + +class ActiveProbeOptimizerTests(unittest.TestCase): + def test_schedules_weakest_source_when_masked(self): + optimizer = ActiveProbeOptimizer(ProbePolicy(trigger_confidence=0.4, min_interval_frames=10)) + + request = optimizer.choose_probe( + 100, + { + "good": SyncState(confidence=0.8), + "weak": SyncState(confidence=0.2), + }, + masked_window=True, + ) + + self.assertIsNotNone(request) + self.assertEqual("weak", request.source_id) + self.assertGreater(request.urgency, 0.0) + + def test_respects_probe_spacing(self): + optimizer = ActiveProbeOptimizer(ProbePolicy(trigger_confidence=0.4, min_interval_frames=10)) + states = {"weak": SyncState(confidence=0.2)} + + self.assertIsNotNone(optimizer.choose_probe(100, states, masked_window=True)) + self.assertIsNone(optimizer.choose_probe(105, states, masked_window=True)) + + def test_waits_for_masked_window_when_configured(self): + optimizer = ActiveProbeOptimizer(ProbePolicy(trigger_confidence=0.4, prefer_masked_windows=True)) + + request = optimizer.choose_probe(100, {"weak": SyncState(confidence=0.2)}, masked_window=False) + + self.assertIsNone(request) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime_sync.py b/tests/test_runtime_sync.py new file mode 100644 index 0000000..8de9268 --- /dev/null +++ b/tests/test_runtime_sync.py @@ -0,0 +1,32 @@ +import unittest + +from audio_field.runtime_sync import ChirpletObservation, RuntimeSyncEstimator + + +class RuntimeSyncTests(unittest.TestCase): + def test_low_score_freezes_previous_state(self): + estimator = RuntimeSyncEstimator(min_score=0.5, smoothing=0.5) + estimator.update(ChirpletObservation("mic", 1, 100.0, 0.25, 0.9, 1.0001)) + + state = estimator.update(ChirpletObservation("mic", 2, 500.0, 2.0, 0.1, 1.2)) + + self.assertEqual(100.0, state.delay_samples) + self.assertEqual(0.25, state.phase_radians) + self.assertEqual(1.0001, state.rate_scale) + self.assertEqual(0.1, state.confidence) + self.assertEqual(1, state.last_frame_index) + + def test_high_score_smooths_delay_phase_and_rate(self): + estimator = RuntimeSyncEstimator(min_score=0.1, smoothing=0.25) + estimator.update(ChirpletObservation("mic", 1, 100.0, 0.0, 0.9, 1.0)) + + state = estimator.update(ChirpletObservation("mic", 2, 108.0, 0.4, 0.8, 1.0004)) + + self.assertAlmostEqual(102.0, state.delay_samples) + self.assertAlmostEqual(0.1, state.phase_radians) + self.assertAlmostEqual(1.0001, state.rate_scale) + self.assertEqual(2, state.last_frame_index) + + +if __name__ == "__main__": + unittest.main()