diff --git a/docs/audio-field.md b/docs/audio-field.md index 857eb38..c4982c0 100644 --- a/docs/audio-field.md +++ b/docs/audio-field.md @@ -104,6 +104,15 @@ Record all locally visible distributed mics for a deadline calibration pass: .\.venv\Scripts\python.exe .\scripts\audio_field.py record-local-calibration --profile .\config\audio-field.json --seconds 10 ``` +For passive runtime sync against ground-truth program output, capture the Scarlett loopback with the local mics: + +```powershell +.\.venv\Scripts\python.exe .\scripts\audio_field.py record-local-calibration --profile .\config\audio-field.json --seconds 20 --record-loopback --loopback-query Scarlett --loopback-rate 48000 +.\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-reference-sync --profile .\config\audio-field.json --run .\calibration\runs\ --window-seconds 4 --hop-seconds 1 --method normalized --min-score 0.08 +``` + +The loopback path is ground-truth content, but not automatically a perfect ground-truth clock. If WASAPI loopback reports discontinuities, use the stable windows and treat the rest as suspect. + Record local mics while playing one speaker sweep. Use the output rate that the device probe says is real: ```powershell diff --git a/requirements-calibration.txt b/requirements-calibration.txt index dd087f7..a420bc4 100644 --- a/requirements-calibration.txt +++ b/requirements-calibration.txt @@ -1,5 +1,6 @@ opencv-contrib-python sounddevice +soundcard scipy imageio-ffmpeg PyOpenGL diff --git a/scripts/audio_field.py b/scripts/audio_field.py index e1f761c..e10b17a 100644 --- a/scripts/audio_field.py +++ b/scripts/audio_field.py @@ -433,6 +433,8 @@ def cmd_record_local_calibration(args): ) stimulus_file = None + loopback_file = None + loopback_rate = None output_device = None output_rate = None if args.play_sweep: @@ -448,7 +450,20 @@ def cmd_record_local_calibration(args): try: for stream in streams: stream.start() - if args.play_sweep: + if args.record_loopback: + loopback_rate = int(args.loopback_rate or input_rate) + loopback = record_loopback( + query=args.loopback_query, + seconds=seconds, + sample_rate=loopback_rate, + channels=args.loopback_channels, + play_data=playback if args.play_sweep else None, + play_rate=output_rate, + play_device=output_device["index"] if output_device else None, + ) + loopback_file = "ground_truth_loopback.wav" + wavfile.write(out / loopback_file, loopback_rate, loopback.astype(np.float32)) + elif args.play_sweep: sd.play(playback, samplerate=output_rate, device=output_device["index"], blocking=True) remaining = max(0.0, seconds - len(playback) / output_rate) time.sleep(remaining) @@ -512,11 +527,44 @@ def cmd_record_local_calibration(args): "outputDevice": summarize_device(output_device) if output_device else None, "outputSampleRate": output_rate, }, + "reference": { + "kind": "wasapi-loopback", + "file": loopback_file, + "sampleRate": loopback_rate, + "query": args.loopback_query if loopback_file else None, + }, }, ) print(out) +def record_loopback(query, seconds, sample_rate, channels=2, play_data=None, play_rate=None, play_device=None): + try: + import soundcard as sc + except Exception as exc: + raise SystemExit(f"soundcard loopback capture unavailable: {exc!r}") from exc + matches = [ + mic + for mic in sc.all_microphones(include_loopback=True) + if "loopback" in repr(mic).lower() and query.lower() in mic.name.lower() + ] + if not matches: + raise SystemExit(f"No loopback device matching {query!r}") + loopback = matches[0] + frame_count = int(seconds * sample_rate) + if play_data is not None: + sd = sounddevice_module() + sd.play(play_data, samplerate=play_rate, device=play_device, blocking=False) + with loopback.recorder(samplerate=sample_rate, channels=channels) as recorder: + data = recorder.record(numframes=frame_count) + if play_data is not None: + try: + sounddevice_module().stop() + except Exception: + pass + return np.asarray(data, dtype=np.float32) + + def make_sweep_at_rate(profile, sample_rate): clone = dict(profile) clone["sampleRate"] = int(sample_rate) @@ -766,6 +814,116 @@ def cmd_assemble_aligned(args): print(json.dumps(result, indent=2)) +def cmd_analyze_reference_sync(args): + profile = load_profile(args.profile) + run = args.run + manifest = read_json(run / "manifest.json") + reference_info = manifest.get("reference") or {} + reference_file = args.reference or (run / reference_info.get("file", "")) + if not reference_file.exists(): + raise SystemExit(f"Reference loopback WAV not found: {reference_file}") + ref_rate, reference = wavfile.read(reference_file) + reference = mono(as_float_matrix(reference)) + reference = resample_if_needed(reference, int(ref_rate), int(profile["sampleRate"])) + field_rate = int(profile["sampleRate"]) + window = int(args.window_seconds * field_rate) + hop = int(args.hop_seconds * field_rate) + max_lag = int(args.max_lag_ms * field_rate / 1000.0) + results = { + "created_utc": datetime.now(timezone.utc).isoformat(), + "sourceRun": str(run), + "reference": str(reference_file), + "sampleRate": field_rate, + "windowSeconds": args.window_seconds, + "hopSeconds": args.hop_seconds, + "sources": [], + } + for source in manifest.get("sources", []): + file_name = source.get("file") + if not file_name or not (run / file_name).exists(): + continue + rec_rate, data = wavfile.read(run / file_name) + mic = resample_if_needed(mono(as_float_matrix(data)), int(rec_rate), field_rate) + delays = [] + scores = [] + for start in range(0, min(len(reference), len(mic)) - window + 1, hop): + ref_win = reference[start : start + window] + mic_win = mic[start : start + window] + if float(np.std(ref_win)) < args.min_reference_std or float(np.std(mic_win)) < args.min_mic_std: + continue + if args.method == "gcc-phat": + delay, score = gcc_phat_delay(mic_win, ref_win, max_lag=max_lag) + else: + delay, score = normalized_delay(mic_win, ref_win, max_lag=max_lag) + if score < args.min_score or abs(delay) < int(args.min_abs_lag_ms * field_rate / 1000.0): + continue + delays.append(delay) + scores.append(score) + if not delays: + continue + delays = np.asarray(delays, dtype=np.float64) + scores = np.asarray(scores, dtype=np.float64) + median = float(np.median(delays)) + mad = float(np.median(np.abs(delays - median))) + results["sources"].append( + { + "micId": source["micId"], + "fieldChannel": source.get("fieldChannel"), + "windows": int(len(delays)), + "medianDelaySamples": median, + "medianDelayMs": 1000.0 * median / field_rate, + "madSamples": mad, + "madMs": 1000.0 * mad / field_rate, + "meanScore": float(np.mean(scores)), + "maxScore": float(np.max(scores)), + "method": args.method, + } + ) + write_json(run / "reference-sync-analysis.json", results) + print(json.dumps(results, indent=2)) + + +def mono(matrix): + if matrix.ndim == 1 or matrix.shape[1] == 1: + return matrix.reshape(-1).astype(np.float32) + return np.mean(matrix, axis=1).astype(np.float32) + + +def gcc_phat_delay(signal_a, signal_b, max_lag): + a = np.asarray(signal_a, dtype=np.float32) + b = np.asarray(signal_b, dtype=np.float32) + a = a - float(np.mean(a)) + b = b - float(np.mean(b)) + n = 1 + while n < len(a) + len(b): + n *= 2 + A = np.fft.rfft(a, n=n) + B = np.fft.rfft(b, n=n) + R = A * np.conj(B) + denom = np.abs(R) + R = np.divide(R, denom, out=np.zeros_like(R), where=denom > 1e-12) + corr = np.fft.irfft(R, n=n) + corr = np.concatenate((corr[-max_lag:], corr[: max_lag + 1])) + index = int(np.argmax(np.abs(corr))) + lag = index - max_lag + score = float(abs(corr[index])) + return lag, score + + +def normalized_delay(signal_a, signal_b, max_lag): + a = np.asarray(signal_a, dtype=np.float32) + b = np.asarray(signal_b, dtype=np.float32) + a = (a - float(np.mean(a))) / (float(np.std(a)) + 1e-9) + b = (b - float(np.mean(b))) / (float(np.std(b)) + 1e-9) + corr = signal.correlate(a, b, mode="full", method="fft") + mid = len(b) - 1 + segment = corr[mid - max_lag : mid + max_lag + 1] + index = int(np.argmax(np.abs(segment))) + lag = index - max_lag + score = float(abs(segment[index]) / max(1, len(a))) + return lag, score + + def as_float_matrix(data): arr = np.asarray(data) if arr.ndim == 1: @@ -1032,6 +1190,10 @@ def main(): p.add_argument("--play-sweep", action="store_true") p.add_argument("--speaker-channel", type=int, default=0) p.add_argument("--output-rate", type=int) + p.add_argument("--record-loopback", action="store_true") + p.add_argument("--loopback-query", default="Scarlett") + p.add_argument("--loopback-rate", type=int) + p.add_argument("--loopback-channels", type=int, default=2) p.set_defaults(func=cmd_record_local_calibration) p = sub.add_parser("play-record", help="Play each speaker calibration sweep while recording all microphones.") @@ -1066,6 +1228,20 @@ def main(): p.add_argument("--seconds", type=float) p.set_defaults(func=cmd_assemble_aligned) + p = sub.add_parser("analyze-reference-sync", help="Estimate live mic delay stability against captured output loopback ground truth.") + p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") + p.add_argument("--run", type=Path, required=True) + p.add_argument("--reference", type=Path) + p.add_argument("--window-seconds", type=float, default=2.0) + p.add_argument("--hop-seconds", type=float, default=1.0) + p.add_argument("--max-lag-ms", type=float, default=500.0) + p.add_argument("--method", choices=["normalized", "gcc-phat"], default="normalized") + p.add_argument("--min-score", type=float, default=0.08) + p.add_argument("--min-reference-std", type=float, default=1e-4) + p.add_argument("--min-mic-std", type=float, default=1e-5) + p.add_argument("--min-abs-lag-ms", type=float, default=0.5) + p.set_defaults(func=cmd_analyze_reference_sync) + p = sub.add_parser("record-field", help="Record the raw synchronized six-channel microphone field.") p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") p.add_argument("--seconds", type=float, default=10.0) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index f0c0327..2486be7 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -36,3 +36,4 @@ {"ts": "2026-05-18T15:21:20+00:00", "type": "decision", "status": "ok", "note": "Calibration delay analysis now uses local chirplet refinement: coarse matched-filter sweep detection initializes a small fractional-delay/chirp-rate atom search, following chirplet parameter-estimation guidance. Plugged-in left/right sweeps produced sub-sample chirplet delays for Kiyo-left, both PS Eye mics, and local Focusrite; Kiyo-right remains missing and neighbor Focusrite still needs import."} {"ts": "2026-05-18T15:22:33+00:00", "type": "decision", "status": "ok", "note": "Runtime mic sync must treat known speaker output as chirplet telemetry every block/frame, updating delay, SRO/rate-scale, phase, and confidence fields dynamically. Static sweep offsets are only initial state; low-confidence runtime chirplet observations should freeze updates rather than corrupt alignment."} {"ts": "2026-05-18T15:23:38+00:00", "type": "decision", "status": "ok", "note": "Active sync probes are now an explicit optimizer responsibility: when passive chirplet telemetry leaves any source below confidence threshold, the system may emit bounded extra chirplets in masked windows, maximizing confidence gain per audible intrusion while respecting spacing and level limits."} +{"ts": "2026-05-18T15:35:24+00:00", "type": "verification", "status": "partial", "note": "Passive sync test using Scarlett output loopback as ground-truth content: 20s music run with normalized 4s/1s window correlation produced local Focusrite median delay 20115.5 samples/419.07 ms with 17 sample/0.35 ms MAD; PS Eye left 23848 samples/496.83 ms with 17 sample/0.35 ms MAD; Kiyo-left and PS Eye-right were noisier at about 3.9-4.2 ms MAD. Soundcard WASAPI loopback reported discontinuities, so loopback clock quality remains suspect."}