From 3e234264458be378ee26ad91ecf1e75bb47aa831 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 18 May 2026 16:12:52 +0100 Subject: [PATCH] Add deadline distributed audio calibration path --- config/audio-field.example.json | 8 +- docs/audio-field.md | 25 +++ scripts/audio_field.py | 355 ++++++++++++++++++++++++++++++++ state/evidence.jsonl | 1 + 4 files changed, 385 insertions(+), 4 deletions(-) diff --git a/config/audio-field.example.json b/config/audio-field.example.json index 5054940..8f6b59f 100644 --- a/config/audio-field.example.json +++ b/config/audio-field.example.json @@ -50,7 +50,7 @@ "role": "spatial-context", "qualityPriority": 30, "machine": "local", - "device": {"query": "Kiyo", "hostApi": "WASAPI", "channel": 0}, + "device": {"query": "Kiyo", "hostApi": "WASAPI", "channel": 0, "matchIndex": 0}, "clockDomain": "kiyo_left_usb", "positionMeters": [-0.65, 0.8, 1.35], "orientationDeg": {"azimuth": 180.0, "elevation": 0.0}, @@ -66,7 +66,7 @@ "role": "spatial-context", "qualityPriority": 30, "machine": "local", - "device": {"query": "Kiyo", "hostApi": "WASAPI", "channel": 0}, + "device": {"query": "Kiyo", "hostApi": "WASAPI", "channel": 0, "matchIndex": 1}, "clockDomain": "kiyo_right_usb", "positionMeters": [0.65, 0.8, 1.35], "orientationDeg": {"azimuth": 180.0, "elevation": 0.0}, @@ -82,7 +82,7 @@ "role": "spatial-context", "qualityPriority": 20, "machine": "local", - "device": {"query": "USB Camera-B4.09.24.1", "hostApi": "WASAPI", "channel": 0}, + "device": {"query": "USB Camera-B4.09.24.1", "hostApi": "WASAPI", "channel": 0, "matchIndex": 0}, "clockDomain": "ps_eye_left_usb", "positionMeters": [-0.9, -0.2, 1.25], "orientationDeg": {"azimuth": 90.0, "elevation": 0.0}, @@ -98,7 +98,7 @@ "role": "spatial-context", "qualityPriority": 20, "machine": "local", - "device": {"query": "USB Camera-B4.09.24.1", "hostApi": "WASAPI", "channel": 0}, + "device": {"query": "USB Camera-B4.09.24.1", "hostApi": "WASAPI", "channel": 0, "matchIndex": 1}, "clockDomain": "ps_eye_right_usb", "positionMeters": [0.9, -0.2, 1.25], "orientationDeg": {"azimuth": -90.0, "elevation": 0.0}, diff --git a/docs/audio-field.md b/docs/audio-field.md index d9ff583..cb4995d 100644 --- a/docs/audio-field.md +++ b/docs/audio-field.md @@ -82,6 +82,31 @@ Generate a calibration sweep without touching hardware: .\.venv\Scripts\python.exe .\scripts\audio_field.py make-stimulus --profile .\config\audio-field.json ``` +Create a distributed run folder that can accept one WAV per mic: + +```powershell +.\.venv\Scripts\python.exe .\scripts\audio_field.py init-run --profile .\config\audio-field.json +``` + +Record all locally visible distributed mics for a deadline calibration pass: + +```powershell +.\.venv\Scripts\python.exe .\scripts\audio_field.py record-local-calibration --profile .\config\audio-field.json --seconds 10 +``` + +Record local mics while playing one speaker sweep. Use the output rate that the device probe says is real: + +```powershell +.\.venv\Scripts\python.exe .\scripts\audio_field.py record-local-calibration --profile .\config\audio-field.json --seconds 10 --play-sweep --speaker-channel 0 --output-rate 44100 +``` + +After dropping the neighbor Focusrite WAV into the run's `sources/` folder, estimate sweep arrivals and assemble the aligned field: + +```powershell +.\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-distributed --profile .\config\audio-field.json --run .\calibration\runs\ +.\.venv\Scripts\python.exe .\scripts\audio_field.py assemble-aligned --profile .\config\audio-field.json --run .\calibration\runs\ +``` + Encode an offline aligned six-channel WAV to FOA AmbiX: ```powershell diff --git a/scripts/audio_field.py b/scripts/audio_field.py index 2f3726d..b3beb86 100644 --- a/scripts/audio_field.py +++ b/scripts/audio_field.py @@ -2,6 +2,7 @@ import argparse import json import math import sys +import time from datetime import datetime, timezone from pathlib import Path @@ -182,6 +183,14 @@ def find_devices(sd, spec, direction: str): return matches +def match_indexed_device(sd, spec, direction: str): + matches = find_devices(sd, spec, direction) + match_index = int(spec.get("matchIndex", 0)) + if match_index < 0 or match_index >= len(matches): + raise SystemExit(f"No {direction} device match {match_index} for {spec!r}; found {len(matches)}") + return matches[match_index] + + def cmd_devices(args): sd = sounddevice_module() hostapis = sd.query_hostapis() @@ -345,6 +354,207 @@ def cmd_make_stimulus(args): print(out) +def cmd_init_run(args): + profile = load_profile(args.profile) + out = args.output or run_dir("audio-distributed") + out.mkdir(parents=True, exist_ok=True) + (out / "sources").mkdir(exist_ok=True) + sweep = make_sweep(profile) + wavfile.write(out / "calibration-sweep.wav", int(profile["sampleRate"]), sweep) + sources = [] + for mic in sorted(profile["microphones"], key=mic_channel): + sources.append( + { + "micId": mic["id"], + "fieldChannel": mic_channel(mic), + "machine": mic.get("machine"), + "clockDomain": mic.get("clockDomain"), + "expectedFile": f"sources/{mic['id']}.wav", + "role": mic.get("role"), + } + ) + write_json( + out / "manifest.json", + { + "kind": "distributed-audio-calibration", + "created_utc": datetime.now(timezone.utc).isoformat(), + "profile": str(args.profile), + "sampleRate": int(profile["sampleRate"]), + "stimulus": "calibration-sweep.wav", + "sources": sources, + "notes": "Drop one mono WAV per mic at expectedFile, or use record-local-calibration for local mics.", + }, + ) + print(out) + + +def cmd_record_local_calibration(args): + profile = load_profile(args.profile) + sd = sounddevice_module() + out = args.output or run_dir("audio-local-calibration") + out.mkdir(parents=True, exist_ok=True) + (out / "sources").mkdir(exist_ok=True) + + input_rate = int(args.input_rate or profile["sampleRate"]) + seconds = float(args.seconds) + frames = int(input_rate * seconds) + local_mics = [mic for mic in profile["microphones"] if mic.get("machine") == args.machine] + grouped = local_device_groups(sd, local_mics) + captures = { + key: np.zeros((frames, group["channels"]), dtype=np.float32) + for key, group in grouped.items() + if key != "_missing" + } + positions = {key: 0 for key in captures} + + streams = [] + + def make_callback(key): + def callback(indata, frame_count, time_info, status): + pos = positions[key] + end = min(pos + frame_count, frames) + if end > pos: + captures[key][pos:end, :] = indata[: end - pos, :] + positions[key] = end + + return callback + + for key, group in grouped.items(): + if key == "_missing": + continue + streams.append( + sd.InputStream( + device=group["device"]["index"], + channels=group["channels"], + samplerate=input_rate, + dtype="float32", + callback=make_callback(key), + ) + ) + + stimulus_file = None + output_device = None + output_rate = None + if args.play_sweep: + output_device = match_device(sd, profile["outputDevice"], "output") + output_rate = int(args.output_rate or output_device["default_samplerate"] or profile["sampleRate"]) + sweep = make_sweep_at_rate(profile, output_rate) + playback = np.zeros((len(sweep), int(profile["outputDevice"]["channels"])), dtype=np.float32) + speaker_channel = int(args.speaker_channel) + playback[:, speaker_channel] = sweep + stimulus_file = "calibration-sweep-played.wav" + wavfile.write(out / stimulus_file, output_rate, sweep) + + try: + for stream in streams: + stream.start() + if args.play_sweep: + sd.play(playback, samplerate=output_rate, device=output_device["index"], blocking=True) + remaining = max(0.0, seconds - len(playback) / output_rate) + time.sleep(remaining) + else: + time.sleep(seconds) + finally: + for stream in streams: + try: + stream.stop() + stream.close() + except Exception: + pass + + sources = [] + for missing in grouped.get("_missing", []): + sources.append( + { + "micId": missing["micId"], + "fieldChannel": missing["fieldChannel"], + "machine": args.machine, + "status": "missing-device", + "error": missing["error"], + } + ) + for mic in sorted(local_mics, key=mic_channel): + if any(missing["micId"] == mic["id"] for missing in grouped.get("_missing", [])): + continue + group_key = device_group_key(grouped, mic) + data = captures[group_key][:, int(mic.get("device", {}).get("channel", 0))] + file_name = f"sources/{mic['id']}.wav" + wavfile.write(out / file_name, input_rate, data.astype(np.float32)) + sources.append( + { + "micId": mic["id"], + "fieldChannel": mic_channel(mic), + "machine": mic.get("machine"), + "clockDomain": mic.get("clockDomain"), + "file": file_name, + "sampleRate": input_rate, + } + ) + + if stimulus_file is None: + sweep = make_sweep(profile) + stimulus_file = "calibration-sweep-reference.wav" + wavfile.write(out / stimulus_file, int(profile["sampleRate"]), sweep) + + write_json( + out / "manifest.json", + { + "kind": "distributed-audio-calibration", + "created_utc": datetime.now(timezone.utc).isoformat(), + "profile": str(args.profile), + "sampleRate": int(profile["sampleRate"]), + "inputSampleRate": input_rate, + "stimulus": stimulus_file, + "sources": sources, + "playback": { + "played": bool(args.play_sweep), + "speakerChannel": int(args.speaker_channel) if args.play_sweep else None, + "outputDevice": summarize_device(output_device) if output_device else None, + "outputSampleRate": output_rate, + }, + }, + ) + print(out) + + +def make_sweep_at_rate(profile, sample_rate): + clone = dict(profile) + clone["sampleRate"] = int(sample_rate) + return make_sweep(clone) + + +def local_device_groups(sd, mics): + groups = {"_missing": []} + for mic in mics: + try: + device = match_indexed_device(sd, mic.get("device", {}), "input") + except SystemExit as exc: + groups["_missing"].append( + { + "micId": mic["id"], + "fieldChannel": mic_channel(mic), + "error": str(exc), + } + ) + continue + channel = int(mic.get("device", {}).get("channel", 0)) + key = str(device["index"]) + if key not in groups: + groups[key] = {"device": device, "mics": [], "channels": channel + 1} + groups[key]["mics"].append(mic) + groups[key]["channels"] = max(groups[key]["channels"], channel + 1) + return groups + + +def device_group_key(groups, mic): + for key, group in groups.items(): + if key == "_missing": + continue + if any(item["id"] == mic["id"] for item in group["mics"]): + return key + raise KeyError(mic["id"]) + + def cmd_play_record(args): profile = load_profile(args.profile) if profile.get("captureMode", "shared-input-device") == "distributed-clocks": @@ -433,6 +643,112 @@ def cmd_analyze_calibration(args): print(json.dumps(output, indent=2)) +def cmd_analyze_distributed(args): + profile = load_profile(args.profile) + run = args.run + manifest = read_json(run / "manifest.json") + stim_rate, stimulus = wavfile.read(run / manifest["stimulus"]) + stimulus = as_float_matrix(stimulus).reshape(-1) + results = { + "created_utc": datetime.now(timezone.utc).isoformat(), + "sourceRun": str(run), + "referenceMicId": args.reference_mic or profile.get("clockModel", {}).get("referenceMicId"), + "sources": [], + } + detections = {} + for source in manifest.get("sources", []): + file_name = source.get("file") or source.get("expectedFile") + if not file_name or not (run / file_name).exists(): + results["sources"].append({"micId": source.get("micId"), "status": "missing", "file": file_name}) + continue + rec_rate, recorded = wavfile.read(run / file_name) + recorded = as_float_matrix(recorded).reshape(-1) + stim_for_rate = resample_if_needed(stimulus, int(stim_rate), int(rec_rate)) + delay, peak, polarity = estimate_delay(recorded, stim_for_rate) + rms = float(np.sqrt(np.mean(recorded * recorded))) if recorded.size else 0.0 + item = { + "micId": source["micId"], + "fieldChannel": source.get("fieldChannel"), + "status": "ok", + "file": file_name, + "sampleRate": int(rec_rate), + "sweepStartSample": int(delay), + "sweepStartMs": 1000.0 * delay / rec_rate, + "peak": float(abs(peak)), + "polarity": int(polarity), + "rms": rms, + } + results["sources"].append(item) + detections[source["micId"]] = item + + reference = detections.get(results["referenceMicId"]) + if reference: + ref_time = reference["sweepStartSample"] / reference["sampleRate"] + for item in results["sources"]: + if item.get("status") != "ok": + continue + item_time = item["sweepStartSample"] / item["sampleRate"] + item["relativeDelaySeconds"] = item_time - ref_time + item["relativeDelaySamplesAtFieldRate"] = int(round((item_time - ref_time) * int(profile["sampleRate"]))) + + write_json(run / "distributed-analysis.json", results) + print(json.dumps(results, indent=2)) + + +def cmd_assemble_aligned(args): + profile = load_profile(args.profile) + run = args.run + manifest = read_json(run / "manifest.json") + analysis_path = args.analysis or run / "distributed-analysis.json" + analysis = read_json(analysis_path) if analysis_path.exists() else {"sources": []} + field_rate = int(profile["sampleRate"]) + seconds = args.seconds + source_by_mic = {item.get("micId"): item for item in manifest.get("sources", [])} + analysis_by_mic = {item.get("micId"): item for item in analysis.get("sources", []) if item.get("status") == "ok"} + + channels = [] + report = [] + max_len = 0 + for mic in sorted(profile["microphones"], key=mic_channel): + source = source_by_mic.get(mic["id"]) + if not source: + channels.append(None) + report.append({"micId": mic["id"], "status": "missing-source-entry"}) + continue + file_name = source.get("file") or source.get("expectedFile") + if not file_name or not (run / file_name).exists(): + channels.append(None) + report.append({"micId": mic["id"], "status": "missing-file", "file": file_name}) + continue + rec_rate, data = wavfile.read(run / file_name) + mono = as_float_matrix(data).reshape(-1) + mono = resample_if_needed(mono, int(rec_rate), field_rate) + delay = int(mic.get("delaySamples", 0)) + if mic["id"] in analysis_by_mic: + delay += int(analysis_by_mic[mic["id"]].get("relativeDelaySamplesAtFieldRate", 0)) + aligned = apply_integer_delay(mono, -delay) + aligned *= db_to_amp(float(mic.get("gainDb", 0.0))) * int(mic.get("polarity", 1)) + channels.append(aligned) + max_len = max(max_len, len(aligned)) + report.append({"micId": mic["id"], "status": "ok", "file": file_name, "appliedDelaySamples": delay}) + + if seconds: + frame_count = int(float(seconds) * field_rate) + else: + frame_count = max_len + field = np.zeros((frame_count, len(channels)), dtype=np.float32) + for index, channel in enumerate(channels): + if channel is None: + continue + count = min(frame_count, len(channel)) + field[:count, index] = channel[:count] + output = args.output or run / "field-aligned.wav" + wavfile.write(output, field_rate, field) + result = {"output": str(output), "sampleRate": field_rate, "shape": list(field.shape), "sources": report} + write_json(run / "aligned-field-manifest.json", result) + print(json.dumps(result, indent=2)) + + def as_float_matrix(data): arr = np.asarray(data) if arr.ndim == 1: @@ -444,6 +760,15 @@ def as_float_matrix(data): return arr +def resample_if_needed(samples, source_rate, target_rate): + if int(source_rate) == int(target_rate): + return samples.astype(np.float32) + gcd = math.gcd(int(source_rate), int(target_rate)) + up = int(target_rate) // gcd + down = int(source_rate) // gcd + return signal.resample_poly(samples.astype(np.float32), up, down).astype(np.float32) + + def estimate_delay(recorded, sweep): corr = signal.fftconvolve(recorded.astype(np.float32), sweep[::-1].astype(np.float32), mode="full") peak_index = int(np.argmax(np.abs(corr))) @@ -621,6 +946,22 @@ def main(): p.add_argument("--output", type=Path) p.set_defaults(func=cmd_make_stimulus) + p = sub.add_parser("init-run", help="Create a distributed calibration run folder and manifest.") + p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") + p.add_argument("--output", type=Path) + p.set_defaults(func=cmd_init_run) + + p = sub.add_parser("record-local-calibration", help="Record local distributed mic sources, optionally while playing one speaker sweep.") + p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") + p.add_argument("--output", type=Path) + p.add_argument("--machine", default="local") + p.add_argument("--seconds", type=float, default=8.0) + p.add_argument("--input-rate", type=int) + p.add_argument("--play-sweep", action="store_true") + p.add_argument("--speaker-channel", type=int, default=0) + p.add_argument("--output-rate", type=int) + p.set_defaults(func=cmd_record_local_calibration) + p = sub.add_parser("play-record", help="Play each speaker calibration sweep while recording all microphones.") p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") p.add_argument("--output", type=Path) @@ -635,6 +976,20 @@ def main(): p.add_argument("--run", type=Path, required=True) p.set_defaults(func=cmd_analyze_calibration) + p = sub.add_parser("analyze-distributed", help="Detect calibration sweep arrival in one WAV per mic and estimate relative delays.") + p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") + p.add_argument("--run", type=Path, required=True) + p.add_argument("--reference-mic") + p.set_defaults(func=cmd_analyze_distributed) + + p = sub.add_parser("assemble-aligned", help="Assemble a distributed run into one aligned six-channel WAV.") + p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") + p.add_argument("--run", type=Path, required=True) + p.add_argument("--analysis", type=Path) + p.add_argument("--output", type=Path) + p.add_argument("--seconds", type=float) + p.set_defaults(func=cmd_assemble_aligned) + p = sub.add_parser("record-field", help="Record the raw synchronized six-channel microphone field.") p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json") p.add_argument("--seconds", type=float, default=10.0) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 373e182..efd3957 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -31,3 +31,4 @@ {"ts": "2026-05-18T14:55:13+00:00", "type": "decision", "status": "ok", "note": "Audio field live output may use bounded buffering: latency is acceptable while delay/SRO estimates settle, but cache depth must converge toward real-time. Core audio code now has testable seams for buffers, latency policy, injectable capture/alignment/encoder/sink ports, and pipeline orchestration."} {"ts": "2026-05-18T14:57:32+00:00", "type": "decision", "status": "ok", "note": "Sensor-fusion sidecar starts with semantic observation authority: calibrated cameras plus timestamped 2D marker observations triangulate into cached 3D point claims, then lower to PLY/brush/splat packets. Capture adapters and render lowerings stay outside the pure fusion core so unit tests can inject synthetic cameras, frames, and observations."} {"ts": "2026-05-18T15:07:24+00:00", "type": "decision", "status": "ok", "note": "OBS visualization boundary: LocalCastBridge should emit typed render-frame packets from the sensor-fusion cache; Aquarium Engine should consume them, render points/brushes/splats to a GPU render target, and publish that texture as Spout sender 'LocalCastBridge Point Cloud'. Render packets carry visual presentation and audio-alignment timestamps so latency can converge with the ambisonic output timeline."} +{"ts": "2026-05-18T15:12:32+00:00", "type": "verification", "status": "partial", "note": "Deadline audio path now supports distributed run manifests, local concurrent mic capture, sweep-arrival analysis, aligned six-channel WAV assembly with missing channels zero-filled, and FOA encoding. Live smoke captured local Kiyo-left, both PS Eye mics, and local Focusrite at 48 kHz; Kiyo-right profile query is currently missing and neighbor Focusrite remains remote/manual import."}