Add passive sync analysis against output loopback
This commit is contained in:
parent
ed7b527b32
commit
f6b32c6356
4 changed files with 188 additions and 1 deletions
|
|
@ -104,6 +104,15 @@ Record all locally visible distributed mics for a deadline calibration pass:
|
|||
.\.venv\Scripts\python.exe .\scripts\audio_field.py record-local-calibration --profile .\config\audio-field.json --seconds 10
|
||||
```
|
||||
|
||||
For passive runtime sync against ground-truth program output, capture the Scarlett loopback with the local mics:
|
||||
|
||||
```powershell
|
||||
.\.venv\Scripts\python.exe .\scripts\audio_field.py record-local-calibration --profile .\config\audio-field.json --seconds 20 --record-loopback --loopback-query Scarlett --loopback-rate 48000
|
||||
.\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-reference-sync --profile .\config\audio-field.json --run .\calibration\runs\<run-folder> --window-seconds 4 --hop-seconds 1 --method normalized --min-score 0.08
|
||||
```
|
||||
|
||||
The loopback path is ground-truth content, but not automatically a perfect ground-truth clock. If WASAPI loopback reports discontinuities, use the stable windows and treat the rest as suspect.
|
||||
|
||||
Record local mics while playing one speaker sweep. Use the output rate that the device probe says is real:
|
||||
|
||||
```powershell
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
opencv-contrib-python
|
||||
sounddevice
|
||||
soundcard
|
||||
scipy
|
||||
imageio-ffmpeg
|
||||
PyOpenGL
|
||||
|
|
|
|||
|
|
@ -433,6 +433,8 @@ def cmd_record_local_calibration(args):
|
|||
)
|
||||
|
||||
stimulus_file = None
|
||||
loopback_file = None
|
||||
loopback_rate = None
|
||||
output_device = None
|
||||
output_rate = None
|
||||
if args.play_sweep:
|
||||
|
|
@ -448,7 +450,20 @@ def cmd_record_local_calibration(args):
|
|||
try:
|
||||
for stream in streams:
|
||||
stream.start()
|
||||
if args.play_sweep:
|
||||
if args.record_loopback:
|
||||
loopback_rate = int(args.loopback_rate or input_rate)
|
||||
loopback = record_loopback(
|
||||
query=args.loopback_query,
|
||||
seconds=seconds,
|
||||
sample_rate=loopback_rate,
|
||||
channels=args.loopback_channels,
|
||||
play_data=playback if args.play_sweep else None,
|
||||
play_rate=output_rate,
|
||||
play_device=output_device["index"] if output_device else None,
|
||||
)
|
||||
loopback_file = "ground_truth_loopback.wav"
|
||||
wavfile.write(out / loopback_file, loopback_rate, loopback.astype(np.float32))
|
||||
elif args.play_sweep:
|
||||
sd.play(playback, samplerate=output_rate, device=output_device["index"], blocking=True)
|
||||
remaining = max(0.0, seconds - len(playback) / output_rate)
|
||||
time.sleep(remaining)
|
||||
|
|
@ -512,11 +527,44 @@ def cmd_record_local_calibration(args):
|
|||
"outputDevice": summarize_device(output_device) if output_device else None,
|
||||
"outputSampleRate": output_rate,
|
||||
},
|
||||
"reference": {
|
||||
"kind": "wasapi-loopback",
|
||||
"file": loopback_file,
|
||||
"sampleRate": loopback_rate,
|
||||
"query": args.loopback_query if loopback_file else None,
|
||||
},
|
||||
},
|
||||
)
|
||||
print(out)
|
||||
|
||||
|
||||
def record_loopback(query, seconds, sample_rate, channels=2, play_data=None, play_rate=None, play_device=None):
|
||||
try:
|
||||
import soundcard as sc
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"soundcard loopback capture unavailable: {exc!r}") from exc
|
||||
matches = [
|
||||
mic
|
||||
for mic in sc.all_microphones(include_loopback=True)
|
||||
if "loopback" in repr(mic).lower() and query.lower() in mic.name.lower()
|
||||
]
|
||||
if not matches:
|
||||
raise SystemExit(f"No loopback device matching {query!r}")
|
||||
loopback = matches[0]
|
||||
frame_count = int(seconds * sample_rate)
|
||||
if play_data is not None:
|
||||
sd = sounddevice_module()
|
||||
sd.play(play_data, samplerate=play_rate, device=play_device, blocking=False)
|
||||
with loopback.recorder(samplerate=sample_rate, channels=channels) as recorder:
|
||||
data = recorder.record(numframes=frame_count)
|
||||
if play_data is not None:
|
||||
try:
|
||||
sounddevice_module().stop()
|
||||
except Exception:
|
||||
pass
|
||||
return np.asarray(data, dtype=np.float32)
|
||||
|
||||
|
||||
def make_sweep_at_rate(profile, sample_rate):
|
||||
clone = dict(profile)
|
||||
clone["sampleRate"] = int(sample_rate)
|
||||
|
|
@ -766,6 +814,116 @@ def cmd_assemble_aligned(args):
|
|||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
def cmd_analyze_reference_sync(args):
|
||||
profile = load_profile(args.profile)
|
||||
run = args.run
|
||||
manifest = read_json(run / "manifest.json")
|
||||
reference_info = manifest.get("reference") or {}
|
||||
reference_file = args.reference or (run / reference_info.get("file", ""))
|
||||
if not reference_file.exists():
|
||||
raise SystemExit(f"Reference loopback WAV not found: {reference_file}")
|
||||
ref_rate, reference = wavfile.read(reference_file)
|
||||
reference = mono(as_float_matrix(reference))
|
||||
reference = resample_if_needed(reference, int(ref_rate), int(profile["sampleRate"]))
|
||||
field_rate = int(profile["sampleRate"])
|
||||
window = int(args.window_seconds * field_rate)
|
||||
hop = int(args.hop_seconds * field_rate)
|
||||
max_lag = int(args.max_lag_ms * field_rate / 1000.0)
|
||||
results = {
|
||||
"created_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"sourceRun": str(run),
|
||||
"reference": str(reference_file),
|
||||
"sampleRate": field_rate,
|
||||
"windowSeconds": args.window_seconds,
|
||||
"hopSeconds": args.hop_seconds,
|
||||
"sources": [],
|
||||
}
|
||||
for source in manifest.get("sources", []):
|
||||
file_name = source.get("file")
|
||||
if not file_name or not (run / file_name).exists():
|
||||
continue
|
||||
rec_rate, data = wavfile.read(run / file_name)
|
||||
mic = resample_if_needed(mono(as_float_matrix(data)), int(rec_rate), field_rate)
|
||||
delays = []
|
||||
scores = []
|
||||
for start in range(0, min(len(reference), len(mic)) - window + 1, hop):
|
||||
ref_win = reference[start : start + window]
|
||||
mic_win = mic[start : start + window]
|
||||
if float(np.std(ref_win)) < args.min_reference_std or float(np.std(mic_win)) < args.min_mic_std:
|
||||
continue
|
||||
if args.method == "gcc-phat":
|
||||
delay, score = gcc_phat_delay(mic_win, ref_win, max_lag=max_lag)
|
||||
else:
|
||||
delay, score = normalized_delay(mic_win, ref_win, max_lag=max_lag)
|
||||
if score < args.min_score or abs(delay) < int(args.min_abs_lag_ms * field_rate / 1000.0):
|
||||
continue
|
||||
delays.append(delay)
|
||||
scores.append(score)
|
||||
if not delays:
|
||||
continue
|
||||
delays = np.asarray(delays, dtype=np.float64)
|
||||
scores = np.asarray(scores, dtype=np.float64)
|
||||
median = float(np.median(delays))
|
||||
mad = float(np.median(np.abs(delays - median)))
|
||||
results["sources"].append(
|
||||
{
|
||||
"micId": source["micId"],
|
||||
"fieldChannel": source.get("fieldChannel"),
|
||||
"windows": int(len(delays)),
|
||||
"medianDelaySamples": median,
|
||||
"medianDelayMs": 1000.0 * median / field_rate,
|
||||
"madSamples": mad,
|
||||
"madMs": 1000.0 * mad / field_rate,
|
||||
"meanScore": float(np.mean(scores)),
|
||||
"maxScore": float(np.max(scores)),
|
||||
"method": args.method,
|
||||
}
|
||||
)
|
||||
write_json(run / "reference-sync-analysis.json", results)
|
||||
print(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
def mono(matrix):
|
||||
if matrix.ndim == 1 or matrix.shape[1] == 1:
|
||||
return matrix.reshape(-1).astype(np.float32)
|
||||
return np.mean(matrix, axis=1).astype(np.float32)
|
||||
|
||||
|
||||
def gcc_phat_delay(signal_a, signal_b, max_lag):
|
||||
a = np.asarray(signal_a, dtype=np.float32)
|
||||
b = np.asarray(signal_b, dtype=np.float32)
|
||||
a = a - float(np.mean(a))
|
||||
b = b - float(np.mean(b))
|
||||
n = 1
|
||||
while n < len(a) + len(b):
|
||||
n *= 2
|
||||
A = np.fft.rfft(a, n=n)
|
||||
B = np.fft.rfft(b, n=n)
|
||||
R = A * np.conj(B)
|
||||
denom = np.abs(R)
|
||||
R = np.divide(R, denom, out=np.zeros_like(R), where=denom > 1e-12)
|
||||
corr = np.fft.irfft(R, n=n)
|
||||
corr = np.concatenate((corr[-max_lag:], corr[: max_lag + 1]))
|
||||
index = int(np.argmax(np.abs(corr)))
|
||||
lag = index - max_lag
|
||||
score = float(abs(corr[index]))
|
||||
return lag, score
|
||||
|
||||
|
||||
def normalized_delay(signal_a, signal_b, max_lag):
|
||||
a = np.asarray(signal_a, dtype=np.float32)
|
||||
b = np.asarray(signal_b, dtype=np.float32)
|
||||
a = (a - float(np.mean(a))) / (float(np.std(a)) + 1e-9)
|
||||
b = (b - float(np.mean(b))) / (float(np.std(b)) + 1e-9)
|
||||
corr = signal.correlate(a, b, mode="full", method="fft")
|
||||
mid = len(b) - 1
|
||||
segment = corr[mid - max_lag : mid + max_lag + 1]
|
||||
index = int(np.argmax(np.abs(segment)))
|
||||
lag = index - max_lag
|
||||
score = float(abs(segment[index]) / max(1, len(a)))
|
||||
return lag, score
|
||||
|
||||
|
||||
def as_float_matrix(data):
|
||||
arr = np.asarray(data)
|
||||
if arr.ndim == 1:
|
||||
|
|
@ -1032,6 +1190,10 @@ def main():
|
|||
p.add_argument("--play-sweep", action="store_true")
|
||||
p.add_argument("--speaker-channel", type=int, default=0)
|
||||
p.add_argument("--output-rate", type=int)
|
||||
p.add_argument("--record-loopback", action="store_true")
|
||||
p.add_argument("--loopback-query", default="Scarlett")
|
||||
p.add_argument("--loopback-rate", type=int)
|
||||
p.add_argument("--loopback-channels", type=int, default=2)
|
||||
p.set_defaults(func=cmd_record_local_calibration)
|
||||
|
||||
p = sub.add_parser("play-record", help="Play each speaker calibration sweep while recording all microphones.")
|
||||
|
|
@ -1066,6 +1228,20 @@ def main():
|
|||
p.add_argument("--seconds", type=float)
|
||||
p.set_defaults(func=cmd_assemble_aligned)
|
||||
|
||||
p = sub.add_parser("analyze-reference-sync", help="Estimate live mic delay stability against captured output loopback ground truth.")
|
||||
p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json")
|
||||
p.add_argument("--run", type=Path, required=True)
|
||||
p.add_argument("--reference", type=Path)
|
||||
p.add_argument("--window-seconds", type=float, default=2.0)
|
||||
p.add_argument("--hop-seconds", type=float, default=1.0)
|
||||
p.add_argument("--max-lag-ms", type=float, default=500.0)
|
||||
p.add_argument("--method", choices=["normalized", "gcc-phat"], default="normalized")
|
||||
p.add_argument("--min-score", type=float, default=0.08)
|
||||
p.add_argument("--min-reference-std", type=float, default=1e-4)
|
||||
p.add_argument("--min-mic-std", type=float, default=1e-5)
|
||||
p.add_argument("--min-abs-lag-ms", type=float, default=0.5)
|
||||
p.set_defaults(func=cmd_analyze_reference_sync)
|
||||
|
||||
p = sub.add_parser("record-field", help="Record the raw synchronized six-channel microphone field.")
|
||||
p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json")
|
||||
p.add_argument("--seconds", type=float, default=10.0)
|
||||
|
|
|
|||
|
|
@ -36,3 +36,4 @@
|
|||
{"ts": "2026-05-18T15:21:20+00:00", "type": "decision", "status": "ok", "note": "Calibration delay analysis now uses local chirplet refinement: coarse matched-filter sweep detection initializes a small fractional-delay/chirp-rate atom search, following chirplet parameter-estimation guidance. Plugged-in left/right sweeps produced sub-sample chirplet delays for Kiyo-left, both PS Eye mics, and local Focusrite; Kiyo-right remains missing and neighbor Focusrite still needs import."}
|
||||
{"ts": "2026-05-18T15:22:33+00:00", "type": "decision", "status": "ok", "note": "Runtime mic sync must treat known speaker output as chirplet telemetry every block/frame, updating delay, SRO/rate-scale, phase, and confidence fields dynamically. Static sweep offsets are only initial state; low-confidence runtime chirplet observations should freeze updates rather than corrupt alignment."}
|
||||
{"ts": "2026-05-18T15:23:38+00:00", "type": "decision", "status": "ok", "note": "Active sync probes are now an explicit optimizer responsibility: when passive chirplet telemetry leaves any source below confidence threshold, the system may emit bounded extra chirplets in masked windows, maximizing confidence gain per audible intrusion while respecting spacing and level limits."}
|
||||
{"ts": "2026-05-18T15:35:24+00:00", "type": "verification", "status": "partial", "note": "Passive sync test using Scarlett output loopback as ground-truth content: 20s music run with normalized 4s/1s window correlation produced local Focusrite median delay 20115.5 samples/419.07 ms with 17 sample/0.35 ms MAD; PS Eye left 23848 samples/496.83 ms with 17 sample/0.35 ms MAD; Kiyo-left and PS Eye-right were noisier at about 3.9-4.2 ms MAD. Soundcard WASAPI loopback reported discontinuities, so loopback clock quality remains suspect."}
|
||||
|
|
|
|||
Loading…
Reference in a new issue