465 lines
20 KiB
Python
465 lines
20 KiB
Python
import argparse
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import queue
|
|
import sys
|
|
import time
|
|
|
|
import numpy as np
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from audio_field.active_probe import ActiveConfidenceMaintainer # noqa: E402
|
|
from audio_field.cultcache_audio import ( # noqa: E402
|
|
DEFAULT_MIC_FIELD_CHANNELS,
|
|
make_audio_phase_field,
|
|
make_mic_field_frame,
|
|
put_live_audio_phase_field,
|
|
put_live_mic_field_frame,
|
|
)
|
|
from audio_field.phase_meaning import LivePhaseMeaningExtractor # noqa: E402
|
|
from audio_field.probe_optimizer import ActiveProbeOptimizer, ProbePolicy # noqa: E402
|
|
from localcast.sensor_fusion import camera_sensor_id_for_microphone, phase_source_id_for_microphone # noqa: E402
|
|
|
|
|
|
HOST_SOURCE_BY_MIC_ID = {
|
|
"mic_focusrite_local": "host-focusrite",
|
|
"mic_focusrite_neighbor": "co-streamer-focusrite",
|
|
}
|
|
|
|
|
|
def load_profile(path: Path) -> dict:
|
|
return json.loads(Path(path).read_text(encoding="utf-8"))
|
|
|
|
|
|
def source_id_for_microphone(mic: dict) -> str | None:
|
|
return str(mic.get("phaseSourceId") or mic.get("sourceId") or HOST_SOURCE_BY_MIC_ID.get(str(mic.get("id", ""))) or phase_source_id_for_microphone(mic) or "")
|
|
|
|
|
|
def local_microphones(profile: dict, machine: str) -> list[dict]:
|
|
return [mic for mic in profile.get("microphones", []) if mic.get("machine") == machine and source_id_for_microphone(mic)]
|
|
|
|
|
|
def summarize_device(device: dict) -> dict:
|
|
return {
|
|
"index": int(device["index"]),
|
|
"name": str(device["name"]),
|
|
"hostapi": str(device.get("hostapi_name", "")),
|
|
"inputs": int(device.get("max_input_channels", 0)),
|
|
"outputs": int(device.get("max_output_channels", 0)),
|
|
"default_samplerate": float(device.get("default_samplerate", 0.0)),
|
|
}
|
|
|
|
|
|
def find_devices(sd, spec: dict, direction: str) -> list[dict]:
|
|
hostapis = sd.query_hostapis()
|
|
devices = sd.query_devices()
|
|
query = str(spec.get("query") or "").lower()
|
|
hostapi_name = str(spec.get("hostApi") or "").lower()
|
|
needed_channels = int(spec.get("channels") or 1)
|
|
channel_key = "max_input_channels" if direction == "input" else "max_output_channels"
|
|
matches = []
|
|
for device in devices:
|
|
if int(device[channel_key]) < needed_channels:
|
|
continue
|
|
hostapi = hostapis[device["hostapi"]]["name"]
|
|
if query and query not in str(device["name"]).lower():
|
|
continue
|
|
if hostapi_name and hostapi_name not in str(hostapi).lower():
|
|
continue
|
|
matches.append({**device, "hostapi_name": hostapi})
|
|
return matches
|
|
|
|
|
|
def match_indexed_device(sd, spec: dict, direction: str) -> dict:
|
|
matches = find_devices(sd, spec, direction)
|
|
match_index = int(spec.get("matchIndex", 0))
|
|
if match_index < 0 or match_index >= len(matches):
|
|
raise RuntimeError(f"No {direction} device match {match_index} for {spec!r}; found {len(matches)}")
|
|
return matches[match_index]
|
|
|
|
|
|
def local_device_groups(sd, mics: list[dict]) -> tuple[dict[str, dict], list[dict]]:
|
|
groups: dict[str, dict] = {}
|
|
missing = []
|
|
for mic in mics:
|
|
try:
|
|
device = match_indexed_device(sd, mic.get("device", {}), "input")
|
|
except RuntimeError as exc:
|
|
missing.append({"micId": mic.get("id"), "sourceId": source_id_for_microphone(mic), "error": str(exc)})
|
|
continue
|
|
channel = int(mic.get("device", {}).get("channel", 0))
|
|
key = str(device["index"])
|
|
groups.setdefault(key, {"device": device, "mics": [], "channels": channel + 1})
|
|
groups[key]["mics"].append(mic)
|
|
groups[key]["channels"] = max(int(groups[key]["channels"]), channel + 1)
|
|
return groups, missing
|
|
|
|
|
|
class InputGroupBuffer:
|
|
def __init__(self) -> None:
|
|
self.queue: queue.Queue[np.ndarray] = queue.Queue()
|
|
self.leftover = np.zeros((0, 0), dtype=np.float32)
|
|
|
|
def push(self, block: np.ndarray) -> None:
|
|
self.queue.put(np.asarray(block, dtype=np.float32).copy())
|
|
|
|
def read(self, frames: int, channels: int, timeout: float = 1.0) -> np.ndarray:
|
|
if self.leftover.shape[1] != channels:
|
|
self.leftover = np.zeros((0, channels), dtype=np.float32)
|
|
chunks = [self.leftover]
|
|
total = int(self.leftover.shape[0])
|
|
deadline = time.monotonic() + float(timeout)
|
|
while total < frames and time.monotonic() < deadline:
|
|
try:
|
|
item = self.queue.get(timeout=max(0.01, min(0.1, deadline - time.monotonic())))
|
|
except queue.Empty:
|
|
continue
|
|
if item.ndim == 1:
|
|
item = item[:, None]
|
|
if item.shape[1] < channels:
|
|
padded = np.zeros((item.shape[0], channels), dtype=np.float32)
|
|
padded[:, : item.shape[1]] = item
|
|
item = padded
|
|
chunks.append(item[:, :channels])
|
|
total += int(item.shape[0])
|
|
merged = np.concatenate(chunks, axis=0) if chunks else np.zeros((0, channels), dtype=np.float32)
|
|
if merged.shape[0] < frames:
|
|
padded = np.zeros((frames, channels), dtype=np.float32)
|
|
padded[: merged.shape[0]] = merged
|
|
self.leftover = np.zeros((0, channels), dtype=np.float32)
|
|
return padded
|
|
block = merged[:frames]
|
|
self.leftover = merged[frames:]
|
|
return block
|
|
|
|
|
|
def make_source_rows(meaning) -> list[dict]:
|
|
return [
|
|
{
|
|
"sourceId": source.source_id,
|
|
"channel": source.channel,
|
|
"delaySamples": source.smoothed_delay_samples,
|
|
"delayMs": source.delay_ms,
|
|
"distanceDeltaMeters": source.distance_delta_m,
|
|
"coherence": source.coherence,
|
|
"fitErrorRadians": source.fit_error_radians,
|
|
"confidence": source.confidence,
|
|
"referenceBleed": source.reference_bleed,
|
|
"suppressionWeight": source.suppression_weight,
|
|
"correctionEnergy": source.correction_energy,
|
|
}
|
|
for source in meaning.sources
|
|
]
|
|
|
|
|
|
def write_status(path: Path, payload: dict) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps({**payload, "updated_monotonic_ns": time.monotonic_ns()}, indent=2, sort_keys=True), encoding="utf-8")
|
|
|
|
|
|
def open_loopback_recorder(query: str, sample_rate: int, channels: int):
|
|
import soundcard as sc
|
|
|
|
matches = [
|
|
mic
|
|
for mic in sc.all_microphones(include_loopback=True)
|
|
if "loopback" in repr(mic).lower() and query.lower() in mic.name.lower()
|
|
]
|
|
if not matches:
|
|
raise RuntimeError(f"No loopback device matching {query!r}")
|
|
return matches[0].recorder(samplerate=sample_rate, channels=channels)
|
|
|
|
|
|
def play_probe_file(path: Path, *, device: int | None = None, output_rate: int | None = None) -> None:
|
|
from scipy.io import wavfile
|
|
from scipy import signal
|
|
import sounddevice as sd
|
|
|
|
rate, data = wavfile.read(path)
|
|
data = np.asarray(data, dtype=np.float32)
|
|
playback_rate = int(output_rate or rate)
|
|
if playback_rate != int(rate):
|
|
divisor = math.gcd(int(rate), playback_rate)
|
|
data = signal.resample_poly(data, playback_rate // divisor, int(rate) // divisor, axis=0).astype(np.float32)
|
|
sd.play(data, playback_rate, device=device, blocking=False)
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Capture local live mics into Faust/phase CultCache fields.")
|
|
parser.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.json")
|
|
parser.add_argument("--fallback-profile", type=Path, default=ROOT / "config" / "audio-field.example.json")
|
|
parser.add_argument("--machine", default="local")
|
|
parser.add_argument("--mic-cache", type=Path, default=ROOT / "calibration" / "runs" / "audio-mic-field.msgpack")
|
|
parser.add_argument("--phase-cache", type=Path, default=ROOT / "calibration" / "runs" / "audio-phase-field.msgpack")
|
|
parser.add_argument("--status", type=Path, default=ROOT / "calibration" / "runs" / "live-audio-field-status.json")
|
|
parser.add_argument("--phase-status", type=Path, default=ROOT / "calibration" / "runs" / "audio-phase-field-status.json")
|
|
parser.add_argument("--sample-rate", type=int)
|
|
parser.add_argument("--chunk-frames", type=int, default=1024)
|
|
parser.add_argument("--duration", type=float)
|
|
parser.add_argument("--loopback-query", default="Scarlett")
|
|
parser.add_argument("--loopback-channels", type=int, default=2)
|
|
parser.add_argument("--no-loopback", action="store_true")
|
|
parser.add_argument("--maintain-confidence", action="store_true")
|
|
parser.add_argument("--target-confidence", type=float, default=0.72)
|
|
parser.add_argument("--trigger-confidence", type=float, default=0.45)
|
|
parser.add_argument("--min-probe-interval-frames", type=int, default=1)
|
|
parser.add_argument("--probe-level-dbfs", type=float, default=-24.0)
|
|
parser.add_argument("--probe-seconds", type=float, default=0.08)
|
|
parser.add_argument("--probe-output-dir", type=Path, default=ROOT / "calibration" / "runs" / "active-probes")
|
|
parser.add_argument("--probe-artifact-slots", type=int, default=512)
|
|
parser.add_argument("--probe-manifest-max-bytes", type=int, default=1_000_000)
|
|
parser.add_argument("--probe-output-query", default="Scarlett")
|
|
parser.add_argument("--probe-output-hostapi", default="WASAPI")
|
|
parser.add_argument("--probe-output-match-index", type=int, default=0)
|
|
parser.add_argument("--probe-start-hz", type=float, default=18500.0)
|
|
parser.add_argument("--probe-end-hz", type=float, default=22000.0)
|
|
parser.add_argument("--harmonic-root-hz", type=float, default=440.0)
|
|
parser.add_argument("--harmonic-voices", type=int, default=48)
|
|
parser.add_argument("--play-probes", action="store_true")
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
profile_path = args.profile if args.profile.exists() else args.fallback_profile
|
|
profile = load_profile(profile_path)
|
|
sample_rate = int(args.sample_rate or profile.get("sampleRate", 48000))
|
|
channels = DEFAULT_MIC_FIELD_CHANNELS
|
|
source_index = {source_id: index for index, source_id in enumerate(channels)}
|
|
mics = local_microphones(profile, args.machine)
|
|
|
|
import sounddevice as sd
|
|
|
|
groups, missing = local_device_groups(sd, mics)
|
|
status_base = {
|
|
"input_mode": "live-local-capture",
|
|
"profile": str(profile_path),
|
|
"sample_rate": sample_rate,
|
|
"channels": list(channels),
|
|
"local_mics": [
|
|
{
|
|
"micId": mic.get("id"),
|
|
"sourceId": source_id_for_microphone(mic),
|
|
"cameraSensorId": camera_sensor_id_for_microphone(mic),
|
|
"device": mic.get("device"),
|
|
}
|
|
for mic in mics
|
|
],
|
|
"missing": missing,
|
|
"device_groups": [
|
|
{"device": summarize_device(group["device"]), "channels": int(group["channels"]), "mics": [mic.get("id") for mic in group["mics"]]}
|
|
for group in groups.values()
|
|
],
|
|
}
|
|
if args.dry_run:
|
|
write_status(args.status, {**status_base, "running": False, "dry_run": True})
|
|
print(json.dumps(status_base, indent=2))
|
|
return
|
|
|
|
buffers = {key: InputGroupBuffer() for key in groups}
|
|
streams = []
|
|
for key, group in groups.items():
|
|
buffer = buffers[key]
|
|
|
|
def callback(indata, frame_count, time_info, status, *, target=buffer):
|
|
target.push(indata)
|
|
|
|
streams.append(
|
|
sd.InputStream(
|
|
device=group["device"]["index"],
|
|
channels=int(group["channels"]),
|
|
samplerate=sample_rate,
|
|
dtype="float32",
|
|
callback=callback,
|
|
)
|
|
)
|
|
|
|
loopback_context = None
|
|
loopback_recorder = None
|
|
loopback_error = ""
|
|
if not args.no_loopback:
|
|
try:
|
|
loopback_context = open_loopback_recorder(args.loopback_query, sample_rate, args.loopback_channels)
|
|
loopback_recorder = loopback_context.__enter__()
|
|
except Exception as exc:
|
|
loopback_error = repr(exc)
|
|
loopback_context = None
|
|
loopback_recorder = None
|
|
probe_output_device = None
|
|
probe_output_error = ""
|
|
if args.play_probes:
|
|
try:
|
|
probe_output_device = match_indexed_device(
|
|
sd,
|
|
{
|
|
"query": args.probe_output_query,
|
|
"hostApi": args.probe_output_hostapi,
|
|
"channels": 2,
|
|
"matchIndex": args.probe_output_match_index,
|
|
},
|
|
"output",
|
|
)
|
|
except Exception as exc:
|
|
probe_output_error = repr(exc)
|
|
|
|
extractor = LivePhaseMeaningExtractor(channels, sample_rate, [250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 12000.0])
|
|
maintainer = None
|
|
if args.maintain_confidence:
|
|
live_source_ids = {source_id_for_microphone(mic) for group in groups.values() for mic in group["mics"]}
|
|
maintainer = ActiveConfidenceMaintainer(
|
|
ActiveProbeOptimizer(
|
|
ProbePolicy(
|
|
target_confidence=args.target_confidence,
|
|
trigger_confidence=args.trigger_confidence,
|
|
min_interval_frames=args.min_probe_interval_frames,
|
|
max_probe_level_dbfs=args.probe_level_dbfs,
|
|
prefer_masked_windows=False,
|
|
)
|
|
),
|
|
sample_rate=sample_rate,
|
|
output_dir=args.probe_output_dir,
|
|
duration_seconds=args.probe_seconds,
|
|
start_hz=args.probe_start_hz,
|
|
end_hz=args.probe_end_hz,
|
|
channels=2,
|
|
channel=0,
|
|
pattern="harmonic-dense",
|
|
harmonic_root_hz=args.harmonic_root_hz,
|
|
harmonic_voices=args.harmonic_voices,
|
|
max_artifacts=args.probe_artifact_slots,
|
|
manifest_max_bytes=args.probe_manifest_max_bytes,
|
|
eligible_source_ids={source_id for source_id in live_source_ids if source_id},
|
|
)
|
|
start = time.monotonic()
|
|
origin_ns = time.monotonic_ns()
|
|
frame_id = 0
|
|
last_status = 0.0
|
|
last_probe = None
|
|
probe_playback_error = ""
|
|
try:
|
|
for stream in streams:
|
|
stream.start()
|
|
while True:
|
|
now = time.monotonic()
|
|
if args.duration is not None and now - start >= args.duration:
|
|
break
|
|
field = np.zeros((args.chunk_frames, len(channels)), dtype=np.float32)
|
|
for key, group in groups.items():
|
|
block = buffers[key].read(args.chunk_frames, int(group["channels"]))
|
|
for mic in group["mics"]:
|
|
source_id = source_id_for_microphone(mic)
|
|
if source_id not in source_index:
|
|
continue
|
|
device_channel = int(mic.get("device", {}).get("channel", 0))
|
|
if device_channel < block.shape[1]:
|
|
field[:, source_index[source_id]] = block[:, device_channel]
|
|
|
|
if loopback_recorder is not None:
|
|
reference_block = loopback_recorder.record(numframes=args.chunk_frames)
|
|
reference = np.asarray(reference_block, dtype=np.float32)
|
|
if reference.ndim == 2:
|
|
reference = np.mean(reference, axis=1)
|
|
else:
|
|
reference = np.zeros(args.chunk_frames, dtype=np.float32)
|
|
|
|
audio_time_ns = origin_ns + int(frame_id * args.chunk_frames * 1_000_000_000 / sample_rate)
|
|
put_live_mic_field_frame(
|
|
args.mic_cache,
|
|
make_mic_field_frame(field, frame_id=frame_id, sample_rate=sample_rate, start_sample=frame_id * args.chunk_frames, audio_time_ns=audio_time_ns, channels=channels),
|
|
)
|
|
meaning = extractor.update(reference, field, frame_id=frame_id, start_sample=frame_id * args.chunk_frames, audio_time_ns=audio_time_ns)
|
|
put_live_audio_phase_field(
|
|
args.phase_cache,
|
|
make_audio_phase_field(
|
|
frame_id=meaning.frame_id,
|
|
sample_rate=meaning.sample_rate,
|
|
start_sample=meaning.start_sample,
|
|
frame_count=meaning.frame_count,
|
|
audio_time_ns=meaning.audio_time_ns,
|
|
reference_id="local-loopback-live",
|
|
sources=make_source_rows(meaning),
|
|
global_confidence=meaning.global_confidence,
|
|
needs_active_probe=meaning.needs_active_probe,
|
|
active_probe_reason=meaning.active_probe_reason,
|
|
),
|
|
)
|
|
emitted = None
|
|
if maintainer is not None:
|
|
emitted = maintainer.update(meaning, reference, force_masked=True)
|
|
if emitted is not None:
|
|
last_probe = {
|
|
"source_id": emitted.request.source_id,
|
|
"reason": emitted.request.reason,
|
|
"urgency": emitted.request.urgency,
|
|
"path": str(emitted.path),
|
|
"emitted_monotonic_ns": emitted.emitted_monotonic_ns,
|
|
}
|
|
if emitted is not None and args.play_probes:
|
|
try:
|
|
play_probe_file(
|
|
emitted.path,
|
|
device=None if probe_output_device is None else int(probe_output_device["index"]),
|
|
output_rate=None
|
|
if probe_output_device is None
|
|
else int(round(float(probe_output_device["default_samplerate"]))),
|
|
)
|
|
probe_playback_error = ""
|
|
except Exception as exc:
|
|
probe_playback_error = repr(exc)
|
|
if now - last_status >= 1.0:
|
|
live_payload = {
|
|
**status_base,
|
|
"running": True,
|
|
"dry_run": False,
|
|
"frame_id": frame_id,
|
|
"closed_loop_probe_capture": loopback_recorder is not None,
|
|
"loopback_query": args.loopback_query,
|
|
"loopback_error": loopback_error,
|
|
"probe_output": None if probe_output_device is None else summarize_device(probe_output_device),
|
|
"probe_output_error": probe_output_error,
|
|
"probe_playback_error": probe_playback_error,
|
|
"global_confidence": meaning.global_confidence,
|
|
"needs_active_probe": meaning.needs_active_probe,
|
|
"maintain_confidence": args.maintain_confidence,
|
|
"play_probes": args.play_probes,
|
|
"last_probe": last_probe,
|
|
}
|
|
write_status(args.status, live_payload)
|
|
write_status(
|
|
args.phase_status,
|
|
{
|
|
"input_mode": "live-local-capture",
|
|
"field": "sounddevice-local-mics",
|
|
"reference": "soundcard-loopback" if loopback_recorder is not None else "missing-loopback-zero-reference",
|
|
"frame_id": frame_id,
|
|
"global_confidence": meaning.global_confidence,
|
|
"needs_active_probe": meaning.needs_active_probe,
|
|
"maintain_confidence": args.maintain_confidence,
|
|
"play_probes": args.play_probes,
|
|
"probe_feedback_mode": "closed-loop-capture" if loopback_recorder is not None else "missing-loopback",
|
|
"closed_loop_probe_capture": loopback_recorder is not None,
|
|
"probe_playback_error": probe_playback_error,
|
|
"last_probe": last_probe,
|
|
},
|
|
)
|
|
last_status = now
|
|
frame_id += 1
|
|
finally:
|
|
for stream in streams:
|
|
try:
|
|
stream.stop()
|
|
stream.close()
|
|
except Exception:
|
|
pass
|
|
if loopback_context is not None:
|
|
try:
|
|
loopback_context.__exit__(None, None, None)
|
|
except Exception:
|
|
pass
|
|
write_status(args.status, {**status_base, "running": False, "stopped": True})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|