diff --git a/audio_field/cultcache_audio.py b/audio_field/cultcache_audio.py index 86ce77a..33aa279 100644 --- a/audio_field/cultcache_audio.py +++ b/audio_field/cultcache_audio.py @@ -25,6 +25,8 @@ LIVE_SOURCE_EVENTS_KEY = "localcast.audio.source-events.live" SOURCE_EVENTS_SCHEMA_ID = "gamecult.localcast.audio.source_events.v1" LIVE_PHASE_FIELD_KEY = "localcast.audio.phase-field.live" PHASE_FIELD_SCHEMA_ID = "gamecult.localcast.audio.phase_field.v1" +LIVE_MIC_FIELD_KEY = "localcast.audio.mic-field.live" +MIC_FIELD_SCHEMA_ID = "gamecult.localcast.audio.mic_field.v1" @dataclass(frozen=True) @@ -82,6 +84,23 @@ class CultAudioPhaseField: active_probe_reason: str +@dataclass(frozen=True) +class CultAudioMicFieldFrame: + schema_version: str + frame_id: int + created_monotonic_ns: int + audio_time_ns: int + sample_rate: int + start_sample: int + frame_count: int + channels: tuple[str, ...] + roles: tuple[dict[str, Any], ...] + graph_id: str + controls: tuple[dict[str, Any], ...] + sample_format: str + interleaved_pcm_f32le: bytes + + def _pack(value: Any) -> bytes: import msgpack @@ -212,6 +231,42 @@ def _decode_phase_field(raw: Any) -> CultAudioPhaseField: ) +def _encode_mic_field(frame: CultAudioMicFieldFrame) -> list[Any]: + return [ + frame.schema_version, + frame.frame_id, + frame.created_monotonic_ns, + frame.audio_time_ns, + frame.sample_rate, + frame.start_sample, + frame.frame_count, + list(frame.channels), + list(frame.roles), + frame.graph_id, + list(frame.controls), + frame.sample_format, + frame.interleaved_pcm_f32le, + ] + + +def _decode_mic_field(raw: Any) -> CultAudioMicFieldFrame: + return CultAudioMicFieldFrame( + schema_version=str(raw[0]), + frame_id=int(raw[1]), + created_monotonic_ns=int(raw[2]), + audio_time_ns=int(raw[3]), + sample_rate=int(raw[4]), + start_sample=int(raw[5]), + frame_count=int(raw[6]), + channels=tuple(str(channel) for channel in raw[7]), + roles=tuple(dict(item) for item in raw[8]), + graph_id=str(raw[9]), + controls=tuple(dict(item) for item in raw[10]), + sample_format=str(raw[11]), + interleaved_pcm_f32le=bytes(raw[12]), + ) + + SPATIAL_AUDIO_FRAME_DOCUMENT = define_document_type( "localcast.audio.spatial_frame", encode=_encode_audio_frame, @@ -248,6 +303,15 @@ PHASE_FIELD_DOCUMENT = define_document_type( payload_decoder=_unpack, ) +MIC_FIELD_DOCUMENT = define_document_type( + "localcast.audio.mic_field", + encode=_encode_mic_field, + decode=_decode_mic_field, + name=lambda frame: LIVE_MIC_FIELD_KEY if isinstance(frame, CultAudioMicFieldFrame) else None, + payload_encoder=_pack, + payload_decoder=_unpack, +) + def _retry_cache_io(operation): last_error: Exception | None = None @@ -268,6 +332,7 @@ def _open_audio_cache_once(path: Path) -> CultCache: .register_document_type(AUDIO_STREAM_STATUS_DOCUMENT) .register_document_type(SOURCE_EVENTS_DOCUMENT) .register_document_type(PHASE_FIELD_DOCUMENT) + .register_document_type(MIC_FIELD_DOCUMENT) .add_generic_store(SingleFileMessagePackBackingStore(path)) .build() ) @@ -422,3 +487,90 @@ def get_live_audio_phase_field(path: Path) -> CultAudioPhaseField | None: return cache.get(PHASE_FIELD_DOCUMENT, LIVE_PHASE_FIELD_KEY) return _retry_cache_io(operation) + + +DEFAULT_MIC_FIELD_CHANNELS = ( + "host-focusrite", + "co-streamer-focusrite", + "kiyo-0", + "kiyo-1", + "ps-eye-0", + "ps-eye-1", +) + + +DEFAULT_MIC_FIELD_ROLES = ( + {"channel": 0, "sourceId": "host-focusrite", "role": "host-voice-anchor"}, + {"channel": 1, "sourceId": "co-streamer-focusrite", "role": "co-streamer-voice-anchor"}, + {"channel": 2, "sourceId": "kiyo-0", "role": "room-witness"}, + {"channel": 3, "sourceId": "kiyo-1", "role": "room-witness"}, + {"channel": 4, "sourceId": "ps-eye-0", "role": "spatial-witness"}, + {"channel": 5, "sourceId": "ps-eye-1", "role": "spatial-witness"}, +) + + +DEFAULT_FAUST_VOICE_CONTROLS = ( + {"path": "/host/gain", "value": 1.0}, + {"path": "/host/witness_reject", "value": 0.35}, + {"path": "/co_streamer/gain", "value": 1.0}, + {"path": "/co_streamer/witness_reject", "value": 0.35}, + {"path": "/ambient/gain", "value": 0.6}, + {"path": "/transients/gain", "value": 0.7}, +) + + +def make_mic_field_frame( + block: np.ndarray, + *, + frame_id: int, + sample_rate: int, + start_sample: int, + audio_time_ns: int, + channels: tuple[str, ...] = DEFAULT_MIC_FIELD_CHANNELS, + roles: tuple[dict[str, Any], ...] = DEFAULT_MIC_FIELD_ROLES, + graph_id: str = "localcast.faust.voice_separation.v1", + controls: tuple[dict[str, Any], ...] = DEFAULT_FAUST_VOICE_CONTROLS, +) -> CultAudioMicFieldFrame: + pcm = np.asarray(block, dtype=np.float32) + if pcm.ndim != 2: + raise ValueError("mic field block must be a 2D frames-by-channels array") + if pcm.shape[1] != len(channels): + raise ValueError(f"expected {len(channels)} channels, got {pcm.shape[1]}") + return CultAudioMicFieldFrame( + schema_version=MIC_FIELD_SCHEMA_ID, + frame_id=int(frame_id), + created_monotonic_ns=time.monotonic_ns(), + audio_time_ns=int(audio_time_ns), + sample_rate=int(sample_rate), + start_sample=int(start_sample), + frame_count=int(pcm.shape[0]), + channels=tuple(channels), + roles=tuple(dict(item) for item in roles), + graph_id=str(graph_id), + controls=tuple(dict(item) for item in controls), + sample_format="f32le-interleaved", + interleaved_pcm_f32le=np.ascontiguousarray(pcm, dtype=" np.ndarray: + if frame.sample_format != "f32le-interleaved": + raise ValueError(f"unsupported sample format: {frame.sample_format}") + pcm = np.frombuffer(frame.interleaved_pcm_f32le, dtype=" None: + def operation() -> None: + cache = open_audio_cache(path) + cache.put(MIC_FIELD_DOCUMENT, LIVE_MIC_FIELD_KEY, frame) + + _retry_cache_io(operation) + + +def get_live_mic_field_frame(path: Path) -> CultAudioMicFieldFrame | None: + def operation() -> CultAudioMicFieldFrame | None: + cache = open_audio_cache(path) + return cache.get(MIC_FIELD_DOCUMENT, LIVE_MIC_FIELD_KEY) + + return _retry_cache_io(operation) diff --git a/docs/aquarium-spatial-audio.md b/docs/aquarium-spatial-audio.md index d8da422..540349d 100644 --- a/docs/aquarium-spatial-audio.md +++ b/docs/aquarium-spatial-audio.md @@ -11,11 +11,15 @@ flowchart TD A["aligned microphone field"] --> B["FOA encoder"] B --> C["AmbiX ACN/SN3D W,Y,Z,X blocks"] C --> D["localcast.audio.spatial_frame"] + A --> M["localcast.audio.mic_field"] + M --> N["Faust voice-separation graph"] A --> H["source-event analysis"] H --> I["localcast.audio.source_events"] D --> E["audio-state.msgpack / CultNet document_put"] + M --> O["audio-mic-field.msgpack / CultNet document_put"] I --> J["audio-events.msgpack / CultNet document_put"] E --> F["Aquarium client"] + O --> F J --> F K["visual-state.msgpack"] --> F F --> G["native Faust spatial DSP + volumetric renderer"] @@ -24,6 +28,7 @@ flowchart TD Live files: - `calibration/runs/audio-state.msgpack`: latest typed AmbiX audio block. +- `calibration/runs/audio-mic-field.msgpack`: latest aligned six-mic block for Faust voice separation. - `calibration/runs/audio-stream-status.msgpack`: audio publisher heartbeat. - `calibration/runs/audio-events.msgpack`: latest typed dialogue-focus and transient vector field. - `calibration/runs/av-sync-status.json`: current Aquarium/Spout publisher view of visual frame id, audio frame id, audio delta, and synchronized event overlay count. @@ -44,12 +49,31 @@ Source event document type: - payload: MessagePack array decoded as `CultAudioSourceEvents` - contents: dialogue anchor weights plus witness-dominant transient events with sample time, estimated room position, direction, energy, confidence, and kind. +Mic-field document type: + +- `localcast.audio.mic_field` +- schema id: `gamecult.localcast.audio.mic_field.v1` +- key: `localcast.audio.mic-field.live` +- payload: MessagePack array decoded as `CultAudioMicFieldFrame` +- audio format: interleaved `float32`, 48 kHz, aligned mic channels +- default channel order: `host-focusrite`, `co-streamer-focusrite`, `kiyo-0`, `kiyo-1`, `ps-eye-0`, `ps-eye-1` +- graph id: `localcast.faust.voice_separation.v1` +- Faust source: `faust/localcast_voice_separation.dsp` + Publisher smoke command: ```powershell .\.venv\Scripts\python.exe .\scripts\stream_spatial_audio.py --input .\calibration\runs\audio-full-sync-20260518-165751\field-foa-ambix.wav --loop --duration 5 --smoke-readback ``` +Faust mic-field publisher: + +```powershell +.\scripts\start-live-faust-mic-field.ps1 +``` + +This publishes the aligned six-channel field directly to Aquarium/Faust while preserving the AmbiX path as a compatibility bed. Stop it with `.\scripts\stop-live-faust-mic-field.ps1`. + Source-event analysis command: ```powershell @@ -65,8 +89,9 @@ Synchronized Aquarium/Spout publisher command: ## Invariants - Aquarium owns rendering and Faust DSP, not microphone capture or clock correction. -- LocalCastBridge publishes declared AmbiX blocks, not undecoded heterogeneous microphone channels. +- LocalCastBridge publishes declared AmbiX blocks for the spatial bed and aligned six-mic blocks for Faust voice separation. Unaligned heterogeneous microphones still do not cross this boundary. - LocalCastBridge publishes source-event geometry separately from AmbiX audio; render effects do not have to infer transient positions from the sound bed. +- Faust owns the voice-separation graph once the aligned mic field has crossed into Aquarium. Python may publish controls and telemetry, but it must not become the hot separation engine. - Aquarium/Spout packaging selects audio source events against each visual frame's `audio_alignment_time_ns`; OBS should receive the already synchronized output rather than trying to align independent media sources. - The audio frame carries `start_sample` and `audio_time_ns` so visual packets can align against the same bounded-latency timeline. - OBS may receive a rendered monitor output later, but OBS is not the spatial bus authority. diff --git a/docs/audio-field.md b/docs/audio-field.md index b6e3d86..f41f762 100644 --- a/docs/audio-field.md +++ b/docs/audio-field.md @@ -195,6 +195,14 @@ Encode an offline aligned six-channel WAV to FOA AmbiX: .\.venv\Scripts\python.exe .\scripts\audio_field.py encode-foa --profile .\config\audio-field.json --input .\calibration\runs\\field-aligned.wav --output .\calibration\runs\\field-foa-ambix.wav ``` +Publish the same aligned six-channel field directly to Aquarium/Faust for voice separation: + +```powershell +.\scripts\start-live-faust-mic-field.ps1 +``` + +This writes `localcast.audio.mic_field` into `calibration/runs/audio-mic-field.msgpack` with channel roles and graph id `localcast.faust.voice_separation.v1`. The Faust graph lives at `faust/localcast_voice_separation.dsp` and has six inputs plus six stem outputs: host voice, co-streamer voice, ambient, transients, local loopback placeholder, and co-streamer loopback placeholder. + Suppress room/transient witness energy before FOA encoding: ```powershell @@ -237,6 +245,7 @@ The `play-record` and `record-field` commands remain for a `shared-input-device` - Ultrasonic probes are allowed only as a bounded optimization strategy. They must stay below Nyquist, remain level-capped, and prove themselves by raising measured confidence; hardware may distort or discard them. - Dense probe textures should be harmonically organized so any lower-band aliases land closer to musical relationships than arbitrary squeal. This is damage control, not a license to trust aliasing. - FOA output is AmbiX: ACN channel order, SN3D normalization, channels `W,Y,Z,X`. +- The Faust voice-separation input is the aligned six-mic field, not the AmbiX collapse. Voice separation needs anchor and witness channels before spatial rendering destroys useful evidence. - Speaker calibration is a measurement path, not a generic monitor output. - Source-event geometry is published beside the AmbiX bed; render effects do not have to infer transient positions from mixed audio. - Phase/chirplet evidence is reduced into alignment, correction, suppression, and probe-control meaning before it crosses the live cache boundary. Raw phase is an estimator detail, not a renderer API. @@ -256,6 +265,7 @@ The audio code is split so the hard parts can be tested without the room, driver - `audio_field.phase_meaning` owns live extraction of meaning from phase/chirplet evidence. It consumes known reference audio plus aligned mic windows, keeps the smoothed phase/frequency state internal, and emits only actionable state for alignment, suppression, and probe control. - Room suppression owns stream-side cleanup: dialogue anchors define desired direct energy, spatial/context mics act as witnesses for room and transient clutter, and the cleaned aligned field feeds FOA/Aquarium/Faust. - Source-event analysis owns non-vocal spatial facts: dialogue focus weights and localized transient vectors are published beside the AmbiX bed, not hidden inside it. +- Aquarium/Faust owns the hot voice-separation graph after `localcast.audio.mic_field`; LocalCastBridge owns alignment, timing, roles, and control publication. - `scripts/audio_field.py` stays as the operator CLI and hardware probe surface. This is the portfolio-piece line: each module owns one invariant, and the tests use mocks/fakes instead of asking a driver stack to please be emotionally available. diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 68cfb9f..923ad35 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -43,6 +43,10 @@ - `scripts/stream_phase_field.py` publishes the live meaning document from an aligned mic field plus known program/loopback reference without exposing raw phase bands as the renderer API. - `audio_field.active_probe` wires low phase-field confidence into `ActiveProbeOptimizer`, emits bounded chirplet WAVs plus `active-probes.jsonl`, and can play probes through the default output device. - `scripts/start-live-audio-phase-field.ps1` starts the live dense harmonic confidence loop with near-ultrasonic probes; `scripts/stop-live-audio-phase-field.ps1` stops the PID recorded in `logs/audio-phase-field.pid`. +- Faust voice-separation boundary: + - `localcast.audio.mic_field` publishes aligned six-mic float32 blocks at `calibration/runs/audio-mic-field.msgpack`. + - `faust/localcast_voice_separation.dsp` is the first Aquarium-hosted graph surface for host voice, co-streamer voice, ambient, transient, and loopback stems. + - `scripts/start-live-faust-mic-field.ps1` starts the mic-field publisher for Aquarium/Faust. ## Temporary @@ -68,4 +72,4 @@ 13. Replace synthetic live-fusion observations with PS3 Eye detector observations and calibrated surface-feature tracks. 14. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present. 15. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video. -16. Wire `stream_phase_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests. +16. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests. diff --git a/faust/localcast_voice_separation.dsp b/faust/localcast_voice_separation.dsp new file mode 100644 index 0000000..cc238d5 --- /dev/null +++ b/faust/localcast_voice_separation.dsp @@ -0,0 +1,38 @@ +declare name "LocalCastBridge Voice Separation"; +declare version "1.0"; +declare author "LocalCastBridge"; +declare license "MIT"; + +// Six aligned microphone inputs: +// 0 host Focusrite cardioid +// 1 co-streamer Focusrite shotgun +// 2-3 Kiyo room/context witnesses +// 4-5 PS Eye spatial witnesses +// +// Outputs: +// 0 host voice +// 1 co-streamer voice +// 2 ambient bed +// 3 transient/witness bed +// 4 local loopback placeholder +// 5 co-streamer loopback placeholder + +hostGain = hslider("host/gain", 1.0, 0.0, 4.0, 0.01); +hostReject = hslider("host/witness_reject", 0.35, 0.0, 1.0, 0.01); +coGain = hslider("co_streamer/gain", 1.0, 0.0, 4.0, 0.01); +coReject = hslider("co_streamer/witness_reject", 0.35, 0.0, 1.0, 0.01); +ambientGain = hslider("ambient/gain", 0.6, 0.0, 2.0, 0.01); +transientGain = hslider("transients/gain", 0.7, 0.0, 2.0, 0.01); + +voice_separation(m0, m1, m2, m3, m4, m5) = host, costreamer, ambient, transients, localLoopback, coLoopback +with { + room = (m2 + m3 + m4 + m5) * 0.25; + ambient = ambientGain * room; + transients = transientGain * ((m4 + m5) * 0.5 - (m2 + m3) * 0.25); + host = hostGain * (m0 - hostReject * room); + costreamer = coGain * (m1 - coReject * room); + localLoopback = 0.0 * m0; + coLoopback = 0.0 * m1; +}; + +process = voice_separation; diff --git a/notes/current-system-map.md b/notes/current-system-map.md index f7b94cf..5fbdca9 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -60,6 +60,9 @@ flowchart TD K --> L["bounded field cache"] L --> M["aligned six-channel blocks"] M --> N["FOA encoder"] + M --> V["Faust mic-field publisher"] + V --> W["localcast.audio.mic_field"] + W --> X["Aquarium Faust voice-separation graph"] N --> O["AmbiX ACN/SN3D bus: W,Y,Z,X"] ``` @@ -73,6 +76,7 @@ Ownership: - Active probe optimization owns extra chirplet emission when confidence drops, bounded by level/spacing/audibility budget. `audio_field.active_probe` is the runtime join that converts phase-field confidence into emitted chirplet WAVs/manifests and optional default-device playback. - The camera/sensor-fusion pipeline may publish world poses later; it does not own audio clocks or channel timing. - OBS may ingest rendered output later; it is not the authority for the Ambisonic field. +- Aquarium/Faust owns hot voice separation after LocalCastBridge publishes `localcast.audio.mic_field`; LocalCastBridge owns alignment, timing, mic roles, graph id, and controls. Invariant: distributed camera/Focusrite microphones must be aligned and resampled into one reference timeline before FOA encoding. Latency is allowed as bounded buffering, but cache depth must converge toward real-time. Speaker output chirplets are live telemetry; delay/SRO/phase state must update during runtime, not only during setup. Extra chirplets may be emitted automatically when confidence drops, but only under the active probe optimizer's budget. The local shielded cardioid and neighbor shotgun are the high-quality dialogue anchors; camera mics provide spatial/context evidence. diff --git a/scripts/live_status.py b/scripts/live_status.py index 2cc602e..c16bebd 100644 --- a/scripts/live_status.py +++ b/scripts/live_status.py @@ -86,6 +86,18 @@ def msgpack_head(path: Path): "needsActiveProbe": payload[9] if len(payload) > 9 else True, } ) + elif summary["type"] == "localcast.audio.mic_field": + summary.update( + { + "frameId": payload[1], + "audioTimeNs": payload[3], + "sampleRate": payload[4], + "startSample": payload[5], + "frameCount": payload[6], + "channels": payload[7] if len(payload) > 7 else [], + "graphId": payload[9] if len(payload) > 9 else "", + } + ) elif summary["type"] in ("localcast.visual.stream_status", "localcast.audio.stream_status"): summary["payloadHead"] = head return summary @@ -110,6 +122,7 @@ def main() -> None: "visual-stream-status.msgpack", "audio-stream-status.msgpack", "audio-phase-field.msgpack", + "audio-mic-field.msgpack", "active-probes/active-probes.jsonl", "stream-spout-status.json", "av-sync-status.json", @@ -124,6 +137,7 @@ def main() -> None: "audioHead": msgpack_head(runs / "audio-state.msgpack"), "eventsHead": msgpack_head(runs / "audio-events.msgpack"), "phaseFieldHead": msgpack_head(runs / "audio-phase-field.msgpack"), + "micFieldHead": msgpack_head(runs / "audio-mic-field.msgpack"), "lastActiveProbe": read_last_jsonl(runs / "active-probes" / "active-probes.jsonl"), } print(json.dumps(payload, indent=2, default=str)) diff --git a/scripts/start-live-faust-mic-field.ps1 b/scripts/start-live-faust-mic-field.ps1 new file mode 100644 index 0000000..99c3791 --- /dev/null +++ b/scripts/start-live-faust-mic-field.ps1 @@ -0,0 +1,35 @@ +param( + [string]$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path, + [string]$Python = ".\.venv\Scripts\python.exe", + [string]$InputPath = ".\calibration\runs\audio-program-live-20260518-180226\field-program-cleaned.wav", + [string]$Cache = ".\calibration\runs\audio-mic-field.msgpack" +) + +$ErrorActionPreference = "Stop" + +Set-Location -LiteralPath $Root + +$logDir = Join-Path $Root "logs" +New-Item -ItemType Directory -Force -Path $logDir | Out-Null + +$resolvedPython = if ([System.IO.Path]::IsPathRooted($Python)) { $Python } else { Join-Path $Root $Python } +$stdout = Join-Path $logDir "faust-mic-field.stdout.log" +$stderr = Join-Path $logDir "faust-mic-field.stderr.log" +$pidPath = Join-Path $logDir "faust-mic-field.pid" + +$arguments = @( + ".\scripts\stream_faust_mic_field.py", + "--input", $InputPath, + "--cache", $Cache, + "--loop", + "--realtime", + "--smoke-readback" +) + +$process = Start-Process -FilePath $resolvedPython -ArgumentList $arguments -RedirectStandardOutput $stdout -RedirectStandardError $stderr -PassThru -WindowStyle Hidden +Set-Content -Path $pidPath -Value ([string]$process.Id) -Encoding ASCII + +Write-Host "Started live Faust mic-field publisher PID $($process.Id)" +Write-Host "stdout: $stdout" +Write-Host "stderr: $stderr" +Write-Host "pid: $pidPath" diff --git a/scripts/stop-live-faust-mic-field.ps1 b/scripts/stop-live-faust-mic-field.ps1 new file mode 100644 index 0000000..af94f6b --- /dev/null +++ b/scripts/stop-live-faust-mic-field.ps1 @@ -0,0 +1,30 @@ +param( + [string]$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +) + +$ErrorActionPreference = "Stop" + +Set-Location -LiteralPath $Root + +$pidPath = Join-Path $Root "logs\faust-mic-field.pid" +if (-not (Test-Path -LiteralPath $pidPath)) { + Write-Host "No live Faust mic-field PID file found." + exit 0 +} + +$pidText = (Get-Content -LiteralPath $pidPath -Raw).Trim() +if ($pidText.Length -eq 0) { + Remove-Item -LiteralPath $pidPath -Force + Write-Host "Empty PID file removed." + exit 0 +} + +$process = Get-Process -Id ([int]$pidText) -ErrorAction SilentlyContinue +if ($null -ne $process) { + Stop-Process -Id $process.Id -Force + Write-Host "Stopped live Faust mic-field PID $($process.Id)" +} else { + Write-Host "Live Faust mic-field PID $pidText is not running." +} + +Remove-Item -LiteralPath $pidPath -Force diff --git a/scripts/stream_faust_mic_field.py b/scripts/stream_faust_mic_field.py new file mode 100644 index 0000000..e0d601b --- /dev/null +++ b/scripts/stream_faust_mic_field.py @@ -0,0 +1,104 @@ +import argparse +from pathlib import Path +import sys +import time + +import numpy as np +from scipy.io import wavfile + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from audio_field.cultcache_audio import ( # noqa: E402 + DEFAULT_MIC_FIELD_CHANNELS, + get_live_mic_field_frame, + make_mic_field_frame, + put_live_mic_field_frame, +) + + +def read_float_wav(path: Path) -> tuple[int, np.ndarray]: + rate, data = wavfile.read(path) + samples = data.astype(np.float32) + if np.issubdtype(data.dtype, np.integer): + samples /= np.iinfo(data.dtype).max + if samples.ndim == 1: + samples = samples[:, None] + return int(rate), samples + + +def source_block(samples: np.ndarray, start_sample: int, frame_count: int, *, loop: bool) -> np.ndarray | None: + if start_sample >= len(samples) and not loop: + return None + block = np.zeros((frame_count, samples.shape[1]), dtype=np.float32) + written = 0 + cursor = start_sample + while written < frame_count: + if cursor >= len(samples): + if not loop: + break + cursor %= len(samples) + count = min(frame_count - written, len(samples) - cursor) + block[written : written + count] = samples[cursor : cursor + count] + written += count + cursor += count + return block + + +def main() -> None: + parser = argparse.ArgumentParser(description="Publish aligned mic channels as a Faust-ready voice-separation input field.") + parser.add_argument("--input", type=Path, default=ROOT / "calibration" / "runs" / "audio-program-live-20260518-180226" / "field-program-cleaned.wav") + parser.add_argument("--cache", type=Path, default=ROOT / "calibration" / "runs" / "audio-mic-field.msgpack") + parser.add_argument("--graph-id", default="localcast.faust.voice_separation.v1") + parser.add_argument("--channel", action="append") + parser.add_argument("--chunk-frames", type=int, default=1024) + parser.add_argument("--duration", type=float) + parser.add_argument("--loop", action="store_true") + parser.add_argument("--realtime", action="store_true") + parser.add_argument("--smoke-readback", action="store_true") + args = parser.parse_args() + + sample_rate, source = read_float_wav(args.input) + channels = tuple(args.channel) if args.channel else DEFAULT_MIC_FIELD_CHANNELS + if len(channels) != source.shape[1]: + raise SystemExit(f"input has {source.shape[1]} channels but {len(channels)} channel ids were provided") + + start_monotonic = time.monotonic() + audio_origin_ns = time.monotonic_ns() + frame_id = 0 + start_sample = 0 + while True: + now = time.monotonic() + if args.duration is not None and now - start_monotonic >= args.duration: + break + block = source_block(source, start_sample, args.chunk_frames, loop=args.loop) + if block is None: + break + audio_time_ns = audio_origin_ns + int(start_sample * 1_000_000_000 / sample_rate) + frame = make_mic_field_frame( + block, + frame_id=frame_id, + sample_rate=sample_rate, + start_sample=start_sample, + audio_time_ns=audio_time_ns, + channels=channels, + graph_id=args.graph_id, + ) + put_live_mic_field_frame(args.cache, frame) + if args.smoke_readback: + loaded = get_live_mic_field_frame(args.cache) + if loaded is None or loaded.frame_id != frame_id: + raise SystemExit("Mic-field CultCache smoke readback failed") + frame_id += 1 + start_sample += args.chunk_frames + if args.realtime: + target_time = start_monotonic + start_sample / sample_rate + sleep_for = target_time - time.monotonic() + if sleep_for > 0: + time.sleep(sleep_for) + + +if __name__ == "__main__": + main() diff --git a/state/map.yaml b/state/map.yaml index d71e475..4657a97 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -139,6 +139,15 @@ research: harmonic near-ultrasonic probe textures so aliases have a musical relationship instead of arbitrary squeal. This is learned audio-field state, not a phase display. + faust_voice_separation: + summary: docs/aquarium-spatial-audio.md + key_decision: > + Voice separation should run in Aquarium's Faust DSP, not as Python hot + audio math after the mic field has already collapsed to AmbiX. + LocalCastBridge now publishes `localcast.audio.mic_field` with six + aligned mic channels, roles, controls, and graph id + `localcast.faust.voice_separation.v1`; the matching Faust graph lives at + `faust/localcast_voice_separation.dsp`. sensor_fusion_architecture: summary: docs/sensor-fusion-architecture.md key_decision: > diff --git a/tests/test_spatial_audio_cultcache.py b/tests/test_spatial_audio_cultcache.py index e641377..d1e6b9a 100644 --- a/tests/test_spatial_audio_cultcache.py +++ b/tests/test_spatial_audio_cultcache.py @@ -9,12 +9,16 @@ from audio_field.cultcache_audio import ( frame_to_numpy, get_live_audio_phase_field, get_live_audio_source_events, + get_live_mic_field_frame, get_live_spatial_audio_frame, make_audio_phase_field, make_audio_source_events, + make_mic_field_frame, make_spatial_audio_frame, + mic_field_to_numpy, put_live_audio_phase_field, put_live_audio_source_events, + put_live_mic_field_frame, put_live_spatial_audio_frame, ) @@ -101,6 +105,27 @@ class SpatialAudioCultCacheTests(unittest.TestCase): self.assertEqual(12.5, loaded.sources[0]["delaySamples"]) self.assertFalse(loaded.needs_active_probe) + def test_live_mic_field_round_trip_through_cultcache(self): + block = np.arange(48, dtype=np.float32).reshape(8, 6) / 48.0 + frame = make_mic_field_frame( + block, + frame_id=11, + sample_rate=48000, + start_sample=512, + audio_time_ns=789, + ) + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "audio-mic-field.msgpack" + put_live_mic_field_frame(path, frame) + loaded = get_live_mic_field_frame(path) + + self.assertIsNotNone(loaded) + self.assertEqual("localcast.faust.voice_separation.v1", loaded.graph_id) + self.assertEqual(6, len(loaded.channels)) + self.assertEqual("host-voice-anchor", loaded.roles[0]["role"]) + np.testing.assert_allclose(block, mic_field_to_numpy(loaded)) + if __name__ == "__main__": unittest.main()