Add audio source event vector field
This commit is contained in:
parent
c0108094a5
commit
0384df9f57
7 changed files with 508 additions and 1 deletions
|
|
@ -21,6 +21,8 @@ LIVE_SPATIAL_AUDIO_KEY = "localcast.audio.spatial-frame.live"
|
|||
SPATIAL_AUDIO_SCHEMA_ID = "gamecult.localcast.audio.spatial_frame.v1"
|
||||
AUDIO_STREAM_STATUS_KEY = "localcast.audio.stream-status.live"
|
||||
AUDIO_STREAM_STATUS_SCHEMA_ID = "gamecult.localcast.audio.stream_status.v1"
|
||||
LIVE_SOURCE_EVENTS_KEY = "localcast.audio.source-events.live"
|
||||
SOURCE_EVENTS_SCHEMA_ID = "gamecult.localcast.audio.source_events.v1"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -51,6 +53,18 @@ class CultAudioStreamStatus:
|
|||
last_error: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CultAudioSourceEvents:
|
||||
schema_version: str
|
||||
frame_id: int
|
||||
audio_time_ns: int
|
||||
sample_rate: int
|
||||
start_sample: int
|
||||
frame_count: int
|
||||
events: tuple[dict[str, Any], ...]
|
||||
voice_focus: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
def _pack(value: Any) -> bytes:
|
||||
import msgpack
|
||||
|
||||
|
|
@ -123,6 +137,32 @@ def _decode_audio_status(raw: Any) -> CultAudioStreamStatus:
|
|||
)
|
||||
|
||||
|
||||
def _encode_source_events(events: CultAudioSourceEvents) -> list[Any]:
|
||||
return [
|
||||
events.schema_version,
|
||||
events.frame_id,
|
||||
events.audio_time_ns,
|
||||
events.sample_rate,
|
||||
events.start_sample,
|
||||
events.frame_count,
|
||||
list(events.events),
|
||||
list(events.voice_focus),
|
||||
]
|
||||
|
||||
|
||||
def _decode_source_events(raw: Any) -> CultAudioSourceEvents:
|
||||
return CultAudioSourceEvents(
|
||||
schema_version=str(raw[0]),
|
||||
frame_id=int(raw[1]),
|
||||
audio_time_ns=int(raw[2]),
|
||||
sample_rate=int(raw[3]),
|
||||
start_sample=int(raw[4]),
|
||||
frame_count=int(raw[5]),
|
||||
events=tuple(dict(item) for item in raw[6]),
|
||||
voice_focus=tuple(dict(item) for item in raw[7]),
|
||||
)
|
||||
|
||||
|
||||
SPATIAL_AUDIO_FRAME_DOCUMENT = define_document_type(
|
||||
"localcast.audio.spatial_frame",
|
||||
encode=_encode_audio_frame,
|
||||
|
|
@ -141,6 +181,15 @@ AUDIO_STREAM_STATUS_DOCUMENT = define_document_type(
|
|||
payload_decoder=_unpack,
|
||||
)
|
||||
|
||||
SOURCE_EVENTS_DOCUMENT = define_document_type(
|
||||
"localcast.audio.source_events",
|
||||
encode=_encode_source_events,
|
||||
decode=_decode_source_events,
|
||||
name=lambda events: LIVE_SOURCE_EVENTS_KEY if isinstance(events, CultAudioSourceEvents) else None,
|
||||
payload_encoder=_pack,
|
||||
payload_decoder=_unpack,
|
||||
)
|
||||
|
||||
|
||||
def _retry_cache_io(operation):
|
||||
last_error: Exception | None = None
|
||||
|
|
@ -159,6 +208,7 @@ def _open_audio_cache_once(path: Path) -> CultCache:
|
|||
CultCache.builder()
|
||||
.register_document_type(SPATIAL_AUDIO_FRAME_DOCUMENT)
|
||||
.register_document_type(AUDIO_STREAM_STATUS_DOCUMENT)
|
||||
.register_document_type(SOURCE_EVENTS_DOCUMENT)
|
||||
.add_generic_store(SingleFileMessagePackBackingStore(path))
|
||||
.build()
|
||||
)
|
||||
|
|
@ -231,3 +281,41 @@ def put_audio_stream_status(path: Path, status: CultAudioStreamStatus) -> None:
|
|||
cache.put(AUDIO_STREAM_STATUS_DOCUMENT, AUDIO_STREAM_STATUS_KEY, status)
|
||||
|
||||
_retry_cache_io(operation)
|
||||
|
||||
|
||||
def make_audio_source_events(
|
||||
*,
|
||||
frame_id: int,
|
||||
sample_rate: int,
|
||||
start_sample: int,
|
||||
frame_count: int,
|
||||
audio_time_ns: int,
|
||||
events: list[dict[str, Any]],
|
||||
voice_focus: list[dict[str, Any]],
|
||||
) -> CultAudioSourceEvents:
|
||||
return CultAudioSourceEvents(
|
||||
schema_version=SOURCE_EVENTS_SCHEMA_ID,
|
||||
frame_id=int(frame_id),
|
||||
audio_time_ns=int(audio_time_ns),
|
||||
sample_rate=int(sample_rate),
|
||||
start_sample=int(start_sample),
|
||||
frame_count=int(frame_count),
|
||||
events=tuple(dict(item) for item in events),
|
||||
voice_focus=tuple(dict(item) for item in voice_focus),
|
||||
)
|
||||
|
||||
|
||||
def put_live_audio_source_events(path: Path, events: CultAudioSourceEvents) -> None:
|
||||
def operation() -> None:
|
||||
cache = open_audio_cache(path)
|
||||
cache.put(SOURCE_EVENTS_DOCUMENT, LIVE_SOURCE_EVENTS_KEY, events)
|
||||
|
||||
_retry_cache_io(operation)
|
||||
|
||||
|
||||
def get_live_audio_source_events(path: Path) -> CultAudioSourceEvents | None:
|
||||
def operation() -> CultAudioSourceEvents | None:
|
||||
cache = open_audio_cache(path)
|
||||
return cache.get(SOURCE_EVENTS_DOCUMENT, LIVE_SOURCE_EVENTS_KEY)
|
||||
|
||||
return _retry_cache_io(operation)
|
||||
|
|
|
|||
168
audio_field/source_events.py
Normal file
168
audio_field/source_events.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
from typing import Iterable
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceEvent:
|
||||
event_id: int
|
||||
start_sample: int
|
||||
duration_samples: int
|
||||
position_m: tuple[float, float, float]
|
||||
direction_m: tuple[float, float, float]
|
||||
energy: float
|
||||
confidence: float
|
||||
kind: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VoiceFocusFrame:
|
||||
start_sample: int
|
||||
duration_samples: int
|
||||
anchor_weights: dict[int, float]
|
||||
witness_energy: float
|
||||
anchor_energy: float
|
||||
noise_ratio: float
|
||||
|
||||
|
||||
def analyze_source_field(
|
||||
field: np.ndarray,
|
||||
sample_rate: int,
|
||||
mic_positions: dict[int, tuple[float, float, float]],
|
||||
anchor_channels: Iterable[int],
|
||||
witness_channels: Iterable[int],
|
||||
*,
|
||||
block_size: int = 1024,
|
||||
hop_size: int = 512,
|
||||
transient_ratio: float = 2.5,
|
||||
min_event_energy: float = 1e-6,
|
||||
onset_ratio: float = 1.8,
|
||||
min_event_spacing_samples: int = 2048,
|
||||
) -> tuple[list[SourceEvent], list[VoiceFocusFrame]]:
|
||||
samples = np.asarray(field, dtype=np.float32)
|
||||
if samples.ndim != 2:
|
||||
raise ValueError("source field must be frames-by-channels")
|
||||
anchors = [int(channel) for channel in anchor_channels if int(channel) < samples.shape[1]]
|
||||
witnesses = [int(channel) for channel in witness_channels if int(channel) < samples.shape[1]]
|
||||
if not witnesses:
|
||||
return [], []
|
||||
block_size = int(block_size)
|
||||
hop_size = int(hop_size)
|
||||
if block_size <= 0 or hop_size <= 0:
|
||||
raise ValueError("block_size and hop_size must be positive")
|
||||
|
||||
events: list[SourceEvent] = []
|
||||
focus: list[VoiceFocusFrame] = []
|
||||
candidates = []
|
||||
window = np.hanning(block_size).astype(np.float32)
|
||||
previous_witness_energy = 0.0
|
||||
for start in range(0, max(0, samples.shape[0] - block_size + 1), hop_size):
|
||||
block = samples[start : start + block_size] * window[:, None]
|
||||
witness_energy_by_channel = {
|
||||
channel: rms_energy(block[:, channel])
|
||||
for channel in witnesses
|
||||
if channel in mic_positions
|
||||
}
|
||||
anchor_energy_by_channel = {channel: rms_energy(block[:, channel]) for channel in anchors}
|
||||
witness_energy = float(sum(witness_energy_by_channel.values()))
|
||||
anchor_energy = float(sum(anchor_energy_by_channel.values()))
|
||||
noise_ratio = witness_energy / max(anchor_energy, 1e-12)
|
||||
focus.append(
|
||||
VoiceFocusFrame(
|
||||
start_sample=start,
|
||||
duration_samples=block_size,
|
||||
anchor_weights=anchor_mix_weights(anchor_energy_by_channel),
|
||||
witness_energy=witness_energy,
|
||||
anchor_energy=anchor_energy,
|
||||
noise_ratio=noise_ratio,
|
||||
)
|
||||
)
|
||||
onset = witness_energy / max(previous_witness_energy, 1e-12)
|
||||
previous_witness_energy = witness_energy
|
||||
if witness_energy < min_event_energy or noise_ratio < transient_ratio or onset < onset_ratio:
|
||||
continue
|
||||
position, confidence = localize_energy_centroid(witness_energy_by_channel, mic_positions)
|
||||
candidates.append(
|
||||
(
|
||||
start,
|
||||
SourceEvent(
|
||||
event_id=-1,
|
||||
start_sample=start,
|
||||
duration_samples=block_size,
|
||||
position_m=position,
|
||||
direction_m=unit_vector(position),
|
||||
energy=witness_energy,
|
||||
confidence=confidence,
|
||||
kind="witness-dominant-transient",
|
||||
),
|
||||
)
|
||||
)
|
||||
events = suppress_nearby_events([event for _, event in candidates], int(min_event_spacing_samples))
|
||||
return events, focus
|
||||
|
||||
|
||||
def rms_energy(samples: np.ndarray) -> float:
|
||||
if samples.size == 0:
|
||||
return 0.0
|
||||
return float(np.mean(np.asarray(samples, dtype=np.float64) ** 2))
|
||||
|
||||
|
||||
def anchor_mix_weights(energies: dict[int, float]) -> dict[int, float]:
|
||||
if not energies:
|
||||
return {}
|
||||
floor = max(max(energies.values()) * 1e-6, 1e-12)
|
||||
total = sum(max(value, floor) for value in energies.values())
|
||||
return {channel: max(value, floor) / total for channel, value in sorted(energies.items())}
|
||||
|
||||
|
||||
def localize_energy_centroid(
|
||||
energies: dict[int, float],
|
||||
mic_positions: dict[int, tuple[float, float, float]],
|
||||
) -> tuple[tuple[float, float, float], float]:
|
||||
usable = [(channel, max(float(energy), 0.0)) for channel, energy in energies.items() if energy > 0.0]
|
||||
if not usable:
|
||||
return (0.0, 0.0, 0.0), 0.0
|
||||
weights = np.asarray([energy for _, energy in usable], dtype=np.float64)
|
||||
positions = np.asarray([mic_positions[channel] for channel, _ in usable], dtype=np.float64)
|
||||
total = float(np.sum(weights))
|
||||
centroid = np.sum(positions * weights[:, None], axis=0) / total
|
||||
dominance = float(np.max(weights) / total)
|
||||
spread = float(np.sqrt(np.average(np.sum((positions - centroid) ** 2, axis=1), weights=weights)))
|
||||
confidence = max(0.0, min(1.0, 0.35 + dominance - 0.15 * spread))
|
||||
return tuple(float(value) for value in centroid), confidence
|
||||
|
||||
|
||||
def suppress_nearby_events(events: list[SourceEvent], min_spacing_samples: int) -> list[SourceEvent]:
|
||||
if not events:
|
||||
return []
|
||||
min_spacing_samples = max(0, int(min_spacing_samples))
|
||||
kept: list[SourceEvent] = []
|
||||
for event in sorted(events, key=lambda item: item.energy, reverse=True):
|
||||
if any(abs(event.start_sample - existing.start_sample) < min_spacing_samples for existing in kept):
|
||||
continue
|
||||
kept.append(event)
|
||||
kept.sort(key=lambda item: item.start_sample)
|
||||
return [
|
||||
SourceEvent(
|
||||
event_id=index,
|
||||
start_sample=event.start_sample,
|
||||
duration_samples=event.duration_samples,
|
||||
position_m=event.position_m,
|
||||
direction_m=event.direction_m,
|
||||
energy=event.energy,
|
||||
confidence=event.confidence,
|
||||
kind=event.kind,
|
||||
)
|
||||
for index, event in enumerate(kept)
|
||||
]
|
||||
|
||||
|
||||
def unit_vector(position: tuple[float, float, float]) -> tuple[float, float, float]:
|
||||
norm = math.sqrt(sum(value * value for value in position))
|
||||
if norm <= 1e-12:
|
||||
return (0.0, 0.0, 0.0)
|
||||
return tuple(float(value / norm) for value in position)
|
||||
|
|
@ -11,15 +11,20 @@ flowchart TD
|
|||
A["aligned microphone field"] --> B["FOA encoder"]
|
||||
B --> C["AmbiX ACN/SN3D W,Y,Z,X blocks"]
|
||||
C --> D["localcast.audio.spatial_frame"]
|
||||
A --> H["source-event analysis"]
|
||||
H --> I["localcast.audio.source_events"]
|
||||
D --> E["audio-state.msgpack / CultNet document_put"]
|
||||
I --> J["audio-events.msgpack / CultNet document_put"]
|
||||
E --> F["Aquarium client"]
|
||||
F --> G["native Faust spatial DSP"]
|
||||
J --> F
|
||||
F --> G["native Faust spatial DSP + volumetric renderer"]
|
||||
```
|
||||
|
||||
Live files:
|
||||
|
||||
- `calibration/runs/audio-state.msgpack`: latest typed AmbiX audio block.
|
||||
- `calibration/runs/audio-stream-status.msgpack`: audio publisher heartbeat.
|
||||
- `calibration/runs/audio-events.msgpack`: latest typed dialogue-focus and transient vector field.
|
||||
|
||||
Document type:
|
||||
|
||||
|
|
@ -29,15 +34,30 @@ Document type:
|
|||
- payload: MessagePack array decoded as `CultSpatialAudioFrame`
|
||||
- audio format: interleaved `float32`, 48 kHz, AmbiX ACN/SN3D channels `W,Y,Z,X`
|
||||
|
||||
Source event document type:
|
||||
|
||||
- `localcast.audio.source_events`
|
||||
- schema id: `gamecult.localcast.audio.source_events.v1`
|
||||
- key: `localcast.audio.source-events.live`
|
||||
- payload: MessagePack array decoded as `CultAudioSourceEvents`
|
||||
- contents: dialogue anchor weights plus witness-dominant transient events with sample time, estimated room position, direction, energy, confidence, and kind.
|
||||
|
||||
Publisher smoke command:
|
||||
|
||||
```powershell
|
||||
.\.venv\Scripts\python.exe .\scripts\stream_spatial_audio.py --input .\calibration\runs\audio-full-sync-20260518-165751\field-foa-ambix.wav --loop --duration 5 --smoke-readback
|
||||
```
|
||||
|
||||
Source-event analysis command:
|
||||
|
||||
```powershell
|
||||
.\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-source-events --profile .\calibration\runs\audio-field-loud-profile.json --input .\calibration\runs\<run-folder>\field-cleaned.wav --cache .\calibration\runs\audio-events.msgpack
|
||||
```
|
||||
|
||||
## Invariants
|
||||
|
||||
- Aquarium owns rendering and Faust DSP, not microphone capture or clock correction.
|
||||
- LocalCastBridge publishes declared AmbiX blocks, not undecoded heterogeneous microphone channels.
|
||||
- LocalCastBridge publishes source-event geometry separately from AmbiX audio; render effects do not have to infer transient positions from the sound bed.
|
||||
- The audio frame carries `start_sample` and `audio_time_ns` so visual packets can align against the same bounded-latency timeline.
|
||||
- OBS may receive a rendered monitor output later, but OBS is not the spatial bus authority.
|
||||
|
|
|
|||
|
|
@ -182,6 +182,14 @@ Suppress known speaker/program bleed using output loopback as ground truth:
|
|||
|
||||
This is the music-as-both-enemy-and-teacher path. The reference program audio is transformed into a time/frequency transfer estimate per mic channel, reconstructed as predicted bleed, subtracted from the aligned field, and exported as phase/frequency mapping evidence. A future live loop should publish the known program stem as its own OBS/Aquarium channel while using the same ground truth to keep learning the room.
|
||||
|
||||
Estimate dialogue focus and volumetric transient events:
|
||||
|
||||
```powershell
|
||||
.\.venv\Scripts\python.exe .\scripts\audio_field.py analyze-source-events --profile .\config\audio-field.json --input .\calibration\runs\<run-folder>\field-cleaned.wav --cache .\calibration\runs\audio-events.msgpack
|
||||
```
|
||||
|
||||
This emits `localcast.audio.source_events` for Aquarium. Dialogue anchors produce voice-focus weights; camera/context mics produce witness-dominant transient events with sample time, estimated room position, direction, energy, and confidence. This is the boundary between a clean vocal bed and renderer-visible clutter: clicks, taps, and other spatial transients should become geometry instead of being smeared into the voice mix.
|
||||
|
||||
The `play-record` and `record-field` commands remain for a `shared-input-device` profile only. They deliberately refuse the distributed rig until an alignment stage emits one coherent six-channel WAV.
|
||||
|
||||
## Invariants
|
||||
|
|
@ -198,6 +206,7 @@ The `play-record` and `record-field` commands remain for a `shared-input-device`
|
|||
- Active chirplet probes are scheduled by an optimization loop, not by panic. It should prefer masked windows, respect minimum spacing and level caps, and maximize expected confidence gain per audible intrusion.
|
||||
- FOA output is AmbiX: ACN channel order, SN3D normalization, channels `W,Y,Z,X`.
|
||||
- Speaker calibration is a measurement path, not a generic monitor output.
|
||||
- Source-event geometry is published beside the AmbiX bed; render effects do not have to infer transient positions from mixed audio.
|
||||
- Camera fusion may publish listener or source pose later, but it does not own audio channel timing.
|
||||
|
||||
## Module Boundaries
|
||||
|
|
@ -211,6 +220,7 @@ The audio code is split so the hard parts can be tested without the room, driver
|
|||
- The next runtime estimator module should own chirplet observations, delay/SRO/phase state, confidence gates, and phase-field updates. It should feed the aligner; it should not be hidden inside the FOA encoder.
|
||||
- The active probe optimizer owns whether to emit extra chirplets. It reads sync confidence and probe budget, then decides when and where an intentional chirplet is worth the cost.
|
||||
- Room suppression owns stream-side cleanup: dialogue anchors define desired direct energy, spatial/context mics act as witnesses for room and transient clutter, and the cleaned aligned field feeds FOA/Aquarium/Faust.
|
||||
- Source-event analysis owns non-vocal spatial facts: dialogue focus weights and localized transient vectors are published beside the AmbiX bed, not hidden inside it.
|
||||
- `scripts/audio_field.py` stays as the operator CLI and hardware probe surface.
|
||||
|
||||
This is the portfolio-piece line: each module owns one invariant, and the tests use mocks/fakes instead of asking a driver stack to please be emotionally available.
|
||||
|
|
|
|||
|
|
@ -60,6 +60,28 @@ _program_reference = load_program_reference_module()
|
|||
suppress_program_reference = _program_reference.suppress_program_reference
|
||||
|
||||
|
||||
def load_source_events_module():
|
||||
spec = importlib.util.spec_from_file_location("localcast_source_events", ROOT / "audio_field" / "source_events.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
_source_events = load_source_events_module()
|
||||
analyze_source_field = _source_events.analyze_source_field
|
||||
|
||||
|
||||
def load_cultcache_audio_module():
|
||||
spec = importlib.util.spec_from_file_location("localcast_cultcache_audio", ROOT / "audio_field" / "cultcache_audio.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def utc_stamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
|
||||
|
|
@ -1818,6 +1840,87 @@ def cmd_suppress_reference(args):
|
|||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
def cmd_analyze_source_events(args):
|
||||
profile = load_profile(args.profile)
|
||||
sample_rate, data = wavfile.read(args.input)
|
||||
if int(sample_rate) != int(profile["sampleRate"]):
|
||||
raise SystemExit(f"Input sample-rate {sample_rate} does not match profile {profile['sampleRate']}")
|
||||
field = as_float_matrix(data)
|
||||
anchors = infer_anchor_channels(profile) if not args.anchor_channel else [int(value) for value in args.anchor_channel]
|
||||
witnesses = infer_witness_channels(profile, anchors) if not args.witness_channel else [int(value) for value in args.witness_channel]
|
||||
mic_positions = {
|
||||
mic_channel(mic): tuple(float(value) for value in mic["positionMeters"])
|
||||
for mic in profile["microphones"]
|
||||
if len(mic.get("positionMeters", [])) == 3
|
||||
}
|
||||
events, focus = analyze_source_field(
|
||||
field,
|
||||
int(sample_rate),
|
||||
mic_positions,
|
||||
anchors,
|
||||
witnesses,
|
||||
block_size=int(args.block_size),
|
||||
hop_size=int(args.hop_size),
|
||||
transient_ratio=float(args.transient_ratio),
|
||||
min_event_energy=float(args.min_event_energy),
|
||||
onset_ratio=float(args.onset_ratio),
|
||||
min_event_spacing_samples=int(args.min_event_spacing_samples),
|
||||
)
|
||||
result = {
|
||||
"input": str(args.input),
|
||||
"sampleRate": int(sample_rate),
|
||||
"anchors": anchors,
|
||||
"witnesses": witnesses,
|
||||
"eventCount": len(events),
|
||||
"events": [
|
||||
{
|
||||
"eventId": event.event_id,
|
||||
"startSample": event.start_sample,
|
||||
"durationSamples": event.duration_samples,
|
||||
"timeSeconds": event.start_sample / float(sample_rate),
|
||||
"positionMeters": list(event.position_m),
|
||||
"direction": list(event.direction_m),
|
||||
"energy": event.energy,
|
||||
"confidence": event.confidence,
|
||||
"kind": event.kind,
|
||||
}
|
||||
for event in events
|
||||
],
|
||||
"voiceFocus": [
|
||||
{
|
||||
"startSample": item.start_sample,
|
||||
"durationSamples": item.duration_samples,
|
||||
"anchorWeights": {str(channel): weight for channel, weight in item.anchor_weights.items()},
|
||||
"witnessEnergy": item.witness_energy,
|
||||
"anchorEnergy": item.anchor_energy,
|
||||
"noiseRatio": item.noise_ratio,
|
||||
}
|
||||
for item in focus
|
||||
],
|
||||
"notes": "Dialogue anchors are the protected voice bed; witness-dominant transients are emitted as estimated volumetric events for the renderer.",
|
||||
}
|
||||
output = args.output or args.input.with_suffix(".source-events.json")
|
||||
write_json(output, result)
|
||||
if args.cache:
|
||||
cultcache_audio = load_cultcache_audio_module()
|
||||
|
||||
start_sample = int(result["voiceFocus"][0]["startSample"]) if result["voiceFocus"] else 0
|
||||
frame_count = int(field.shape[0])
|
||||
cultcache_audio.put_live_audio_source_events(
|
||||
args.cache,
|
||||
cultcache_audio.make_audio_source_events(
|
||||
frame_id=int(time.monotonic_ns() % (2**31)),
|
||||
sample_rate=int(sample_rate),
|
||||
start_sample=start_sample,
|
||||
frame_count=frame_count,
|
||||
audio_time_ns=time.monotonic_ns(),
|
||||
events=result["events"][: int(args.max_cache_events)],
|
||||
voice_focus=result["voiceFocus"][-int(args.max_cache_focus) :],
|
||||
),
|
||||
)
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
def infer_anchor_channels(profile):
|
||||
anchors = []
|
||||
for mic in profile["microphones"]:
|
||||
|
|
@ -2110,6 +2213,23 @@ def main():
|
|||
p.add_argument("--subtraction-strength", type=float, default=0.85)
|
||||
p.set_defaults(func=cmd_suppress_reference)
|
||||
|
||||
p = sub.add_parser("analyze-source-events", help="Estimate dialogue focus and witness-dominant volumetric transient events from an aligned field.")
|
||||
p.add_argument("--profile", type=Path, default=ROOT / "config" / "audio-field.example.json")
|
||||
p.add_argument("--input", type=Path, required=True)
|
||||
p.add_argument("--output", type=Path)
|
||||
p.add_argument("--cache", type=Path)
|
||||
p.add_argument("--anchor-channel", type=int, action="append")
|
||||
p.add_argument("--witness-channel", type=int, action="append")
|
||||
p.add_argument("--block-size", type=int, default=1024)
|
||||
p.add_argument("--hop-size", type=int, default=512)
|
||||
p.add_argument("--transient-ratio", type=float, default=2.5)
|
||||
p.add_argument("--min-event-energy", type=float, default=1e-6)
|
||||
p.add_argument("--onset-ratio", type=float, default=1.8)
|
||||
p.add_argument("--min-event-spacing-samples", type=int, default=2048)
|
||||
p.add_argument("--max-cache-events", type=int, default=128)
|
||||
p.add_argument("--max-cache-focus", type=int, default=64)
|
||||
p.set_defaults(func=cmd_analyze_source_events)
|
||||
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
||||
|
|
|
|||
77
tests/test_source_events.py
Normal file
77
tests/test_source_events.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from audio_field.source_events import analyze_source_field
|
||||
|
||||
|
||||
class SourceEventTests(unittest.TestCase):
|
||||
def test_witness_transient_becomes_positioned_event(self):
|
||||
field = np.zeros((4096, 4), dtype=np.float32)
|
||||
field[1024:1280, 2] = 0.8
|
||||
field[1024:1280, 3] = 0.2
|
||||
positions = {
|
||||
0: (0.0, 0.0, 1.2),
|
||||
1: (0.0, 2.0, 1.2),
|
||||
2: (-1.0, 0.0, 1.0),
|
||||
3: (1.0, 0.0, 1.0),
|
||||
}
|
||||
|
||||
events, focus = analyze_source_field(
|
||||
field,
|
||||
48000,
|
||||
positions,
|
||||
anchor_channels=[0, 1],
|
||||
witness_channels=[2, 3],
|
||||
block_size=512,
|
||||
hop_size=256,
|
||||
transient_ratio=2.0,
|
||||
)
|
||||
|
||||
self.assertTrue(events)
|
||||
strongest = max(events, key=lambda event: event.energy)
|
||||
self.assertLess(strongest.position_m[0], -0.5)
|
||||
self.assertEqual("witness-dominant-transient", strongest.kind)
|
||||
self.assertGreater(strongest.confidence, 0.5)
|
||||
self.assertTrue(focus)
|
||||
|
||||
def test_anchor_weights_follow_voice_energy(self):
|
||||
field = np.zeros((2048, 3), dtype=np.float32)
|
||||
field[:, 0] = 0.2
|
||||
field[:, 1] = 0.6
|
||||
positions = {0: (0, 0, 1), 1: (0, 2, 1), 2: (1, 1, 1)}
|
||||
|
||||
_, focus = analyze_source_field(
|
||||
field,
|
||||
48000,
|
||||
positions,
|
||||
anchor_channels=[0, 1],
|
||||
witness_channels=[2],
|
||||
block_size=512,
|
||||
hop_size=512,
|
||||
)
|
||||
|
||||
self.assertGreater(focus[0].anchor_weights[1], focus[0].anchor_weights[0])
|
||||
self.assertLess(focus[0].noise_ratio, 0.01)
|
||||
|
||||
def test_steady_witness_texture_does_not_become_event_spam(self):
|
||||
field = np.zeros((4096, 2), dtype=np.float32)
|
||||
field[:, 1] = 0.2
|
||||
positions = {0: (0, 0, 1), 1: (1, 0, 1)}
|
||||
|
||||
events, _ = analyze_source_field(
|
||||
field,
|
||||
48000,
|
||||
positions,
|
||||
anchor_channels=[0],
|
||||
witness_channels=[1],
|
||||
block_size=512,
|
||||
hop_size=256,
|
||||
transient_ratio=2.0,
|
||||
)
|
||||
|
||||
self.assertLessEqual(len(events), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -7,8 +7,11 @@ import numpy as np
|
|||
from audio_field.cultcache_audio import (
|
||||
SPATIAL_AUDIO_SCHEMA_ID,
|
||||
frame_to_numpy,
|
||||
get_live_audio_source_events,
|
||||
get_live_spatial_audio_frame,
|
||||
make_audio_source_events,
|
||||
make_spatial_audio_frame,
|
||||
put_live_audio_source_events,
|
||||
put_live_spatial_audio_frame,
|
||||
)
|
||||
|
||||
|
|
@ -44,6 +47,27 @@ class SpatialAudioCultCacheTests(unittest.TestCase):
|
|||
self.assertEqual(2048, loaded.start_sample)
|
||||
np.testing.assert_allclose(block, frame_to_numpy(loaded))
|
||||
|
||||
def test_live_source_events_round_trip_through_cultcache(self):
|
||||
events = make_audio_source_events(
|
||||
frame_id=7,
|
||||
sample_rate=48000,
|
||||
start_sample=4096,
|
||||
frame_count=1024,
|
||||
audio_time_ns=123,
|
||||
events=[{"eventId": 1, "positionMeters": [1.0, 2.0, 3.0], "confidence": 0.8}],
|
||||
voice_focus=[{"anchorChannel": 4, "weight": 1.0}],
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "audio-events.msgpack"
|
||||
put_live_audio_source_events(path, events)
|
||||
loaded = get_live_audio_source_events(path)
|
||||
|
||||
self.assertIsNotNone(loaded)
|
||||
self.assertEqual(48000, loaded.sample_rate)
|
||||
self.assertEqual(1, loaded.events[0]["eventId"])
|
||||
self.assertEqual(4, loaded.voice_focus[0]["anchorChannel"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in a new issue