Add online Realtek Move sync worker

This commit is contained in:
Meta Vens 2026-06-01 20:36:20 +01:00
commit 54104250e7

View file

@ -0,0 +1,386 @@
#!/usr/bin/env python3
"""Drive all Nightwing PS Moves from Starfire Realtek loopback music.
This is an online calibration worker. Starfire owns the loopback-derived music
state; Nightwing owns local PS Move HID actuation. The worker emits unique,
smooth RGB contours for every connected Move and writes a JSONL trace that can
be scored offline against Eye observations.
"""
from __future__ import annotations
import argparse
import base64
import colorsys
import json
import math
import os
import subprocess
import sys
import time
from collections import deque
from dataclasses import dataclass
from pathlib import Path
import numpy as np
REMOTE_MULTI_RECEIVER = r"""
import os
import sys
import time
moves = {}
for spec in sys.argv[1].split(","):
if not spec:
continue
name, path = spec.split("=", 1)
moves[name] = path
log = os.path.expanduser("~/.local/state/gamecult/codex-ssh-activity.log")
os.makedirs(os.path.dirname(log), exist_ok=True)
with open(log, "a", encoding="utf-8") as f:
f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S%z')} Codex: online Move music sync receiver armed for {len(moves)} Moves.\n")
last = {}
def write_rgb(path, r, g, b):
with open(path, "wb", buffering=0) as device:
device.write(bytes([0x06, 0, max(0, min(255, r)), max(0, min(255, g)), max(0, min(255, b)), 0, 0, 0, 0]))
for path in moves.values():
write_rgb(path, 0, 0, 0)
for line in sys.stdin:
parts = line.strip().split()
if len(parts) != 4:
continue
name = parts[0]
path = moves.get(name)
if not path:
continue
try:
rgb = tuple(max(0, min(255, int(part))) for part in parts[1:4])
except ValueError:
continue
if last.get(name) == rgb:
continue
write_rgb(path, rgb[0], rgb[1], rgb[2])
last[name] = rgb
for path in moves.values():
write_rgb(path, 0, 0, 0)
"""
@dataclass(frozen=True)
class Move:
name: str
hidraw: str
base_rgb: tuple[int, int, int]
def parse_move(value: str) -> Move:
name, rest = value.split("=", 1)
hidraw, color = rest.split(":", 1)
color = color.lstrip("#")
if len(color) != 6:
raise ValueError(f"invalid move color: {value}")
rgb = (int(color[0:2], 16), int(color[2:4], 16), int(color[4:6], 16))
return Move(name=name, hidraw=hidraw, base_rgb=rgb)
def rgb_to_hsv01(rgb: tuple[int, int, int]) -> tuple[float, float, float]:
return colorsys.rgb_to_hsv(rgb[0] / 255.0, rgb[1] / 255.0, rgb[2] / 255.0)
class OnlineAnalyzer:
def __init__(self, rate: int, fps: float, fft_size: int, args: argparse.Namespace) -> None:
self.rate = rate
self.fps = fps
self.fft_size = fft_size
self.args = args
self.window = np.hanning(fft_size).astype(np.float32)
bins = fft_size // 2 + 1
self.fast = np.zeros(bins, dtype=np.float32)
self.slow = np.ones(bins, dtype=np.float32) * 1e-5
self.delta_fast = np.zeros(bins, dtype=np.float32)
self.delta_slow = np.ones(bins, dtype=np.float32) * 1e-5
self.minv = np.zeros(bins, dtype=np.float32)
self.maxv = np.ones(bins, dtype=np.float32) * 1e-4
self.flux_history: deque[float] = deque(maxlen=max(8, int(args.onset_history_seconds * fps)))
self.onset_series: deque[float] = deque(maxlen=max(32, int(args.tempo_window_seconds * fps)))
self.pending = np.zeros(0, dtype=np.float32)
self.last_peak = 0.0
self.onset_env = 0.0
self.level_env = 0.0
self.bpm = 140.0
self.bpm_confidence = 0.0
self.beat_phase = 0.0
self.last_t = time.monotonic()
def analyze(self, mono: np.ndarray, now: float) -> dict[str, float | tuple[float, float, float]]:
if mono.size < self.fft_size:
mono = np.pad(mono, (self.fft_size - mono.size, 0))
frame = mono[-self.fft_size :].astype(np.float32) * self.window
mag = np.abs(np.fft.rfft(frame)).astype(np.float32)
mag[0:2] *= 0.1
a = self.args
self.fast = a.whiten_fast * mag + (1.0 - a.whiten_fast) * self.fast
self.slow = a.whiten_slow * mag + (1.0 - a.whiten_slow) * self.slow
positive = np.maximum(0.0, self.fast - self.slow)
self.delta_fast = a.whiten_delta_fast * positive + (1.0 - a.whiten_delta_fast) * self.delta_fast
self.delta_slow = a.whiten_delta_slow * positive + (1.0 - a.whiten_delta_slow) * self.delta_slow
delta = np.maximum(0.0, self.delta_fast - self.delta_slow)
decay = a.whiten_decay
self.minv = np.minimum(delta, self.minv * decay + delta * (1.0 - decay))
self.maxv = np.maximum(delta, self.maxv * decay + delta * (1.0 - decay))
norm = np.clip((delta - self.minv) / np.maximum(1e-6, self.maxv - self.minv), 0.0, 1.0)
shaped = np.power(norm, max(0.5, a.whiten_contrast))
flux = float(np.mean(shaped))
body = float(np.sqrt(np.mean(np.square(mono)))) if mono.size else 0.0
self.flux_history.append(flux)
ordered = sorted(self.flux_history)
rank = 0.0
if ordered:
below = sum(1 for value in ordered if value <= flux)
rank = below / len(ordered)
warm = len(self.flux_history) >= min(self.flux_history.maxlen or 1, int(a.warmup_seconds * self.fps))
hit = 0.0
if warm and rank >= a.onset_threshold and (now - self.last_peak) >= a.onset_cooldown_seconds:
hit = min(1.0, max(0.0, rank) ** a.onset_exponent)
self.last_peak = now
self.beat_phase = 0.0
self.onset_env = max(hit, self.onset_env * a.onset_decay)
self.level_env = max(self.onset_env, body * a.body_gain, self.level_env * a.level_decay)
self.onset_series.append(flux)
if len(self.onset_series) >= int(self.fps * 4) and int(now * 2) != int(self.last_t * 2):
self._estimate_tempo()
dt = max(0.0, now - self.last_t)
self.last_t = now
self.beat_phase = (self.beat_phase + dt * self.bpm / 60.0) % 1.0
f0, color = self._estimate_fundamental_and_color(mag)
return {
"flux": flux,
"percentile": rank,
"hit": hit,
"body": body,
"level": self.level_env,
"bpm": self.bpm,
"bpm_confidence": self.bpm_confidence,
"beat_phase": self.beat_phase,
"fundamental_hz": f0,
"color_balance": color,
}
def _estimate_tempo(self) -> None:
x = np.asarray(self.onset_series, dtype=np.float32)
x = x - float(np.mean(x))
if float(np.max(np.abs(x))) < 1e-6:
return
best_bpm = self.bpm
best_score = 0.0
for bpm in np.linspace(self.args.tempo_min_bpm, self.args.tempo_max_bpm, 161):
lag = int(round((60.0 / float(bpm)) * self.fps))
if lag <= 1 or lag >= len(x) - 4:
continue
a = x[lag:]
b = x[:-lag]
denom = float(np.linalg.norm(a) * np.linalg.norm(b))
score = 0.0 if denom <= 1e-9 else float(np.dot(a, b) / denom)
if score > best_score:
best_score = score
best_bpm = float(bpm)
self.bpm = 0.88 * self.bpm + 0.12 * best_bpm
self.bpm_confidence = max(0.0, min(1.0, best_score))
def _estimate_fundamental_and_color(self, mag: np.ndarray) -> tuple[float, tuple[float, float, float]]:
freqs = np.fft.rfftfreq(self.fft_size, 1.0 / self.rate)
lo = max(1, int(np.searchsorted(freqs, self.args.fundamental_min)))
hi = min(len(freqs) - 1, int(np.searchsorted(freqs, self.args.fundamental_max)))
best_f0 = 110.0
best_score = 0.0
for idx in range(lo, hi):
f0 = freqs[idx]
score = 0.0
h = 1
while f0 * h < self.rate * 0.45:
j = int(np.searchsorted(freqs, f0 * h))
if 0 <= j < len(mag):
score += float(mag[j]) / h
h += 1
if score > best_score:
best_score = score
best_f0 = float(f0)
bands = np.zeros(3, dtype=np.float32)
for f, m in zip(freqs[2:], mag[2:]):
if f <= 0:
continue
octave = (math.log2(max(f, 1e-6) / max(best_f0, 1e-6)) % 1.0)
bands[int(octave * 3.0) % 3] += float(m)
total = float(np.sum(bands))
if total <= 1e-6:
return best_f0, (0.25, 0.20, 0.55)
bands /= total
bands = np.power(np.maximum(bands, 0.0), self.args.color_contrast)
bands /= max(1e-6, float(np.max(bands)))
return best_f0, (float(bands[0]), float(bands[1]), float(bands[2]))
def move_rgb(move: Move, move_index: int, move_count: int, state: dict[str, object], args: argparse.Namespace) -> tuple[int, int, int]:
level = float(state["level"])
beat = float(state["beat_phase"])
hit = float(state["hit"])
base_h, base_s, _ = rgb_to_hsv01(move.base_rgb)
spectral = state["color_balance"]
assert isinstance(spectral, tuple)
phase = (beat + move_index / max(1, move_count)) % 1.0
gesture = 0.52 + 0.48 * (0.5 + 0.5 * math.sin(2.0 * math.pi * phase)) ** 1.7
accent = min(1.0, level * gesture + hit * 0.55)
hue = (base_h + 0.045 * math.sin(2.0 * math.pi * phase)) % 1.0
sat = min(1.0, max(0.72, base_s) + 0.18 * hit)
val = min(1.0, accent ** 0.55)
rr, gg, bb = colorsys.hsv_to_rgb(hue, sat, val)
r = rr * (0.45 + 0.55 * float(spectral[0]))
g = gg * (0.45 + 0.55 * float(spectral[1]))
b = bb * (0.45 + 0.55 * float(spectral[2]))
white = max(0.0, hit - 0.92) / 0.08
return (
int(min(255, 255 * r + 18 * white)),
int(min(255, 255 * g + 18 * white)),
int(min(255, 255 * b + 18 * white)),
)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--duration", type=float, default=1800.0)
parser.add_argument("--out-dir", default="artifacts/runtime/online-move-music-sync")
parser.add_argument("--wasapi", default="src/Mimir.WasapiLoopback/bin/Debug/net10.0-windows/Mimir.WasapiLoopback.exe")
parser.add_argument("--device", default="Realtek")
parser.add_argument("--sample-rate", type=int, default=48000)
parser.add_argument("--channels", type=int, default=2)
parser.add_argument("--fps", type=float, default=60.0)
parser.add_argument("--fft-size", type=int, default=512)
parser.add_argument("--ssh-target", default="nightwing")
parser.add_argument("--move", action="append", required=True)
parser.add_argument("--body-gain", type=float, default=0.22)
parser.add_argument("--level-decay", type=float, default=0.80)
parser.add_argument("--onset-decay", type=float, default=0.18)
parser.add_argument("--onset-threshold", type=float, default=0.62)
parser.add_argument("--onset-exponent", type=float, default=2.0)
parser.add_argument("--onset-history-seconds", type=float, default=4.5)
parser.add_argument("--onset-cooldown-seconds", type=float, default=0.11)
parser.add_argument("--warmup-seconds", type=float, default=0.5)
parser.add_argument("--whiten-fast", type=float, default=0.78)
parser.add_argument("--whiten-slow", type=float, default=0.12)
parser.add_argument("--whiten-delta-fast", type=float, default=0.62)
parser.add_argument("--whiten-delta-slow", type=float, default=0.08)
parser.add_argument("--whiten-decay", type=float, default=0.78)
parser.add_argument("--whiten-contrast", type=float, default=3.6)
parser.add_argument("--color-contrast", type=float, default=3.4)
parser.add_argument("--tempo-window-seconds", type=float, default=12.0)
parser.add_argument("--tempo-min-bpm", type=float, default=70.0)
parser.add_argument("--tempo-max-bpm", type=float, default=260.0)
parser.add_argument("--fundamental-min", type=float, default=55.0)
parser.add_argument("--fundamental-max", type=float, default=880.0)
args = parser.parse_args()
out_dir = Path(args.out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
moves = [parse_move(value) for value in args.move]
(out_dir / "moves.json").write_text(json.dumps([move.__dict__ for move in moves], indent=2), encoding="utf-8")
remote_b64 = base64.b64encode(REMOTE_MULTI_RECEIVER.encode("utf-8")).decode("ascii")
move_specs = ",".join(f"{move.name}={move.hidraw}" for move in moves)
remote_cmd = (
"tmp=$(mktemp); "
f"printf '%s' '{remote_b64}' | base64 -d > \"$tmp\"; "
f"python3 \"$tmp\" '{move_specs}'; "
"status=$?; rm -f \"$tmp\"; exit $status"
)
ssh = subprocess.Popen(["ssh", "-o", "BatchMode=yes", args.ssh_target, remote_cmd], stdin=subprocess.PIPE)
wasapi = subprocess.Popen(
[
args.wasapi,
"--device",
args.device,
"--output",
"stdout",
"--sample-rate",
str(args.sample_rate),
"--channels",
str(args.channels),
"--seconds",
str(args.duration),
],
stdout=subprocess.PIPE,
stderr=(out_dir / "wasapi.err.log").open("w", encoding="utf-8", errors="replace"),
)
assert wasapi.stdout is not None and ssh.stdin is not None
analyzer = OnlineAnalyzer(args.sample_rate, args.fps, args.fft_size, args)
frame_samples = max(args.fft_size, int(args.sample_rate / args.fps))
frame_bytes = frame_samples * args.channels * 4
trace_path = out_dir / "online-sync.jsonl"
start = time.monotonic()
frames = 0
try:
with trace_path.open("w", encoding="utf-8") as trace:
while time.monotonic() - start < args.duration:
data = wasapi.stdout.read(frame_bytes)
if len(data) < frame_bytes:
break
block = np.frombuffer(data, dtype=np.float32).reshape((-1, args.channels))
mono = np.mean(block, axis=1)
now = time.monotonic()
state = analyzer.analyze(mono, now)
rgbs = {}
for index, move in enumerate(moves):
rgb = move_rgb(move, index, len(moves), state, args)
rgbs[move.name] = rgb
ssh.stdin.write(f"{move.name} {rgb[0]} {rgb[1]} {rgb[2]}\n".encode("ascii"))
ssh.stdin.flush()
frames += 1
record = {
"kind": "mimir.online_move_music_sync_frame.v1",
"t_monotonic": now,
"elapsed_seconds": now - start,
"frame": frames,
**{key: value for key, value in state.items() if key != "color_balance"},
"color_balance": list(state["color_balance"]), # type: ignore[index]
"moves": {name: list(rgb) for name, rgb in rgbs.items()},
}
trace.write(json.dumps(record, sort_keys=True) + "\n")
if frames % max(1, int(args.fps * 2)) == 0:
trace.flush()
print(
f"t={record['elapsed_seconds']:.1f}s level={record['level']:.3f} "
f"hit={record['hit']:.3f} pct={record['percentile']:.3f} "
f"bpm={record['bpm']:.1f}/{record['bpm_confidence']:.2f} "
f"f0={record['fundamental_hz']:.1f}",
flush=True,
)
finally:
try:
for move in moves:
ssh.stdin.write(f"{move.name} 0 0 0\n".encode("ascii"))
ssh.stdin.close()
except Exception:
pass
for proc in (wasapi, ssh):
if proc.poll() is None:
proc.terminate()
time.sleep(0.2)
for proc in (wasapi, ssh):
if proc.poll() is None:
proc.kill()
return 0
if __name__ == "__main__":
raise SystemExit(main())