From 12deb8fc8e8c4aeb3a32c36767e4fa9a872abc2f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 31 May 2026 19:12:37 +0100 Subject: [PATCH] Encode PS Move latency probe events --- docs/implementation-plan.md | 14 + state/evidence.jsonl | 1 + state/map.yaml | 15 +- tools/move_latency_probe.py | 609 ++++++++++++++++++++++++++ tools/nightwing_psmove_music_pulse.py | 49 ++- 5 files changed, 670 insertions(+), 18 deletions(-) create mode 100644 tools/move_latency_probe.py diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 7a1753e..6108338 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -454,6 +454,20 @@ a named invariant that the native runtime cannot protect yet. `percentile^exponent` so exponent `2` maps a median notable frame to 25% brightness while top recent-window rises approach full brightness. Steady-state material falls back toward black between emitted peak frames. + The ASIO reader drains the native capture queue to the newest loopback blocks + before each LED frame; stale queued audio was the plausible cause of + multi-second music-pulse lag, not Bluetooth or the hidraw LED command path. + `tools/move_latency_probe.py` is the current diagnostic latency receipt: + PS3 Eye is the default fast visual witness, Scarlett cardioid is the default + chirp timing witness, and the event train uses an order-3 de Bruijn schedule + with symbol-coded chirp bands, LED colors, and small variable gaps. It writes + `event-schedule.json`, arms the Nightwing LED script before playback, fits + detected audio/video events back to the known schedule, and marks receipts + invalid when the Move sphere is not sufficiently visible. A live encoded + PS3 Eye run recovered all eight visual events with schedule residuals within + about 4 ms; `ffplay`/DirectShow audio startup is still too soft for final + physical latency authority, so native timestamped Scarlett playback/capture + remains the production timing cut. Next useful cut is direct report decoding plus optical sphere observations from PS3 Eyes/Kiyos entering the same calibration-evidence path as LED spline constraints. diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 60585b0..c35783c 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -273,3 +273,4 @@ {"ts": "2026-05-31T18:39:59+01:00", "type": "implementation", "status": "ok", "note": "PS Move music pulse now mirrors Perlines' FFT whitening pattern: fast/slow scrubbed spectrum, positive delta scrubbed against slow delta, rapidly decaying adaptive min/max normalization, warm-up gating, and high-saturation dark-idle LED bursts."} {"ts": "2026-05-31T18:43:07+01:00", "type": "implementation", "status": "ok", "note": "PS Move music pulse range-hit behavior now treats a Perlines-style adaptive delta-range strike as full brightness: idle remains near black, but when the whitened FFT delta reaches the top of its adaptive range the LED onset is forced to max."} {"ts": "2026-05-31T18:47:32+01:00", "type": "implementation", "status": "ok", "note": "PS Move onset amplitude now ranks current whitened-delta broadband flux against a recent rolling history and emits brightness as percentile^exponent; exponent 2 maps median notable frames to 25% and top recent-window rises to full brightness."} +{"ts": "2026-05-31T19:11:18+01:00", "type": "implementation", "status": "ok", "note": "PS Move latency probe now uses PS3 Eye visual witnesses and Scarlett cardioid audio by default, emits an order-3 de Bruijn event schedule with symbol-coded chirp bands, LED colors, and variable gaps, arms the Nightwing LED script before playback, writes event-schedule.json, and marks receipts invalid when audio/video event recovery is insufficient. Live encoded run recovered all eight PS3 Eye visual events with residuals within about 4 ms; ffplay/DirectShow audio start remains too soft for final physical latency authority."} diff --git a/state/map.yaml b/state/map.yaml index 0c14c30..449e51d 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -178,7 +178,20 @@ invariants: Onset amplitude ranks the current whitened-delta broadband flux against a recent rolling history and emits `percentile^exponent`, so exponent `2` makes a median notable frame 25% brightness while the strongest recent - rises approach full brightness. + rises approach full brightness. The ASIO reader now drains the native + capture queue to the newest loopback blocks before each LED frame, because + stale queued audio was the plausible seconds-late music-pulse cause. + `tools/move_latency_probe.py` is the diagnostic PS Move latency receipt: + PS3 Eyes are the default fast visual witnesses, Scarlett cardioid is the + default chirp timing witness, and the event train uses an order-3 de Bruijn + schedule with symbol-coded chirp bands, LED colors, and small variable + gaps. The probe writes `event-schedule.json`, arms Nightwing's LED script + before playback, fits audio/video detections back to the known schedule, + and marks receipts invalid when the Move sphere is not visible enough. + Live encoded Eye/Scarlett testing recovered all eight visual events with + schedule residuals within about 4 ms; `ffplay`/DirectShow audio startup is + still too soft for final physical latency authority, so native timestamped + Scarlett playback/capture remains the production timing cut. - PS3 Eyes now have a first sparse tracking surface instead of only cadence proof. `MimirSparseFeatureTracker` detects Bayer/luma corner features, links them into sensor-local tracks with clipspace position, velocity, age, diff --git a/tools/move_latency_probe.py b/tools/move_latency_probe.py new file mode 100644 index 0000000..87ec8d3 --- /dev/null +++ b/tools/move_latency_probe.py @@ -0,0 +1,609 @@ +#!/usr/bin/env python3 +"""Estimate PS Move light latency from a camera plus Scarlett mic witness. + +This is a diagnostic receipt generator, not a runtime timing authority. +It records a camera while Starfire plays a chirp train into the room and +Nightwing flashes the PS Move sphere on the same cadence. The analysis detects +chirp peaks in the Scarlett cardioid mic and temporal brightening peaks in the +camera feed, then reports the visual-minus-audio offset seen by those witnesses. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import shutil +import struct +import subprocess +import sys +import time +import wave +import re +from pathlib import Path + +import numpy as np +from PIL import Image + + +DEFAULT_FFMPEG = ( + r"C:\Users\Meta\AppData\Local\Microsoft\WinGet\Packages" + r"\Gyan.FFmpeg_Microsoft.Winget.Source_8wekyb3d8bbwe" + r"\ffmpeg-8.1.1-full_build\bin\ffmpeg.exe" +) +DEFAULT_FFPLAY = str(Path(DEFAULT_FFMPEG).with_name("ffplay.exe")) +DEFAULT_FFPROBE = str(Path(DEFAULT_FFMPEG).with_name("ffprobe.exe")) +DEFAULT_KIYO_PRO_VIDEO = ( + r"@device_pnp_\\?\usb#vid_1532&pid_0e05&mi_00#9&3c07f79&0&0000" + r"#{65e8773d-8f56-11d0-a3b9-00a0c9223196}\global" +) +DEFAULT_PS3_EYE_0_VIDEO = "PS3 Eye Universal" +DEFAULT_PS3_EYE_1_VIDEO = "PS3 Eye Universal2" +DEFAULT_SCARLETT_CARDIOID = "Analogue 1 + 2 (2- Focusrite USB Audio)" +DEFAULT_KIYO_PRO_MIC = "Microphone (7- Razer Kiyo Pro)" + +VIDEO_PRESETS = { + "ps3-eye-0": (DEFAULT_PS3_EYE_0_VIDEO, 320, 240, 60), + "ps3-eye-1": (DEFAULT_PS3_EYE_1_VIDEO, 320, 240, 60), + "kiyo-pro": (DEFAULT_KIYO_PRO_VIDEO, 640, 480, 30), +} + +AUDIO_PRESETS = { + "scarlett-cardioid": DEFAULT_SCARLETT_CARDIOID, + "kiyo-pro": DEFAULT_KIYO_PRO_MIC, +} + + +REMOTE_LED = r""" +import os +import sys +import time + +hidraw = sys.argv[1] +events = [] +for part in sys.argv[2].split(","): + if not part: + continue + fields = part.split(":") + events.append((float(fields[0]), int(fields[1]), int(fields[2]), int(fields[3]))) +pulse_seconds = float(sys.argv[3]) +start_delay = float(sys.argv[4]) +log = os.path.expanduser("~/.local/state/gamecult/codex-ssh-activity.log") +os.makedirs(os.path.dirname(log), exist_ok=True) +with open(log, "a", encoding="utf-8") as f: + f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S%z')} Codex: PS Move latency LED pulse train armed.\n") + +def write_rgb(r, g, b): + with open(hidraw, "wb", buffering=0) as device: + device.write(bytes([0x06, 0, r, g, b, 0, 0, 0, 0])) + +write_rgb(0, 0, 0) +sys.stdin.readline() +start = time.perf_counter() + start_delay +for offset, r, g, b in events: + wait = start + offset - time.perf_counter() + if wait > 0: + time.sleep(wait) + write_rgb(r, g, b) + time.sleep(pulse_seconds) + write_rgb(0, 0, 0) +write_rgb(0, 0, 0) +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ffmpeg", default=DEFAULT_FFMPEG) + parser.add_argument("--ffplay", default=DEFAULT_FFPLAY) + parser.add_argument("--ffprobe", default=DEFAULT_FFPROBE) + parser.add_argument("--video-preset", choices=sorted(VIDEO_PRESETS), default="ps3-eye-0") + parser.add_argument("--audio-preset", choices=sorted(AUDIO_PRESETS), default="scarlett-cardioid") + parser.add_argument("--video-device") + parser.add_argument("--audio-device") + parser.add_argument("--ssh-target", default="nightwing") + parser.add_argument("--hidraw", default="/dev/hidraw1") + parser.add_argument("--out-dir", default="artifacts/runtime/move-latency") + parser.add_argument("--duration", type=float, default=8.0) + parser.add_argument("--lead", type=float, default=1.2) + parser.add_argument("--interval", type=float, default=0.8) + parser.add_argument("--pulses", type=int, default=6) + parser.add_argument("--schedule", choices=("uniform", "debruijn"), default="debruijn") + parser.add_argument("--alphabet-size", type=int, default=8) + parser.add_argument("--order", type=int, default=3) + parser.add_argument("--chirp-ms", type=float, default=80.0) + parser.add_argument("--pulse-ms", type=float, default=90.0) + parser.add_argument("--led-start-delay-ms", type=float, default=520.0) + parser.add_argument("--sample-rate", type=int, default=48000) + parser.add_argument("--width", type=int) + parser.add_argument("--height", type=int) + parser.add_argument("--framerate", type=int) + args = parser.parse_args() + preset_device, preset_width, preset_height, preset_framerate = VIDEO_PRESETS[args.video_preset] + args.video_device = args.video_device or preset_device + args.audio_device = args.audio_device or AUDIO_PRESETS[args.audio_preset] + args.width = args.width or preset_width + args.height = args.height or preset_height + args.framerate = args.framerate or preset_framerate + return args + + +def build_debruijn(alphabet_size: int, order: int) -> list[int]: + alphabet_size = max(2, int(alphabet_size)) + order = max(1, int(order)) + sequence: list[int] = [] + a = [0] * (alphabet_size * order) + + def db(t: int, p: int) -> None: + if t > order: + if order % p == 0: + sequence.extend(a[1 : p + 1]) + return + a[t] = a[t - p] + db(t + 1, p) + for j in range(a[t - p] + 1, alphabet_size): + a[t] = j + db(t + 1, t) + + db(1, 1) + return sequence + + +def rotate_to_distinct_opening(symbols: list[int], order: int) -> list[int]: + if len(symbols) <= order: + return symbols + windows: set[tuple[int, ...]] = set() + for index in range(len(symbols)): + window = tuple(symbols[(index + offset) % len(symbols)] for offset in range(order)) + if window in windows: + continue + windows.add(window) + if len(set(window)) == min(order, len(set(symbols))): + return symbols[index:] + symbols[:index] + return symbols + + +def symbol_chirp(symbol: int, args: argparse.Namespace) -> tuple[float, float, tuple[int, int, int]]: + # Eight log-spaced-ish lanes in the PS Move/camera-friendly audio band. + lane = symbol % max(2, args.alphabet_size) + base = 1450.0 * (2.0 ** (lane / 7.0)) + glide = 1.20 if ((symbol // max(2, args.alphabet_size)) & 1) == 0 else 0.84 + hue = (lane / max(1, args.alphabet_size)) % 1.0 + sector = int(hue * 6.0) + x = int(255 * (1.0 - abs((hue * 6.0) % 2.0 - 1.0))) + colors = [(255, x, 0), (x, 255, 0), (0, 255, x), (0, x, 255), (x, 0, 255), (255, 0, x)] + return base, base * glide, colors[sector % 6] + + +def build_events(args: argparse.Namespace) -> list[dict[str, float | int | tuple[int, int, int]]]: + if args.schedule == "uniform": + symbols = list(range(args.pulses)) + offsets = [args.lead + index * args.interval for index in range(args.pulses)] + else: + sequence = rotate_to_distinct_opening(build_debruijn(args.alphabet_size, args.order), args.order) + symbols = sequence[: args.pulses] + offsets = [] + cursor = args.lead + for symbol in symbols: + offsets.append(cursor) + # Time-domain coding: symbol-dependent gap rides on top of the cadence. + cursor += args.interval + 0.035 * ((symbol % 4) - 1.5) + events = [] + for index, (symbol, offset) in enumerate(zip(symbols, offsets)): + start_hz, end_hz, color = symbol_chirp(int(symbol), args) + events.append({"index": index, "symbol": int(symbol), "offset": float(offset), "start_hz": start_hz, "end_hz": end_hz, "color": color}) + return events + + +def render_chirp_template(start_hz: float, end_hz: float, args: argparse.Namespace) -> np.ndarray: + chirp_len = int(args.chirp_ms * 0.001 * args.sample_rate) + t = np.arange(chirp_len, dtype=np.float32) / args.sample_rate + k = (end_hz - start_hz) / max(1e-6, args.chirp_ms * 0.001) + phase = 2.0 * math.pi * (start_hz * t + 0.5 * k * t * t) + chirp = np.sin(phase).astype(np.float32) + chirp *= np.hanning(chirp_len).astype(np.float32) + chirp *= 0.45 + return chirp + + +def generate_chirp(path: Path, args: argparse.Namespace, events: list[dict[str, float | int | tuple[int, int, int]]]) -> np.ndarray: + sample_rate = args.sample_rate + total = int(args.duration * sample_rate) + audio = np.zeros(total, dtype=np.float32) + chirp_len = int(args.chirp_ms * 0.001 * sample_rate) + prototype = np.zeros(chirp_len, dtype=np.float32) + for event in events: + chirp = render_chirp_template(float(event["start_hz"]), float(event["end_hz"]), args) + if int(event["index"]) == 0: + prototype = chirp.copy() + offset = float(event["offset"]) + start = int(offset * sample_rate) + end = min(total, start + chirp_len) + audio[start:end] += chirp[: end - start] + pcm = np.clip(audio, -1.0, 1.0) + with wave.open(str(path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + wav.writeframes((pcm * 32767).astype(" Path: + capture_path = out_dir / "capture.mkv" + ffmpeg_cmd = [ + args.ffmpeg, + "-hide_banner", + "-y", + "-thread_queue_size", + "1024", + "-rtbufsize", + "512M", + "-f", + "dshow", + "-video_size", + f"{args.width}x{args.height}", + "-framerate", + str(args.framerate), + "-i", + f"video={args.video_device}", + "-thread_queue_size", + "1024", + "-f", + "dshow", + "-i", + f"audio={args.audio_device}", + "-t", + f"{args.duration:.3f}", + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c:v", + "ffv1", + "-level", + "3", + "-c:a", + "pcm_s16le", + "-ac", + "1", + "-ar", + str(args.sample_rate), + str(capture_path), + ] + ffmpeg = subprocess.Popen(ffmpeg_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + time.sleep(1.0) + + remote_b64 = base64.b64encode(REMOTE_LED.encode("utf-8")).decode("ascii") + event_csv = ",".join( + f"{float(event['offset']):.6f}:{int(event['color'][0])}:{int(event['color'][1])}:{int(event['color'][2])}" + for event in events + ) + pulse_seconds = args.pulse_ms * 0.001 + remote_cmd = ( + "tmp=$(mktemp); " + f"printf '%s' '{remote_b64}' | base64 -d > \"$tmp\"; " + f"python3 \"$tmp\" '{args.hidraw}' '{event_csv}' '{pulse_seconds:.6f}' '{args.led_start_delay_ms * 0.001:.6f}'; " + "status=$?; rm -f \"$tmp\"; exit $status" + ) + led = subprocess.Popen(["ssh", "-o", "BatchMode=yes", args.ssh_target, remote_cmd], stdin=subprocess.PIPE, text=True) + time.sleep(0.5) + audio = subprocess.Popen([args.ffplay, "-nodisp", "-autoexit", "-loglevel", "quiet", str(chirp_path)]) + if led.stdin is not None: + led.stdin.write("go\n") + led.stdin.flush() + led.stdin.close() + + audio.wait(timeout=args.duration + 3) + led.wait(timeout=args.duration + 5) + _, stderr = ffmpeg.communicate(timeout=args.duration + 10) + (out_dir / "ffmpeg-capture.err.log").write_text(stderr, encoding="utf-8", errors="replace") + if ffmpeg.returncode != 0: + raise RuntimeError(f"ffmpeg capture failed with {ffmpeg.returncode}; see {out_dir / 'ffmpeg-capture.err.log'}") + return capture_path + + +def extract_capture(args: argparse.Namespace, out_dir: Path, capture_path: Path) -> tuple[Path, Path, list[float]]: + audio_path = out_dir / "timing-mic.wav" + frames_dir = out_dir / "frames" + if frames_dir.exists(): + shutil.rmtree(frames_dir) + frames_dir.mkdir(parents=True) + subprocess.run( + [args.ffmpeg, "-hide_banner", "-y", "-i", str(capture_path), "-map", "0:a:0", "-ac", "1", "-ar", str(args.sample_rate), str(audio_path)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + [args.ffmpeg, "-hide_banner", "-y", "-i", str(capture_path), "-map", "0:v:0", "-vsync", "0", str(frames_dir / "frame_%06d.png")], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + probe = subprocess.run( + [ + args.ffprobe, + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "frame=best_effort_timestamp_time", + "-of", + "json", + str(capture_path), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + data = json.loads(probe.stdout) + times = [float(frame["best_effort_timestamp_time"]) for frame in data.get("frames", []) if "best_effort_timestamp_time" in frame] + return audio_path, frames_dir, times + + +def detect_audio_peaks( + audio_path: Path, + chirp: np.ndarray, + args: argparse.Namespace, + events: list[dict[str, float | int | tuple[int, int, int]]] | None = None, +) -> list[float]: + with wave.open(str(audio_path), "rb") as wav: + sample_rate = wav.getframerate() + pcm = wav.readframes(wav.getnframes()) + samples = np.frombuffer(pcm, dtype=" tuple[list[float], list[float]]: + frame_paths = sorted(frames_dir.glob("frame_*.png")) + scores: list[float] = [] + previous: np.ndarray | None = None + for path in frame_paths: + image = Image.open(path).convert("L") + arr = np.asarray(image, dtype=np.float32) + if previous is None: + scores.append(0.0) + else: + positive_delta = np.maximum(arr - previous, 0.0) + compact_flash = float(np.percentile(positive_delta, 99.90)) + bright_tail = float(np.percentile(arr, 99.85)) + scores.append(compact_flash + bright_tail * 0.02) + previous = arr + if not scores: + return [], [] + times = frame_times[: len(scores)] + if len(times) < len(scores): + times = [index / args.framerate for index in range(len(scores))] + baseline = float(np.percentile(scores, 20)) + signal = np.asarray(scores, dtype=np.float32) - baseline + signal = np.maximum(signal, 0.0) + threshold = max(float(np.percentile(signal, 92)), float(np.max(signal)) * 0.40) + min_gap = max(1, int(args.interval * 0.55 * args.framerate)) + peaks: list[int] = [] + work = signal.copy() + for _ in range(args.pulses): + index = int(np.argmax(work)) + if work[index] <= threshold: + break + peaks.append(index) + lo = max(0, index - min_gap) + hi = min(work.size, index + min_gap) + work[lo:hi] = 0.0 + return sorted(times[index] for index in peaks), scores + + +def pair_peaks(audio_peaks: list[float], video_peaks: list[float], args: argparse.Namespace) -> list[tuple[float, float, float]]: + if not audio_peaks or not video_peaks: + return [] + candidates = sorted(video - audio for audio in audio_peaks for video in video_peaks) + tolerance = max(0.06, args.interval * 0.16) + best_offset = candidates[0] + best_key: tuple[int, float, float] | None = None + for candidate in candidates: + residuals = [min(abs((audio + candidate) - video) for video in video_peaks) for audio in audio_peaks] + inliers = [residual for residual in residuals if residual <= tolerance] + if not inliers: + continue + key = (len(inliers), -float(np.median(inliers)), -abs(candidate)) + if best_key is None or key > best_key: + best_key = key + best_offset = candidate + pairs: list[tuple[float, float, float]] = [] + used_video: set[int] = set() + for audio_peak in audio_peaks: + nearest_index = min( + (index for index in range(len(video_peaks)) if index not in used_video), + key=lambda index: abs(video_peaks[index] - (audio_peak + best_offset)), + default=None, + ) + if nearest_index is None: + break + video_peak = video_peaks[nearest_index] + if abs(video_peak - (audio_peak + best_offset)) <= tolerance: + used_video.add(nearest_index) + pairs.append((audio_peak, video_peak, video_peak - audio_peak)) + return pairs + + +def fit_schedule( + peaks: list[float], + events: list[dict[str, float | int | tuple[int, int, int]]], + args: argparse.Namespace, +) -> tuple[float | None, list[dict[str, float | int | None]]]: + if not peaks or not events: + return None, [] + offsets = [float(event["offset"]) for event in events] + candidates = sorted(peak - offset for peak in peaks for offset in offsets) + tolerance = max(0.08, args.interval * 0.18) + best_shift: float | None = None + best_key: tuple[int, float, float] | None = None + for candidate in candidates: + residuals = [min(abs((offset + candidate) - peak) for peak in peaks) for offset in offsets] + inliers = [residual for residual in residuals if residual <= tolerance] + if not inliers: + continue + key = (len(inliers), -float(np.median(inliers)), -abs(candidate)) + if best_key is None or key > best_key: + best_key = key + best_shift = candidate + if best_shift is None: + return None, [] + used_peaks: set[int] = set() + matches: list[dict[str, float | int | None]] = [] + for event in events: + expected = float(event["offset"]) + best_shift + nearest_index = min( + (index for index in range(len(peaks)) if index not in used_peaks), + key=lambda index: abs(peaks[index] - expected), + default=None, + ) + if nearest_index is None or abs(peaks[nearest_index] - expected) > tolerance: + matches.append({"index": int(event["index"]), "symbol": int(event["symbol"]), "expected_s": expected, "observed_s": None, "residual_ms": None}) + continue + used_peaks.add(nearest_index) + matches.append( + { + "index": int(event["index"]), + "symbol": int(event["symbol"]), + "expected_s": expected, + "observed_s": peaks[nearest_index], + "residual_ms": (peaks[nearest_index] - expected) * 1000.0, + } + ) + return best_shift, matches + + +def parse_dshow_start_delta_ms(out_dir: Path) -> float | None: + log_path = out_dir / "ffmpeg-capture.err.log" + if not log_path.exists(): + return None + text = log_path.read_text(encoding="utf-8", errors="replace") + starts = [float(match.group(1)) for match in re.finditer(r"Duration: N/A, start: ([0-9.]+)", text)] + if len(starts) < 2: + return None + return (starts[0] - starts[1]) * 1000.0 + + +def summarize( + audio_peaks: list[float], + video_peaks: list[float], + out_dir: Path, + scores: list[float], + args: argparse.Namespace, + events: list[dict[str, float | int | tuple[int, int, int]]], +) -> dict: + pairs = pair_peaks(audio_peaks, video_peaks, args) + offsets_ms = [pair[2] * 1000.0 for pair in pairs] + audio_shift, audio_matches = fit_schedule(audio_peaks, events, args) + video_shift, video_matches = fit_schedule(video_peaks, events, args) + schedule_visual_minus_audio_ms = None + if audio_shift is not None and video_shift is not None: + schedule_visual_minus_audio_ms = (video_shift - audio_shift) * 1000.0 + enough_visual = len(video_peaks) >= max(2, int(math.ceil(args.pulses * 0.65))) + enough_audio = len(audio_peaks) >= max(2, int(math.ceil(args.pulses * 0.65))) + enough_pairs = len(pairs) >= max(2, int(math.ceil(min(len(audio_peaks), len(video_peaks), args.pulses) * 0.50))) + receipt_valid = enough_visual and enough_audio and enough_pairs + summary = { + "video_preset": args.video_preset, + "video_device": args.video_device, + "video_mode": f"{args.width}x{args.height}@{args.framerate}", + "audio_preset": args.audio_preset, + "audio_device": args.audio_device, + "led_start_delay_ms": args.led_start_delay_ms, + "schedule": args.schedule, + "alphabet_size": args.alphabet_size, + "order": args.order, + "events": events, + "audio_peaks_s": audio_peaks, + "video_peaks_s": video_peaks, + "paired_peaks": [ + {"audio_s": audio, "video_s": video, "visual_minus_audio_ms": offset * 1000.0} + for audio, video, offset in pairs + ], + "paired_count": len(pairs), + "audio_schedule_shift_ms": audio_shift * 1000.0 if audio_shift is not None else None, + "video_schedule_shift_ms": video_shift * 1000.0 if video_shift is not None else None, + "schedule_visual_minus_audio_ms": schedule_visual_minus_audio_ms, + "audio_schedule_matches": audio_matches, + "video_schedule_matches": video_matches, + "receipt_valid": receipt_valid, + "validity": { + "enough_visual_events": enough_visual, + "enough_audio_events": enough_audio, + "enough_paired_events": enough_pairs, + }, + "dshow_video_start_minus_audio_start_ms": parse_dshow_start_delta_ms(out_dir), + "note": "Offsets are diagnostic receipt values. DirectShow stream start offsets are reported separately; native timestamped capture should own final calibration timing.", + "visual_minus_audio_ms": offsets_ms, + "median_visual_minus_audio_ms": float(np.median(offsets_ms)) if offsets_ms else None, + "mean_visual_minus_audio_ms": float(np.mean(offsets_ms)) if offsets_ms else None, + "score_min": float(np.min(scores)) if scores else None, + "score_max": float(np.max(scores)) if scores else None, + } + (out_dir / "latency-summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + return summary + + +def main() -> int: + args = parse_args() + out_dir = Path(args.out_dir) + if not out_dir.is_absolute(): + out_dir = Path.cwd() / out_dir + out_dir.mkdir(parents=True, exist_ok=True) + events = build_events(args) + chirp_path = out_dir / "chirp-train.wav" + (out_dir / "event-schedule.json").write_text(json.dumps(events, indent=2), encoding="utf-8") + chirp = generate_chirp(chirp_path, args, events) + capture_path = run_capture(args, out_dir, chirp_path, events) + audio_path, frames_dir, frame_times = extract_capture(args, out_dir, capture_path) + audio_peaks = detect_audio_peaks(audio_path, chirp, args, events) + video_peaks, scores = detect_video_peaks(frames_dir, frame_times, args) + summary = summarize(audio_peaks, video_peaks, out_dir, scores, args, events) + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/nightwing_psmove_music_pulse.py b/tools/nightwing_psmove_music_pulse.py index b743c5c..5f6705a 100644 --- a/tools/nightwing_psmove_music_pulse.py +++ b/tools/nightwing_psmove_music_pulse.py @@ -102,6 +102,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--gain", type=float, default=0.25, help="Sustain/body gain under onset bursts.") parser.add_argument("--floor", type=float, default=0.008, help="Noise floor.") parser.add_argument("--fft-size", type=int, default=512, help="Tiny FFT size for onset flux.") + parser.add_argument("--drain-blocks", type=int, default=4096, help="Maximum ASIO blocks to drain per LED frame.") parser.add_argument("--onset-gain", type=float, default=1.0, help="Scale applied after percentile onset amplitude.") parser.add_argument("--onset-decay", type=float, default=0.22, help="Per-frame onset decay.") parser.add_argument("--onset-threshold", type=float, default=0.58, help="Minimum recent-window percentile that can emit a peak.") @@ -204,32 +205,45 @@ class AsioOnsetReader: onset_history_ms: float, onset_cooldown_ms: float, warmup_ms: float, + drain_blocks: int, ) -> tuple[float, float, tuple[float, float, float], float, float, float] | None: channel = ctypes.c_int() timestamp_ns = ctypes.c_longlong() sequence = ctypes.c_ulonglong() frame_count = ctypes.c_int() while time.monotonic() < deadline: - ok = self.dll.mimir_asio_read( - self.handle, - ctypes.byref(channel), - ctypes.byref(timestamp_ns), - ctypes.byref(sequence), - ctypes.byref(frame_count), - self.buffer, - self.max_frames, - ) - if not ok: + drained: list[float] = [] + reads = 0 + while reads < max(1, drain_blocks): + ok = self.dll.mimir_asio_read( + self.handle, + ctypes.byref(channel), + ctypes.byref(timestamp_ns), + ctypes.byref(sequence), + ctypes.byref(frame_count), + self.buffer, + self.max_frames, + ) + if not ok: + break + reads += 1 + frames = max(0, min(int(frame_count.value), self.max_frames)) + if int(channel.value) not in self.channels or frames == 0: + continue + drained.extend(float(self.buffer[index]) for index in range(frames)) + keep = self.fft_size * 4 + if len(drained) > keep: + del drained[: len(drained) - keep] + if not drained: time.sleep(0.001) continue - frames = max(0, min(int(frame_count.value), self.max_frames)) - if int(channel.value) not in self.channels or frames == 0: - continue - for index in range(frames): - self.pending.append(float(self.buffer[index])) + self.pending.extend(drained) + keep_pending = self.fft_size * 6 + if len(self.pending) > keep_pending: + del self.pending[: len(self.pending) - keep_pending] if len(self.pending) >= self.fft_size: - frame = np.asarray(self.pending[: self.fft_size], dtype=np.float32) - del self.pending[: self.fft_size // 2] + frame = np.asarray(self.pending[-self.fft_size :], dtype=np.float32) + self.pending.clear() rms = float(np.sqrt(np.mean(frame * frame))) spectrum = np.abs(np.fft.rfft(frame * self.window)).astype(np.float32) spectrum = np.sqrt(np.maximum(spectrum, 0.0)).astype(np.float32) @@ -452,6 +466,7 @@ def main() -> int: args.onset_history_ms, args.onset_cooldown_ms, args.warmup_ms, + args.drain_blocks, ) if onset is None: continue