Encode PS Move latency probe events

This commit is contained in:
Meta Vens 2026-05-31 19:12:37 +01:00
commit 12deb8fc8e
5 changed files with 670 additions and 18 deletions

View file

@ -454,6 +454,20 @@ a named invariant that the native runtime cannot protect yet.
`percentile^exponent` so exponent `2` maps a median notable frame to 25%
brightness while top recent-window rises approach full brightness.
Steady-state material falls back toward black between emitted peak frames.
The ASIO reader drains the native capture queue to the newest loopback blocks
before each LED frame; stale queued audio was the plausible cause of
multi-second music-pulse lag, not Bluetooth or the hidraw LED command path.
`tools/move_latency_probe.py` is the current diagnostic latency receipt:
PS3 Eye is the default fast visual witness, Scarlett cardioid is the default
chirp timing witness, and the event train uses an order-3 de Bruijn schedule
with symbol-coded chirp bands, LED colors, and small variable gaps. It writes
`event-schedule.json`, arms the Nightwing LED script before playback, fits
detected audio/video events back to the known schedule, and marks receipts
invalid when the Move sphere is not sufficiently visible. A live encoded
PS3 Eye run recovered all eight visual events with schedule residuals within
about 4 ms; `ffplay`/DirectShow audio startup is still too soft for final
physical latency authority, so native timestamped Scarlett playback/capture
remains the production timing cut.
Next useful cut is direct report decoding plus optical sphere observations
from PS3 Eyes/Kiyos entering the same calibration-evidence path as LED
spline constraints.

View file

@ -273,3 +273,4 @@
{"ts": "2026-05-31T18:39:59+01:00", "type": "implementation", "status": "ok", "note": "PS Move music pulse now mirrors Perlines' FFT whitening pattern: fast/slow scrubbed spectrum, positive delta scrubbed against slow delta, rapidly decaying adaptive min/max normalization, warm-up gating, and high-saturation dark-idle LED bursts."}
{"ts": "2026-05-31T18:43:07+01:00", "type": "implementation", "status": "ok", "note": "PS Move music pulse range-hit behavior now treats a Perlines-style adaptive delta-range strike as full brightness: idle remains near black, but when the whitened FFT delta reaches the top of its adaptive range the LED onset is forced to max."}
{"ts": "2026-05-31T18:47:32+01:00", "type": "implementation", "status": "ok", "note": "PS Move onset amplitude now ranks current whitened-delta broadband flux against a recent rolling history and emits brightness as percentile^exponent; exponent 2 maps median notable frames to 25% and top recent-window rises to full brightness."}
{"ts": "2026-05-31T19:11:18+01:00", "type": "implementation", "status": "ok", "note": "PS Move latency probe now uses PS3 Eye visual witnesses and Scarlett cardioid audio by default, emits an order-3 de Bruijn event schedule with symbol-coded chirp bands, LED colors, and variable gaps, arms the Nightwing LED script before playback, writes event-schedule.json, and marks receipts invalid when audio/video event recovery is insufficient. Live encoded run recovered all eight PS3 Eye visual events with residuals within about 4 ms; ffplay/DirectShow audio start remains too soft for final physical latency authority."}

View file

@ -178,7 +178,20 @@ invariants:
Onset amplitude ranks the current whitened-delta broadband flux against a
recent rolling history and emits `percentile^exponent`, so exponent `2`
makes a median notable frame 25% brightness while the strongest recent
rises approach full brightness.
rises approach full brightness. The ASIO reader now drains the native
capture queue to the newest loopback blocks before each LED frame, because
stale queued audio was the plausible seconds-late music-pulse cause.
`tools/move_latency_probe.py` is the diagnostic PS Move latency receipt:
PS3 Eyes are the default fast visual witnesses, Scarlett cardioid is the
default chirp timing witness, and the event train uses an order-3 de Bruijn
schedule with symbol-coded chirp bands, LED colors, and small variable
gaps. The probe writes `event-schedule.json`, arms Nightwing's LED script
before playback, fits audio/video detections back to the known schedule,
and marks receipts invalid when the Move sphere is not visible enough.
Live encoded Eye/Scarlett testing recovered all eight visual events with
schedule residuals within about 4 ms; `ffplay`/DirectShow audio startup is
still too soft for final physical latency authority, so native timestamped
Scarlett playback/capture remains the production timing cut.
- PS3 Eyes now have a first sparse tracking surface instead of only cadence
proof. `MimirSparseFeatureTracker` detects Bayer/luma corner features,
links them into sensor-local tracks with clipspace position, velocity, age,

609
tools/move_latency_probe.py Normal file
View file

@ -0,0 +1,609 @@
#!/usr/bin/env python3
"""Estimate PS Move light latency from a camera plus Scarlett mic witness.
This is a diagnostic receipt generator, not a runtime timing authority.
It records a camera while Starfire plays a chirp train into the room and
Nightwing flashes the PS Move sphere on the same cadence. The analysis detects
chirp peaks in the Scarlett cardioid mic and temporal brightening peaks in the
camera feed, then reports the visual-minus-audio offset seen by those witnesses.
"""
from __future__ import annotations
import argparse
import base64
import json
import math
import os
import shutil
import struct
import subprocess
import sys
import time
import wave
import re
from pathlib import Path
import numpy as np
from PIL import Image
DEFAULT_FFMPEG = (
r"C:\Users\Meta\AppData\Local\Microsoft\WinGet\Packages"
r"\Gyan.FFmpeg_Microsoft.Winget.Source_8wekyb3d8bbwe"
r"\ffmpeg-8.1.1-full_build\bin\ffmpeg.exe"
)
DEFAULT_FFPLAY = str(Path(DEFAULT_FFMPEG).with_name("ffplay.exe"))
DEFAULT_FFPROBE = str(Path(DEFAULT_FFMPEG).with_name("ffprobe.exe"))
DEFAULT_KIYO_PRO_VIDEO = (
r"@device_pnp_\\?\usb#vid_1532&pid_0e05&mi_00#9&3c07f79&0&0000"
r"#{65e8773d-8f56-11d0-a3b9-00a0c9223196}\global"
)
DEFAULT_PS3_EYE_0_VIDEO = "PS3 Eye Universal"
DEFAULT_PS3_EYE_1_VIDEO = "PS3 Eye Universal2"
DEFAULT_SCARLETT_CARDIOID = "Analogue 1 + 2 (2- Focusrite USB Audio)"
DEFAULT_KIYO_PRO_MIC = "Microphone (7- Razer Kiyo Pro)"
VIDEO_PRESETS = {
"ps3-eye-0": (DEFAULT_PS3_EYE_0_VIDEO, 320, 240, 60),
"ps3-eye-1": (DEFAULT_PS3_EYE_1_VIDEO, 320, 240, 60),
"kiyo-pro": (DEFAULT_KIYO_PRO_VIDEO, 640, 480, 30),
}
AUDIO_PRESETS = {
"scarlett-cardioid": DEFAULT_SCARLETT_CARDIOID,
"kiyo-pro": DEFAULT_KIYO_PRO_MIC,
}
REMOTE_LED = r"""
import os
import sys
import time
hidraw = sys.argv[1]
events = []
for part in sys.argv[2].split(","):
if not part:
continue
fields = part.split(":")
events.append((float(fields[0]), int(fields[1]), int(fields[2]), int(fields[3])))
pulse_seconds = float(sys.argv[3])
start_delay = float(sys.argv[4])
log = os.path.expanduser("~/.local/state/gamecult/codex-ssh-activity.log")
os.makedirs(os.path.dirname(log), exist_ok=True)
with open(log, "a", encoding="utf-8") as f:
f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S%z')} Codex: PS Move latency LED pulse train armed.\n")
def write_rgb(r, g, b):
with open(hidraw, "wb", buffering=0) as device:
device.write(bytes([0x06, 0, r, g, b, 0, 0, 0, 0]))
write_rgb(0, 0, 0)
sys.stdin.readline()
start = time.perf_counter() + start_delay
for offset, r, g, b in events:
wait = start + offset - time.perf_counter()
if wait > 0:
time.sleep(wait)
write_rgb(r, g, b)
time.sleep(pulse_seconds)
write_rgb(0, 0, 0)
write_rgb(0, 0, 0)
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--ffmpeg", default=DEFAULT_FFMPEG)
parser.add_argument("--ffplay", default=DEFAULT_FFPLAY)
parser.add_argument("--ffprobe", default=DEFAULT_FFPROBE)
parser.add_argument("--video-preset", choices=sorted(VIDEO_PRESETS), default="ps3-eye-0")
parser.add_argument("--audio-preset", choices=sorted(AUDIO_PRESETS), default="scarlett-cardioid")
parser.add_argument("--video-device")
parser.add_argument("--audio-device")
parser.add_argument("--ssh-target", default="nightwing")
parser.add_argument("--hidraw", default="/dev/hidraw1")
parser.add_argument("--out-dir", default="artifacts/runtime/move-latency")
parser.add_argument("--duration", type=float, default=8.0)
parser.add_argument("--lead", type=float, default=1.2)
parser.add_argument("--interval", type=float, default=0.8)
parser.add_argument("--pulses", type=int, default=6)
parser.add_argument("--schedule", choices=("uniform", "debruijn"), default="debruijn")
parser.add_argument("--alphabet-size", type=int, default=8)
parser.add_argument("--order", type=int, default=3)
parser.add_argument("--chirp-ms", type=float, default=80.0)
parser.add_argument("--pulse-ms", type=float, default=90.0)
parser.add_argument("--led-start-delay-ms", type=float, default=520.0)
parser.add_argument("--sample-rate", type=int, default=48000)
parser.add_argument("--width", type=int)
parser.add_argument("--height", type=int)
parser.add_argument("--framerate", type=int)
args = parser.parse_args()
preset_device, preset_width, preset_height, preset_framerate = VIDEO_PRESETS[args.video_preset]
args.video_device = args.video_device or preset_device
args.audio_device = args.audio_device or AUDIO_PRESETS[args.audio_preset]
args.width = args.width or preset_width
args.height = args.height or preset_height
args.framerate = args.framerate or preset_framerate
return args
def build_debruijn(alphabet_size: int, order: int) -> list[int]:
alphabet_size = max(2, int(alphabet_size))
order = max(1, int(order))
sequence: list[int] = []
a = [0] * (alphabet_size * order)
def db(t: int, p: int) -> None:
if t > order:
if order % p == 0:
sequence.extend(a[1 : p + 1])
return
a[t] = a[t - p]
db(t + 1, p)
for j in range(a[t - p] + 1, alphabet_size):
a[t] = j
db(t + 1, t)
db(1, 1)
return sequence
def rotate_to_distinct_opening(symbols: list[int], order: int) -> list[int]:
if len(symbols) <= order:
return symbols
windows: set[tuple[int, ...]] = set()
for index in range(len(symbols)):
window = tuple(symbols[(index + offset) % len(symbols)] for offset in range(order))
if window in windows:
continue
windows.add(window)
if len(set(window)) == min(order, len(set(symbols))):
return symbols[index:] + symbols[:index]
return symbols
def symbol_chirp(symbol: int, args: argparse.Namespace) -> tuple[float, float, tuple[int, int, int]]:
# Eight log-spaced-ish lanes in the PS Move/camera-friendly audio band.
lane = symbol % max(2, args.alphabet_size)
base = 1450.0 * (2.0 ** (lane / 7.0))
glide = 1.20 if ((symbol // max(2, args.alphabet_size)) & 1) == 0 else 0.84
hue = (lane / max(1, args.alphabet_size)) % 1.0
sector = int(hue * 6.0)
x = int(255 * (1.0 - abs((hue * 6.0) % 2.0 - 1.0)))
colors = [(255, x, 0), (x, 255, 0), (0, 255, x), (0, x, 255), (x, 0, 255), (255, 0, x)]
return base, base * glide, colors[sector % 6]
def build_events(args: argparse.Namespace) -> list[dict[str, float | int | tuple[int, int, int]]]:
if args.schedule == "uniform":
symbols = list(range(args.pulses))
offsets = [args.lead + index * args.interval for index in range(args.pulses)]
else:
sequence = rotate_to_distinct_opening(build_debruijn(args.alphabet_size, args.order), args.order)
symbols = sequence[: args.pulses]
offsets = []
cursor = args.lead
for symbol in symbols:
offsets.append(cursor)
# Time-domain coding: symbol-dependent gap rides on top of the cadence.
cursor += args.interval + 0.035 * ((symbol % 4) - 1.5)
events = []
for index, (symbol, offset) in enumerate(zip(symbols, offsets)):
start_hz, end_hz, color = symbol_chirp(int(symbol), args)
events.append({"index": index, "symbol": int(symbol), "offset": float(offset), "start_hz": start_hz, "end_hz": end_hz, "color": color})
return events
def render_chirp_template(start_hz: float, end_hz: float, args: argparse.Namespace) -> np.ndarray:
chirp_len = int(args.chirp_ms * 0.001 * args.sample_rate)
t = np.arange(chirp_len, dtype=np.float32) / args.sample_rate
k = (end_hz - start_hz) / max(1e-6, args.chirp_ms * 0.001)
phase = 2.0 * math.pi * (start_hz * t + 0.5 * k * t * t)
chirp = np.sin(phase).astype(np.float32)
chirp *= np.hanning(chirp_len).astype(np.float32)
chirp *= 0.45
return chirp
def generate_chirp(path: Path, args: argparse.Namespace, events: list[dict[str, float | int | tuple[int, int, int]]]) -> np.ndarray:
sample_rate = args.sample_rate
total = int(args.duration * sample_rate)
audio = np.zeros(total, dtype=np.float32)
chirp_len = int(args.chirp_ms * 0.001 * sample_rate)
prototype = np.zeros(chirp_len, dtype=np.float32)
for event in events:
chirp = render_chirp_template(float(event["start_hz"]), float(event["end_hz"]), args)
if int(event["index"]) == 0:
prototype = chirp.copy()
offset = float(event["offset"])
start = int(offset * sample_rate)
end = min(total, start + chirp_len)
audio[start:end] += chirp[: end - start]
pcm = np.clip(audio, -1.0, 1.0)
with wave.open(str(path), "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(sample_rate)
wav.writeframes((pcm * 32767).astype("<i2").tobytes())
return prototype
def run_capture(args: argparse.Namespace, out_dir: Path, chirp_path: Path, events: list[dict[str, float | int | tuple[int, int, int]]]) -> Path:
capture_path = out_dir / "capture.mkv"
ffmpeg_cmd = [
args.ffmpeg,
"-hide_banner",
"-y",
"-thread_queue_size",
"1024",
"-rtbufsize",
"512M",
"-f",
"dshow",
"-video_size",
f"{args.width}x{args.height}",
"-framerate",
str(args.framerate),
"-i",
f"video={args.video_device}",
"-thread_queue_size",
"1024",
"-f",
"dshow",
"-i",
f"audio={args.audio_device}",
"-t",
f"{args.duration:.3f}",
"-map",
"0:v:0",
"-map",
"1:a:0",
"-c:v",
"ffv1",
"-level",
"3",
"-c:a",
"pcm_s16le",
"-ac",
"1",
"-ar",
str(args.sample_rate),
str(capture_path),
]
ffmpeg = subprocess.Popen(ffmpeg_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
time.sleep(1.0)
remote_b64 = base64.b64encode(REMOTE_LED.encode("utf-8")).decode("ascii")
event_csv = ",".join(
f"{float(event['offset']):.6f}:{int(event['color'][0])}:{int(event['color'][1])}:{int(event['color'][2])}"
for event in events
)
pulse_seconds = args.pulse_ms * 0.001
remote_cmd = (
"tmp=$(mktemp); "
f"printf '%s' '{remote_b64}' | base64 -d > \"$tmp\"; "
f"python3 \"$tmp\" '{args.hidraw}' '{event_csv}' '{pulse_seconds:.6f}' '{args.led_start_delay_ms * 0.001:.6f}'; "
"status=$?; rm -f \"$tmp\"; exit $status"
)
led = subprocess.Popen(["ssh", "-o", "BatchMode=yes", args.ssh_target, remote_cmd], stdin=subprocess.PIPE, text=True)
time.sleep(0.5)
audio = subprocess.Popen([args.ffplay, "-nodisp", "-autoexit", "-loglevel", "quiet", str(chirp_path)])
if led.stdin is not None:
led.stdin.write("go\n")
led.stdin.flush()
led.stdin.close()
audio.wait(timeout=args.duration + 3)
led.wait(timeout=args.duration + 5)
_, stderr = ffmpeg.communicate(timeout=args.duration + 10)
(out_dir / "ffmpeg-capture.err.log").write_text(stderr, encoding="utf-8", errors="replace")
if ffmpeg.returncode != 0:
raise RuntimeError(f"ffmpeg capture failed with {ffmpeg.returncode}; see {out_dir / 'ffmpeg-capture.err.log'}")
return capture_path
def extract_capture(args: argparse.Namespace, out_dir: Path, capture_path: Path) -> tuple[Path, Path, list[float]]:
audio_path = out_dir / "timing-mic.wav"
frames_dir = out_dir / "frames"
if frames_dir.exists():
shutil.rmtree(frames_dir)
frames_dir.mkdir(parents=True)
subprocess.run(
[args.ffmpeg, "-hide_banner", "-y", "-i", str(capture_path), "-map", "0:a:0", "-ac", "1", "-ar", str(args.sample_rate), str(audio_path)],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
subprocess.run(
[args.ffmpeg, "-hide_banner", "-y", "-i", str(capture_path), "-map", "0:v:0", "-vsync", "0", str(frames_dir / "frame_%06d.png")],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
probe = subprocess.run(
[
args.ffprobe,
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"frame=best_effort_timestamp_time",
"-of",
"json",
str(capture_path),
],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
data = json.loads(probe.stdout)
times = [float(frame["best_effort_timestamp_time"]) for frame in data.get("frames", []) if "best_effort_timestamp_time" in frame]
return audio_path, frames_dir, times
def detect_audio_peaks(
audio_path: Path,
chirp: np.ndarray,
args: argparse.Namespace,
events: list[dict[str, float | int | tuple[int, int, int]]] | None = None,
) -> list[float]:
with wave.open(str(audio_path), "rb") as wav:
sample_rate = wav.getframerate()
pcm = wav.readframes(wav.getnframes())
samples = np.frombuffer(pcm, dtype="<i2").astype(np.float32) / 32768.0
if args.schedule == "debruijn" and events:
peaks: list[float] = []
search_radius = int(max(0.45, args.interval * 0.65) * sample_rate)
timing_bias = int(0.32 * sample_rate)
for event in events:
template = render_chirp_template(float(event["start_hz"]), float(event["end_hz"]), args)
template -= float(np.mean(template))
template /= max(1e-6, float(np.linalg.norm(template)))
predicted = int((float(event["offset"]) + timing_bias / sample_rate) * sample_rate)
lo = max(0, predicted - search_radius)
hi = min(samples.size, predicted + search_radius + template.size)
if hi - lo <= template.size:
continue
corr = np.correlate(samples[lo:hi], template, mode="valid")
corr = np.maximum(corr, 0.0)
index = int(np.argmax(corr))
if corr[index] <= max(0.01, float(np.percentile(corr, 95)) * 0.35):
continue
peaks.append((lo + index) / sample_rate)
return sorted(peaks)
template = chirp.astype(np.float32)
template -= float(np.mean(template))
template /= max(1e-6, float(np.linalg.norm(template)))
corr = np.correlate(samples, template, mode="valid")
corr = np.maximum(corr, 0.0)
min_gap = int(args.interval * 0.55 * sample_rate)
peaks: list[int] = []
work = corr.copy()
for _ in range(args.pulses):
index = int(np.argmax(work))
if work[index] <= max(0.02, float(np.percentile(corr, 99)) * 0.2):
break
peaks.append(index)
lo = max(0, index - min_gap)
hi = min(work.size, index + min_gap)
work[lo:hi] = 0.0
return sorted(index / sample_rate for index in peaks)
def detect_video_peaks(frames_dir: Path, frame_times: list[float], args: argparse.Namespace) -> tuple[list[float], list[float]]:
frame_paths = sorted(frames_dir.glob("frame_*.png"))
scores: list[float] = []
previous: np.ndarray | None = None
for path in frame_paths:
image = Image.open(path).convert("L")
arr = np.asarray(image, dtype=np.float32)
if previous is None:
scores.append(0.0)
else:
positive_delta = np.maximum(arr - previous, 0.0)
compact_flash = float(np.percentile(positive_delta, 99.90))
bright_tail = float(np.percentile(arr, 99.85))
scores.append(compact_flash + bright_tail * 0.02)
previous = arr
if not scores:
return [], []
times = frame_times[: len(scores)]
if len(times) < len(scores):
times = [index / args.framerate for index in range(len(scores))]
baseline = float(np.percentile(scores, 20))
signal = np.asarray(scores, dtype=np.float32) - baseline
signal = np.maximum(signal, 0.0)
threshold = max(float(np.percentile(signal, 92)), float(np.max(signal)) * 0.40)
min_gap = max(1, int(args.interval * 0.55 * args.framerate))
peaks: list[int] = []
work = signal.copy()
for _ in range(args.pulses):
index = int(np.argmax(work))
if work[index] <= threshold:
break
peaks.append(index)
lo = max(0, index - min_gap)
hi = min(work.size, index + min_gap)
work[lo:hi] = 0.0
return sorted(times[index] for index in peaks), scores
def pair_peaks(audio_peaks: list[float], video_peaks: list[float], args: argparse.Namespace) -> list[tuple[float, float, float]]:
if not audio_peaks or not video_peaks:
return []
candidates = sorted(video - audio for audio in audio_peaks for video in video_peaks)
tolerance = max(0.06, args.interval * 0.16)
best_offset = candidates[0]
best_key: tuple[int, float, float] | None = None
for candidate in candidates:
residuals = [min(abs((audio + candidate) - video) for video in video_peaks) for audio in audio_peaks]
inliers = [residual for residual in residuals if residual <= tolerance]
if not inliers:
continue
key = (len(inliers), -float(np.median(inliers)), -abs(candidate))
if best_key is None or key > best_key:
best_key = key
best_offset = candidate
pairs: list[tuple[float, float, float]] = []
used_video: set[int] = set()
for audio_peak in audio_peaks:
nearest_index = min(
(index for index in range(len(video_peaks)) if index not in used_video),
key=lambda index: abs(video_peaks[index] - (audio_peak + best_offset)),
default=None,
)
if nearest_index is None:
break
video_peak = video_peaks[nearest_index]
if abs(video_peak - (audio_peak + best_offset)) <= tolerance:
used_video.add(nearest_index)
pairs.append((audio_peak, video_peak, video_peak - audio_peak))
return pairs
def fit_schedule(
peaks: list[float],
events: list[dict[str, float | int | tuple[int, int, int]]],
args: argparse.Namespace,
) -> tuple[float | None, list[dict[str, float | int | None]]]:
if not peaks or not events:
return None, []
offsets = [float(event["offset"]) for event in events]
candidates = sorted(peak - offset for peak in peaks for offset in offsets)
tolerance = max(0.08, args.interval * 0.18)
best_shift: float | None = None
best_key: tuple[int, float, float] | None = None
for candidate in candidates:
residuals = [min(abs((offset + candidate) - peak) for peak in peaks) for offset in offsets]
inliers = [residual for residual in residuals if residual <= tolerance]
if not inliers:
continue
key = (len(inliers), -float(np.median(inliers)), -abs(candidate))
if best_key is None or key > best_key:
best_key = key
best_shift = candidate
if best_shift is None:
return None, []
used_peaks: set[int] = set()
matches: list[dict[str, float | int | None]] = []
for event in events:
expected = float(event["offset"]) + best_shift
nearest_index = min(
(index for index in range(len(peaks)) if index not in used_peaks),
key=lambda index: abs(peaks[index] - expected),
default=None,
)
if nearest_index is None or abs(peaks[nearest_index] - expected) > tolerance:
matches.append({"index": int(event["index"]), "symbol": int(event["symbol"]), "expected_s": expected, "observed_s": None, "residual_ms": None})
continue
used_peaks.add(nearest_index)
matches.append(
{
"index": int(event["index"]),
"symbol": int(event["symbol"]),
"expected_s": expected,
"observed_s": peaks[nearest_index],
"residual_ms": (peaks[nearest_index] - expected) * 1000.0,
}
)
return best_shift, matches
def parse_dshow_start_delta_ms(out_dir: Path) -> float | None:
log_path = out_dir / "ffmpeg-capture.err.log"
if not log_path.exists():
return None
text = log_path.read_text(encoding="utf-8", errors="replace")
starts = [float(match.group(1)) for match in re.finditer(r"Duration: N/A, start: ([0-9.]+)", text)]
if len(starts) < 2:
return None
return (starts[0] - starts[1]) * 1000.0
def summarize(
audio_peaks: list[float],
video_peaks: list[float],
out_dir: Path,
scores: list[float],
args: argparse.Namespace,
events: list[dict[str, float | int | tuple[int, int, int]]],
) -> dict:
pairs = pair_peaks(audio_peaks, video_peaks, args)
offsets_ms = [pair[2] * 1000.0 for pair in pairs]
audio_shift, audio_matches = fit_schedule(audio_peaks, events, args)
video_shift, video_matches = fit_schedule(video_peaks, events, args)
schedule_visual_minus_audio_ms = None
if audio_shift is not None and video_shift is not None:
schedule_visual_minus_audio_ms = (video_shift - audio_shift) * 1000.0
enough_visual = len(video_peaks) >= max(2, int(math.ceil(args.pulses * 0.65)))
enough_audio = len(audio_peaks) >= max(2, int(math.ceil(args.pulses * 0.65)))
enough_pairs = len(pairs) >= max(2, int(math.ceil(min(len(audio_peaks), len(video_peaks), args.pulses) * 0.50)))
receipt_valid = enough_visual and enough_audio and enough_pairs
summary = {
"video_preset": args.video_preset,
"video_device": args.video_device,
"video_mode": f"{args.width}x{args.height}@{args.framerate}",
"audio_preset": args.audio_preset,
"audio_device": args.audio_device,
"led_start_delay_ms": args.led_start_delay_ms,
"schedule": args.schedule,
"alphabet_size": args.alphabet_size,
"order": args.order,
"events": events,
"audio_peaks_s": audio_peaks,
"video_peaks_s": video_peaks,
"paired_peaks": [
{"audio_s": audio, "video_s": video, "visual_minus_audio_ms": offset * 1000.0}
for audio, video, offset in pairs
],
"paired_count": len(pairs),
"audio_schedule_shift_ms": audio_shift * 1000.0 if audio_shift is not None else None,
"video_schedule_shift_ms": video_shift * 1000.0 if video_shift is not None else None,
"schedule_visual_minus_audio_ms": schedule_visual_minus_audio_ms,
"audio_schedule_matches": audio_matches,
"video_schedule_matches": video_matches,
"receipt_valid": receipt_valid,
"validity": {
"enough_visual_events": enough_visual,
"enough_audio_events": enough_audio,
"enough_paired_events": enough_pairs,
},
"dshow_video_start_minus_audio_start_ms": parse_dshow_start_delta_ms(out_dir),
"note": "Offsets are diagnostic receipt values. DirectShow stream start offsets are reported separately; native timestamped capture should own final calibration timing.",
"visual_minus_audio_ms": offsets_ms,
"median_visual_minus_audio_ms": float(np.median(offsets_ms)) if offsets_ms else None,
"mean_visual_minus_audio_ms": float(np.mean(offsets_ms)) if offsets_ms else None,
"score_min": float(np.min(scores)) if scores else None,
"score_max": float(np.max(scores)) if scores else None,
}
(out_dir / "latency-summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
return summary
def main() -> int:
args = parse_args()
out_dir = Path(args.out_dir)
if not out_dir.is_absolute():
out_dir = Path.cwd() / out_dir
out_dir.mkdir(parents=True, exist_ok=True)
events = build_events(args)
chirp_path = out_dir / "chirp-train.wav"
(out_dir / "event-schedule.json").write_text(json.dumps(events, indent=2), encoding="utf-8")
chirp = generate_chirp(chirp_path, args, events)
capture_path = run_capture(args, out_dir, chirp_path, events)
audio_path, frames_dir, frame_times = extract_capture(args, out_dir, capture_path)
audio_peaks = detect_audio_peaks(audio_path, chirp, args, events)
video_peaks, scores = detect_video_peaks(frames_dir, frame_times, args)
summary = summarize(audio_peaks, video_peaks, out_dir, scores, args, events)
print(json.dumps(summary, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -102,6 +102,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--gain", type=float, default=0.25, help="Sustain/body gain under onset bursts.")
parser.add_argument("--floor", type=float, default=0.008, help="Noise floor.")
parser.add_argument("--fft-size", type=int, default=512, help="Tiny FFT size for onset flux.")
parser.add_argument("--drain-blocks", type=int, default=4096, help="Maximum ASIO blocks to drain per LED frame.")
parser.add_argument("--onset-gain", type=float, default=1.0, help="Scale applied after percentile onset amplitude.")
parser.add_argument("--onset-decay", type=float, default=0.22, help="Per-frame onset decay.")
parser.add_argument("--onset-threshold", type=float, default=0.58, help="Minimum recent-window percentile that can emit a peak.")
@ -204,32 +205,45 @@ class AsioOnsetReader:
onset_history_ms: float,
onset_cooldown_ms: float,
warmup_ms: float,
drain_blocks: int,
) -> tuple[float, float, tuple[float, float, float], float, float, float] | None:
channel = ctypes.c_int()
timestamp_ns = ctypes.c_longlong()
sequence = ctypes.c_ulonglong()
frame_count = ctypes.c_int()
while time.monotonic() < deadline:
ok = self.dll.mimir_asio_read(
self.handle,
ctypes.byref(channel),
ctypes.byref(timestamp_ns),
ctypes.byref(sequence),
ctypes.byref(frame_count),
self.buffer,
self.max_frames,
)
if not ok:
drained: list[float] = []
reads = 0
while reads < max(1, drain_blocks):
ok = self.dll.mimir_asio_read(
self.handle,
ctypes.byref(channel),
ctypes.byref(timestamp_ns),
ctypes.byref(sequence),
ctypes.byref(frame_count),
self.buffer,
self.max_frames,
)
if not ok:
break
reads += 1
frames = max(0, min(int(frame_count.value), self.max_frames))
if int(channel.value) not in self.channels or frames == 0:
continue
drained.extend(float(self.buffer[index]) for index in range(frames))
keep = self.fft_size * 4
if len(drained) > keep:
del drained[: len(drained) - keep]
if not drained:
time.sleep(0.001)
continue
frames = max(0, min(int(frame_count.value), self.max_frames))
if int(channel.value) not in self.channels or frames == 0:
continue
for index in range(frames):
self.pending.append(float(self.buffer[index]))
self.pending.extend(drained)
keep_pending = self.fft_size * 6
if len(self.pending) > keep_pending:
del self.pending[: len(self.pending) - keep_pending]
if len(self.pending) >= self.fft_size:
frame = np.asarray(self.pending[: self.fft_size], dtype=np.float32)
del self.pending[: self.fft_size // 2]
frame = np.asarray(self.pending[-self.fft_size :], dtype=np.float32)
self.pending.clear()
rms = float(np.sqrt(np.mean(frame * frame)))
spectrum = np.abs(np.fft.rfft(frame * self.window)).astype(np.float32)
spectrum = np.sqrt(np.maximum(spectrum, 0.0)).astype(np.float32)
@ -452,6 +466,7 @@ def main() -> int:
args.onset_history_ms,
args.onset_cooldown_ms,
args.warmup_ms,
args.drain_blocks,
)
if onset is None:
continue