From 7e3c1fb5ffb36229a8b8b558f57f71f35e31ff59 Mon Sep 17 00:00:00 2001 From: Aqua Date: Tue, 10 Nov 2026 23:16:28 +0000 Subject: [PATCH 1/4] aqua-feel sync: audio-visual onset lag against perceptual windows Zero-dependency check: audio spectral-flux onsets and visual luma/flux onsets, paired per logged event, lag in ms, verdict against default/BT.1359-1/EBU R37/ATSC IS-191 windows. Synthetic known-offset fixture and 19 checks. SYNC.md says what the metric cannot tell. Co-Authored-By: Claude Sonnet 5.5 --- tools/aqua-feel/README.md | 2 + tools/aqua-feel/SYNC.md | 88 +++++++++++++ tools/aqua-feel/feel.mjs | 21 ++++ tools/aqua-feel/package.json | 2 +- tools/aqua-feel/sync-fixture.mjs | 38 ++++++ tools/aqua-feel/sync.mjs | 208 +++++++++++++++++++++++++++++++ tools/aqua-feel/sync.test.mjs | 108 ++++++++++++++++ 7 files changed, 466 insertions(+), 1 deletion(-) create mode 100644 tools/aqua-feel/SYNC.md create mode 100644 tools/aqua-feel/sync-fixture.mjs create mode 100644 tools/aqua-feel/sync.mjs create mode 100644 tools/aqua-feel/sync.test.mjs diff --git a/tools/aqua-feel/README.md b/tools/aqua-feel/README.md index c4c0e65..fa0e04a 100644 --- a/tools/aqua-feel/README.md +++ b/tools/aqua-feel/README.md @@ -9,6 +9,8 @@ The WAVs and the gallery exist so a person can press play. To look at a finished audio file (wav/ogg/flac: spectrogram, onsets at 1 ms, level envelope, click/clip/double-hit/overlap/abrupt-end detectors, two-file compare) rather than a rendered patch, use `../aqua-ear`. +To check that sound and picture of a capture line up (video or WAV plus a frame-diff track), use `node feel.mjs sync`: see `SYNC.md`. + ## Run ``` diff --git a/tools/aqua-feel/SYNC.md b/tools/aqua-feel/SYNC.md new file mode 100644 index 0000000..e65b1c8 --- /dev/null +++ b/tools/aqua-feel/SYNC.md @@ -0,0 +1,88 @@ +# aqua-feel sync + +Checks audio-visual synchrony of a capture: finds audio onsets and visual onsets, pairs them, reports the lag per event in ms and +compares it with published detectability windows. Standalone: Node 18+, no npm dependencies (`feel.mjs` imports `sync.mjs`, which +imports one FFT from `metrics.js`). ffmpeg and ffprobe are needed only to read a video file directly. + +**The metric is not the verdict.** It compares two onset lists extracted from signals. A human ear and eye still decide whether a +capture feels in sync. Nothing here heard or saw anything. + +## Run + +``` +# WAV + a per-frame track (no ffmpeg needed) +node feel.mjs sync --wav capture.wav --track frames.csv [--events hits.txt] [--fps 60] [--kind luma|flux] +# a video with one audio and one video stream (needs ffmpeg + ffprobe on PATH) +node feel.mjs sync --video capture.mp4 [--events hits.txt] [--kind luma|flux] +# options: --profile default|bt1359|ebu-r37|atsc-is191 --lead-ms N --lag-ms N --json out.json --no-gate +node sync.test.mjs # 19 checks on synthetic fixtures +node sync-fixture.mjs DIR 80 60 # write a fixture (audio 80 ms late, 60 fps) to try the CLI on +``` + +Exit 0 when every pair is `inside-window` or `borderline`; 1 when any pair is outside the window, an event has no onset, or nothing +paired; 2 on a tool error. `--no-gate` always exits 0. + +**Inputs.** `frames.csv`: one row per video frame, `t,value` in seconds (a header line is skipped), or one value per line with `--fps`. +`value` is mean frame luminance (`--kind luma`, default: a flash or cut is a positive step) or a frame-difference magnitude +(`--kind flux`, motion flux: a start of movement is a rise). `hits.txt`: seconds, optionally `,label` per line (`#` comments), or JSON +`[0.5, {"t":1.3,"label":"hit"}]`. With a video, luminance and flux are computed on a 64x36 gray downscale; stream `start_time` +offsets are applied so both clocks are the container's. + +**Lag convention.** `lag = audio onset - visual onset`. Positive means the sound comes after the picture (audio late); negative means +the sound comes first (audio early). With `--events`, each logged event is matched to the nearest audio and nearest visual onset +within 300 ms, and `a-ev` / `v-ev` give each side's offset from the logged time, so you can tell which side is late. Without an +event log, onsets pair by proximity (within 300 ms, one-to-one). + +## How onsets are found + +- **Audio:** log-magnitude spectral flux (1024-sample window, 128-sample hop, 60 Hz and up), half-wave rectified; peaks over + median + 8 robust sigmas, at least 50 ms apart; each is then refined on the samples to the first one within 20 dB of the local peak. +- **Visual:** the positive step (luma) or the value (flux) of the track; peaks over median + 8 robust sigmas and at least 10% of the + track's range, at least 100 ms apart; the onset is the first frame of the rise reaching 50% of the peak. + +## Verdict + +Per pair, against a window of how far audio may lead (`lead`) or trail (`lag`) the picture before the offset is detectable: + +| profile | lead ms | lag ms | source | +|---|---|---|---| +| `default` | 25 | 100 | the conservative range this tool was asked for; it is not one standard: it sits between the ATSC and ITU figures. Treat as a tripwire | +| `bt1359` | 45 | 125 | ITU-R BT.1359-1 (1998), "Relative timing of sound and vision for broadcasting": detectability threshold +45 / -125 ms (acceptability +90 / -185) | +| `ebu-r37` | 40 | 60 | EBU R37 (2007) limit for emission, sound before / after picture. Figure taken from a secondary summary, the EBU text was not read | +| `atsc-is191` | 15 | 45 | ATSC IS-191 (2003) limit at the DTV encoder input. Secondary summary; the same document gives +30 / -90 ms end to end. The document itself was not read | + +Verdict words: `inside-window`; `audio-late-detectable` / `audio-early-detectable` (outside the window by more than the measurement +uncertainty); `borderline` (the window edge lies within the uncertainty); `no-audio-onset` / `no-visual-onset` / `no-onsets` (the +logged event had no onset within 300 ms). `pass` means every pair is `inside-window` or `borderline` and none is missing. +Sources: [ITU-R BT.1359-1](https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.1359-1-199811-I!!PDF-E.pdf) (the PDF text was +fragmentary in the search; figures confirmed against secondary summaries), and the TV Technology articles +[AV synchronization, how bad is bad](https://www.tvtechnology.com/opinions/av-synchronization-how-bad-is-bad) and +[ATSC recommends AV sync rules](https://tvtechnology.com/opinions/atsc-recommends-av-sync-rules). Windows depend on content (speech +lip-sync is more sensitive than a game impact) and on the viewer; they are population thresholds, not a property of your capture. + +## Uncertainty + +Reported as `uncertaintyMs`: half a frame period plus half an audio hop (about +/-9.7 ms at 60 fps, +/-18 ms at 30 fps). A visual +onset can only land on a frame, and a frame stamps when the picture started to be displayed, not when the light reached the eye. +Treat lags closer together than the uncertainty as equal. + +## What the metric cannot tell you + +- **Whether it feels right.** Two onsets agreeing says the energy rose at the same time, not that the sound belongs to the picture. + A person must watch and listen. +- **That the pairing is the right pairing.** Onsets are matched by nearness. A sound that belongs to an earlier or later visual + event can be paired with the wrong one; with an event log, the log is trusted, with none, nothing checks the pairing but you. +- **Which onset the viewer attends to.** A swell, a reverb tail, a ramped attack or a low-frequency thump has a perceptual onset + later than the first sample above the threshold; a flash with a fade reads late the same way. A decaying visual (afterimage, a bloom + that outlives the event) is not modelled. +- **Anything about the chain you did not capture.** If the capture is of the app's output, display latency, speaker/Bluetooth + latency and a player's own sync are not in it. A capture can be in sync and the playback chain still be 150 ms off. +- **Variable frame rate.** Constant fps is assumed from `avg_frame_rate` (video) or the track's own timestamps (CSV, which can be + exact). A VFR video drifts. +- **Content that is not onset-shaped.** Continuous music against continuous motion has no onsets to compare; the tool returns + few or none, and silence in either track is reported as no pass, not as sync. +- **Measurements from downscaled luminance.** A flash that is local (a small icon) moves the mean luminance little and may fall + below the 10%-of-range floor; use a cropped frame-diff track in that case. +- **The video path is untested.** `--video` (ffmpeg/ffprobe decode) was written without ffmpeg on the machine it was built on and has never run; the WAV + track path is what the fixtures exercise. +- **The tuned constants.** The 8-sigma threshold, window sizes, 300 ms pairing window and 10% floor were set against synthetic + fixtures only (`sync-fixture.mjs`: tone-burst hits against 4-frame flashes, known offset). They have not been tuned on a real capture. diff --git a/tools/aqua-feel/feel.mjs b/tools/aqua-feel/feel.mjs index f5d220c..9e5835f 100644 --- a/tools/aqua-feel/feel.mjs +++ b/tools/aqua-feel/feel.mjs @@ -5,6 +5,7 @@ // Usage: node feel.mjs run [--out DIR] [--only SUBSTR] [--no-gate] render, measure, write report + gallery, gate // node feel.mjs gate REPORT.json [THRESHOLDS.json] [--partial] re-gate a report (--partial: a subset; skips id-rule and waiver bookkeeping) // node feel.mjs compare BEFORE.json AFTER.json [--only SUBSTR] before/after table +// node feel.mjs sync --wav A.wav --track T.csv [--events E] | --video V [--fps N] [--kind luma|flux] [--profile P] [--json OUT] A/V synchrony (SYNC.md) // node feel.mjs correlate TRIAL_EXPORT.json ... feel metrics vs listening-trial ratings import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -53,7 +54,27 @@ async function run() { process.exit(!noGate && !report.gate.pass ? 1 : 0); } +async function sync() { + const { syncAnalyze, readWav, readTrack, readEvents, readVideo, formatSync } = await import("./sync.mjs"); + const wav = flag("--wav"), trk = flag("--track"), vid = flag("--video"), evs = flag("--events"), fps = Number(flag("--fps") ?? 0) || undefined; + const kind = flag("--kind") ?? "luma", profile = flag("--profile") ?? "default", json = flag("--json"), noGate = bool("--no-gate"); + const lead = flag("--lead-ms"), lag = flag("--lag-ms"); + let audio, sampleRate, audioStartS = 0, visual; + try { + if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; } + else if (wav && trk) { const w = readWav(readFileSync(wav)); audio = w.samples; sampleRate = w.sampleRate; visual = readTrack(readFileSync(trk, "utf8"), fps); } + else throw new Error("give --video FILE, or --wav FILE with --track FILE"); + const r = syncAnalyze({ audio, sampleRate, audioStartS, visual, visualKind: kind, profile, events: evs ? readEvents(readFileSync(evs, "utf8")) : undefined, + leadMs: lead === undefined ? undefined : Number(lead), lagMs: lag === undefined ? undefined : Number(lag) }); + r.caveat = "Nothing here was heard or seen: two onset lists compared. The metric is not the verdict; see SYNC.md."; + console.log(formatSync(r)); + if (json) writeFileSync(json, JSON.stringify(r, null, 1)); + process.exit(!noGate && !r.summary.pass ? 1 : 0); + } catch (e) { console.error(`sync: ${e.message}`); process.exit(2); } +} + const cmds = { + sync, run, gate() { const partial = bool("--partial"), report = readJson(args[0]), g = gate(report, readJson(args[1] ?? join(here, "thresholds.json")), { partial }); diff --git a/tools/aqua-feel/package.json b/tools/aqua-feel/package.json index 6707152..b0d2271 100644 --- a/tools/aqua-feel/package.json +++ b/tools/aqua-feel/package.json @@ -3,5 +3,5 @@ "private": true, "type": "module", "description": "Headless feel meter for AquaSynth: renders every shipped patch and Ding offline and measures them. Measures; does not listen.", - "scripts": { "test": "node test.mjs", "feel": "node feel.mjs", "shelf": "node shelf.mjs build", "shelf:check": "node shelf.mjs check" } + "scripts": { "test": "node test.mjs && node sync.test.mjs", "feel": "node feel.mjs", "shelf": "node shelf.mjs build", "shelf:check": "node shelf.mjs check" } } diff --git a/tools/aqua-feel/sync-fixture.mjs b/tools/aqua-feel/sync-fixture.mjs new file mode 100644 index 0000000..73897bb --- /dev/null +++ b/tools/aqua-feel/sync-fixture.mjs @@ -0,0 +1,38 @@ +// Synthetic A/V fixture with a known offset: event times, a mono WAV of tone-burst "hits", and a per-frame luminance track of flashes. +// The audio is the visual delayed by offsetMs (positive = audio late), so syncAnalyze must recover lag = offsetMs. Nothing here is a real capture. +// CLI: node sync-fixture.mjs OUTDIR [offsetMs=80] [fps=60] writes OUTDIR/{fixture.wav,track.csv,events.txt,truth.json} +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { wavFloat32 } from "./render.js"; + +export const FIXTURE_EVENTS_S = [0.5, 1.3, 2.2, 3.4, 4.1, 5.0]; + +export function makeFixture({ offsetMs = 80, fps = 60, sampleRate = 48000, seconds = 6, noise = 0.003, events = FIXTURE_EVENTS_S, seed = 7 } = {}) { + let s = seed >>> 0; const rand = () => { s = (Math.imul(s, 1664525) + 1013904223) >>> 0; return s / 2 ** 32 - 0.5; }; + const audio = new Float32Array(Math.round(sampleRate * seconds)); + for (let i = 0; i < audio.length; i++) audio[i] = noise * 2 * rand(); + for (const e of events) { // 40 ms decaying 880 Hz burst with a 1 ms ramp: a hit with a sharp but not instantaneous attack + const start = Math.round((e + offsetMs / 1000) * sampleRate); + for (let j = 0; j < 0.04 * sampleRate && start + j < audio.length; j++) { + const t = j / sampleRate; audio[start + j] += 0.6 * Math.min(1, t / 0.001) * Math.exp(-t / 0.012) * Math.sin(2 * Math.PI * 880 * t); + } + } + const n = Math.round(fps * seconds), t = new Float64Array(n), v = new Float64Array(n); + for (let f = 0; f < n; f++) { // frame f is on screen from f/fps; a flash is bright for 4 frames starting at the first frame at or after the event + t[f] = f / fps; v[f] = 0.1 + 0.01 * 2 * rand(); + for (const e of events) { const f0 = Math.ceil(e * fps - 1e-9); if (f >= f0 && f < f0 + 4) v[f] = 0.9; } + } + return { audio, sampleRate, visual: { t, v }, events: events.map((e, i) => ({ t: e, label: `hit${i}` })), offsetMs, fps }; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const [dir, off = "80", fps = "60"] = process.argv.slice(2); + if (!dir) { console.error("usage: node sync-fixture.mjs OUTDIR [offsetMs=80] [fps=60]"); process.exit(2); } + const fx = makeFixture({ offsetMs: Number(off), fps: Number(fps) }); mkdirSync(dir, { recursive: true }); + const wav = wavFloat32(fx.audio); wav.writeUInt32LE(fx.sampleRate, 24); wav.writeUInt32LE(fx.sampleRate * 4, 28); // wavFloat32 writes 48 kHz; same rate here + writeFileSync(join(dir, "fixture.wav"), wav); + writeFileSync(join(dir, "track.csv"), "t,luma\n" + Array.from(fx.visual.t, (x, i) => `${x.toFixed(6)},${fx.visual.v[i].toFixed(5)}`).join("\n") + "\n"); + writeFileSync(join(dir, "events.txt"), fx.events.map((e) => `${e.t},${e.label}`).join("\n") + "\n"); + writeFileSync(join(dir, "truth.json"), JSON.stringify({ offsetMs: fx.offsetMs, fps: fx.fps, note: "audio = visual delayed by offsetMs; synthetic" }, null, 1)); + console.log(`wrote fixture to ${dir} (audio ${fx.offsetMs} ms late)`); +} diff --git a/tools/aqua-feel/sync.mjs b/tools/aqua-feel/sync.mjs new file mode 100644 index 0000000..bf31ae6 --- /dev/null +++ b/tools/aqua-feel/sync.mjs @@ -0,0 +1,208 @@ +// aqua-feel sync: audio-visual synchrony check. Zero npm dependencies (node >= 18); ffmpeg/ffprobe only if you hand it a video. +// NOTHING HERE HEARS OR SEES ANYTHING. It compares two onset lists extracted from signals; see SYNC.md for what that cannot tell you. +// Library: syncAnalyze({ audio, sampleRate, visual:{t,v}, events, profile }) -> report. CLI wiring is in feel.mjs (`node feel.mjs sync ...`). +import { readFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { fft } from "./metrics.js"; + +// Lag convention: lag = audio onset time - visual onset time. Positive = audio LATE (after the picture); negative = audio EARLY. +// Windows are [leadMs, lagMs]: audio may lead the picture by up to leadMs and trail it by up to lagMs before the offset is "detectable". +// Sources: see SYNC.md. 'default' is the requested conservative profile; the rest are the published ones, secondary-source figures noted there. +export const PROFILES = { + default: { leadMs: 25, lagMs: 100, note: "conservative; between ATSC IS-191 and ITU-R BT.1359-1" }, + "bt1359": { leadMs: 45, lagMs: 125, note: "ITU-R BT.1359-1 (1998) detectability threshold" }, + "ebu-r37": { leadMs: 40, lagMs: 60, note: "EBU R37 (2007) emission limit, per secondary source" }, + "atsc-is191": { leadMs: 15, lagMs: 45, note: "ATSC IS-191 limit at the encoder input, per secondary source" }, +}; + +// ---------- input readers ---------- +export function readWav(buf) { + if (buf.toString("latin1", 0, 4) !== "RIFF" || buf.toString("latin1", 8, 12) !== "WAVE") throw new Error("not a RIFF/WAVE file"); + let fmt, data; + for (let p = 12; p + 8 <= buf.length;) { + const id = buf.toString("latin1", p, p + 4), sz = buf.readUInt32LE(p + 4), body = p + 8; + if (id === "fmt ") fmt = { tag: buf.readUInt16LE(body), ch: buf.readUInt16LE(body + 2), sr: buf.readUInt32LE(body + 4), bits: buf.readUInt16LE(body + 14) }; + if (id === "data") { data = buf.subarray(body, Math.min(buf.length, body + sz)); break; } + p = body + sz + (sz & 1); + } + if (!fmt || !data) throw new Error("WAV has no fmt or data chunk"); + if (fmt.tag === 0xfffe) fmt.tag = buf.readUInt16LE(buf.indexOf("fmt ") + 8 + 24); // WAVE_FORMAT_EXTENSIBLE: sub-format tag + const bps = fmt.bits / 8, frames = Math.floor(data.length / (bps * fmt.ch)), out = new Float32Array(frames); + const get = fmt.tag === 3 && fmt.bits === 32 ? (o) => data.readFloatLE(o) : fmt.tag === 3 && fmt.bits === 64 ? (o) => data.readDoubleLE(o) + : fmt.tag === 1 && fmt.bits === 16 ? (o) => data.readInt16LE(o) / 32768 : fmt.tag === 1 && fmt.bits === 24 ? (o) => data.readIntLE(o, 3) / 8388608 + : fmt.tag === 1 && fmt.bits === 32 ? (o) => data.readInt32LE(o) / 2147483648 : null; + if (!get) throw new Error(`unsupported WAV format tag ${fmt.tag} / ${fmt.bits} bit`); + for (let i = 0; i < frames; i++) { let s = 0; for (let c = 0; c < fmt.ch; c++) s += get((i * fmt.ch + c) * bps); out[i] = s / fmt.ch; } // mono mix + return { samples: out, sampleRate: fmt.sr, channels: fmt.ch }; +} + +// Frame-diff track: lines of "t,value" (seconds) or a single "value" column with fps given. A non-numeric first line is a header. +export function readTrack(text, fps) { + const t = [], v = []; + for (const line of text.split(/\r?\n/)) { + if (!line.trim()) continue; + const f = line.trim().split(/[\s,;]+/).filter(Boolean).map(Number); + if (f.some(Number.isNaN)) { if (t.length || v.length) throw new Error(`bad track line: ${line}`); continue; } + if (f.length >= 2) { t.push(f[0]); v.push(f[1]); } + else { if (!fps) throw new Error("single-column track needs --fps"); t.push(v.length / fps); v.push(f[0]); } + } + if (v.length < 3) throw new Error("track has fewer than 3 frames"); + return { t: Float64Array.from(t), v: Float64Array.from(v) }; +} + +// Events: JSON array of numbers or {t,label}, or text lines "t[,label]" (seconds). +export function readEvents(text) { + const s = text.trim(); + if (s.startsWith("[") || s.startsWith("{")) { + const j = JSON.parse(s), a = Array.isArray(j) ? j : j.events; + return a.map((e, i) => (typeof e === "number" ? { t: e, label: `e${i}` } : { t: Number(e.t), label: e.label ?? `e${i}` })); + } + return s.split(/\r?\n/).map((l, i) => l.trim()).filter((l) => l && !l.startsWith("#") && !Number.isNaN(Number(l.split(/[,\s]+/)[0]))) + .map((l, i) => { const [t, ...r] = l.split(/[,\s]+/); return { t: Number(t), label: r.join(" ") || `e${i}` }; }); +} + +// Video via ffmpeg: audio as mono f32 at 48 kHz, video as 64x36 gray frames; per-frame mean luminance and mean |frame diff| (motion flux). +// Stream start_time offsets are applied so both clocks are the container's clock. Constant frame rate is assumed (avg_frame_rate). +export function readVideo(path, { ffmpeg = "ffmpeg", ffprobe = "ffprobe" } = {}) { + let probe; + try { probe = JSON.parse(execFileSync(ffprobe, ["-v", "error", "-show_streams", "-of", "json", path], { encoding: "utf8" })); } + catch (e) { throw new Error(`ffprobe failed (${e.code === "ENOENT" ? "not on PATH; give --wav and --track instead" : e.message})`); } + const vs = probe.streams.find((s) => s.codec_type === "video"), as = probe.streams.find((s) => s.codec_type === "audio"); + if (!vs || !as) throw new Error("file needs one video and one audio stream"); + const [n, d] = vs.avg_frame_rate.split("/").map(Number), fps = n / d, W = 64, H = 36; + const audioRaw = execFileSync(ffmpeg, ["-v", "error", "-i", path, "-map", `0:${as.index}`, "-vn", "-ac", "1", "-ar", "48000", "-f", "f32le", "-"], { maxBuffer: 1 << 30 }); + const videoRaw = execFileSync(ffmpeg, ["-v", "error", "-i", path, "-map", `0:${vs.index}`, "-an", "-vf", `scale=${W}:${H}:flags=area,format=gray`, "-vsync", "passthrough", "-f", "rawvideo", "-"], { maxBuffer: 1 << 30 }); + const frames = Math.floor(videoRaw.length / (W * H)), t = new Float64Array(frames), lum = new Float64Array(frames), flux = new Float64Array(frames); + const vOff = Number(vs.start_time ?? 0) || 0, aOff = Number(as.start_time ?? 0) || 0; + for (let f = 0; f < frames; f++) { + let s = 0, df = 0; for (let i = 0; i < W * H; i++) { const p = videoRaw[f * W * H + i]; s += p; if (f) df += Math.abs(p - videoRaw[(f - 1) * W * H + i]); } + t[f] = vOff + f / fps; lum[f] = s / (W * H) / 255; flux[f] = f ? df / (W * H) / 255 : 0; + } + const a = new Float32Array(audioRaw.buffer, audioRaw.byteOffset, Math.floor(audioRaw.length / 4)); + return { audio: { samples: a, sampleRate: 48000, startS: aOff }, luma: { t, v: lum }, flux: { t, v: flux }, fps }; +} + +// ---------- onset detection ---------- +const median = (a) => { const s = Float64Array.from(a).sort(); const n = s.length; return n ? (n % 2 ? s[(n - 1) / 2] : (s[n / 2 - 1] + s[n / 2]) / 2) : 0; }; + +// Pick onsets from a nonnegative novelty curve nov[i] at times tt[i]. A peak is a local maximum above max(floor, med + k*1.4826*MAD) and +// at least minGapS from the previous kept peak (the larger wins). The onset is the first sample of the rise that reaches riseFrac of the peak. +function pickOnsets(nov, tt, { k, floor, minGapS, riseFrac }) { + const med = median(nov), mad = median(nov.map((x) => Math.abs(x - med))), thr = Math.max(floor, med + k * 1.4826 * mad); + const peaks = []; + for (let i = 0; i < nov.length; i++) { + if (nov[i] < thr || (i && nov[i] < nov[i - 1]) || (i + 1 < nov.length && nov[i] <= nov[i + 1])) continue; + const last = peaks[peaks.length - 1]; + if (last && tt[i] - tt[last] < minGapS) { if (nov[i] > nov[last]) peaks[peaks.length - 1] = i; continue; } + peaks.push(i); + } + return { + threshold: thr, + onsets: peaks.map((p) => { + let i = p; while (i > 0 && nov[i - 1] >= riseFrac * nov[p] && tt[p] - tt[i - 1] < 0.1) i--; + return { t: tt[i], peakT: tt[p], strength: nov[p] }; + }), + }; +} + +// Audio: log-magnitude spectral flux, 1024 window / 128 hop (2.7 ms at 48 kHz), half-wave rectified, summed over 60 Hz..Nyquist. +export function audioOnsets(x, sr, { startS = 0, k = 8, minGapS = 0.05 } = {}) { + const N = 1024, HOP = 128, win = Float64Array.from({ length: N }, (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / N)); + const lo = Math.ceil((60 * N) / sr), hi = N / 2, frames = Math.max(0, Math.floor((x.length - N) / HOP) + 1); + const flux = new Float64Array(frames), tt = new Float64Array(frames), prev = new Float64Array(hi + 1), re = new Float64Array(N), im = new Float64Array(N); + for (let f = 0; f < frames; f++) { + for (let i = 0; i < N; i++) { re[i] = x[f * HOP + i] * win[i]; im[i] = 0; } + fft(re, im); + let s = 0; + for (let b = lo; b <= hi; b++) { const m = Math.log1p(1e3 * Math.hypot(re[b], im[b]) / N); if (f) s += Math.max(0, m - prev[b]); prev[b] = m; } + flux[f] = s; tt[f] = startS + (f * HOP + N / 2) / sr; // a frame is stamped at its centre + } + // The window centre sits half a window after the first audible sample's frame start; the rise-start pick lands within one window of the true onset. + const r = pickOnsets(flux, tt, { k, floor: 1e-3 * hi, minGapS, riseFrac: 0.25 }); + // The flux frame is only good to a window; refine each onset on the samples: the first |x| within 20 dB (10%) of the local peak, searched + // from one window before the flux time to one window after it (the window centre can sit up to N/2 early of a click inside it). + for (const o of r.onsets) { + const c = Math.round((o.t - startS) * sr), a0 = Math.max(0, c - N), a1 = Math.min(x.length, c + N); + let pk = 0; for (let i = a0; i < a1; i++) pk = Math.max(pk, Math.abs(x[i])); + for (let i = a0; i < a1; i++) if (Math.abs(x[i]) >= 0.1 * pk) { o.t = startS + i / sr; break; } + } + r.resolutionMs = (1000 * HOP) / sr; r.windowMs = (1000 * N) / sr; + return r; +} + +// Visual: onset = frame where the track rises. Luminance uses the positive frame-to-frame step; motion flux is already a change measure. +export function visualOnsets(track, { kind = "luma", k = 8, minGapS = 0.1 } = {}) { + const { t, v } = track, nov = new Float64Array(v.length); + for (let i = 1; i < v.length; i++) nov[i] = kind === "luma" ? Math.max(0, v[i] - v[i - 1]) : Math.max(0, v[i]); + const range = Math.max(...v) - Math.min(...v); + const r = pickOnsets(nov, t, { k, floor: 0.1 * range + 1e-12, minGapS, riseFrac: 0.5 }); + const dts = []; for (let i = 1; i < t.length; i++) dts.push(t[i] - t[i - 1]); + r.framePeriodMs = 1000 * median(dts); + return r; +} + +// ---------- pairing and verdict ---------- +export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 300 }) { + const prof = { ...(PROFILES[profile] ?? (() => { throw new Error(`unknown profile ${profile}`); })()) }; + if (leadMs !== undefined) prof.leadMs = leadMs; + if (lagMs !== undefined) prof.lagMs = lagMs; + const a = audioOnsets(audio, sampleRate, { startS: audioStartS }), v = visualOnsets(visual, { kind: visualKind }); + const uncertaintyMs = Math.max(a.resolutionMs, v.framePeriodMs) / 2 + a.resolutionMs / 2; // half a frame period, plus half an audio hop + const used = new Set(), pairs = []; + const nearest = (list, t) => { let best = -1, bd = Infinity; list.forEach((o, i) => { const d = Math.abs(o.t - t); if (d < bd) { bd = d; best = i; } }); return best; }; + const classify = (lag) => { + const excess = lag < 0 ? -lag - prof.leadMs : lag - prof.lagMs; // >0: outside the window by that many ms + if (excess > uncertaintyMs) return lag < 0 ? "audio-early-detectable" : "audio-late-detectable"; + if (excess > -uncertaintyMs) return "borderline"; // the window edge lies inside the measurement uncertainty + return "inside-window"; + }; + if (events?.length) { + for (const e of events) { + const ai = nearest(a.onsets, e.t), vi = nearest(v.onsets, e.t), ao = a.onsets[ai], vo = v.onsets[vi]; + const okA = ao && Math.abs(ao.t - e.t) * 1e3 <= pairWindowMs, okV = vo && Math.abs(vo.t - e.t) * 1e3 <= pairWindowMs; + const row = { label: e.label, eventS: e.t, audioS: okA ? ao.t : null, visualS: okV ? vo.t : null, audioVsEventMs: okA ? (ao.t - e.t) * 1e3 : null, visualVsEventMs: okV ? (vo.t - e.t) * 1e3 : null }; + row.lagMs = okA && okV ? (ao.t - vo.t) * 1e3 : null; + row.verdict = row.lagMs === null ? (okA ? "no-visual-onset" : okV ? "no-audio-onset" : "no-onsets") : classify(row.lagMs); + if (okA) used.add(`a${ai}`); if (okV) used.add(`v${vi}`); + pairs.push(row); + } + } else { + a.onsets.forEach((ao, ai) => { + const vi = nearest(v.onsets, ao.t), vo = v.onsets[vi]; + if (vo && Math.abs(vo.t - ao.t) * 1e3 <= pairWindowMs && !used.has(`v${vi}`)) { + used.add(`a${ai}`); used.add(`v${vi}`); + const lagMs = (ao.t - vo.t) * 1e3; + pairs.push({ label: `a${ai}`, eventS: null, audioS: ao.t, visualS: vo.t, audioVsEventMs: null, visualVsEventMs: null, lagMs, verdict: classify(lagMs) }); + } + }); + } + const lags = pairs.map((p) => p.lagMs).filter((x) => x !== null); + const summary = { + pairs: lags.length, audioOnsets: a.onsets.length, visualOnsets: v.onsets.length, + unpairedAudio: a.onsets.filter((_, i) => !used.has(`a${i}`)).length, unpairedVisual: v.onsets.filter((_, i) => !used.has(`v${i}`)).length, + medianLagMs: lags.length ? median(lags) : null, minLagMs: lags.length ? Math.min(...lags) : null, maxLagMs: lags.length ? Math.max(...lags) : null, + // Spread of the lags: a constant offset is a latency (fixable by a delay), a wide spread is jitter (not fixable by one number). + spreadMs: lags.length ? Math.max(...lags) - Math.min(...lags) : null, + outsideWindow: pairs.filter((p) => /detectable$/.test(p.verdict)).length, borderline: pairs.filter((p) => p.verdict === "borderline").length, + missing: pairs.filter((p) => p.lagMs === null).length, + }; + summary.pass = summary.pairs > 0 && summary.outsideWindow === 0 && summary.missing === 0; + return { + schema: "aqua-feel.sync.v1", lagConvention: "lagMs = audio onset - visual onset; positive = audio late, negative = audio early", + profile: { name: profile, ...prof }, uncertaintyMs, visualKind, audioResolutionMs: a.resolutionMs, framePeriodMs: v.framePeriodMs, + audioOnsets: a.onsets, visualOnsets: v.onsets, pairs, summary, + }; +} + +export function formatSync(r) { + const f = (x, d = 1) => (x === null || x === undefined ? " - " : x.toFixed(d).padStart(7)); + const L = [`profile ${r.profile.name}: audio may lead by ${r.profile.leadMs} ms, lag by ${r.profile.lagMs} ms (${r.profile.note ?? "custom"})`, + `lag = audio - visual (+ = audio late). measurement uncertainty about +/-${r.uncertaintyMs.toFixed(1)} ms (frame period ${r.framePeriodMs.toFixed(1)} ms, audio hop ${r.audioResolutionMs.toFixed(1)} ms)`, + "", "label event s audio s visual s a-ev ms v-ev ms lag ms verdict"]; + for (const p of r.pairs) L.push(`${String(p.label).padEnd(10).slice(0, 10)} ${f(p.eventS, 3)} ${f(p.audioS, 3)} ${f(p.visualS, 3)} ${f(p.audioVsEventMs)} ${f(p.visualVsEventMs)} ${f(p.lagMs)} ${p.verdict}`); + const s = r.summary; + L.push("", `${s.pairs} paired, ${s.unpairedAudio} audio and ${s.unpairedVisual} visual onsets unpaired; median lag ${s.medianLagMs === null ? "-" : s.medianLagMs.toFixed(1)} ms, range ${s.minLagMs === null ? "-" : `${s.minLagMs.toFixed(1)}..${s.maxLagMs.toFixed(1)}`} ms (spread ${s.spreadMs === null ? "-" : s.spreadMs.toFixed(1)} ms)`, + `${s.outsideWindow} outside the window, ${s.borderline} borderline, ${s.missing} without both onsets. ${s.pass ? "WITHIN WINDOW" : "NOT WITHIN WINDOW"} by this metric.`, + "A metric over two onset lists, not a verdict: a person still has to watch and listen (SYNC.md)."); + return L.join("\n"); +} diff --git a/tools/aqua-feel/sync.test.mjs b/tools/aqua-feel/sync.test.mjs new file mode 100644 index 0000000..3ac1fa3 --- /dev/null +++ b/tools/aqua-feel/sync.test.mjs @@ -0,0 +1,108 @@ +// Checks of `aqua-feel sync` on synthetic fixtures with known offsets. No real capture is used. Usage: node sync.test.mjs +import assert from "node:assert/strict"; +import { syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, PROFILES } from "./sync.mjs"; +import { wavFloat32 } from "./render.js"; +import { makeFixture, FIXTURE_EVENTS_S } from "./sync-fixture.mjs"; + +let n = 0; +const t = (name, fn) => { fn(); n++; console.log(`ok ${name}`); }; +const run = (o, extra = {}) => { const fx = makeFixture(o); return { fx, r: syncAnalyze({ audio: fx.audio, sampleRate: fx.sampleRate, visual: fx.visual, events: fx.events, ...extra }) }; }; + +for (const off of [-60, -20, 0, 40, 80, 150, 250]) { + t(`recovers a ${off} ms offset at 60 fps within the stated uncertainty, every event`, () => { + const { r } = run({ offsetMs: off }); + assert.equal(r.pairs.length, FIXTURE_EVENTS_S.length); + for (const p of r.pairs) assert.ok(Math.abs(p.lagMs - off) <= r.uncertaintyMs, `${p.label}: ${p.lagMs} vs ${off} (+/-${r.uncertaintyMs})`); + assert.equal(r.summary.unpairedVisual, 0, "no spurious visual onsets"); + assert.equal(r.summary.unpairedAudio, 0, "no spurious audio onsets"); + }); +} + +t("30 fps: the visual onset lands on a frame at or after the event, so the lag error stays under one frame period", () => { + for (const off of [0, 80]) { + const { r } = run({ offsetMs: off, fps: 30 }); + for (const p of r.pairs) assert.ok(Math.abs(p.lagMs - off) <= 1000 / 30, `${p.lagMs} vs ${off}`); + } +}); + +t("verdict classes follow the default window (lead 25, lag 100) beyond the uncertainty", () => { + const verdictOf = (off) => run({ offsetMs: off }).r.summary; + assert.equal(verdictOf(0).pass, true); + assert.equal(verdictOf(80).pass, true, "80 ms late is inside the 100 ms lag window"); + assert.equal(verdictOf(150).pass, false); + assert.equal(run({ offsetMs: 150 }).r.pairs[0].verdict, "audio-late-detectable"); + assert.equal(run({ offsetMs: -60 }).r.pairs[0].verdict, "audio-early-detectable"); + assert.equal(run({ offsetMs: 100 }).r.pairs[0].verdict, "borderline", "the window edge sits inside the uncertainty"); +}); + +t("the profile changes the verdict, not the measurement", () => { + const a = run({ offsetMs: 70 }, { profile: "default" }).r, b = run({ offsetMs: 70 }, { profile: "ebu-r37" }).r; + assert.equal(a.pairs[0].lagMs, b.pairs[0].lagMs); + assert.equal(a.summary.pass, true); assert.equal(b.summary.pass, false); + assert.ok(PROFILES.default.leadMs === 25 && PROFILES.default.lagMs === 100); +}); + +t("without an events log, onsets pair by proximity and the offset is still recovered", () => { + const fx = makeFixture({ offsetMs: 60 }), r = syncAnalyze({ audio: fx.audio, sampleRate: fx.sampleRate, visual: fx.visual }); + assert.equal(r.pairs.length, FIXTURE_EVENTS_S.length); + for (const p of r.pairs) assert.ok(Math.abs(p.lagMs - 60) <= r.uncertaintyMs); +}); + +t("jitter shows as spread, not as a median: alternate events 0 and 90 ms late", () => { + const fx = makeFixture({ offsetMs: 0 }), clean = makeFixture({ offsetMs: 90 }), a = Float32Array.from(fx.audio); + const s0 = Math.round(FIXTURE_EVENTS_S[1] * 48000) - 100, s1 = Math.round(FIXTURE_EVENTS_S[1] * 48000) + 4800; + for (let i = s0; i < s1; i++) a[i] = clean.audio[i]; // event 1 comes from the 90 ms-late render + const r = syncAnalyze({ audio: a, sampleRate: 48000, visual: fx.visual, events: fx.events }); + assert.ok(r.summary.spreadMs > 80 && r.summary.spreadMs < 100, `spread ${r.summary.spreadMs}`); +}); + +t("an event with no sound is reported, not paired to a neighbour", () => { + const fx = makeFixture({ offsetMs: 0 }), a = Float32Array.from(fx.audio); + for (let i = Math.round(2.0 * 48000); i < Math.round(2.6 * 48000); i++) a[i] = 0; // silence the 2.2 s hit + const r = syncAnalyze({ audio: a, sampleRate: 48000, visual: fx.visual, events: fx.events }); + const p = r.pairs.find((x) => x.eventS === 2.2); + assert.equal(p.verdict, "no-audio-onset"); assert.equal(p.lagMs, null); assert.equal(r.summary.pass, false); +}); + +t("an offset past the 300 ms pairing window is reported as missing, not as a 300+ ms lag", () => { + const { r } = run({ offsetMs: 400 }); + assert.equal(r.pairs[0].verdict, "no-audio-onset"); assert.equal(r.summary.pass, false); +}); + +t("silence and a flat track yield no onsets and no pass", () => { + const r = syncAnalyze({ audio: new Float32Array(48000 * 2), sampleRate: 48000, visual: { t: Float64Array.from({ length: 120 }, (_, i) => i / 60), v: new Float64Array(120).fill(0.5) } }); + assert.equal(r.summary.audioOnsets, 0); assert.equal(r.summary.visualOnsets, 0); assert.equal(r.summary.pass, false); +}); + +t("a sine-bed with a swell rather than a click still gives one audio onset near its start", () => { + const sr = 48000, x = new Float32Array(sr * 3); + for (let i = Math.round(1 * sr); i < x.length; i++) { const tt = (i - sr) / sr; x[i] = 0.5 * Math.min(1, tt / 0.01) * Math.sin(2 * Math.PI * 440 * tt); } + const o = audioOnsets(x, sr).onsets; + assert.equal(o.length, 1); assert.ok(Math.abs(o[0].t - 1.0) < 0.012, `${o[0].t}`); +}); + +t("motion-flux track: a step in flux is an onset at its frame", () => { + const tt = Float64Array.from({ length: 90 }, (_, i) => i / 30), v = new Float64Array(90); v[30] = 0.4; v[31] = 0.3; + const o = visualOnsets({ t: tt, v }, { kind: "flux" }).onsets; + assert.equal(o.length, 1); assert.ok(Math.abs(o[0].t - 1.0) < 1e-9); +}); + +t("readWav reads 16-bit PCM and 32-bit float, mixes stereo to mono, and rejects non-WAV", () => { + const f = wavFloat32(Float32Array.from([0, 0.5, -0.5])), w = readWav(f); + assert.equal(w.sampleRate, 48000); assert.deepEqual(Array.from(w.samples), [0, 0.5, -0.5]); + const b = Buffer.alloc(44 + 8); b.write("RIFF", 0); b.writeUInt32LE(44, 4); b.write("WAVEfmt ", 8); b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(2, 22); + b.writeUInt32LE(44100, 24); b.writeUInt32LE(176400, 28); b.writeUInt16LE(4, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(8, 40); + b.writeInt16LE(16384, 44); b.writeInt16LE(-16384, 46); b.writeInt16LE(8192, 48); b.writeInt16LE(8192, 50); + const s = readWav(b); assert.equal(s.sampleRate, 44100); assert.deepEqual(Array.from(s.samples), [0, 0.25]); + assert.throws(() => readWav(Buffer.from("not a wav file at all"))); +}); + +t("readTrack takes header + t,value rows or a single column with fps; readEvents takes lines or JSON", () => { + const a = readTrack("t,luma\n0,0.1\n0.1,0.2\n0.2,0.3\n"); assert.deepEqual(Array.from(a.t), [0, 0.1, 0.2]); + const b = readTrack("0.1\n0.2\n0.3\n", 10); assert.ok(Math.abs(b.t[2] - 0.2) < 1e-12); + assert.throws(() => readTrack("0.1\n0.2\n0.3\n"), /fps/); + assert.deepEqual(readEvents("# hits\n0.5,kick\n1.25\n").map((e) => e.t), [0.5, 1.25]); + assert.deepEqual(readEvents('[{"t":2,"label":"x"},3]').map((e) => e.label), ["x", "e1"]); +}); + +console.log(`${n} sync checks passed. Synthetic fixtures only; nothing was heard or seen.`); -- 2.47.3 From c33187fda8cf796ba206ccf4994f05b64e4a9e1f Mon Sep 17 00:00:00 2001 From: Aqua Date: Tue, 10 Nov 2026 23:24:47 +0000 Subject: [PATCH 2/4] aqua-feel sync: fix Soul findings (bed/DC refinement, frame-quantisation interval, ambiguous pairing, strict parsers); real browser clip - onset refinement is relative to the steady bed and DC; searched from the flux rise start to the flux peak - lag is judged on [lag - hop/2, lag + frame period + hop/2], not a point: visual onsets are quantised up to the next frame - events and onsets pair one-to-one; another onset inside the pairing window is ambiguous-pairing and fails - unpaired onsets fail unless --allow-unpaired; bad numbers/kinds/profiles/options are errors; parsers refuse unreadable lines - readVideo: works without ffprobe (ffmpeg -i), resamples VFR to CFR by pts; fixtures/browser-capture.webm + recipe Co-Authored-By: Claude Sonnet 5.5 --- tools/aqua-feel/SYNC.md | 47 +++-- tools/aqua-feel/feel.mjs | 19 +- .../fixtures/browser-capture.events.txt | 7 + tools/aqua-feel/fixtures/browser-capture.webm | Bin 0 -> 104659 bytes .../fixtures/record-browser-capture.cjs | 38 ++++ tools/aqua-feel/sync-fixture.mjs | 7 +- tools/aqua-feel/sync.mjs | 187 +++++++++++------- tools/aqua-feel/sync.test.mjs | 135 ++++++++++--- 8 files changed, 321 insertions(+), 119 deletions(-) create mode 100644 tools/aqua-feel/fixtures/browser-capture.events.txt create mode 100644 tools/aqua-feel/fixtures/browser-capture.webm create mode 100644 tools/aqua-feel/fixtures/record-browser-capture.cjs diff --git a/tools/aqua-feel/SYNC.md b/tools/aqua-feel/SYNC.md index e65b1c8..4426cca 100644 --- a/tools/aqua-feel/SYNC.md +++ b/tools/aqua-feel/SYNC.md @@ -14,13 +14,16 @@ capture feels in sync. Nothing here heard or saw anything. node feel.mjs sync --wav capture.wav --track frames.csv [--events hits.txt] [--fps 60] [--kind luma|flux] # a video with one audio and one video stream (needs ffmpeg + ffprobe on PATH) node feel.mjs sync --video capture.mp4 [--events hits.txt] [--kind luma|flux] -# options: --profile default|bt1359|ebu-r37|atsc-is191 --lead-ms N --lag-ms N --json out.json --no-gate -node sync.test.mjs # 19 checks on synthetic fixtures +# options: --profile default|bt1359|ebu-r37|atsc-is191 --lead-ms N --lag-ms N --pair-window-ms N (default 300) +# --min-gap-ms N --allow-unpaired --json out.json --no-gate +# ffmpeg/ffprobe: on PATH, or FFMPEG=/path FFPROBE=/path (a static ffmpeg without ffprobe works) +node sync.test.mjs # 34 checks on synthetic fixtures (35 with ffmpeg: FFMPEG=/path/to/ffmpeg also runs a real browser capture) node sync-fixture.mjs DIR 80 60 # write a fixture (audio 80 ms late, 60 fps) to try the CLI on ``` -Exit 0 when every pair is `inside-window` or `borderline`; 1 when any pair is outside the window, an event has no onset, or nothing -paired; 2 on a tool error. `--no-gate` always exits 0. +Exit 0 when no pair is detectably outside the window; 1 when any pair is outside, an event or onset is unpaired or has no partner, a pairing is ambiguous, +or nothing paired; 2 on a tool error (bad option, unreadable or malformed input). `--no-gate` always exits 0. `--allow-unpaired` stops stray +onsets (background hits that have no counterpart) from failing the check. **Inputs.** `frames.csv`: one row per video frame, `t,value` in seconds (a header line is skipped), or one value per line with `--fps`. `value` is mean frame luminance (`--kind luma`, default: a flash or cut is a positive step) or a frame-difference magnitude @@ -31,14 +34,18 @@ offsets are applied so both clocks are the container's. **Lag convention.** `lag = audio onset - visual onset`. Positive means the sound comes after the picture (audio late); negative means the sound comes first (audio early). With `--events`, each logged event is matched to the nearest audio and nearest visual onset within 300 ms, and `a-ev` / `v-ev` give each side's offset from the logged time, so you can tell which side is late. Without an -event log, onsets pair by proximity (within 300 ms, one-to-one). +event log, audio onsets pair with visual onsets by proximity (within 300 ms, one-to-one, nearest first). **A pair is `ambiguous-pairing` and fails the check when +another onset of the same kind lies inside the pairing window**: with hits closer together than about twice the lag, the nearest onset is not +necessarily the right one and no amount of arithmetic can tell. Narrow `--pair-window-ms` if you know the lag is small, or space your events out. +Parsers refuse input they cannot read (timecodes, a third column, empty cells, non-increasing times) rather than guess. ## How onsets are found - **Audio:** log-magnitude spectral flux (1024-sample window, 128-sample hop, 60 Hz and up), half-wave rectified; peaks over - median + 8 robust sigmas, at least 50 ms apart; each is then refined on the samples to the first one within 20 dB of the local peak. + median + 8 robust sigmas, at least 50 ms apart; each is then refined on the samples to the first one that rises above both 10% of the local peak and + 2.5 times the steady bed (the median block peak of the 0.1 s before, after removing the median as DC). A bed louder than a tenth of the hit still defeats it. - **Visual:** the positive step (luma) or the value (flux) of the track; peaks over median + 8 robust sigmas and at least 10% of the - track's range, at least 100 ms apart; the onset is the first frame of the rise reaching 50% of the peak. + track's range, at least 100 ms apart (`--min-gap-ms` changes it, for both trains; flashes closer than that merge into one onset and the events beyond the first go unpaired); the onset is the first frame of the rise reaching 50% of the peak. ## Verdict @@ -51,9 +58,10 @@ Per pair, against a window of how far audio may lead (`lead`) or trail (`lag`) t | `ebu-r37` | 40 | 60 | EBU R37 (2007) limit for emission, sound before / after picture. Figure taken from a secondary summary, the EBU text was not read | | `atsc-is191` | 15 | 45 | ATSC IS-191 (2003) limit at the DTV encoder input. Secondary summary; the same document gives +30 / -90 ms end to end. The document itself was not read | -Verdict words: `inside-window`; `audio-late-detectable` / `audio-early-detectable` (outside the window by more than the measurement -uncertainty); `borderline` (the window edge lies within the uncertainty); `no-audio-onset` / `no-visual-onset` / `no-onsets` (the -logged event had no onset within 300 ms). `pass` means every pair is `inside-window` or `borderline` and none is missing. +Verdict words: `inside-window` (the whole truth interval, below, lies inside the window); `audio-late-detectable` / `audio-early-detectable` (the whole interval lies outside); +`borderline` (the interval straddles a window edge: not shown to be outside, so it does not fail the gate, and the report says "NOT DETECTABLY OUTSIDE" rather than "WITHIN"); +`ambiguous-pairing`; `no-audio-onset` / `no-visual-onset` / `no-onsets` (the logged event had no onset within the pairing window, so a sound further than 300 ms from its event +reads as absent). `pass` means no pair is detectably outside, ambiguous or missing, and nothing is unpaired (unless `--allow-unpaired`). Sources: [ITU-R BT.1359-1](https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.1359-1-199811-I!!PDF-E.pdf) (the PDF text was fragmentary in the search; figures confirmed against secondary summaries), and the TV Technology articles [AV synchronization, how bad is bad](https://www.tvtechnology.com/opinions/av-synchronization-how-bad-is-bad) and @@ -62,9 +70,11 @@ lip-sync is more sensitive than a game impact) and on the viewer; they are popul ## Uncertainty -Reported as `uncertaintyMs`: half a frame period plus half an audio hop (about +/-9.7 ms at 60 fps, +/-18 ms at 30 fps). A visual -onset can only land on a frame, and a frame stamps when the picture started to be displayed, not when the light reached the eye. -Treat lags closer together than the uncertainty as equal. +A visual onset can only land on a frame, and a frame is stamped when its picture started to be displayed. A change at time tau first shows in the first frame +stamped at or after tau, so the measured visual time is late by 0 to one frame period, and the measured lag (audio minus visual) is **biased low** by that much. The audio +onset is good to about half a hop (1.4 ms). Each pair therefore reports `lagLowMs = lag - 1.4` and `lagHighMs = lag + frame period + 1.4`: the truth lies in that interval +(16.7 + 2.7 ms wide at 60 fps; 33.3 + 2.7 ms at 30 fps), and the verdict is read off the interval. This holds for events on and off the frame grid in the fixtures +(`sync.test.mjs`). The display's own latency (the time from stamp to light) is not in it. ## What the metric cannot tell you @@ -77,12 +87,13 @@ Treat lags closer together than the uncertainty as equal. that outlives the event) is not modelled. - **Anything about the chain you did not capture.** If the capture is of the app's output, display latency, speaker/Bluetooth latency and a player's own sync are not in it. A capture can be in sync and the playback chain still be 150 ms off. -- **Variable frame rate.** Constant fps is assumed from `avg_frame_rate` (video) or the track's own timestamps (CSV, which can be - exact). A VFR video drifts. +- **Variable frame rate.** A video is resampled by presentation time to constant frame rate (ffmpeg `fps` filter), so a VFR browser capture works, but the frame period + used in the interval is the resampled one, not the source's real gaps. A CSV track keeps its own timestamps. +- **Stereo is mixed to mono**, so an antiphase stereo track cancels to silence and reads as no onsets. - **Content that is not onset-shaped.** Continuous music against continuous motion has no onsets to compare; the tool returns few or none, and silence in either track is reported as no pass, not as sync. - **Measurements from downscaled luminance.** A flash that is local (a small icon) moves the mean luminance little and may fall below the 10%-of-range floor; use a cropped frame-diff track in that case. -- **The video path is untested.** `--video` (ffmpeg/ffprobe decode) was written without ffmpeg on the machine it was built on and has never run; the WAV + track path is what the fixtures exercise. -- **The tuned constants.** The 8-sigma threshold, window sizes, 300 ms pairing window and 10% floor were set against synthetic - fixtures only (`sync-fixture.mjs`: tone-burst hits against 4-frame flashes, known offset). They have not been tuned on a real capture. +- **One real clip.** `fixtures/browser-capture.webm` is a headless-Chromium MediaRecorder capture (WebAudio beeps and canvas flashes scheduled at the same instants, vp8 + opus, recipe in `fixtures/record-browser-capture.cjs`). It is a test that the video path runs and pairs, not a validation of the lag it reports: nothing independent says what the true lag of that clip is. No Playground UI or Perlines clip has been run. +- **The tuned constants.** The 8-sigma threshold, window sizes, 300 ms pairing window, 10% floors and 2.5x bed factor were set against synthetic + fixtures (`sync-fixture.mjs`: tone-burst hits against 4-frame flashes, known offset, with noise, tone and DC beds) and one browser clip. They have not been tuned on game or music content. diff --git a/tools/aqua-feel/feel.mjs b/tools/aqua-feel/feel.mjs index 9e5835f..d94f3ab 100644 --- a/tools/aqua-feel/feel.mjs +++ b/tools/aqua-feel/feel.mjs @@ -5,7 +5,7 @@ // Usage: node feel.mjs run [--out DIR] [--only SUBSTR] [--no-gate] render, measure, write report + gallery, gate // node feel.mjs gate REPORT.json [THRESHOLDS.json] [--partial] re-gate a report (--partial: a subset; skips id-rule and waiver bookkeeping) // node feel.mjs compare BEFORE.json AFTER.json [--only SUBSTR] before/after table -// node feel.mjs sync --wav A.wav --track T.csv [--events E] | --video V [--fps N] [--kind luma|flux] [--profile P] [--json OUT] A/V synchrony (SYNC.md) +// node feel.mjs sync --wav A.wav --track T.csv [--events E] | --video V [--fps N] [--kind luma|flux] [--profile P] [--lead-ms N --lag-ms N] [--pair-window-ms N] [--min-gap-ms N] [--allow-unpaired] [--json OUT] A/V synchrony (SYNC.md) // node feel.mjs correlate TRIAL_EXPORT.json ... feel metrics vs listening-trial ratings import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -55,17 +55,22 @@ async function run() { } async function sync() { - const { syncAnalyze, readWav, readTrack, readEvents, readVideo, formatSync } = await import("./sync.mjs"); - const wav = flag("--wav"), trk = flag("--track"), vid = flag("--video"), evs = flag("--events"), fps = Number(flag("--fps") ?? 0) || undefined; - const kind = flag("--kind") ?? "luma", profile = flag("--profile") ?? "default", json = flag("--json"), noGate = bool("--no-gate"); - const lead = flag("--lead-ms"), lag = flag("--lag-ms"); - let audio, sampleRate, audioStartS = 0, visual; + const { syncAnalyze, readWav, readTrack, readEvents, readVideo, formatSync, PROFILES } = await import("./sync.mjs"); try { + const val = (n) => { const i = args.indexOf(n); if (i < 0) return undefined; if (i + 1 >= args.length || args[i + 1].startsWith("--")) throw new Error(`${n} needs a value`); return args.splice(i, 2)[1]; }; + const num = (n) => { const x = val(n); if (x === undefined) return undefined; if (!Number.isFinite(Number(x)) || x.trim() === "") throw new Error(`${n} needs a number, got "${x}"`); return Number(x); }; + const wav = val("--wav"), trk = val("--track"), vid = val("--video"), evs = val("--events"), fps = num("--fps"), json = val("--json"); + const kind = val("--kind") ?? "luma", profile = val("--profile") ?? "default", lead = num("--lead-ms"), lag = num("--lag-ms"), pw = num("--pair-window-ms"), gap = num("--min-gap-ms"); + const noGate = bool("--no-gate"), allowUnpaired = bool("--allow-unpaired"); + if (args.length) throw new Error(`unknown argument ${args[0]}`); + if (!["luma", "flux"].includes(kind)) throw new Error(`--kind is luma or flux, got "${kind}"`); + if (!PROFILES[profile]) throw new Error(`--profile is one of ${Object.keys(PROFILES).join(", ")}, got "${profile}"`); + let audio, sampleRate, audioStartS = 0, visual; if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; } else if (wav && trk) { const w = readWav(readFileSync(wav)); audio = w.samples; sampleRate = w.sampleRate; visual = readTrack(readFileSync(trk, "utf8"), fps); } else throw new Error("give --video FILE, or --wav FILE with --track FILE"); const r = syncAnalyze({ audio, sampleRate, audioStartS, visual, visualKind: kind, profile, events: evs ? readEvents(readFileSync(evs, "utf8")) : undefined, - leadMs: lead === undefined ? undefined : Number(lead), lagMs: lag === undefined ? undefined : Number(lag) }); + leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 300, minGapMs: gap, allowUnpaired }); r.caveat = "Nothing here was heard or seen: two onset lists compared. The metric is not the verdict; see SYNC.md."; console.log(formatSync(r)); if (json) writeFileSync(json, JSON.stringify(r, null, 1)); diff --git a/tools/aqua-feel/fixtures/browser-capture.events.txt b/tools/aqua-feel/fixtures/browser-capture.events.txt new file mode 100644 index 0000000..91263c9 --- /dev/null +++ b/tools/aqua-feel/fixtures/browser-capture.events.txt @@ -0,0 +1,7 @@ +1,hit0 +1.9,hit1 +2.8,hit2 +3.4,hit3 +4.5,hit4 +5.2,hit5 +6.3,hit6 diff --git a/tools/aqua-feel/fixtures/browser-capture.webm b/tools/aqua-feel/fixtures/browser-capture.webm new file mode 100644 index 0000000000000000000000000000000000000000..4d0abcf1fee8cc4e7d5497495357c0e5e19a2247 GIT binary patch literal 104659 zcmeF41wd8Xw)fYjyQGnlluiMqkyJubx*G%p1w?5P5Jc$?38h0ox>LGK0TBrm>5lzv zP()FmUeCMl>UZVf0d}ml_LyUiIp$pBH~)m9V~KIXUTz@ab;!RZHyGh%Hw0mKPkTc> zOW`|^KX1E1;VZZrgTNoYa3oWWBIpKkJ(rCq&5Os-m48mb6=`~_Dr%X}xRbifT{ zTYm==&&sQ*`nrc3Na?*B$m_PUwz3@ecx;Y4NU+2$U!e-}c+P#LT*!}NHz=hNH&mrk zH&9mjfEyI#&1Hz9Qqtm69&S)xUP9Wka!LyN4})Z_uh~i%>KcHc0Kl~r*#COzU*IWz z&J~DrgVHl_BO?_lymol0({4b#5M_d(pBvu_!capogYG~7=A#&0HIZ*S|p{ax2<34#w&1n)r%k}T-Wf_`{W(BdC23dEyaH(%1S{|gw07oIS(d$xBw!oTXq)@z7yEF2 z%Kwe?PY6QJAD^KqGzRuJ z-f`@pI^TH7wwqYIkX(WgjT_(tT!Mid2o(cZw=T`%P}Bl<;#IkT!)neeb*Kxl#(JneABRCh$&Qx=th;hfxp!=-eI z?)-%Ame2r@VWQSa?kBY{`Ml8nbiAh=Om%gK?|9Vs(@POE6q_vIYDbe`o2tpp8n%eL z=5=MGom744u6C;6>q916F;A*p5R-2){_NeCt&f>h;+rNl`LdfQ>^B>5tl1DhG#err zPzBUfyTk+M0rJRQ^T3-K$9;k2o9FO}7ZOPjB6I`jj}8QK6M`J*6o65$#_mSkOJcX| zM^H>EoT2$jSoa+^qQb3`4xXYyMm<8i);)TSb^nYaXzUx5>Wps-G;OFB2yHv}=ycXY zES_fd-hgwIxv-0>hs7PZfyy)Eg2UqUtWbTxj1Sr`fofPe7xARzO}OY40pD=G*XoS# z(6MJ@6_s;Q&lI`_Bo5whXeKMNeA;95cHZ(c z9c6M$M5gWn?*bCHScSkj(X+-OI2j@`5pnhAu}kDV>>HW3qi6EQQ7+9q_;^vdccv8J z%aIMIgR3%xhp{x1%`qO-NDm(WL+U8u)%&!b-nYfgg{azXlc?Bwl=hv^;c5974=$4j8 z&GLix{Z`N2cANXchulFJ$DBy8B|$L34Il%$_OEr{jV$#1i~XOwVpIVd9p@%8(dJg7 zrM=S6fBYYkQ4dFR+{Hno5Kh&^ixWn19FkG3OGfD&J~0>jjQl@7$KD*7F=!OQJ{yJ7 zo#J@-Ln-q7{$G$E?I~tYlp;CWsZE#9N0hZGn>~?;=xtLr>EVvVZc8x@;*ZE_qceLV z7E#y6ZxTc&c&rg1vE?go-*yL~#S1=55R7*NgpQ1M%$MhI@NJ-WvUGQ^oy^^c+XI7L zw!ypL1sWG{U&`1;8#l;62^>!$-FRDi2?~bG%=EtLs8O49k8)i5|IB!+wy~JNjTh@P_Xk z_0lnS@Hy@d9}I|mbe`h|{QbP)J0JCG{DH|Bjy}q-nJjqUMCXngqJF<`OMmFwC%^8w z$6WP07guoTs#Zsrcihr{jH~v4=~PgN$DHb*9WahQ=&!ZIM!b+{f)J4#z_Mg;(BAQa%?Aq$D&PGuv%ke`?Tb`Z?pvuU zblNdz7*7NFBqhn+lIhv$DEv(_yODA#4wFj2}btFKO__j%7y zpVV}f6s#8QZ?yKJc1oiU0WZIsV1i;f$YxuWuwB~Pd~s>7J8wy*oLEJfCilkV*n4i@ zA`ZT7>cZw3C2|kZ*MMqP(OmdqW~R*gc)$mG{mmdS<-wxT_@4NiI4!F!0Jx zW=qyNZgZ)>YDN%(Z>#V71lbeFYO6|)JJIYb0l2eOCX>~o zkY)C7eL0g(8H4Yz@Nvz-EcQuqJQ1g>zMMd3?Oj(zed{WHc;Ob+D+J)-QbP(yTfcM! zA+3`onZ>x|JnG=dIU{9(O;WBs{SwN{w^2_6+i~>8gcJM*mS)%my0*O9PbFX~sq&-Z zv)4e(d+P|s0t4|PjR_)Erxs*4 zW`6>?9|07{+SjX{gqVJJXQy#}g#^7ae&Nr`1Te${8*<0|DH$nPd_<5 zN$SE{wY_-{eyr7+e^1|SefcDMW%&55okr;SDej+t(pl(N-}&V^|6b6K4FA_c|2G!m z{kip5=w~064;hal{;v&-?-E67Kg_pcv~0ldl4?ga+=MRYDXiLDc%OD2x8t3|_0)yg zml6ujtWc)xSFoEGYvZ}rxu7YgmS)e^_*SQwT-<;fNZ`2{*F82?vEb<-fC2;0q-fYu zce5GYEQR2_7K<|?PGM}=vV1_PCWY0N`Py`E`N!`in4*b13c85~Qr^aeyxd^$yvH1w zqm(ny=Wv1c-tF>jxEAezdT65rg|`Ig+uP0bppo6SF=pZOPPWU~QfJ-lM2Sqi`KP&Y z``@kPG*2&wMFvSdpHNq(%_w=nD1s*B&_3c)+x2oj+jM%PBGj|+dS!8;9bVn@vXi&b zFsH_gwIa0aAFyyBGKR2<@v~9CtH)i_xhSvUWKmUE?!7EVRlW`mm&b$L2aG#SE?ubi z%4a>2UPFDBVYsSxM-a#^3Y5QKwWcOnXplr6T)AGJ2_B4ISUGW1BM&d&Q5N+)krJW+ zJLoQ0+rMLvE0eu%3P;4fo^4GcV2aaVNrUuljN`1AU3%1PbysJ=$hdsS%jH)z$%(^I zfZNs7Uelx~bKRKh$|1QXA?0yV3cX>0r5BW*J>{!o!}1T{mKuK4QNzTJGA;S4-pSh{ zRR7tHGq2GKa|9!HYOb7^eouP!Y$&NVi$}X!Mx;W4AwS)9I@kK;kdb@NcSPh*<}dF^ zj5wT zdsZ=cPbl?Iyu3_%S4DqCQ{E29wWtArXn^ZRy5-Z|DDA+(x&|Xp80!4#w85Bj8~#m| z%VAt-ccLlrPAFwqVLpl^u6U)z`ATDw)b#Sr=arPz5@t6Yp}~#x4YZG*b2^KA2wN6* z#W6)TQA^JtBqbXchfH@yEn7UayETXdt7kS<$ESHatVReyO(MO>wSu+3C+xWh?D$4x zjhn-6nE}GfckgVM^*{#HO0=ukd$LHx)%`q^O0`qatzBXDRbxFgLN-xBKwKv;nVRKG zu9TP8ay<$o(O3%n@(gU1-U?VKin*Th+coB$iEC9(c#6&L>|Z&W6A5fG+cKYy1rMFa zY&w;mFgU*=?_(eog?>&xv9ZK@`jPn?fhYUME<>bEm# zePXpy3r|};?6=3}K<&QaXzkKDe=kB}q*M)e@d?B5(tTzYg0)D|Boq_T7SfyA15Cfr>nuXvIG*7FYH!}K$0wEv-?3zR~oTJ%22eiNo z%&vCDV3@%qLvOtYRB|$Wd&>7lSZMIW5w< z63NCvak+IVW`lcWuveXWkFU>jUH!esZKGuala`mb{e?GcOy6?JJt$y|8p>^e5d*nB z8_z;jr>D7{^aQ%{Zn?Cu?)c5NR3%0)L0hcrx6&4Qn_Eqlgr7UMFWG_}$-MSto&0Ph zF*7EVP~`xI(5YMaxesu$p{guMwYCZ7M7@O?m=dO4iFDrO4bu2vzJ}Ec6VCCqxU4p#y68+RxLT`Q39Q`` zVWvMx{{owb=$e^ygyZ?qFxGLTm4K&GINTe)cmPPE-N*FBgA%{xK3mP9yIFEfDi%bt zfe|EOfgrtfIHTl{`fHA9mo#Qs!IrdZLo!xgydLh2nl77(7N9)>w06 z!>Ip#;;%{mzj6l2pG(+blakdkRU-UwS;auO2nhnb{Qg^?CDt(iJY|T(vc8#R!rY4_VfSI~d8`I-5<<4yT zsR)EYM!9EQ$o!hSy;tA)qM>baY`ZUys`rvZCE`TSnK+->Es*Ng=T^XRu*uLDAvLF?SGXaKOv@B3A(DkLWw_Jl?qj5B%FU^gWGE$5)%%Dg@QytOdyCFyshT=<^j=Q z>dxhdED;8;1|B%&cQZn(`%Y|#WiFiNmLIj(zjeXR3;KC|wBi#C+OgPpc*Ryfnu6u% zAcr_*_Zh+^F=AzxruE1Eo0d73M~L;Qc*C$uVVEqUuNJ+jGCCJF-XkF^{9GeZ>knb<`KiwAE2OHDJl^H-& z%Sfw#FO}}ZnMwWxR7*bfjL>TrwCSl^{d!gS-U^!Qa}KM+N6&hnC?KLZ!6jvQ)u+)u zw(QKA2X}~{4%DnYwGmP}M`;}~NVO1|jBuGe#iC&cv_ZxjT-R$$k!?~Q%)f5-T!#7P zyp3q;R$#;|INdc$mmm_~FH*7;7Fiq|#c#dB!J$)sYD)8>9{ zBp2_t`l!FYewQmg>-kx|P&zafNdLRrjijaDTED7xntF`Bz^}Hz* zWs%%lqhC2}@-CRse*H*(YFTkDC z!P8Z>@QD@22vjy=1DoxsvD9YH>Y+i^?l<(6u1D7t+xo~nim}DSwxW~*wKdQ?_3HVR4!RW zp^rf1B)J1vC_ZYL(Q^~Kv#!T4+G&_bz^Wsj?`%fOy|^HN$#80%763qCFXM=qdDVol zS`6=Sl}B+aCNuVn{-aeE!7f{MTwu-U#q+OT>0ISb4={TuUyyc+xU=|qp@#KY?`1xi zg16SbG5%|B*y*U&$`nu}8ee&{;S%72wl@8Q)iWL%;;r~1nMJGYu1V8ZX**+`t|}`s zvvm&T5E|?{Svzm93axZqO18=ZCRK?_%A?lM-eq2@{zUUk%%qDED#%oB=_+l`6O_xU zHtW@`K~vMPPT20O4dR`GORCcC7cZ!$X1=WuQ7{sAYW8=RRb!Q*v7~?aif}mg2Jky9R6VR5%4Q4TK}Yd^AS)f+t$l$u*XfscfR); zDV@@NziVE|_?nUsA7Fc!@Dg6NFj#z7H-0vxf1Hh#_faRVY5mPjWW1};Xm$Q9mAK?- zQkTizWQJ+1I`5r&r>W{@sIV5U5=uEswy3^~Wf{*Qa}iw{g>d`S_U7CAxrlI~XtJHh zDb=zO8Hg}1+HfC8UDTsOM|Dwp6tx{+AV|}Kxo)ISw5l=56nzhOGrIVUo#~w+*8uH@ zgywX_edLL*8@0{#a@BlP&L@;w*B=M+JM_Zee0^^OT&kH5f}=VW`BI1gsA2&DC!TlM z0wjP_`|6qyx#QGiZ)QBgY3(PCjjoqXwH2%2T|x7<*x`bpaZtgk?5rPP2_QY3mwuV% z4V-ZWDe9)(_3!2ea0CG*(D)_^7+fVtle{(uZJ9)4iuXozk4)d_{ipPk_`B)nA9KeT zrtcsm?H38UERS*ejWb;jto0$z#yeN*z%9b?f@|`XRd%}(E!;&InL90rrEO*yA=ps>0BT9v^cJ*!M9R3X~E zC@C3gM@(Ybpyc&A>M@yzBe*MLCP~dCUhYA#2_98vX_fYGalPt?0Ms}(xNP8=Y?dxgH<{)<0c36YS^)=OIURpYh-X!KdOE-a3dI-BVfQ4|=+rF* zh`+#>x611R@UC#(L8vY%5MO}3WS_FocOD=T-1iEfz6xtV2>>Y~lEk8e!61N_^ywxR z^tK?lc5PuO%C0@hWgV$=S`&|b&X)38INVx=2DyTNpTSs3!Q~8JZdV{*m4JO|!#z~_ zbmKL6_s1c)QQzu(qvckhLDndTCtvFEwXvT+(kt|I; zyr3E+Y>Vuqzi+-5cHL}9*aJu~Xs)076goh%Ty8Cz&7bysXK}ShqAA`u02^R@Z`{WM z8V8)LaHugLCbm4yGO(osZ2~Ca-LRNjtAgtUl<>_SUbBh@=v;v`m>Umfn8AQ6R#@`| za{~)RA|yq3pB-Nc@V@zGqPwIeFRb0rjN-187eg=)C8=3Ga|b3GpcYYioY}G$S32ND zVjl%7UsDvPVO|E%#A0JAx+Qm(YG1cbo2X(MRKU|FY(2~6jp8+TVb1qj!`Lee);o4j zT@?b{pM<_z=##@4@J80pm^XrMu!>VO%z2Q}0eaqk#gED51nbGF@)9Bo+;>+5?>^&4 z+hI``oEA?7_Y>mv4hOlH%Ss}LHfk~RZE(_RGx412>6xgr)kz&bA+K^%qbDKcqwP%D z;&zAMmzcs5AerBWf|oIE12BLnhq(!eVb){by5W{>rQ0qyMs_y`hIEk70_yulJ1`z; z3d_FnZn1tco)xO%KHP^RG%{E6wEZ9=LGekfz*u8~0YL%yk$iM;0rQ;pu;>=m*sl6v z`YCXfHvO%bK7zQu!mz%Ru&o`eEI=q3og+IQr(}MQ(xxx!Ke)eD{{f5tW%dax@hSUs z*%TCck3(tEMZwU1npLW{zEcw2sp28O1Ntc=SB$4)uZdf4V_(!gZfZ1V7^Rec!k&2bW2F5ltkG{@8Osh_lvwP$-tB^ zk!IhCF84hjE`w@mP@XagdT`=JBdEuK8{jn0y=J4!HJj65Y#l3-cPb}*D@4ztnv0Hg zH9fW3+2r+ZX=n{AivO#((+XScN@%9I#4Rk%W*uDH!TO!=`Yh2S?2}VVvTwq_b*s#I z7{fF4TBS(Yetc831!eobC()oElR8oKIv;k(n1}rB-KYW+=5ZMQu`@^^93XWupBN~N zXUQ2&mJKXV3xiI-%at}YXrdJjMS($!?ui7*KxY<5>L4-5vCq$3j)p&N^y1=FexAB~ zc}6{eY&nrs1MmXqw-mPYAtNHR`Ru*Q9>_0HkGl~a%pL-$@jx2dnC1Q;nFc^T=6*pD zasx^SVH~cb3Ip-<&EUeq{;GnUO(Jx3>UUT!TTRd42g)cErmG(=SUtc_*ZYlrCMPrbRY^CY9A_II zTZ|Z)c#{V#&E&5}6~do&PK5E_lGh$%-!(w0KJ~`wT8*?5_=Vjn$4QYU#FG|E&U3Kd zHl7vqo#4}e3y1+b$a`{eJ)hUcWtGl)ybO-?Mn$z2GdY>0F`czUhcEI5$>2zGoM@iO zeWVm`VTkXUvk}?&G0uh8-D|<_4;A+FRMn~aj_*y@i@IIA-k_AV`_K;U5USEx$L6ni(IT54!&0 zz9QC9?)gjA3a~tfi>Ti(js0V?SXi}Bi>Uj~m9UG~mttq3-U4EI%9E^xR@HGezdawI zOTTJzm7=#b#p0Tg78(qfH!lPGLY;UH&36Q5PI_^6b7NlQZ7Xl;Ps<6O0OO<~*9=YHZ zJ2uZJ-@1$2+8=t+9pYQnCW~$M;~)U>>Ag`h=ql*voLr284-E435;YTeETz_-K=QAr z0X_J@0x&GEo4vB1Z4Gbfm-guFXAWZbFVrewKsqEy@f*I?Z3PK!h^Jk4kku_vqo8Nv zcAd*rx?Q#hUm>kaC|?aKtvaD|JMT@gIFtYE62`OxpfC!?+LGXkD&aSc>9v`$OlVaR zt?uwFKEHk!PvN`_tI=tkP#^#C8TY#dHlT+vXio8HPyymA$!cp=CJ&cmt*k->$~+ZU zTO9_mr=L~ z{W!LqHwJzCTP+u!jA#5IiCrvq4HPFBs8dQy26zH3%vasW`ay39G|eZ;q!yu?-xnj+ z%p|c1AYvFxvNU6px&~p3hnFRg^VpoxD0Dqz_i)yM#0r_|HvYB7tLcjGD10wBNu&2} zzmRY}7i~fCgfnPaOKDn^iamsz5|M9zU>J4MML&ZS=NYF3+v$$M&MFf6gxd2JvXd!W z?0#gs=zcc~I(MAmpPSDTG8~+{^;CJY>Wj$s(fgc@jjF(=z_7>ePQQ?50de zL!%YF!S?E{>R4-GAM{wnRYhq997LEK%A10f)-M(-v7xG4EQ2X%pn<3ShkbbZ_o~XFpM9B+7H9H#B$}iFJnb$yhNTV>!qyD=)zjo0hhdzelN#W45U3(NeflO4Wnh6m0zFH{?@=8!d0@VHGfKR-mzEkh zPRFUp6Ctg~U2zbm2Hg=>A7UKP+EM{<>kax_dy*xUz;MP(gfHc8=U>Nb@`vUn*pV{M z1Bie9`;d6eOm)hPeV*EE*0b3(YD z70D=?rtyri52EQLZN*X$!^$@AJ-&tq6=)ZH-us@;4ACPp5Gt#HqT7B7bRAu$yL`)~ z5F_dJoPr?U`NBI4=z_F1Ns{cYkVs*~daC|OR^)}H!MhA69gyv(odE=ZrcWSE=Ww73 zmj5aT`oelDTlP0BAS*40NhD;^jn3og$l+1}8kfZ64bb=r^x$O{INuxEN>;mt_hXXK z3kA1ehXf(Jx*Q-vB>&XOE56OP2p)x?-*W=^6Ze*3#}0Em-9sxS0&uXfd#->tAcAAk z4r4F($K-f$pf95l9QU`_md~~`_@%TM?$qHJ#CYTYf5)JIc?^OR*>4>_$g=W2X#(l2 zA8+f)99i{`?N)Gxhmg?kzNNz#i$s*)S|oz=I@YUlj`gaK85zM*yZ!AA|9WqD^kJ|% zdl-hl-qhx&HWPts{`O$?sp#PAo8`a9YUNK1R&Z~=^h9{NW1jetGK4;|;jx;pKV~R{ zM?M^vDZl?={uF=~p78)HJnIVnh!v(?!mDsy8y$VHm(V$uiC z<&<8=9#x+WGG)4hv5U<^;avA5-4KM)nuy<)No_su|13awD?yA&nBtP_>PaJ!zQMb} zvPIXK44$KKk*B7ehyg~h+Zk77KoZuUd&**91*f_ZCQ|6(ipqOm-G@95;p#WjoqJs+$#(f5goTtw6a^i{ zHd#d(;)xsGG1+;V0|b{HYUQ}f!YTUUmvK&3WZ$#NjU;}p-}Z7vDkBJZYgwQUup(o3rQR}%B0BWuIg zE)S`66Dl`6it5pME0FJPvPKx@wk(R-bk756faOt?VfU#QHFPP1s2Nkmnr}Pjw#oaW zmXHvlx`Ad|r_v<+NQa6RgDfZw<&Kh8>x^vST0$p*r2<(NA=BQCD3-^;`P)8EsUz-W zQ6tW(lX_VimcH)so8oay)e~-u-=eC!om?RT>vTnqi?qquATO}-zZP1#V}>di;&1C^*7L zY%3jP&lX9{FRrs*+kPKVxl?mr*-gP`Da>xfh)H1GG41q%nI^$j6w5x?#{gpVp|M&+ z+6VgpzgyGge&E5>Q&!q2Z1gRNJud_zsxVlCJH@SKDtU%-LyV^E2@>bDkSSZc$Kn^9Pa`8jRYtFu~LMH2(j-vH6 zlP=Dd3FFp{3KWg&ctx96ak)x(G8yb?-knJ5?M%ncZ*je4PzD>C=GF7cL^Z)Oc~(PW zYVJW%DA1g^@w)xq!*EQj8hAZB+Pdje$c@`{>WeQ}u5O6YTa#%%!r+yhfA1$9+mGaa zhQAW#NrXdk0Ow0IO?>jlNiYfSD3~kn90PZ73t1>Koj}Q>wM$RcC=z zaRJapQ0jx|BM`7putCWa3YP@}_N9t1#^N@*bLXnwIhW7*=v5pGg3!ZzYdpG%kvxeZ za#EsqWhllXua1?v%HN5U@xx|+8L07Y{nBjM?WDIJ2(oUM*mgv(^oZce%zIz0YE_+_ zm%~-XK(Up+69vB>2PGuCl?J*_=%&y~BhPMf2JDN zC~QYz?;CS06y6QvxZvHJA>s&@M`-rnF^113qCq;vLXggO@6lPMiP8cQWuN*Bt^?UE zCpnVtEWMR7LBP;=sIFpFS=W0#!ei{7^wL>gs?H1MMiRFes4e1+oe}$-nI-2mH#0O_ z>)z87nSrO&&#b~f)}qu-jap*{PV+X+-aXsyPY`M;)|K-b|J(JB-(~1tuF2$xE2;BX&XR#*lzO6zYi#KgQ|OgxSKdD3GG!s|Bbj=Z!{Bj7M%Ez2^(1n1>l66Gk)Gi)JTn`Z7KIdg zM(LP!W#v2ePwm*eedcmtUkVGD2I?$c$XRvBP-^H8HztC8yRZh`x*C3`ie!MokA#yz z8$o1cwF!E@jB|4$h!~?XXmmC!0p2h161AS=fHf5ROnq9rYGCgf^z2(fFdf5Wukhqn zS0eG2o^+8-u4Bp5o0+%0+6VbmW23Rs@URa+zOb;#4D${D;)AoE|Gl+J(-oe`uQy)u#!bY=!< zbC=GmcQ{M0gG;K$dh48T$<8HK+Kg2@GaiQ*b-ANx$int}iSCwmW!fSN7tMyD6tpK4 z`g1bgG_XBe=P!sx;R*||&xx!(&+gQ4?*x*rMQ2Ljb!!7+qj0zTCmPpH?vY9~*Xl~T z)e&#iWKh9FZzUq#FAYPELN&>aWzZL1qo$C^S`xi6D=UKnid=%*?zHr5zdaePpXUly z@Ngt5*GKXWH#-@X6jaY=Am3v|C)mK#b3|W*e7S;$Am6D72*_7Z@;`ul@QXw@R|D&P zT6s$QR<@Ft>2`Whqz4i4%Fmo}Xqq7I@z}O(N(k6dmJ1H!B^2R6%6PljHn z7WAodnc^1+J$4p2Er!?etxwaeBFd)b1XEa!LwgH$L|zuEQiCdut0Fi-U@~H*)*@1~ zklnc4AsK^I-mXJ65mMXT{OXsp1_b=0S;K!2?CZOM``$~Bv2?*|gNW6*p2IVll@{FXBe0GbYteU% zLAc!V6s2JY3Uf%&mZB`UZYKK_ZpA*>$Mmw+?b~2quFD?*_MOx+4?F_<%GoJi)?JzG zOXLZc9iLdSs*A_+I0%01+RQ|ikw}v!C-->5r_pZfs<&N-LqvbVRE*dF*nlH{*M|V}qAN#dE2dVW7o3LB0v@QuCocY5SBDDX$rH#nnbj z@QbWrzys@nv~`B7s}4Tkium5R*cBb%o4&D4cE4;{Ise~4zJ1(6`unmQf49^9aWLfm zu!8hYB|Z>Tk7b+>K4qN$r!3!O;iVWxmT5RjJ{vvnT&8t;q=cI!T$-gMp>Bdjbt~)8 zv}P`_j$9`4I~gxn7dlEvw>+^U!DoZkjgMHqknE2vU;ck3%a@}cR{N3V>*h}q{lxNB z?P%y@BlOoEOu)p8glHe3QU*qDbz~vwTA5+BwxBEMMXh)XTO5 zmaj|l#Xid?H89cB(>Vw!CI1e~XLb8O!18^_IFbHloIj(}^=*_7;l^Rf8N<=4#_^J~ zKW1NQgs4yX^Y6^=T}JrJ?tRJb)nA$M`VPCNxJ5p#0_&3Yws;Fq7dq#@X%FUTmakMY z^7eum@&<^4ABxnHTC4BBncZ_d(HZI$lQ?5vW|M{68-$G?L)Wy3K`wp0;_ahQ6^wIF z#e&Jl6ctO)IIB4?Wn;}GO8$r-Cp@VRbK$@8yf(YF{;4+1~)@>-T zT`-i%5CMy)0k2=B`b}>1tGf07HFob~5u5CHL;D|7#D-9QSSfwwFu;Dj<jAG z<6ma+i10^Q{BOiEQpMesP9?}?4ezElT5K04GdpA%ffd*rT|dDxR!RJ28KZ-W<4yucibqk#A=YVxJ1%-td z&Ep4U+60U$y;7Q3YF8Y;PuU5YK_pnf*fwQb>&UJ#5r?|(7V#MhDsJfe5e3!lMEJjE z89x#agon}b53bEd03VQlkXjXwzvSP?%_IH*^6xDXOrZEc(>TbJ^4i-Fy$-$(Q%Nd@+>kUa!)jZyW<{O`bG~s;D8fPyX5Fp*Nt*3>0EOu*pJytQ6PN zGiaBwxw|R7o*V6_P-hrp?7A7VTe6@_xc_71pYTx*^-HK1BKctz{wF#Ps5(}<@yBFi zh?2*$2-jou#Qwr9Q+z;LRvtjd?ul_Nsp-2ITzN&xJq{lwY9}}yPa*uh#%rTk1-;Jj`#r0lS z_JF5j;*(+S=DmOruPf}J)BiXFpiD^&;(CwRjfT|OZ&1=NWfSYWWY<2vGM|knhLxfqxYOe)j8Seijw|4_t9zW7C>JeA>-QtZ0I`73y zI$ztZG0v@KrN&lZV>9|@Ls9u^6;N-S?PYvq!;Vm2EmXR~Lq-|O>0VuOk#8$WjHNPI z5;fU^O0&#eA)L!*dmxSZ`Z_!vYe2XNL4;WO`G|n4U99?q$(gwi)DVreD(;aBPc?$c z`mQh~6S=>3WXsCjyfiL+HPx5pO6x|`}l$|Y%Go1GsQj|;hRBqH8!TGdt^K!RsSOIAq zH*Ah)&H^uR;C0cqB`I)Wi&wZ@9U0q1&X+O2xJdDWEfwB;LVMUSy1F6)pR@JKlZ$G8 zpyadaG^=!7&?P~P3r|w9Ei?S>I$zj&K5iV;VNZ32ZPN&Wa}n-1y)Vf3iQCg(z^A)d zNOHx+v*+QO)$hygnO<6Z{ZriD_KuEMPYrutytl$|Cia;?@g)4+JRO!8O4SlB<@ymz zT(*}N(To=<@zb~0Wci|>#JY<(y{YECsHIi7eIEC+hE?=k0m~kTG&4)Jr%kSTEwiw; znSu12${)Esmo+%SpWyZqgg?FfwaOGbZhAx_^DdF-<-)_3x;Ma*4LHUg+c zVn-sGF1?4jJBe$Fl78n&u|m}imv=bV6c`^XJsG>#_9UaLAm_%cWFajQX_bT)$ba?? z>}@nHYoE;PXhjq}7%3+@#Ii7199TPQUdNgapXH8IdyiR5v$W2iXCOYjNM?$zA5(T_ z)J5XVnmAkHDMtkBe$D#OD=?CgeRZ~zPN0Sxt<*-|;pSNCW@R!04c>TW^w)ys?Nl{Z z_89i0ZoG{3d>`cZMhP`(vkIXj5f|CAAnRopw|ZR);wHkrAm7`{;U%ZMf!Z#zv1ctG z1^JkpW1}&KzvB3q=9m5x9AAX0HBaScWJ-3$tuY2F`hNI(w=}8C`UHmrX;(v0qpter9A;Za)CzvC3p-FYsyEC&*$`w z1CGznH8|2(D1Eb&m}TObrV}eSoYn!y_f0{*BaV+fqTmM{-@ikUF95BvLfgitg|C?e zRbN)N4{uZgOPu6>uxFRNbnu5Z47vzRhMq%p$d2}ta_P^>ZF``U;IlVG zI}>8dN+-R$w^vnsT3QczMO%zRzG`D@oyQH&SA|BQSZ@g%Ng<6x)(dId6IuK8W;6FrCdJHggqJpfP1n*SbJrRw94t6FiN0`?hKy{e6ya2g`*p zRLy&MqNJ>kHq9;^Tp0X;<8z1X&i_e}4{iJ#f_!-iBHnHQ-gl>z`}=Od;62^IXb2b= zArx>!G1-G*P5cLco>(9k)pkRo+d;UbGA`^}35Bh+f$=cA>Nbuu3~l;PKPiGK@Qd#} zc23s67xV+e|FzJMT=Rd$LQ>;DwH+3+$^p4Z=mv0qx6}TGbADKXg6#OIxbQcYAhajW zl){}o+rG-7A4Fn`-day3!nu-^U6NP7hoz@(mT8t~cPEg@fa_7ibIZ#a;<)Rz$rnAl z!<8Wtgy`hb?}dIOLFhN|FOrJX7k^6$!Y@cg%Y6yLyaQ4ZB0+eThv#IatGIiJfjlXa3irCpgAH0Ha86gd;m-YVeE^Ev3mo3M!wB#As$)fAH7rL?~B zkyIqFrfhTrEq)>u`9cp!#s7T?Ld;SIG^*WV4>t|O`&ZP{p$Gd`%udqQdSXH2UlV*_ zeSf}kci91lhgLs?RiMqO2+YK<;VddCaMpTJ(phr$V#qvnqe4_6f_klKy8&IfV}eDv zejgNUb3O_PQT?IC>sbJWiwT+H_nWpBZ_mlIFwgU7P|3 zO}OAJX9DIVj2n;Gg5_ybX(JfiQA-x6Gu@nT+Fk5kS_o>A^?7n9`=0ZOyX0@!-)!B$ zL#)~?AiN&6;0;ac24u>4>ctizCR`kO5T^X##n@^pg zUmzEoo^$*ETQ&Ri-IMV?*X#?jy!K;qu@j^G{2{q`zWvhkPvjzSY48ss7k^E&@BfNi zr2oS;`x??tiVr|vD#q>JHtC^C6?yv*CeV}&w5&jIIxVNgQ33a9QG23o>mcH_+_+NG`%22FI^hi3e$Wa{r2^J6o< zI2cp*cgOyZfSTfuPZe7dp~9_Zs3I4u&6vJj{+plwZ0pgF^ac z;*KKsDRKYpkjbGwwao9z9g>m%cgr3A56I-_a))RMF*!Xc6y=)OG2f6or2eklA?|;z z+~N7mcS5298O~X@)#BHL7{ z60cyBs6&~j8&z~_FWv%FxyiOJ(#7%;nwtMXbn*XhP;vd0qO6t0n1_?T`b!S9$5d_5 z4){{ZNOut(-o0I9-{e1=Gt&@pD!H(i)B!L5gJvZn)CC>|cgQvwpczMZ5(^y0#PhC7 zoDj`Q0fzrN(Z&A>RGb>78_LzVf+H4+WYWniel9L@Df{ecT1rHQ*D~>yjzgs?BM6Q; zioYMI_@OZT`y1H)mhtuL?Yp<#Q9M4>`217lLzI+bx#Y9oR{79}&XD_C%7@VqviLLQ z!=djhA6A?Ew3P%`K; zJ$dNhLX27Ct~hN^h|28n;sICunqQn4La%hS;kSVvdo)7WV z{yV>|fjw~Y>yi4ZYYBn@DT03Q-9X8LUJGc4R|OLPcvbLF030sNI-Hw9ocTx;4Y$+3 z@tXk_X6060o%=}O9JnM!JD1xjXVeLwqW{2sW(OkU8IKGIk-4guJdYf9m0TbDzl7)+va6ZIBoss?hVkIV55{z>XO6s$u+;@;8a>N7Z4y16vXSO~`;e0Bo|4pTEZqTYQ zlWh3&wye+?Nq{1a3Vf@xZX#>rS;jLh`eXMFiU;aYTy2>rTseI~h&9@lx0|Ao;ZnY6 zMXZD|MolUUpT!OC+%43Mm_WVcDYuyoIqV5BD}G2x{YJ8M^pg>J>q_c2{pVWpg*Df6 zn&vu}J$-iN>rYchmT^H!>Yrhh2JK7X@QYpXo?+2Wjl8Pl%4P1Po&L|Ia1Ki9|4B5k z{H&z@pB(4jv4N-f)6(siCH4Pn(m3~`s#tGh@L;eEtGrQXyXVR9jEa?_^wr)uY~XE3 zZ-?q?gfCJLjYA9>@KlQ!&SXM#hg_+xe{?wd0+A$k*hYSEZC_tusFFv-8{UrE?z?^o z9o%NQVFlax#uYp3OoDR3qyoum*{q>_uFHHz)u11{RKW&%|D|c5pPF zXD+45B3L6Z-)sIPjpN>a*}CvLdbXYd%)`2UX`JOM)Tc{%6vYOgOXDDZA&oP0-n9w@ z%*ikp{YRv6eppg(622Nn`yfDrZ=Hjf^C90m%x%KW1S|Tk9usAin-0}N>JrSB@h6|Z zwRt(dK9=wz_0ft=)6EzR`?p9r!Kd4$%1iZSG?+}Y?9bSm|4=``h##od|P@Co{y-sCbtIy>+MWVncZJgKtt+OHlQNaKi3gf!>F!udj^al|(4 zFZn18c})zqq3fq4cB8ZiqOy&la! zsM!Ck?%wO~1^obc_-mmbAtC=O7DCkc$*DwkykJ{`V4@qq17Jdb=+^b<7uB4wA~yx0 z$jC4-M|V2Ty8d?C;Cy#V({+$+>~9wGpR9gA(&<^ROAq);r$@#{AnVej+y`Z~rt{RT zF=**dV_La4&~(Lf4|IA2DA#5ju|a10Iz7*F?GJQ%$h(=EZ<30v5x@FOr$_svPLKVC z^1VW;^djw#Iz9h#^*f#4f4utr+d4hC$o3-;ogRoDLdBV=xcBo(&|Vm|fwqZGX3uyq zL->n>OE-N8-o2>B^wpVvb`3sl!(8vG;jo%!?0fGM1y{09;FsXSy`d-Ot^jwI59Jq! zKF@1++fE0@SB04mbb82BEE;w!!pRPGdia9*_jP){U;X~8Iz9h~)$i_VrN^q@)jYHC z1!3;KsXv2HYw8nNAW)0OolBHu=8=+ZWnF{49_p|`UVBQ=3|Xmz=~+So*E3`%3BKWWdU9@f0^-~S{WFr25pa#8t zs&MIAkp0_}ZzrRh?(^k}s@bLx5HK~bDHpJS3LIvmEfp@o=gOFE`l~}1(|C{c2h2sH zgsnqU;SuHHUbl`vSLKy@-~Vx8Vb||7-8O=q%|s|2l6|u(yw8Cl$b-l=vRY zupmNl-aKL8T1z)SpLl8`h4quO(>o{~(Fh)g@|IIDW~dxu_?#7WWZZj!Y0%gZbX;d5 z+8~l;fZ?0bJc(U`pt>62LA;%_OtaHpo(#oA%UoxeOEc^uqFt*7{Vv{t7h9gnZz!Sxb00#f)bj4TK{B9{bhne>xD2o~F6RWzerLKnNaUBey) z(PmSMJvy^rcF$Yk-X8nxdduCIYUt_#wKyd&P8QGb^nZ}(h!yASmub*}prC8wayDJYFv6!(d-b*U$H zQjB$-NFEFdNbKXj`IZJx7}IY44r^@oU`NhuumvF+8dkf$E=4$g74Er?QuMEm>B?U? zTf@KAB5J{}WkYJgNfdBx=9Zy7|Aks`Da!WZseLA({38t2Ye9Qf)PgTVdj#6rC<`KI z{Kx3#c1G~vP=Ay0LHTb%dwND*Ym&Ea4BCTZ&@w3_MppxdKzo`y!taQ~YQar-B$l8( z3TKZ(p*;(v+6HHjyh^;LY8ZaN=^w2HUrIma;pyipVa%pXKi8_FA<&1D9jp)mc6*{) zd0TM8?VnFnZb)f-+HxBIPgJDqndBHAbSW8{KO> z^s72bT8ff1J`_KxJ(f**6B?#qWo!D|73u3jd|(ynQy=m1nR;0gYmfX$L zjV{@>5BJ(1LVW(^73mU$p}6l@uE;*a*=4KeJy_4s{N=6@^)33@yJA(8RPAVZ$q6iu zaz!6vD$-M{PF%sg9eBW*Ki!6(=JdtAL(z{f)snS@qh35` zXjW-9@HazzV5JnWJD-2!(32nruBC+4f-km&RK~R}ANqNaErA79Xhq83ScP6ZVDW{{ zXhgsCz%kL73l=65&rAbbi9;oyimmnr4(ZewglGyyDxSBFs4wM44B`X8-J$6Gp$z?Zril5K2_zxoCMrYc-*Fh1WJ39nmcQ7esibww&`AV7 zz(y>DB$J+i+W{*>=i_g&s7HfM740E(*p1CQvCe}I` zBQ!^fh>2^VGtdyd_J{I;bxr06-vIexhk1JgudchJ(`FJ$)lGd5jDc6krs)Ya`3LTuSkTom==j59_5!DfQSQX zB6%dYGJ?0dI&N%8+CZd&)Db~Q9RYJ*bojTde!z`=i_M}9Sbw*jQ0QOx1L0`NO%xFA zBD={}nXriN!Y8Z~>pu)R)oIkZyXNyQyY5ZJFMz>5v*3B~2)hrsd>#^OCc$&LUGP^& z>6c!e0&skmUR?k-}1@mF}tFxEO z;h&vbT`jlT@n-#q8diE0F%FVfE%h7D`j5J0`t94IYjYwV<(GyxF$vT}Kg=3SCQ?JR zi|pE)(Hm9=h*>{k0k~_QTK{3TN`DjUA-VP$B-fS|Nt7i70Wgr+nrSEWLUlk6RJP!6 z>LZg~p3vIWXfymtXmV`l>aMzD z8+l{4?9cwjkKeKV<=bZeGY&_$deT@kOt>vZQW->$__atH4ci-sW=fkEBDeQ4O^+Ua zzWm9o>nt-@dDFYnjdwB&L!`Ss3FVB-(%p3-a%ZnEv;fKs^UDx9=2?an5VUCKUm<21hBTh>zWbxyW%Mno|*uTQi6 zhQYbUu8BwO-p7lgb?UH+d)F-c-KY~Q^IZfx)X;rznczVoZ6{88p z`17go9RDor&M=5_8W<_b@!t_i&+2m#`+iNf>KKE@xjT3!g;|{OPTWLZ9%Xn(Tq%pD z9Vvv1MY8s|u@90aH#goP%d7SEvUs>s?Akwcwa}YX{DsOZXJ+0T=oKmnGVkgxQ+Qmq z@Oo>XPIFk}f%_oVyIsOnJ5eokr7vlq5WZy+d-omxt8G~=`FnyR0WZXo(G$b!)5iuE zsP`#q?k5cB)hf6z^hz#{i=8Xg%qsGh>-k|57RyRBd~cZ8wf~#wZ-`wNqXCR{37tn* z=b0N?dJsQAny%EY}K_4$pcX`Osy*rC- z_t~ju*Bx*!?{Yk7Z;V}b`%RN#Y;xD7Zq&BNPkKbZyd5)dr`iAf1;aBC4t4N;6P6s+ z^SKg4xZi7;c~HxmCTJ&by85@iW14Uh)lj4B#;6i#G3UG8{OC~F%O0)p52i}VuV%1D zf`p%8kG;K1>Up~j-8qiYo@tr;bVtOBHBAf3eWILR#e+-NWB_gAVXz zX&Rv27@tg|y5ADhdmSoL%};`GpGwgRUI{R;t`wNtvdFyzU-9q zQI%rb>o`AhpW2bRPm9rE$M5C}oe8qJiLyZ;U=Lzf4SCNKny_yZkT$?;+D0<{jA*rO zp+B?`5&>zw-DBc8*c`qnUKB3c-PUkC+r8#zl^1dN#2x-1mv4I(!)4W{z?ov-h-l4|+P;=cy|sln!_a_wbScQCcT< z62W(Gjir(0SJ$}xNrO)N8#^8#%DaTZ{q)uPljNgKs{L#}*EJ9| zrQK5WLCxvYshji^PAN|-FpY`AUfr?cx4VPd&kVYF*{ZNjSk{AX{AzW;t9?DfQ0=D? z)>9p~OELE9SKCfg5C)0S?v-|<+GS2~l_2@^RSeycC-RMLtRt~0*oSH#={}u9C2kjw zc--v7oP8pR459tJ!+%h->)?)%lVt7__mY)4{4Mv~s@tgc6J82<5HX7*?Qw3G0{*lV zkc1X~Fn9`)!XOKSHf0gf>J{CrABp73Lp#B%L8K>Gdo^FBJ951}mQd17%NRPENx zc-dn`!$X{l>Bl{!h1IL^pezqd`x<6m$svA^AoujPq6K!hV_XqL_Rr!z^Vs8L(458a z4#+UL;9~VnjKC^|ULp3Q&BYgVY9Vz&Mt!a!!U=cZh#rmfqrLQ*0!vE2uTgR6q}y(P z?DoB~yG(4tWac{#*JsE1smYWIF{?8Xuc=*2DeUzWnUfVKy^@o) z88x&=fhZ}chPM70AxuNNF^^hZaa94*OxX|j_P9A{h(eahZ8f>7SBW&ta)#>#D}3jWiZGP;2y*%o8_cpx%BZq zYb8ctnQjpC8{$!~k)f_OszdP?jI{esRREoUG!YIEe|o+eB9mpSqnh@`ovz_B_4WL(v6Zot}$>Cumf^@q;UP(T{gAX(4!22?kg~7tbBn{!t}2Q@V)GDSzez@{3rcA$F?Dsw_)x*!rbB7E zvrbev5bwf}>QJ!D)2<#{z5wS!U!BW-xskoV$6K=ZRtja)I!t)$*Muf;d**rgabvKhuQf&+ z4V>?6DpBM+tJtXszFkO0eqyhUec>J;S6sdC(-uE34Q_3HKvE7fUzRtPPM-tkcU&etc>(%Mpzq}qW|46 zp2h4727z2GN%^(x>(OF6Fbd-Dw9^NRD$oOZsCW&l^&zv)$I8^vw^Ws>dq{W>;gJ9PiU2 zPDD4<6HZ*^tzqnWrxX3W@k5lPm$qq_z`O-+CDUh5m#Ty7YBHnJ164Pq$=qa=pnp>P zSJZvKpY);jqz;zME(RyGab9lIJ-XyhyLtGMUM&-7*jTE`C`ZH?z_aEA5v9efdD{e< zB?8R{+mGKa*YD7fD$v=}VbGOVnBvEM@E&8!h}>nEbP}%pgGA%as{Ml}J}?mAVmv@v zi*1=AZBk75!&t+!xYZY_ZIvvJlzn9uM{0=3;%3`=^bN0C=$E{0E3qs^CFwpdN6deG zCig=$VZsY9thg`5Q7c5!yh-5yOymsp)uk%x8(miaqONv`X!fb8E zK@2S@pQ(*)dKpx{q1JqTfK&A;nO_Qr$V!>FhK$}foJT~!7Rp?BKe7iNopF#7UpddA?swuX9`_R#$T!JO#Cy%=pgYv+A# zJErh>8IyZ4**{-df-d=z2y`<}G&xQzYY*q{uc1AxQ2Li5a8_v#e?Z`TPtO{vy12{f zsp!Or;bU)cQOs^x(l;Ff2Ogq+*hSm2@m6^;mE?y< z+tmf38>;RiJ&7a;fyq^Lf49{TxDw6B!<>HAFnhFRdTRh*!D;nbOuEErmxM;FeJ5 zX%bh(PB1@k#;heRbCI;vHbvnb=h6|4U$^HW>vxV6MgP<}7Teab)ug2(3(e;XZh5U& zu5IM0@qZ}SPLREqC>sD{EvYer~RMa*U3!JYmFZ9?O6DlQ_MPO!w>zAy!x8 z8Oa6z$_>Byu@|B`shFLmPdDc%~hif_wV2Mo8B_N`kO1ibNdha znc=rB^!Hdud}ERqGXBNL`n!NKG9B0o4^|MjDF9`DtnRA-W%?IFhjWXV8>=rwKId-u zPE*5HFSAn}q#u7PUClfDg=6+$WzbA*u=&xx)ctm+T2-$ld<&dpMCDNdO6NUCquYuK zJGpt7H6J9G;v)gda8DutWhD0znhjN>%&IH;pe+n2>mRc6EY!@;cwQ9sLS5K#n4V7| z{wt{Z<7LuM^>{8JNx`%JWI|<@8FS_z8uOAr^{LosVcC2z?XWWV@7PRcg z87!{Fu3m&H-I6RWsKw59cVEr%6@SFhN;PvU3UD<(X2N17K}@&^DEm>fp%qDYNKJYDAU0{`JN2!Cjb~3+Hv*O;b9Q z?7iX)K8L4UTqeoa2KfrxxA}zd zNQP2J+9|xu>7x?Uihrts-l%n@ihg0TT7$uoHQyl;(>QyNFF^DNWerKiVl?<&KpC0K zVmtQmd~J1ZzM;hxStL@vR=K>7^cQJA+yH?zz1*%#FLEQL;rLRyjK?xoB zjB%d8#33llm7s`Ogy=SDgTh^6-ZOj+Red+DJ4E-#AMUA?ys11j^O|)}6!r|rcOm5~ zo3o9_R~OvCqUSnals_`)IR;eXe3fi+z(#SX8;`z0dY-q1rQkJTo=+%wo@cBj_5_}> zi<)b8KAy+>o9DiU2{SWEB~MpiH{7CtYd`t~T32X49=&AfdS1Jv{otutq5bHcTLhH- z)7p>nT@{B}xmvI;c^R`kd@bEBy?Zoo{ z1 zON>K9f*5yRr-)yBf4whs#(*-|K(^_b&KK2-s5raIMl4u&parG{0`ymrS) zJ=_g^#*>zczD<2GSn8FbMc-PGqHn61uhY^MqN6_)ecw*G_S2$oA0u!rfN9Y|IEhaNW+6BZKC)6)*n}#S2*>yjEUp z-9fuuE5Bkxa`le#tRGoBVx45ZUt#CH3^dm#8FB`9ZK+Sl@Qlh;4zvamplh91lhP5; zY+i^S37N$|(pnR^zpQ;{Vb=ssH+QSZ{i>Ax&iP~0%&^9btMn#z|W&*Z4} z6vnuSsd{lop7?>xyu8~(l!Rm1kAkVqQ#w2zk?CNM7Pis1-RKLKG@6~}h$C-{2r|WN zfGJ--G_0w7`FB#!_%xx^Gs*?1@`aIi1kV@zro;1#pY1o5@tVL=+MyvrbtY{-_2P$< zpj0!v_eph6ZSo2T9b0vWI7Qgm`1CLCx=S%E;Yr?FL?C9Loj7*C*e~Oh5`=oT$J3k~ zqI@aDgDGF&)HA6clrQNsDZj3KN&n}x3^fe#OtHPKvS%rasC9qz8edqQGnGiL=lnTxgi`yJ2DcT6EBm)NdYwc%l2 z_MiEJZDXwJ&c!RgLrH+3ohv{qqD{Z}8y;5-ju2e{V5^&2`{^i1Apiy(JARA$mw;n( zJv|7q1QTr|LM)O0vaC8eA3PXWi22Dwuy9{Xfi?cJTIVUTZ#`LLM@jr3+m zlnEUL=jp!F{el-4WJ;f1pRYk1;xK&n%fPYr-vEvk$!!wg*wc(%imhl`4=65BlZ>wg z94joUzx5+DRE~#2Kv3r)I{DIY6o^sS|Na2L1X_(wZt00t-e|UnUBvID=nGaUDYb)jvOOO zG=I0-diaZ7YT_7===y32YW9t)C7gi2RxR<@>j?e~aEoB>M;!rez&>aEHyYT!|4Z!mR8SSe86V;d#z7f8U-S(K(?s*S(%N=HRCle5sit8m%$@NK*>xJIgKt)euJD>2qaVjbIMI2SqMxP)1%V)BCQfN~+?+J>}qO9`QR#;jpsH zn=d-@mZ(=Ld23UzB4N0Uiq*2Cd{n2;S-npbN)9-7TpP074H((e--hV=mgS`Q*|$Z( zYEWJ2EhSdCu9UW{^Um3B6kPyA^K`Ky=NL*i@uMz4nk8i_Gz@K4d!YkADPmzf% z6V(vxsS2fkGka>XM_0q%w#(mhNq_PJ?F^Ml3T_C_nZYmL1g4$!92!o=Fr^eU)1Sc; zgwR+rzPaO|s*328UVf~J@@b-q&X#;{xSX$^P=1eq)Smp9C3up2h$TetCv#v=jv33D z{*n>E7rnFEo(MAhzV3|k4IeYy6P?z2LfuHaH1i3t!>(uSDIgB)i-`7)hU#lOW{#D- zGK1;>6|Sg0(jx*g02JLdhGvsJL==d_;YD0NQEQZc@Jdx=3OwtJ3lLw3~(Mbs;x znBZN@=KIIkQ)d-|;2}v55!{d}RJTs{)dPPRSNOxMlJ9Jpa$5?Fm8m2J$3MltGUy>< zxId&Y7^W6)7F>5R{Y z_*bZIAH4x=fc>>ph8bD~+8ZI^z0widtFoaVZ#S?rm~r#4g*(#Q3Kt#uU~_!8jgC!XhJ=D6qij`uWsO8*Noe~y26ZW?Lkr8f%+zt2@Hl5a6uz=ZK8A{2+%-80AzX9mdWra zUcl!ahjm?K%dDU&zOD=^l+;K;1*T&Bp*-)GHXSGl;ZuMJ!U2${fbHn?(_ux)2kjkE z#IRZl`OE0FRUv;nR{h+l+DJp-D$)>Gie2b%*WJQkWJ>47UBus|-%(zJ+UN)R9V|@z zz`pynFi`-HtXm8`x7HL$*}BrLsIZXT>UpAc0tg^oe-hb@w|uOXG*M2&`%Q4&huJdA z6{?-ke)=(WS?nkM^&f3B_S0G}NhJpFC%5$E{y*0*{GGHs4>WD6*fV9d7I#sP z=r!Zc(l_~>U+KhuZ(r`)OB3;ntrVPXCk7C&t5=d;&_dUQ;Hs=av6&!W+6K* zc{Y^i@<;avN&=HEdYmF}>a`k3`3zYTP2=L}P7_CI(ANei+zXuKoE%IVu6^9hrEW*6 zwYO)$LRqrkv6WIg6iUl8ah*#WVRY1m)AHJV-#0BPulX)V5^I#tZE@xGCmjGc5~$^N zC((b^@!UaDqI>*?KW--GyoppmjCO12%~rY0dQ+9G<6iAno+HLDCS*FMq9(3d2|L*z z94Ug{oHB5tiU(YJ;~+)8ky zuwG|x!#RNZ&}knv+}gaMCYI7qhD8_rR4DKzJ(DZnREiPo|E|SzTqZW{(uL6G8>BDl z@%9$QF9@JimIU7}&f3oj(TE=sYh?&kIahfcaygz$d<&T~me9V3OVmBmtnW-loRmRE z&2*;eJ{zO%B_3Foi0hU5vQJd#p)HKtIQdZ&dX_e|Z4%|Re`(LL-If^z;4=(hj@4RV)fW5`E&((3E}HpmU1Kz!5jEiC%Wknp5($&e;?92C<^2u8OkbPCUS| z%a~RI?-V)3@cgpV`|L-d0|loiEhi7UP94u?JC{MDJnBmvudr9Y^OUM)n`G&P8tHc; z|BJ@5txYEmDal2IY(#lIyppX9 z1}RIXf9JKJm&hRW6P=ed&yb}Ovc49VpYitlBPFV=k)YB&7f5$Vs`VbsM2q|+&GPP4 zXJ+`mPrwVmgZH1D5#=(Ypg+v#B^G1oAW}XZdpo<#t8336DX$rGjL&|v!-rG%vI`G+ zj}Im%->-Z#9uaRf4esydNsUNV(t7;Gr{-1sXrPos<2?U)!Hy$4p1ReZe{oFQJd2p} z$q4=v`CQIiEIBuZ3ANbom;*9=x8c;i&?boTI!L)-zYYWIMwHhZr0(Iuqr>WT>4t^-LLwX;;Ri-UNw%+0k`}P%obG zP@A=Y+)Vm=Vpb}tQj9$(;I}D$ue+T^T`BjP? zZcqs@72p-wa+97?_`ZDKBcYGkF*wgJc)xl2iq2#x^JY@Zl?#-3;|V!beVn{J$GX|s zjFa=*3P^z)dpAU>V7l`WzU{at@VuCEw0#k5lxV8NtDh6p>P4;&=z}a(xD`4_Xzq3J-BZq^g-wwU!n@&NQ2miEP|nKJz-8v z?7}z=a`8i+L!4IOLWap|pDpc_V>6rBTm)+(U>@G%R7QWHz03?Vs+>?u$|0RifOLvSZe5w;0ej5+wPEt^u0+Y|Ig9j1~=~_PYl1TFO39><}#>&(~zq_v+@dYR5t@4`PILfy==@ z3*dS10ta{&@^2XOuOe#sMU<{t@JGl?@!$%KGVmni16`1xqROGavjF*NP6zVhBrdB`_GTB%s&Hmf}X;x;!Er0Wu;lRzk! z*MFKV)XC-W&dFK-akft9ykB(Lm+)s zz=z7}O6KiZWyMfgD_5kQSahzB2$%RJ=c1!rn&i>35iKD(*LLJu(H(2KHJu3Dj2!El zaoekZZpOoKGpep@#%*u>xfwlS!2#VCVZ>hx4maT45sGLDDL8C<_0PMb3~t73xEWVn bqOFb+lG_4dt3SOvLXYx4k~MZ3 literal 0 HcmV?d00001 diff --git a/tools/aqua-feel/fixtures/record-browser-capture.cjs b/tools/aqua-feel/fixtures/record-browser-capture.cjs new file mode 100644 index 0000000..b6c1591 --- /dev/null +++ b/tools/aqua-feel/fixtures/record-browser-capture.cjs @@ -0,0 +1,38 @@ +// Recipe for fixtures/browser-capture.webm (needs playwright-core and a chromium headless shell; edit the executablePath). Records a real browser A/V clip: WebAudio beeps + canvas flashes at the same scheduled instants, muxed by MediaRecorder. Not the Playground UI. +const { chromium } = require('playwright-core'); +const fs = require('fs'); +(async () => { + // CHROME=/path/to/chrome-headless-shell node record-browser-capture.cjs (writes clip.webm and events.txt in the cwd) + const browser = await chromium.launch({ executablePath: process.env.CHROME, args: ['--autoplay-policy=no-user-gesture-required'] }); + const pg = await (await browser.newContext({ viewport: { width: 320, height: 180 } })).newPage(); + await pg.setContent(''); + const res = await pg.evaluate(async () => { + const c = document.getElementById('c'), g = c.getContext('2d'), ctx = new AudioContext(), dest = ctx.createMediaStreamDestination(); + await ctx.resume(); + const stream = new MediaStream([...c.captureStream(30).getVideoTracks(), ...dest.stream.getAudioTracks()]); + const rec = new MediaRecorder(stream, { mimeType: 'video/webm;codecs=vp8,opus' }), chunks = []; + rec.ondataavailable = e => chunks.push(e.data); + const hits = [1.0, 1.9, 2.8, 3.4, 4.5, 5.2, 6.3]; // seconds after recording start (intended instants) + let flashes = [], t0, on = 0; + const draw = (now) => { + const t = (now - t0) / 1000; let lit = false; + for (const h of hits) if (t >= h && t < h + 0.12) lit = true; + g.fillStyle = lit ? '#fff' : `rgb(${20 + (Math.random() * 3) | 0},20,20)`; g.fillRect(0, 0, 320, 180); + if (lit && !on) flashes.push(t); on = lit ? 1 : 0; + if (t < 8) requestAnimationFrame(draw); else rec.stop(); + }; + const done = new Promise(r => rec.onstop = r); + rec.start(100); + t0 = performance.now(); const a0 = ctx.currentTime; + for (const h of hits) { const o = ctx.createOscillator(), gn = ctx.createGain(); o.frequency.value = 880; gn.gain.setValueAtTime(0, a0 + h); gn.gain.linearRampToValueAtTime(0.5, a0 + h + 0.002); gn.gain.exponentialRampToValueAtTime(0.001, a0 + h + 0.15); o.connect(gn); gn.connect(dest); o.start(a0 + h); o.stop(a0 + h + 0.2); } + requestAnimationFrame(draw); + await done; + const blob = new Blob(chunks, { type: 'video/webm' }), buf = new Uint8Array(await blob.arrayBuffer()); + let s = ''; for (let i = 0; i < buf.length; i += 8192) s += String.fromCharCode(...buf.subarray(i, i + 8192)); + return { b64: btoa(s), hits, flashes, baseLatency: ctx.baseLatency, outputLatency: ctx.outputLatency }; + }); + fs.writeFileSync('clip.webm', Buffer.from(res.b64, 'base64')); + fs.writeFileSync('events.txt', res.hits.map((h, i) => `${h},hit${i}`).join('\n') + '\n'); + console.log(JSON.stringify({ hits: res.hits, flashes: res.flashes, base: res.baseLatency, out: res.outputLatency, bytes: fs.statSync('clip.webm').size })); + await browser.close(); +})(); diff --git a/tools/aqua-feel/sync-fixture.mjs b/tools/aqua-feel/sync-fixture.mjs index 73897bb..142f729 100644 --- a/tools/aqua-feel/sync-fixture.mjs +++ b/tools/aqua-feel/sync-fixture.mjs @@ -7,14 +7,15 @@ import { wavFloat32 } from "./render.js"; export const FIXTURE_EVENTS_S = [0.5, 1.3, 2.2, 3.4, 4.1, 5.0]; -export function makeFixture({ offsetMs = 80, fps = 60, sampleRate = 48000, seconds = 6, noise = 0.003, events = FIXTURE_EVENTS_S, seed = 7 } = {}) { +// bed: steady tones [[hz, amp], ...] under the hits; dc: constant offset; gain: hit amplitude. Events need not sit on frame boundaries. +export function makeFixture({ offsetMs = 80, fps = 60, sampleRate = 48000, seconds = 6, noise = 0.003, events = FIXTURE_EVENTS_S, seed = 7, bed = [], dc = 0, gain = 0.6 } = {}) { let s = seed >>> 0; const rand = () => { s = (Math.imul(s, 1664525) + 1013904223) >>> 0; return s / 2 ** 32 - 0.5; }; const audio = new Float32Array(Math.round(sampleRate * seconds)); - for (let i = 0; i < audio.length; i++) audio[i] = noise * 2 * rand(); + for (let i = 0; i < audio.length; i++) { audio[i] = dc + noise * 2 * rand(); for (const [hz, amp] of bed) audio[i] += amp * Math.sin((2 * Math.PI * hz * i) / sampleRate); } for (const e of events) { // 40 ms decaying 880 Hz burst with a 1 ms ramp: a hit with a sharp but not instantaneous attack const start = Math.round((e + offsetMs / 1000) * sampleRate); for (let j = 0; j < 0.04 * sampleRate && start + j < audio.length; j++) { - const t = j / sampleRate; audio[start + j] += 0.6 * Math.min(1, t / 0.001) * Math.exp(-t / 0.012) * Math.sin(2 * Math.PI * 880 * t); + const t = j / sampleRate; audio[start + j] += gain * Math.min(1, t / 0.001) * Math.exp(-t / 0.012) * Math.sin(2 * Math.PI * 880 * t); } } const n = Math.round(fps * seconds), t = new Float64Array(n), v = new Float64Array(n); diff --git a/tools/aqua-feel/sync.mjs b/tools/aqua-feel/sync.mjs index bf31ae6..4e61596 100644 --- a/tools/aqua-feel/sync.mjs +++ b/tools/aqua-feel/sync.mjs @@ -36,50 +36,73 @@ export function readWav(buf) { return { samples: out, sampleRate: fmt.sr, channels: fmt.ch }; } -// Frame-diff track: lines of "t,value" (seconds) or a single "value" column with fps given. A non-numeric first line is a header. +// Frame-diff track: lines of "t,value" (seconds, strictly increasing) or a single "value" column with fps given. Only the first line may be a header. +// Anything else that does not parse is an error: a silently dropped or shifted row would give a plausible wrong lag. export function readTrack(text, fps) { const t = [], v = []; - for (const line of text.split(/\r?\n/)) { - if (!line.trim()) continue; - const f = line.trim().split(/[\s,;]+/).filter(Boolean).map(Number); - if (f.some(Number.isNaN)) { if (t.length || v.length) throw new Error(`bad track line: ${line}`); continue; } - if (f.length >= 2) { t.push(f[0]); v.push(f[1]); } + text.split(/\r?\n/).forEach((raw, ln) => { + const line = raw.trim(); if (!line) return; + const tok = line.split(/[\s,;]+/), f = tok.map(Number); + if (tok.some((x) => x === "") || f.some((x) => !Number.isFinite(x))) { if (ln === 0 && !t.length && !v.length) return; throw new Error(`track line ${ln + 1} is not numeric: ${raw}`); } + if (f.length > 2) throw new Error(`track line ${ln + 1} has ${f.length} columns; expected "t,value" or one value per line: ${raw}`); + if (f.length === 2) { if (t.length && f[0] <= t[t.length - 1]) throw new Error(`track line ${ln + 1}: time ${f[0]} does not increase`); t.push(f[0]); v.push(f[1]); } else { if (!fps) throw new Error("single-column track needs --fps"); t.push(v.length / fps); v.push(f[0]); } - } + }); if (v.length < 3) throw new Error("track has fewer than 3 frames"); return { t: Float64Array.from(t), v: Float64Array.from(v) }; } -// Events: JSON array of numbers or {t,label}, or text lines "t[,label]" (seconds). +// Events: JSON array of numbers or {t,label}, or text lines "t[,label]" in seconds ('#' comments; the first line may be a header). Anything else is an error. export function readEvents(text) { - const s = text.trim(); + const s = text.trim(), fin = (x, where) => { if (!Number.isFinite(x)) throw new Error(`event ${where} has no numeric time`); return x; }; + let out; if (s.startsWith("[") || s.startsWith("{")) { const j = JSON.parse(s), a = Array.isArray(j) ? j : j.events; - return a.map((e, i) => (typeof e === "number" ? { t: e, label: `e${i}` } : { t: Number(e.t), label: e.label ?? `e${i}` })); + if (!Array.isArray(a)) throw new Error("events JSON must be an array or {events:[...]}"); + out = a.map((e, i) => (typeof e === "number" ? { t: fin(e, i), label: `e${i}` } : e && typeof e === "object" ? { t: fin(Number(e.t), i), label: String(e.label ?? `e${i}`) } : fin(NaN, i))); + } else { + out = []; + s.split(/\r?\n/).forEach((raw, ln) => { + const l = raw.trim(); if (!l || l.startsWith("#")) return; + const m = l.match(/^([^,\s]+)(?:[,\s]+(.*))?$/), t = Number(m[1]); + if (!Number.isFinite(t)) { if (ln === 0 && !out.length) return; throw new Error(`events line ${ln + 1} does not start with a time in seconds: ${raw}`); } + out.push({ t, label: m[2]?.trim() || `e${out.length}` }); + }); } - return s.split(/\r?\n/).map((l, i) => l.trim()).filter((l) => l && !l.startsWith("#") && !Number.isNaN(Number(l.split(/[,\s]+/)[0]))) - .map((l, i) => { const [t, ...r] = l.split(/[,\s]+/); return { t: Number(t), label: r.join(" ") || `e${i}` }; }); + if (!out.length) throw new Error("the events log has no events"); + return out; } // Video via ffmpeg: audio as mono f32 at 48 kHz, video as 64x36 gray frames; per-frame mean luminance and mean |frame diff| (motion flux). -// Stream start_time offsets are applied so both clocks are the container's clock. Constant frame rate is assumed (avg_frame_rate). -export function readVideo(path, { ffmpeg = "ffmpeg", ffprobe = "ffprobe" } = {}) { - let probe; - try { probe = JSON.parse(execFileSync(ffprobe, ["-v", "error", "-show_streams", "-of", "json", path], { encoding: "utf8" })); } - catch (e) { throw new Error(`ffprobe failed (${e.code === "ENOENT" ? "not on PATH; give --wav and --track instead" : e.message})`); } - const vs = probe.streams.find((s) => s.codec_type === "video"), as = probe.streams.find((s) => s.codec_type === "audio"); - if (!vs || !as) throw new Error("file needs one video and one audio stream"); - const [n, d] = vs.avg_frame_rate.split("/").map(Number), fps = n / d, W = 64, H = 36; +// Stream start_time offsets are applied so both clocks are the container's clock; video is resampled to constant frame rate by pts. +export function readVideo(path, { ffmpeg = process.env.FFMPEG ?? "ffmpeg", ffprobe = process.env.FFPROBE ?? "ffprobe", targetFps } = {}) { + // Stream facts from ffprobe when present, else from `ffmpeg -i` (static ffmpeg builds ship without ffprobe; -i prints them to stderr and exits 1). + let vs, as; + try { + const probe = JSON.parse(execFileSync(ffprobe, ["-v", "error", "-show_streams", "-of", "json", path], { encoding: "utf8" })); + const sv = probe.streams.find((x) => x.codec_type === "video"), sa = probe.streams.find((x) => x.codec_type === "audio"); + if (sv && sa) { const [n, d] = sv.avg_frame_rate.split("/").map(Number); vs = { index: sv.index, fps: n / d, start: Number(sv.start_time ?? 0) || 0 }; as = { index: sa.index, start: Number(sa.start_time ?? 0) || 0 }; } + } catch (e) { if (e.code !== "ENOENT") throw new Error(`ffprobe failed: ${e.message}`); } + if (!vs) { + let err = ""; try { execFileSync(ffmpeg, ["-hide_banner", "-i", path], { encoding: "utf8", stdio: ["ignore", "ignore", "pipe"] }); } catch (e) { if (e.code === "ENOENT") throw new Error("neither ffprobe nor ffmpeg is on PATH (set FFMPEG/FFPROBE); give --wav and --track instead"); err = String(e.stderr ?? ""); } + const line = (kind) => err.split("\n").find((l) => new RegExp(`Stream #\\d+:(\\d+).*: ${kind}:`).test(l)); + const vl = line("Video"), al = line("Audio"); + if (!vl || !al) throw new Error("file needs one video and one audio stream"); + const idx = (l) => Number(l.match(/Stream #\d+:(\d+)/)[1]), st = (l) => Number(l.match(/\bstart (\d+(?:\.\d+)?)/)?.[1] ?? 0); + const fpsm = vl.match(/(\d+(?:\.\d+)?) fps/) ?? vl.match(/(\d+(?:\.\d+)?) tbr/); + vs = { index: idx(vl), fps: Number(fpsm?.[1] ?? 30), start: st(vl) }; as = { index: idx(al), start: st(al) }; + } + // Resample to constant frame rate by presentation time (a variable-frame-rate source, e.g. a browser capture, repeats frames), so frame f is at start + f/fps. + const fps = targetFps ?? vs.fps, W = 64, H = 36; const audioRaw = execFileSync(ffmpeg, ["-v", "error", "-i", path, "-map", `0:${as.index}`, "-vn", "-ac", "1", "-ar", "48000", "-f", "f32le", "-"], { maxBuffer: 1 << 30 }); - const videoRaw = execFileSync(ffmpeg, ["-v", "error", "-i", path, "-map", `0:${vs.index}`, "-an", "-vf", `scale=${W}:${H}:flags=area,format=gray`, "-vsync", "passthrough", "-f", "rawvideo", "-"], { maxBuffer: 1 << 30 }); + const videoRaw = execFileSync(ffmpeg, ["-v", "error", "-i", path, "-map", `0:${vs.index}`, "-an", "-vf", `fps=${fps},scale=${W}:${H}:flags=area,format=gray`, "-f", "rawvideo", "-"], { maxBuffer: 1 << 30 }); const frames = Math.floor(videoRaw.length / (W * H)), t = new Float64Array(frames), lum = new Float64Array(frames), flux = new Float64Array(frames); - const vOff = Number(vs.start_time ?? 0) || 0, aOff = Number(as.start_time ?? 0) || 0; for (let f = 0; f < frames; f++) { let s = 0, df = 0; for (let i = 0; i < W * H; i++) { const p = videoRaw[f * W * H + i]; s += p; if (f) df += Math.abs(p - videoRaw[(f - 1) * W * H + i]); } - t[f] = vOff + f / fps; lum[f] = s / (W * H) / 255; flux[f] = f ? df / (W * H) / 255 : 0; + t[f] = vs.start + f / fps; lum[f] = s / (W * H) / 255; flux[f] = f ? df / (W * H) / 255 : 0; } - const a = new Float32Array(audioRaw.buffer, audioRaw.byteOffset, Math.floor(audioRaw.length / 4)); - return { audio: { samples: a, sampleRate: 48000, startS: aOff }, luma: { t, v: lum }, flux: { t, v: flux }, fps }; + const a = new Float32Array(audioRaw.buffer.slice(audioRaw.byteOffset, audioRaw.byteOffset + audioRaw.length - (audioRaw.length % 4))); + return { audio: { samples: a, sampleRate: 48000, startS: as.start }, luma: { t, v: lum }, flux: { t, v: flux }, fps }; } // ---------- onset detection ---------- @@ -119,12 +142,18 @@ export function audioOnsets(x, sr, { startS = 0, k = 8, minGapS = 0.05 } = {}) { } // The window centre sits half a window after the first audible sample's frame start; the rise-start pick lands within one window of the true onset. const r = pickOnsets(flux, tt, { k, floor: 1e-3 * hi, minGapS, riseFrac: 0.25 }); - // The flux frame is only good to a window; refine each onset on the samples: the first |x| within 20 dB (10%) of the local peak, searched - // from one window before the flux time to one window after it (the window centre can sit up to N/2 early of a click inside it). + // The flux frame is only good to a window; refine each onset on the samples. DC and the steady bed are estimated from the stretch before the + // search window (block peaks of |x - median|, their median, so a previous hit's tail in one block does not count); the onset is the first sample + // whose deviation exceeds max(10% of the local peak, 2.5x the bed). Search: from one window before the rise start (the window centre can sit up to + // N/2 early of a click inside it) to one window after the flux peak (over a loud bed the rise start can sit far before the real onset). If nothing clears the bar the flux time stands. for (const o of r.onsets) { - const c = Math.round((o.t - startS) * sr), a0 = Math.max(0, c - N), a1 = Math.min(x.length, c + N); - let pk = 0; for (let i = a0; i < a1; i++) pk = Math.max(pk, Math.abs(x[i])); - for (let i = a0; i < a1; i++) if (Math.abs(x[i]) >= 0.1 * pk) { o.t = startS + i / sr; break; } + const c = Math.round((o.t - startS) * sr), cp = Math.round((o.peakT - startS) * sr), a0 = Math.max(0, c - N), a1 = Math.min(x.length, cp + N), b0 = Math.max(0, a0 - 8 * 512), blocks = []; + const dc = median(Array.from(x.subarray(b0, a1))); + for (let q = b0; q + 512 <= a0; q += 512) { let m = 0; for (let i = q; i < q + 512; i++) m = Math.max(m, Math.abs(x[i] - dc)); blocks.push(m); } + const bed = blocks.length ? median(blocks) : 0; + let pk = 0; for (let i = a0; i < a1; i++) pk = Math.max(pk, Math.abs(x[i] - dc)); + const bar = Math.max(0.1 * pk, 2.5 * bed); + for (let i = a0; i < a1; i++) if (Math.abs(x[i] - dc) >= bar && bar < pk) { o.t = startS + i / sr; break; } } r.resolutionMs = (1000 * HOP) / sr; r.windowMs = (1000 * N) / sr; return r; @@ -142,67 +171,89 @@ export function visualOnsets(track, { kind = "luma", k = 8, minGapS = 0.1 } = {} } // ---------- pairing and verdict ---------- -export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 300 }) { - const prof = { ...(PROFILES[profile] ?? (() => { throw new Error(`unknown profile ${profile}`); })()) }; +// One-to-one greedy matching of onsets to anchors (events, or audio onsets) by distance, within windowS. Returns Map anchorIndex -> onsetIndex. +function match(anchors, onsets, windowS) { + const edges = []; + anchors.forEach((a, i) => onsets.forEach((o, j) => { const d = Math.abs(o.t - a); if (d <= windowS) edges.push([d, i, j]); })); + edges.sort((x, y) => x[0] - y[0]); + const ai = new Map(), oj = new Set(); + for (const [, i, j] of edges) if (!ai.has(i) && !oj.has(j)) { ai.set(i, j); oj.add(j); } + return ai; +} +const within = (list, t, windowS) => list.filter((o) => Math.abs(o.t - t) <= windowS).length; + +// Frame quantisation: a visual change at time tau first shows in the frame stamped at or after tau, so visual - tau is in [0, framePeriod) and the +// measured lag (audio - visual) is biased LOW by up to a frame period. The truth therefore lies in [lag - audioHop/2, lag + framePeriod + audioHop/2]; +// a pair is judged on that interval, not on a point. +export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 300, minGapMs, allowUnpaired = false }) { + if (!PROFILES[profile]) throw new Error(`unknown profile ${profile}`); + const prof = { ...PROFILES[profile] }, custom = leadMs !== undefined || lagMs !== undefined; if (leadMs !== undefined) prof.leadMs = leadMs; if (lagMs !== undefined) prof.lagMs = lagMs; - const a = audioOnsets(audio, sampleRate, { startS: audioStartS }), v = visualOnsets(visual, { kind: visualKind }); - const uncertaintyMs = Math.max(a.resolutionMs, v.framePeriodMs) / 2 + a.resolutionMs / 2; // half a frame period, plus half an audio hop - const used = new Set(), pairs = []; - const nearest = (list, t) => { let best = -1, bd = Infinity; list.forEach((o, i) => { const d = Math.abs(o.t - t); if (d < bd) { bd = d; best = i; } }); return best; }; + if (custom) { prof.note = `profile ${profile} with the window overridden on the command line`; } + for (const [n, x] of [["leadMs", prof.leadMs], ["lagMs", prof.lagMs], ["pairWindowMs", pairWindowMs]]) if (!Number.isFinite(x) || x <= 0) throw new Error(`${n} must be a positive number`); + const gap = minGapMs === undefined ? {} : { minGapS: minGapMs / 1000 }; + const a = audioOnsets(audio, sampleRate, { startS: audioStartS, ...gap }), v = visualOnsets(visual, { kind: visualKind, ...gap }); + const hopMs = a.resolutionMs, periodMs = v.framePeriodMs, win = pairWindowMs / 1000; const classify = (lag) => { - const excess = lag < 0 ? -lag - prof.leadMs : lag - prof.lagMs; // >0: outside the window by that many ms - if (excess > uncertaintyMs) return lag < 0 ? "audio-early-detectable" : "audio-late-detectable"; - if (excess > -uncertaintyMs) return "borderline"; // the window edge lies inside the measurement uncertainty - return "inside-window"; + const lo = lag - hopMs / 2, hi = lag + periodMs + hopMs / 2; // truth interval, ms + if (lo >= -prof.leadMs && hi <= prof.lagMs) return "inside-window"; + if (lo > prof.lagMs) return "audio-late-detectable"; + if (hi < -prof.leadMs) return "audio-early-detectable"; + return "borderline"; }; - if (events?.length) { - for (const e of events) { - const ai = nearest(a.onsets, e.t), vi = nearest(v.onsets, e.t), ao = a.onsets[ai], vo = v.onsets[vi]; - const okA = ao && Math.abs(ao.t - e.t) * 1e3 <= pairWindowMs, okV = vo && Math.abs(vo.t - e.t) * 1e3 <= pairWindowMs; - const row = { label: e.label, eventS: e.t, audioS: okA ? ao.t : null, visualS: okV ? vo.t : null, audioVsEventMs: okA ? (ao.t - e.t) * 1e3 : null, visualVsEventMs: okV ? (vo.t - e.t) * 1e3 : null }; - row.lagMs = okA && okV ? (ao.t - vo.t) * 1e3 : null; - row.verdict = row.lagMs === null ? (okA ? "no-visual-onset" : okV ? "no-audio-onset" : "no-onsets") : classify(row.lagMs); - if (okA) used.add(`a${ai}`); if (okV) used.add(`v${vi}`); - pairs.push(row); - } + const pairs = [], usedA = new Set(), usedV = new Set(); + const row = (label, eventS, ao, vo, ambiguous) => { + const r = { label, eventS, audioS: ao?.t ?? null, visualS: vo?.t ?? null, audioVsEventMs: ao && eventS !== null ? (ao.t - eventS) * 1e3 : null, visualVsEventMs: vo && eventS !== null ? (vo.t - eventS) * 1e3 : null, lagMs: ao && vo ? (ao.t - vo.t) * 1e3 : null }; + if (r.lagMs !== null) { r.lagLowMs = r.lagMs - hopMs / 2; r.lagHighMs = r.lagMs + periodMs + hopMs / 2; } + r.verdict = r.lagMs === null ? (ao ? "no-visual-onset" : vo ? "no-audio-onset" : "no-onsets") : ambiguous ? "ambiguous-pairing" : classify(r.lagMs); + return r; + }; + if (events) { + const et = events.map((e) => e.t), ma = match(et, a.onsets, win), mv = match(et, v.onsets, win); + events.forEach((e, i) => { + const ao = ma.has(i) ? a.onsets[ma.get(i)] : undefined, vo = mv.has(i) ? v.onsets[mv.get(i)] : undefined; + if (ao) usedA.add(ma.get(i)); if (vo) usedV.add(mv.get(i)); + // another onset of the same train inside the window: the nearest need not be the right one + pairs.push(row(e.label, e.t, ao, vo, within(a.onsets, e.t, win) > 1 || within(v.onsets, e.t, win) > 1)); + }); } else { - a.onsets.forEach((ao, ai) => { - const vi = nearest(v.onsets, ao.t), vo = v.onsets[vi]; - if (vo && Math.abs(vo.t - ao.t) * 1e3 <= pairWindowMs && !used.has(`v${vi}`)) { - used.add(`a${ai}`); used.add(`v${vi}`); - const lagMs = (ao.t - vo.t) * 1e3; - pairs.push({ label: `a${ai}`, eventS: null, audioS: ao.t, visualS: vo.t, audioVsEventMs: null, visualVsEventMs: null, lagMs, verdict: classify(lagMs) }); - } + const m = match(a.onsets.map((o) => o.t), v.onsets, win); + a.onsets.forEach((ao, i) => { + if (!m.has(i)) return; + const vo = v.onsets[m.get(i)]; usedA.add(i); usedV.add(m.get(i)); + pairs.push(row(`a${i}`, null, ao, vo, within(v.onsets, ao.t, win) > 1 || within(a.onsets, vo.t, win) > 1)); }); } const lags = pairs.map((p) => p.lagMs).filter((x) => x !== null); + const count = (f) => pairs.filter(f).length; const summary = { pairs: lags.length, audioOnsets: a.onsets.length, visualOnsets: v.onsets.length, - unpairedAudio: a.onsets.filter((_, i) => !used.has(`a${i}`)).length, unpairedVisual: v.onsets.filter((_, i) => !used.has(`v${i}`)).length, + unpairedAudio: a.onsets.length - usedA.size, unpairedVisual: v.onsets.length - usedV.size, medianLagMs: lags.length ? median(lags) : null, minLagMs: lags.length ? Math.min(...lags) : null, maxLagMs: lags.length ? Math.max(...lags) : null, // Spread of the lags: a constant offset is a latency (fixable by a delay), a wide spread is jitter (not fixable by one number). spreadMs: lags.length ? Math.max(...lags) - Math.min(...lags) : null, - outsideWindow: pairs.filter((p) => /detectable$/.test(p.verdict)).length, borderline: pairs.filter((p) => p.verdict === "borderline").length, - missing: pairs.filter((p) => p.lagMs === null).length, + outsideWindow: count((p) => /detectable$/.test(p.verdict)), borderline: count((p) => p.verdict === "borderline"), ambiguous: count((p) => p.verdict === "ambiguous-pairing"), + missing: count((p) => p.lagMs === null), }; - summary.pass = summary.pairs > 0 && summary.outsideWindow === 0 && summary.missing === 0; + const unpaired = summary.unpairedAudio + summary.unpairedVisual; + summary.pass = summary.pairs > 0 && summary.outsideWindow === 0 && summary.missing === 0 && summary.ambiguous === 0 && (allowUnpaired || unpaired === 0); return { - schema: "aqua-feel.sync.v1", lagConvention: "lagMs = audio onset - visual onset; positive = audio late, negative = audio early", - profile: { name: profile, ...prof }, uncertaintyMs, visualKind, audioResolutionMs: a.resolutionMs, framePeriodMs: v.framePeriodMs, + schema: "aqua-feel.sync.v1", lagConvention: "lagMs = audio onset - visual onset; positive = audio late, negative = audio early. Truth lies in [lagLowMs, lagHighMs] (visual onsets are quantised up to the next frame)", + profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualKind, audioResolutionMs: hopMs, framePeriodMs: periodMs, audioOnsets: a.onsets, visualOnsets: v.onsets, pairs, summary, }; } export function formatSync(r) { const f = (x, d = 1) => (x === null || x === undefined ? " - " : x.toFixed(d).padStart(7)); - const L = [`profile ${r.profile.name}: audio may lead by ${r.profile.leadMs} ms, lag by ${r.profile.lagMs} ms (${r.profile.note ?? "custom"})`, - `lag = audio - visual (+ = audio late). measurement uncertainty about +/-${r.uncertaintyMs.toFixed(1)} ms (frame period ${r.framePeriodMs.toFixed(1)} ms, audio hop ${r.audioResolutionMs.toFixed(1)} ms)`, + const L = [`profile ${r.profile.name}: audio may lead by ${r.profile.leadMs} ms, lag by ${r.profile.lagMs} ms (${r.profile.note})`, + `lag = audio - visual (+ = audio late). The visual onset is quantised up to the next frame (period ${r.framePeriodMs.toFixed(1)} ms), so the true lag lies in [lag - ${(r.audioResolutionMs / 2).toFixed(1)}, lag + ${(r.framePeriodMs + r.audioResolutionMs / 2).toFixed(1)}] ms`, "", "label event s audio s visual s a-ev ms v-ev ms lag ms verdict"]; for (const p of r.pairs) L.push(`${String(p.label).padEnd(10).slice(0, 10)} ${f(p.eventS, 3)} ${f(p.audioS, 3)} ${f(p.visualS, 3)} ${f(p.audioVsEventMs)} ${f(p.visualVsEventMs)} ${f(p.lagMs)} ${p.verdict}`); - const s = r.summary; - L.push("", `${s.pairs} paired, ${s.unpairedAudio} audio and ${s.unpairedVisual} visual onsets unpaired; median lag ${s.medianLagMs === null ? "-" : s.medianLagMs.toFixed(1)} ms, range ${s.minLagMs === null ? "-" : `${s.minLagMs.toFixed(1)}..${s.maxLagMs.toFixed(1)}`} ms (spread ${s.spreadMs === null ? "-" : s.spreadMs.toFixed(1)} ms)`, - `${s.outsideWindow} outside the window, ${s.borderline} borderline, ${s.missing} without both onsets. ${s.pass ? "WITHIN WINDOW" : "NOT WITHIN WINDOW"} by this metric.`, + const s = r.summary, rng = s.minLagMs === null ? "-" : `${s.minLagMs.toFixed(1)}..${s.maxLagMs.toFixed(1)}`; + L.push("", `${s.pairs} paired, ${s.unpairedAudio} audio and ${s.unpairedVisual} visual onsets unpaired${r.allowUnpaired ? " (allowed)" : " (fails the check unless --allow-unpaired)"}; median lag ${s.medianLagMs === null ? "-" : s.medianLagMs.toFixed(1)} ms, range ${rng} ms (spread ${s.spreadMs === null ? "-" : s.spreadMs.toFixed(1)} ms)`, + `${s.outsideWindow} outside the window, ${s.borderline} borderline, ${s.ambiguous} ambiguous (another onset within ${r.pairWindowMs} ms), ${s.missing} without both onsets. ${s.pass ? (s.borderline ? `NOT DETECTABLY OUTSIDE THE WINDOW (${s.borderline} borderline: the interval straddles the edge)` : "WITHIN WINDOW") : "NOT WITHIN WINDOW"} by this metric.`, "A metric over two onset lists, not a verdict: a person still has to watch and listen (SYNC.md)."); return L.join("\n"); } diff --git a/tools/aqua-feel/sync.test.mjs b/tools/aqua-feel/sync.test.mjs index 3ac1fa3..01985bd 100644 --- a/tools/aqua-feel/sync.test.mjs +++ b/tools/aqua-feel/sync.test.mjs @@ -1,6 +1,7 @@ // Checks of `aqua-feel sync` on synthetic fixtures with known offsets. No real capture is used. Usage: node sync.test.mjs import assert from "node:assert/strict"; -import { syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, PROFILES } from "./sync.mjs"; +import { formatSync, syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, PROFILES } from "./sync.mjs"; +import { readFileSync } from "node:fs"; import { wavFloat32 } from "./render.js"; import { makeFixture, FIXTURE_EVENTS_S } from "./sync-fixture.mjs"; @@ -8,44 +9,52 @@ let n = 0; const t = (name, fn) => { fn(); n++; console.log(`ok ${name}`); }; const run = (o, extra = {}) => { const fx = makeFixture(o); return { fx, r: syncAnalyze({ audio: fx.audio, sampleRate: fx.sampleRate, visual: fx.visual, events: fx.events, ...extra }) }; }; -for (const off of [-60, -20, 0, 40, 80, 150, 250]) { - t(`recovers a ${off} ms offset at 60 fps within the stated uncertainty, every event`, () => { - const { r } = run({ offsetMs: off }); - assert.equal(r.pairs.length, FIXTURE_EVENTS_S.length); - for (const p of r.pairs) assert.ok(Math.abs(p.lagMs - off) <= r.uncertaintyMs, `${p.label}: ${p.lagMs} vs ${off} (+/-${r.uncertaintyMs})`); - assert.equal(r.summary.unpairedVisual, 0, "no spurious visual onsets"); - assert.equal(r.summary.unpairedAudio, 0, "no spurious audio onsets"); +// The truth must lie in the interval each pair reports ([lagLowMs, lagHighMs]); the interval is the claim, a point estimate is not. +const inInterval = (p, truth) => p.lagLowMs - 1e-9 <= truth && truth <= p.lagHighMs + 1e-9; +const OFFGRID = [0.5042, 1.3117, 2.2209, 3.4033, 4.1161, 5.0087]; // events between frame boundaries at 30 and 60 fps + +for (const fps of [30, 60]) for (const off of [-60, -20, 0, 40, 80, 150, 250]) { + t(`${fps} fps, ${off} ms offset: the truth is inside every pair's interval, events on and off the frame grid`, () => { + for (const events of [FIXTURE_EVENTS_S, OFFGRID]) { + const { r } = run({ offsetMs: off, fps, events }); + assert.equal(r.pairs.length, events.length); + for (const p of r.pairs) assert.ok(inInterval(p, off), `${p.label}: [${p.lagLowMs}, ${p.lagHighMs}] vs ${off}`); + assert.equal(r.summary.unpairedVisual, 0, "no spurious visual onsets"); assert.equal(r.summary.unpairedAudio, 0, "no spurious audio onsets"); + } }); } -t("30 fps: the visual onset lands on a frame at or after the event, so the lag error stays under one frame period", () => { - for (const off of [0, 80]) { - const { r } = run({ offsetMs: off, fps: 30 }); - for (const p of r.pairs) assert.ok(Math.abs(p.lagMs - off) <= 1000 / 30, `${p.lagMs} vs ${off}`); - } +t("the interval is no wider than one frame period plus one audio hop (it is a measurement, not a shrug)", () => { + const { r } = run({ offsetMs: 80, fps: 60 }), p = r.pairs[0]; + assert.ok(p.lagHighMs - p.lagLowMs <= r.framePeriodMs + r.audioResolutionMs + 1e-9); }); -t("verdict classes follow the default window (lead 25, lag 100) beyond the uncertainty", () => { - const verdictOf = (off) => run({ offsetMs: off }).r.summary; - assert.equal(verdictOf(0).pass, true); - assert.equal(verdictOf(80).pass, true, "80 ms late is inside the 100 ms lag window"); - assert.equal(verdictOf(150).pass, false); - assert.equal(run({ offsetMs: 150 }).r.pairs[0].verdict, "audio-late-detectable"); - assert.equal(run({ offsetMs: -60 }).r.pairs[0].verdict, "audio-early-detectable"); - assert.equal(run({ offsetMs: 100 }).r.pairs[0].verdict, "borderline", "the window edge sits inside the uncertainty"); +t("verdict classes follow the default window (lead 25, lag 100) judged on the interval", () => { + const v = (off, o = {}) => run({ offsetMs: off, ...o }).r; + assert.equal(v(0).summary.pass, true); + assert.equal(v(60).summary.pass, true, "60 ms late is inside the 100 ms lag window"); + assert.equal(v(150).pairs[0].verdict, "audio-late-detectable"); + assert.equal(v(-60).pairs[0].verdict, "audio-early-detectable"); + assert.equal(v(95).pairs[0].verdict, "borderline", "the interval straddles the 100 ms edge"); + assert.equal(v(95).summary.pass, true, "borderline is not detectably outside, so it does not fail the gate"); + assert.match(formatSync(v(95)), /borderline/, "but the report says so"); + assert.match(formatSync(v(0)), /WITHIN WINDOW/); assert.doesNotMatch(formatSync(v(95)), /WITHIN WINDOW/); + assert.equal(v(85, { fps: 120 }).pairs[0].verdict, "inside-window", "a finer frame clock narrows the interval"); + assert.equal(v(85, { fps: 30 }).pairs[0].verdict, "borderline", "a coarser one widens it"); }); t("the profile changes the verdict, not the measurement", () => { const a = run({ offsetMs: 70 }, { profile: "default" }).r, b = run({ offsetMs: 70 }, { profile: "ebu-r37" }).r; assert.equal(a.pairs[0].lagMs, b.pairs[0].lagMs); assert.equal(a.summary.pass, true); assert.equal(b.summary.pass, false); + assert.match(run({ offsetMs: 70 }, { lagMs: 200 }).r.profile.note, /overridden/, "an overridden window is not reported as the default profile"); assert.ok(PROFILES.default.leadMs === 25 && PROFILES.default.lagMs === 100); }); t("without an events log, onsets pair by proximity and the offset is still recovered", () => { const fx = makeFixture({ offsetMs: 60 }), r = syncAnalyze({ audio: fx.audio, sampleRate: fx.sampleRate, visual: fx.visual }); assert.equal(r.pairs.length, FIXTURE_EVENTS_S.length); - for (const p of r.pairs) assert.ok(Math.abs(p.lagMs - 60) <= r.uncertaintyMs); + for (const p of r.pairs) assert.ok(inInterval(p, 60)); }); t("jitter shows as spread, not as a median: alternate events 0 and 90 ms late", () => { @@ -69,6 +78,69 @@ t("an offset past the 300 ms pairing window is reported as missing, not as a 300 assert.equal(r.pairs[0].verdict, "no-audio-onset"); assert.equal(r.summary.pass, false); }); +// ---- regressions from the Soul pass on PR #32 ---- +t("a noise bed, a steady-tone bed or a DC offset under the hits does not move the audio onset (refinement is relative to the bed)", () => { + for (const o of [{ noise: 0.03 }, { bed: [[220, 0.05], [330, 0.03]] }, { dc: 0.25 }, { gain: 0.03, noise: 0.003 }, { dc: -0.3, noise: 0.02 }]) { + const { r } = run({ offsetMs: 50, ...o }); + assert.equal(r.pairs.length, 6, JSON.stringify(o)); + for (const p of r.pairs) assert.ok(inInterval(p, 50), `${JSON.stringify(o)} ${p.label}: ${p.lagMs}`); + } +}); + +t("dense hits: events 120 ms apart with a +80 ms lag are not silently paired to the previous hit", () => { + const ev = [0.5, 0.62, 0.74, 0.86, 0.98, 1.1, 1.22]; + const { r } = run({ offsetMs: 80, events: ev }); + // Either the pairing is right, or it is flagged; it is never a confident wrong lag with a pass. + for (const p of r.pairs) assert.ok(inInterval(p, 80) || p.verdict === "ambiguous-pairing" || p.lagMs === null, `${p.label}: ${p.lagMs} ${p.verdict}`); + assert.equal(r.summary.pass, false, "overlapping candidates inside the pairing window cannot pass"); + assert.ok(r.summary.ambiguous > 0); +}); + +t("an audio onset is never reused by two events", () => { + const { r } = run({ offsetMs: 200, events: [0.5, 0.58, 0.66, 0.74] }); + const used = r.pairs.map((p) => p.audioS).filter((x) => x !== null); + assert.equal(new Set(used).size, used.length); +}); + +t("a narrower --pair-window-ms resolves dense hits that the default calls ambiguous", () => { + const ev = [0.5, 0.62, 0.74, 0.86, 0.98]; + const { r } = run({ offsetMs: 30, events: ev }, { pairWindowMs: 50 }); + for (const p of r.pairs) assert.ok(inInterval(p, 30), `${p.label}: ${p.lagMs}`); + assert.equal(r.summary.ambiguous, 0); +}); + +t("flashes closer than the visual minimum gap merge, and then an event goes unpaired rather than getting another flash's lag", () => { + const { r } = run({ offsetMs: 0, events: [0.5, 0.56, 0.62, 1.5] }); + assert.equal(r.summary.pass, false); + for (const p of r.pairs) assert.ok(p.lagMs === null || inInterval(p, 0) || p.verdict === "ambiguous-pairing", `${p.label}: ${p.lagMs} ${p.verdict}`); +}); + +t("a stray sound or flash fails the check unless --allow-unpaired is given", () => { + const fx = makeFixture({ offsetMs: 0 }), a = Float32Array.from(fx.audio), click = makeFixture({ offsetMs: 0, events: [0.1] }).audio; + for (let i = 4000; i < 7000; i++) a[i] += click[i]; + const run2 = (o) => syncAnalyze({ audio: a, sampleRate: 48000, visual: fx.visual, events: fx.events, ...o }); + assert.equal(run2({}).summary.unpairedAudio, 1); assert.equal(run2({}).summary.pass, false); + assert.equal(run2({ allowUnpaired: true }).summary.pass, true); +}); + +t("bad options fail loudly: NaN window, unknown profile", () => { + const fx = makeFixture({}); + assert.throws(() => syncAnalyze({ audio: fx.audio, sampleRate: 48000, visual: fx.visual, lagMs: NaN }), /positive/); + assert.throws(() => syncAnalyze({ audio: fx.audio, sampleRate: 48000, visual: fx.visual, profile: "nope" }), /unknown profile/); +}); + +t("parsers refuse input that would give a plausible wrong answer", () => { + assert.throws(() => readEvents("0:00.5\n0:01.2\n"), /does not start with a time/); + assert.throws(() => readEvents("time,label\nfoo,bar\n"), /line 2/); + assert.deepEqual(readEvents("time,label\n0.5,a\n1.5,b, loud\n").map((e) => e.label), ["a", "b, loud"], "a header on line 1 is skipped; commas stay in the label"); + assert.throws(() => readEvents("[0.5, null]"), /no numeric time/); + assert.throws(() => readEvents('[{"time":2}]'), /no numeric time/); + assert.throws(() => readEvents("# only a comment\n"), /no events/); + assert.throws(() => readTrack("0,0,0.1\n1,0.1,0.2\n2,0.2,0.3\n"), /3 columns/); + assert.throws(() => readTrack("0.0,0.1\n0.0167,\n0.0333,0.3\n"), /not numeric/); + assert.throws(() => readTrack("0,0.1\n0.5,0.2\n0.4,0.3\n"), /does not increase/); +}); + t("silence and a flat track yield no onsets and no pass", () => { const r = syncAnalyze({ audio: new Float32Array(48000 * 2), sampleRate: 48000, visual: { t: Float64Array.from({ length: 120 }, (_, i) => i / 60), v: new Float64Array(120).fill(0.5) } }); assert.equal(r.summary.audioOnsets, 0); assert.equal(r.summary.visualOnsets, 0); assert.equal(r.summary.pass, false); @@ -105,4 +177,21 @@ t("readTrack takes header + t,value rows or a single column with fps; readEvents assert.deepEqual(readEvents('[{"t":2,"label":"x"},3]').map((e) => e.label), ["x", "e1"]); }); -console.log(`${n} sync checks passed. Synthetic fixtures only; nothing was heard or seen.`); +// The video path needs ffmpeg (FFMPEG=/path or on PATH); skipped, and said so, when there is none. +{ + const { readVideo } = await import("./sync.mjs"), { execFileSync } = await import("node:child_process"); + const ff = process.env.FFMPEG ?? "ffmpeg"; + let have = true; try { execFileSync(ff, ["-version"], { stdio: "ignore" }); } catch { have = false; } + if (!have) console.log("skip video path: no ffmpeg (set FFMPEG)"); + else t("a real browser capture (VFR vp8+opus webm): every logged hit pairs, and the lag is a constant (spread under one frame)", () => { + const v = readVideo(new URL("./fixtures/browser-capture.webm", import.meta.url).pathname, { ffmpeg: ff }); + const ev = readEvents(readFileSync(new URL("./fixtures/browser-capture.events.txt", import.meta.url), "utf8")); + const r = syncAnalyze({ audio: v.audio.samples, sampleRate: v.audio.sampleRate, audioStartS: v.audio.startS, visual: v.luma, events: ev }); + assert.equal(r.pairs.length, 7); assert.equal(r.summary.missing, 0); + assert.ok(r.summary.spreadMs < 1000 / v.fps, `spread ${r.summary.spreadMs}`); + assert.equal(r.summary.unpairedAudio + r.summary.unpairedVisual, 0); + for (const p of r.pairs) assert.ok(p.visualVsEventMs >= 0 && p.visualVsEventMs <= 1000 / v.fps, `flash ${p.visualVsEventMs} ms off its logged instant`); + }); +} + +console.log(`${n} sync checks passed. Fixtures only (synthetic, plus one browser capture when ffmpeg is present); nothing was heard or seen.`); -- 2.47.3 From b6a806ac9646f21cdea1979f0490e6c590fe0d20 Mon Sep 17 00:00:00 2001 From: Aqua Date: Tue, 10 Nov 2026 23:36:47 +0000 Subject: [PATCH 3/4] aqua-feel sync: second Soul pass fixes (video clock, ambiguity rule, parsers) - readVideo: ffmpeg only (no ffprobe); stream starts read from the first decoded frame; the padded cfr video is no longer offset twice (video starting late read 2x its offset). Known-offset clips built with ffmpeg are now a test. - ambiguity: a pair is ambiguous when a same-kind neighbour is closer than twice the pairing window (default 200 ms), not when anything is inside it - audio onsets that refine to the same sample are de-duplicated - parsers: strict numbers (no null/""/true/hex), header on the first data line, semicolons, trailing commas, column-count consistency; zero windows allowed - SYNC.md: limits for pulsing beds, noise gates, hit-rate cost of the ambiguity rule Co-Authored-By: Claude Sonnet 5.5 --- tools/aqua-feel/SYNC.md | 26 +++++----- tools/aqua-feel/feel.mjs | 2 +- tools/aqua-feel/sync.mjs | 93 ++++++++++++++++++++--------------- tools/aqua-feel/sync.test.mjs | 55 ++++++++++++++++++--- 4 files changed, 116 insertions(+), 60 deletions(-) diff --git a/tools/aqua-feel/SYNC.md b/tools/aqua-feel/SYNC.md index 4426cca..1248b57 100644 --- a/tools/aqua-feel/SYNC.md +++ b/tools/aqua-feel/SYNC.md @@ -2,7 +2,7 @@ Checks audio-visual synchrony of a capture: finds audio onsets and visual onsets, pairs them, reports the lag per event in ms and compares it with published detectability windows. Standalone: Node 18+, no npm dependencies (`feel.mjs` imports `sync.mjs`, which -imports one FFT from `metrics.js`). ffmpeg and ffprobe are needed only to read a video file directly. +imports one FFT from `metrics.js`). ffmpeg (no ffprobe) is needed only to read a video file directly. **The metric is not the verdict.** It compares two onset lists extracted from signals. A human ear and eye still decide whether a capture feels in sync. Nothing here heard or saw anything. @@ -12,12 +12,11 @@ capture feels in sync. Nothing here heard or saw anything. ``` # WAV + a per-frame track (no ffmpeg needed) node feel.mjs sync --wav capture.wav --track frames.csv [--events hits.txt] [--fps 60] [--kind luma|flux] -# a video with one audio and one video stream (needs ffmpeg + ffprobe on PATH) +# a video with one audio and one video stream (needs ffmpeg on PATH, or FFMPEG=/path/to/ffmpeg) node feel.mjs sync --video capture.mp4 [--events hits.txt] [--kind luma|flux] -# options: --profile default|bt1359|ebu-r37|atsc-is191 --lead-ms N --lag-ms N --pair-window-ms N (default 300) +# options: --profile default|bt1359|ebu-r37|atsc-is191 --lead-ms N --lag-ms N --pair-window-ms N (default 200) # --min-gap-ms N --allow-unpaired --json out.json --no-gate -# ffmpeg/ffprobe: on PATH, or FFMPEG=/path FFPROBE=/path (a static ffmpeg without ffprobe works) -node sync.test.mjs # 34 checks on synthetic fixtures (35 with ffmpeg: FFMPEG=/path/to/ffmpeg also runs a real browser capture) +node sync.test.mjs # 37 checks on synthetic fixtures; 43 with FFMPEG=/path/to/ffmpeg (also builds clips with known stream offsets and runs a real browser capture) node sync-fixture.mjs DIR 80 60 # write a fixture (audio 80 ms late, 60 fps) to try the CLI on ``` @@ -33,10 +32,13 @@ offsets are applied so both clocks are the container's. **Lag convention.** `lag = audio onset - visual onset`. Positive means the sound comes after the picture (audio late); negative means the sound comes first (audio early). With `--events`, each logged event is matched to the nearest audio and nearest visual onset -within 300 ms, and `a-ev` / `v-ev` give each side's offset from the logged time, so you can tell which side is late. Without an -event log, audio onsets pair with visual onsets by proximity (within 300 ms, one-to-one, nearest first). **A pair is `ambiguous-pairing` and fails the check when -another onset of the same kind lies inside the pairing window**: with hits closer together than about twice the lag, the nearest onset is not -necessarily the right one and no amount of arithmetic can tell. Narrow `--pair-window-ms` if you know the lag is small, or space your events out. +within the pairing window (200 ms), and `a-ev` / `v-ev` give each side's offset from the logged time, so you can tell which side is late. Without an +event log, audio onsets pair with visual onsets by proximity (within the pairing window, one-to-one, nearest first). **A pair is `ambiguous-pairing` and fails the check when +another onset of the same kind lies closer than twice the pairing window** (400 ms by default). Nearest-onset pairing is right only if the true lag is under half +the spacing of the onsets, and the tool cannot know the lag, so it takes the pairing window as the largest lag you consider plausible. The cost is real: material +faster than 2.5 hits per second cannot pass at the default, however good the sync. Narrow `--pair-window-ms` if the lag is known to be small (`50` clears 100 ms +spacing), or space the events out. A lag beyond the window reads as `no-audio-onset`; a lag beyond half the hit spacing but inside the window can read as the +previous hit's (wrong sign), which the ambiguity rule is there to catch. Parsers refuse input they cannot read (timecodes, a third column, empty cells, non-increasing times) rather than guess. ## How onsets are found @@ -60,7 +62,7 @@ Per pair, against a window of how far audio may lead (`lead`) or trail (`lag`) t Verdict words: `inside-window` (the whole truth interval, below, lies inside the window); `audio-late-detectable` / `audio-early-detectable` (the whole interval lies outside); `borderline` (the interval straddles a window edge: not shown to be outside, so it does not fail the gate, and the report says "NOT DETECTABLY OUTSIDE" rather than "WITHIN"); -`ambiguous-pairing`; `no-audio-onset` / `no-visual-onset` / `no-onsets` (the logged event had no onset within the pairing window, so a sound further than 300 ms from its event +`ambiguous-pairing`; `no-audio-onset` / `no-visual-onset` / `no-onsets` (the logged event had no onset within the pairing window, so a sound further than 200 ms from its event reads as absent). `pass` means no pair is detectably outside, ambiguous or missing, and nothing is unpaired (unless `--allow-unpaired`). Sources: [ITU-R BT.1359-1](https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.1359-1-199811-I!!PDF-E.pdf) (the PDF text was fragmentary in the search; figures confirmed against secondary summaries), and the TV Technology articles @@ -94,6 +96,8 @@ onset is good to about half a hop (1.4 ms). Each pair therefore reports `lagLowM few or none, and silence in either track is reported as no pass, not as sync. - **Measurements from downscaled luminance.** A flash that is local (a small icon) moves the mean luminance little and may fall below the 10%-of-range floor; use a cropped frame-diff track in that case. +- **Known stream offsets.** `sync.test.mjs` builds clips with ffmpeg whose video or audio stream starts late (`-itsoffset`, stream copy) and checks that the known lag lies in every pair's interval, so the container start-time bookkeeping has a ground truth; the clips are synthetic flashes and tones. +- **A pulsing bed.** The bed estimate is a median of block peaks and cannot follow dynamics: a 4 Hz tremolo bed at 10% depth under the hits pulled some lags off by up to 28 ms in testing. Beds louder than a tenth of the hit and noise gates that cut the sound to digital silence (the noise resuming reads as an onset) can also give a wrong onset that passes when the true lag is well inside the window. - **One real clip.** `fixtures/browser-capture.webm` is a headless-Chromium MediaRecorder capture (WebAudio beeps and canvas flashes scheduled at the same instants, vp8 + opus, recipe in `fixtures/record-browser-capture.cjs`). It is a test that the video path runs and pairs, not a validation of the lag it reports: nothing independent says what the true lag of that clip is. No Playground UI or Perlines clip has been run. -- **The tuned constants.** The 8-sigma threshold, window sizes, 300 ms pairing window, 10% floors and 2.5x bed factor were set against synthetic +- **The tuned constants.** The 8-sigma threshold, window sizes, 200 ms pairing window, 10% floors and 2.5x bed factor were set against synthetic fixtures (`sync-fixture.mjs`: tone-burst hits against 4-frame flashes, known offset, with noise, tone and DC beds) and one browser clip. They have not been tuned on game or music content. diff --git a/tools/aqua-feel/feel.mjs b/tools/aqua-feel/feel.mjs index d94f3ab..5c659d6 100644 --- a/tools/aqua-feel/feel.mjs +++ b/tools/aqua-feel/feel.mjs @@ -70,7 +70,7 @@ async function sync() { else if (wav && trk) { const w = readWav(readFileSync(wav)); audio = w.samples; sampleRate = w.sampleRate; visual = readTrack(readFileSync(trk, "utf8"), fps); } else throw new Error("give --video FILE, or --wav FILE with --track FILE"); const r = syncAnalyze({ audio, sampleRate, audioStartS, visual, visualKind: kind, profile, events: evs ? readEvents(readFileSync(evs, "utf8")) : undefined, - leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 300, minGapMs: gap, allowUnpaired }); + leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 200, minGapMs: gap, allowUnpaired }); r.caveat = "Nothing here was heard or seen: two onset lists compared. The metric is not the verdict; see SYNC.md."; console.log(formatSync(r)); if (json) writeFileSync(json, JSON.stringify(r, null, 1)); diff --git a/tools/aqua-feel/sync.mjs b/tools/aqua-feel/sync.mjs index 4e61596..d6bcd51 100644 --- a/tools/aqua-feel/sync.mjs +++ b/tools/aqua-feel/sync.mjs @@ -2,7 +2,7 @@ // NOTHING HERE HEARS OR SEES ANYTHING. It compares two onset lists extracted from signals; see SYNC.md for what that cannot tell you. // Library: syncAnalyze({ audio, sampleRate, visual:{t,v}, events, profile }) -> report. CLI wiring is in feel.mjs (`node feel.mjs sync ...`). import { readFileSync } from "node:fs"; -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { fft } from "./metrics.js"; // Lag convention: lag = audio onset time - visual onset time. Positive = audio LATE (after the picture); negative = audio EARLY. @@ -36,15 +36,23 @@ export function readWav(buf) { return { samples: out, sampleRate: fmt.sr, channels: fmt.ch }; } +const NUM = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/, num = (x) => (typeof x === "number" ? x : typeof x === "string" && NUM.test(x.trim()) ? Number(x) : NaN); +// A header is a first data line with no digit in it; a line with digits that fails to parse (a timecode, a hex number) is an error. +const isHeader = (line, first) => first && !/\d/.test(line); + // Frame-diff track: lines of "t,value" (seconds, strictly increasing) or a single "value" column with fps given. Only the first line may be a header. // Anything else that does not parse is an error: a silently dropped or shifted row would give a plausible wrong lag. export function readTrack(text, fps) { const t = [], v = []; - text.split(/\r?\n/).forEach((raw, ln) => { - const line = raw.trim(); if (!line) return; - const tok = line.split(/[\s,;]+/), f = tok.map(Number); - if (tok.some((x) => x === "") || f.some((x) => !Number.isFinite(x))) { if (ln === 0 && !t.length && !v.length) return; throw new Error(`track line ${ln + 1} is not numeric: ${raw}`); } + let first = true, ncols = 0; + text.replace(/^/, "").split(/\r?\n/).forEach((raw, ln) => { + const line = raw.trim().replace(/[\s,;]+$/, ""); if (!line) return; + const wasFirst = first; first = false; + const tok = line.split(/[\s,;]+/), f = tok.map(num); + if (f.some((x) => !Number.isFinite(x))) { if (isHeader(line, wasFirst)) return; throw new Error(`track line ${ln + 1} is not numeric: ${raw}`); } if (f.length > 2) throw new Error(`track line ${ln + 1} has ${f.length} columns; expected "t,value" or one value per line: ${raw}`); + if (ncols && f.length !== ncols) throw new Error(`track line ${ln + 1} has ${f.length} column(s), earlier lines had ${ncols} (an empty cell?): ${raw}`); + ncols = f.length; if (f.length === 2) { if (t.length && f[0] <= t[t.length - 1]) throw new Error(`track line ${ln + 1}: time ${f[0]} does not increase`); t.push(f[0]); v.push(f[1]); } else { if (!fps) throw new Error("single-column track needs --fps"); t.push(v.length / fps); v.push(f[0]); } }); @@ -54,18 +62,24 @@ export function readTrack(text, fps) { // Events: JSON array of numbers or {t,label}, or text lines "t[,label]" in seconds ('#' comments; the first line may be a header). Anything else is an error. export function readEvents(text) { - const s = text.trim(), fin = (x, where) => { if (!Number.isFinite(x)) throw new Error(`event ${where} has no numeric time`); return x; }; + const s = text.replace(/^/, "").trim(), fin = (x, where) => { if (!Number.isFinite(x)) throw new Error(`event ${where} has no numeric time`); return x; }; let out; if (s.startsWith("[") || s.startsWith("{")) { const j = JSON.parse(s), a = Array.isArray(j) ? j : j.events; if (!Array.isArray(a)) throw new Error("events JSON must be an array or {events:[...]}"); - out = a.map((e, i) => (typeof e === "number" ? { t: fin(e, i), label: `e${i}` } : e && typeof e === "object" ? { t: fin(Number(e.t), i), label: String(e.label ?? `e${i}`) } : fin(NaN, i))); + out = a.map((e, i) => { + if (typeof e === "number") return { t: fin(e, i), label: `e${i}` }; + if (!e || typeof e !== "object") return fin(NaN, i); + if (e.label !== undefined && typeof e.label !== "string") throw new Error(`event ${i} label must be a string`); + return { t: fin(num(e.t), i), label: e.label ?? `e${i}` }; + }); } else { - out = []; + out = []; let first = true; s.split(/\r?\n/).forEach((raw, ln) => { const l = raw.trim(); if (!l || l.startsWith("#")) return; - const m = l.match(/^([^,\s]+)(?:[,\s]+(.*))?$/), t = Number(m[1]); - if (!Number.isFinite(t)) { if (ln === 0 && !out.length) return; throw new Error(`events line ${ln + 1} does not start with a time in seconds: ${raw}`); } + const wasFirst = first; first = false; + const m = l.match(/^([^,;\s]+)(?:[,;\s]+(.*))?$/), t = num(m[1]); + if (!Number.isFinite(t)) { if (isHeader(l, wasFirst)) return; throw new Error(`events line ${ln + 1} does not start with a time in seconds: ${raw}`); } out.push({ t, label: m[2]?.trim() || `e${out.length}` }); }); } @@ -75,34 +89,30 @@ export function readEvents(text) { // Video via ffmpeg: audio as mono f32 at 48 kHz, video as 64x36 gray frames; per-frame mean luminance and mean |frame diff| (motion flux). // Stream start_time offsets are applied so both clocks are the container's clock; video is resampled to constant frame rate by pts. -export function readVideo(path, { ffmpeg = process.env.FFMPEG ?? "ffmpeg", ffprobe = process.env.FFPROBE ?? "ffprobe", targetFps } = {}) { - // Stream facts from ffprobe when present, else from `ffmpeg -i` (static ffmpeg builds ship without ffprobe; -i prints them to stderr and exits 1). - let vs, as; - try { - const probe = JSON.parse(execFileSync(ffprobe, ["-v", "error", "-show_streams", "-of", "json", path], { encoding: "utf8" })); - const sv = probe.streams.find((x) => x.codec_type === "video"), sa = probe.streams.find((x) => x.codec_type === "audio"); - if (sv && sa) { const [n, d] = sv.avg_frame_rate.split("/").map(Number); vs = { index: sv.index, fps: n / d, start: Number(sv.start_time ?? 0) || 0 }; as = { index: sa.index, start: Number(sa.start_time ?? 0) || 0 }; } - } catch (e) { if (e.code !== "ENOENT") throw new Error(`ffprobe failed: ${e.message}`); } - if (!vs) { - let err = ""; try { execFileSync(ffmpeg, ["-hide_banner", "-i", path], { encoding: "utf8", stdio: ["ignore", "ignore", "pipe"] }); } catch (e) { if (e.code === "ENOENT") throw new Error("neither ffprobe nor ffmpeg is on PATH (set FFMPEG/FFPROBE); give --wav and --track instead"); err = String(e.stderr ?? ""); } - const line = (kind) => err.split("\n").find((l) => new RegExp(`Stream #\\d+:(\\d+).*: ${kind}:`).test(l)); - const vl = line("Video"), al = line("Audio"); - if (!vl || !al) throw new Error("file needs one video and one audio stream"); - const idx = (l) => Number(l.match(/Stream #\d+:(\d+)/)[1]), st = (l) => Number(l.match(/\bstart (\d+(?:\.\d+)?)/)?.[1] ?? 0); - const fpsm = vl.match(/(\d+(?:\.\d+)?) fps/) ?? vl.match(/(\d+(?:\.\d+)?) tbr/); - vs = { index: idx(vl), fps: Number(fpsm?.[1] ?? 30), start: st(vl) }; as = { index: idx(al), start: st(al) }; - } - // Resample to constant frame rate by presentation time (a variable-frame-rate source, e.g. a browser capture, repeats frames), so frame f is at start + f/fps. - const fps = targetFps ?? vs.fps, W = 64, H = 36; - const audioRaw = execFileSync(ffmpeg, ["-v", "error", "-i", path, "-map", `0:${as.index}`, "-vn", "-ac", "1", "-ar", "48000", "-f", "f32le", "-"], { maxBuffer: 1 << 30 }); - const videoRaw = execFileSync(ffmpeg, ["-v", "error", "-i", path, "-map", `0:${vs.index}`, "-an", "-vf", `fps=${fps},scale=${W}:${H}:flags=area,format=gray`, "-f", "rawvideo", "-"], { maxBuffer: 1 << 30 }); +export function readVideo(path, { ffmpeg = process.env.FFMPEG ?? "ffmpeg", targetFps } = {}) { + // ffmpeg only (no ffprobe: static builds ship without it). `ffmpeg -i` lists the streams; the first decoded frame of each stream says when it starts. + const run = (a) => { const r = spawnSync(ffmpeg, a, { encoding: "utf8", maxBuffer: 1 << 26 }); if (r.error?.code === "ENOENT") throw new Error("ffmpeg is not on PATH (set FFMPEG=/path/to/ffmpeg); give --wav and --track instead"); return String(r.stderr ?? ""); }; + const lines = run(["-hide_banner", "-i", path]).split("\n"), find = (kind) => lines.find((l) => new RegExp(`Stream #\\d+:\\d+.*: ${kind}:`).test(l)); + const vl = find("Video"), al = find("Audio"); + if (!vl || !al) throw new Error("file needs one video and one audio stream"); + const idx = (l) => Number(l.match(/Stream #\d+:(\d+)/)[1]), fpsm = vl.match(/(\d+(?:\.\d+)?) fps/) ?? vl.match(/(\d+(?:\.\d+)?) tbr/); + const fps = targetFps ?? Number(fpsm?.[1] ?? 30), W = 64, H = 36, vi = idx(vl), ai = idx(al); + // Time of a stream's first frame on the container clock (ffmpeg shifts every stream by the earliest start, so these are relative to that). + const first = (flags) => { const m = run(["-hide_banner", "-i", path, ...flags, "-f", "null", "-"]).match(/pts_time:\s*(-?\d+(?:\.\d+)?)/); if (!m) throw new Error("could not read a stream start time from ffmpeg"); return Number(m[1]); }; + const _vStart = first(["-map", `0:${vi}`, "-an", "-frames:v", "1", "-vf", "showinfo"]), aStart = first(["-map", `0:${ai}`, "-vn", "-frames:a", "1", "-af", "ashowinfo"]); + const out = (a) => execFileSync(ffmpeg, ["-v", "error", "-i", path, ...a, "-"], { maxBuffer: 1 << 30 }); + const audioRaw = out(["-map", `0:${ai}`, "-vn", "-ac", "1", "-ar", "48000", "-f", "f32le"]); + // Resample to constant frame rate by presentation time (a variable-frame-rate source, e.g. a browser capture, repeats frames). ffmpeg pads the cfr + // video from the container start (the earliest stream start), so frame f is at f/fps whatever the video's own start_time: the video start is + // already in the padding and is not added again (adding it was a 2x-offset bug); the audio, written unpadded, carries its start as startS. + const videoRaw = out(["-map", `0:${vi}`, "-an", "-vf", `fps=${fps},scale=${W}:${H}:flags=area,format=gray`, "-f", "rawvideo"]); const frames = Math.floor(videoRaw.length / (W * H)), t = new Float64Array(frames), lum = new Float64Array(frames), flux = new Float64Array(frames); for (let f = 0; f < frames; f++) { let s = 0, df = 0; for (let i = 0; i < W * H; i++) { const p = videoRaw[f * W * H + i]; s += p; if (f) df += Math.abs(p - videoRaw[(f - 1) * W * H + i]); } - t[f] = vs.start + f / fps; lum[f] = s / (W * H) / 255; flux[f] = f ? df / (W * H) / 255 : 0; + t[f] = f / fps; lum[f] = s / (W * H) / 255; flux[f] = f ? df / (W * H) / 255 : 0; } const a = new Float32Array(audioRaw.buffer.slice(audioRaw.byteOffset, audioRaw.byteOffset + audioRaw.length - (audioRaw.length % 4))); - return { audio: { samples: a, sampleRate: 48000, startS: as.start }, luma: { t, v: lum }, flux: { t, v: flux }, fps }; + return { audio: { samples: a, sampleRate: 48000, startS: aStart }, luma: { t, v: lum }, flux: { t, v: flux }, fps, videoStartS: _vStart }; } // ---------- onset detection ---------- @@ -155,6 +165,8 @@ export function audioOnsets(x, sr, { startS = 0, k = 8, minGapS = 0.05 } = {}) { const bar = Math.max(0.1 * pk, 2.5 * bed); for (let i = a0; i < a1; i++) if (Math.abs(x[i] - dc) >= bar && bar < pk) { o.t = startS + i / sr; break; } } + // Two flux peaks can refine to the same sample onset (a step in the bed plus the hit): keep one per refined time within the minimum gap. + r.onsets = r.onsets.reduce((acc, o) => { const l = acc[acc.length - 1]; if (l && o.t - l.t < minGapS) { if (o.strength > l.strength) acc[acc.length - 1] = { ...o, t: Math.min(o.t, l.t) }; } else acc.push(o); return acc; }, []); r.resolutionMs = (1000 * HOP) / sr; r.windowMs = (1000 * N) / sr; return r; } @@ -180,18 +192,20 @@ function match(anchors, onsets, windowS) { for (const [, i, j] of edges) if (!ai.has(i) && !oj.has(j)) { ai.set(i, j); oj.add(j); } return ai; } -const within = (list, t, windowS) => list.filter((o) => Math.abs(o.t - t) <= windowS).length; +// Nearest-onset pairing is right only if the true lag is under half the spacing of that train's onsets. The caller declares the largest lag it +// considers plausible (the pairing window); a pair whose neighbouring onset lies closer than twice that cannot be told from a swap. +const crowded = (list, o, windowS) => list.some((x) => x !== o && Math.abs(x.t - o.t) < 2 * windowS); // Frame quantisation: a visual change at time tau first shows in the frame stamped at or after tau, so visual - tau is in [0, framePeriod) and the // measured lag (audio - visual) is biased LOW by up to a frame period. The truth therefore lies in [lag - audioHop/2, lag + framePeriod + audioHop/2]; // a pair is judged on that interval, not on a point. -export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 300, minGapMs, allowUnpaired = false }) { +export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false }) { if (!PROFILES[profile]) throw new Error(`unknown profile ${profile}`); const prof = { ...PROFILES[profile] }, custom = leadMs !== undefined || lagMs !== undefined; if (leadMs !== undefined) prof.leadMs = leadMs; if (lagMs !== undefined) prof.lagMs = lagMs; if (custom) { prof.note = `profile ${profile} with the window overridden on the command line`; } - for (const [n, x] of [["leadMs", prof.leadMs], ["lagMs", prof.lagMs], ["pairWindowMs", pairWindowMs]]) if (!Number.isFinite(x) || x <= 0) throw new Error(`${n} must be a positive number`); + for (const [n, x] of [["leadMs", prof.leadMs], ["lagMs", prof.lagMs], ["pairWindowMs", pairWindowMs]]) if (!Number.isFinite(x) || x < 0 || (n === "pairWindowMs" && x === 0)) throw new Error(`${n} must be a ${n === "pairWindowMs" ? "positive" : "non-negative"} number`); const gap = minGapMs === undefined ? {} : { minGapS: minGapMs / 1000 }; const a = audioOnsets(audio, sampleRate, { startS: audioStartS, ...gap }), v = visualOnsets(visual, { kind: visualKind, ...gap }); const hopMs = a.resolutionMs, periodMs = v.framePeriodMs, win = pairWindowMs / 1000; @@ -214,15 +228,14 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual events.forEach((e, i) => { const ao = ma.has(i) ? a.onsets[ma.get(i)] : undefined, vo = mv.has(i) ? v.onsets[mv.get(i)] : undefined; if (ao) usedA.add(ma.get(i)); if (vo) usedV.add(mv.get(i)); - // another onset of the same train inside the window: the nearest need not be the right one - pairs.push(row(e.label, e.t, ao, vo, within(a.onsets, e.t, win) > 1 || within(v.onsets, e.t, win) > 1)); + pairs.push(row(e.label, e.t, ao, vo, (ao && crowded(a.onsets, ao, win)) || (vo && crowded(v.onsets, vo, win)))); }); } else { const m = match(a.onsets.map((o) => o.t), v.onsets, win); a.onsets.forEach((ao, i) => { if (!m.has(i)) return; const vo = v.onsets[m.get(i)]; usedA.add(i); usedV.add(m.get(i)); - pairs.push(row(`a${i}`, null, ao, vo, within(v.onsets, ao.t, win) > 1 || within(a.onsets, vo.t, win) > 1)); + pairs.push(row(`a${i}`, null, ao, vo, crowded(a.onsets, ao, win) || crowded(v.onsets, vo, win))); }); } const lags = pairs.map((p) => p.lagMs).filter((x) => x !== null); @@ -253,7 +266,7 @@ export function formatSync(r) { for (const p of r.pairs) L.push(`${String(p.label).padEnd(10).slice(0, 10)} ${f(p.eventS, 3)} ${f(p.audioS, 3)} ${f(p.visualS, 3)} ${f(p.audioVsEventMs)} ${f(p.visualVsEventMs)} ${f(p.lagMs)} ${p.verdict}`); const s = r.summary, rng = s.minLagMs === null ? "-" : `${s.minLagMs.toFixed(1)}..${s.maxLagMs.toFixed(1)}`; L.push("", `${s.pairs} paired, ${s.unpairedAudio} audio and ${s.unpairedVisual} visual onsets unpaired${r.allowUnpaired ? " (allowed)" : " (fails the check unless --allow-unpaired)"}; median lag ${s.medianLagMs === null ? "-" : s.medianLagMs.toFixed(1)} ms, range ${rng} ms (spread ${s.spreadMs === null ? "-" : s.spreadMs.toFixed(1)} ms)`, - `${s.outsideWindow} outside the window, ${s.borderline} borderline, ${s.ambiguous} ambiguous (another onset within ${r.pairWindowMs} ms), ${s.missing} without both onsets. ${s.pass ? (s.borderline ? `NOT DETECTABLY OUTSIDE THE WINDOW (${s.borderline} borderline: the interval straddles the edge)` : "WITHIN WINDOW") : "NOT WITHIN WINDOW"} by this metric.`, + `${s.outsideWindow} outside the window, ${s.borderline} borderline, ${s.ambiguous} ambiguous (neighbouring onset closer than ${2 * r.pairWindowMs} ms), ${s.missing} without both onsets. ${s.pass ? (s.borderline ? `NOT DETECTABLY OUTSIDE THE WINDOW (${s.borderline} borderline: the interval straddles the edge)` : "WITHIN WINDOW") : "NOT WITHIN WINDOW"} by this metric.`, "A metric over two onset lists, not a verdict: a person still has to watch and listen (SYNC.md)."); return L.join("\n"); } diff --git a/tools/aqua-feel/sync.test.mjs b/tools/aqua-feel/sync.test.mjs index 01985bd..8091280 100644 --- a/tools/aqua-feel/sync.test.mjs +++ b/tools/aqua-feel/sync.test.mjs @@ -13,7 +13,7 @@ const run = (o, extra = {}) => { const fx = makeFixture(o); return { fx, r: sync const inInterval = (p, truth) => p.lagLowMs - 1e-9 <= truth && truth <= p.lagHighMs + 1e-9; const OFFGRID = [0.5042, 1.3117, 2.2209, 3.4033, 4.1161, 5.0087]; // events between frame boundaries at 30 and 60 fps -for (const fps of [30, 60]) for (const off of [-60, -20, 0, 40, 80, 150, 250]) { +for (const fps of [30, 60]) for (const off of [-60, -20, 0, 40, 80, 150, 190]) { t(`${fps} fps, ${off} ms offset: the truth is inside every pair's interval, events on and off the frame grid`, () => { for (const events of [FIXTURE_EVENTS_S, OFFGRID]) { const { r } = run({ offsetMs: off, fps, events }); @@ -73,11 +73,18 @@ t("an event with no sound is reported, not paired to a neighbour", () => { assert.equal(p.verdict, "no-audio-onset"); assert.equal(p.lagMs, null); assert.equal(r.summary.pass, false); }); -t("an offset past the 300 ms pairing window is reported as missing, not as a 300+ ms lag", () => { - const { r } = run({ offsetMs: 400 }); +t("an offset past the 200 ms pairing window is reported as missing, not as a longer lag", () => { + const { r } = run({ offsetMs: 250 }); assert.equal(r.pairs[0].verdict, "no-audio-onset"); assert.equal(r.summary.pass, false); }); +t("sparse events (0.7 s or more apart) pair without ambiguity at a +80 ms lag; the same lag at 0.3 s spacing is ambiguous", () => { + assert.equal(run({ offsetMs: 80 }).r.summary.ambiguous, 0); + const ev = [0.5, 0.8, 1.1, 1.4, 1.7]; + const { r } = run({ offsetMs: 80, events: ev }); + assert.equal(r.summary.ambiguous, ev.length); assert.equal(r.summary.pass, false); +}); + // ---- regressions from the Soul pass on PR #32 ---- t("a noise bed, a steady-tone bed or a DC offset under the hits does not move the audio onset (refinement is relative to the bed)", () => { for (const o of [{ noise: 0.03 }, { bed: [[220, 0.05], [330, 0.03]] }, { dc: 0.25 }, { gain: 0.03, noise: 0.003 }, { dc: -0.3, noise: 0.02 }]) { @@ -116,8 +123,8 @@ t("flashes closer than the visual minimum gap merge, and then an event goes unpa }); t("a stray sound or flash fails the check unless --allow-unpaired is given", () => { - const fx = makeFixture({ offsetMs: 0 }), a = Float32Array.from(fx.audio), click = makeFixture({ offsetMs: 0, events: [0.1] }).audio; - for (let i = 4000; i < 7000; i++) a[i] += click[i]; + const fx = makeFixture({ offsetMs: 0 }), a = Float32Array.from(fx.audio), click = makeFixture({ offsetMs: 0, events: [2.8] }).audio; + for (let i = 134000; i < 137500; i++) a[i] += click[i]; const run2 = (o) => syncAnalyze({ audio: a, sampleRate: 48000, visual: fx.visual, events: fx.events, ...o }); assert.equal(run2({}).summary.unpairedAudio, 1); assert.equal(run2({}).summary.pass, false); assert.equal(run2({ allowUnpaired: true }).summary.pass, true); @@ -125,7 +132,7 @@ t("a stray sound or flash fails the check unless --allow-unpaired is given", () t("bad options fail loudly: NaN window, unknown profile", () => { const fx = makeFixture({}); - assert.throws(() => syncAnalyze({ audio: fx.audio, sampleRate: 48000, visual: fx.visual, lagMs: NaN }), /positive/); + assert.throws(() => syncAnalyze({ audio: fx.audio, sampleRate: 48000, visual: fx.visual, lagMs: NaN }), /non-negative/); assert.throws(() => syncAnalyze({ audio: fx.audio, sampleRate: 48000, visual: fx.visual, profile: "nope" }), /unknown profile/); }); @@ -137,8 +144,22 @@ t("parsers refuse input that would give a plausible wrong answer", () => { assert.throws(() => readEvents('[{"time":2}]'), /no numeric time/); assert.throws(() => readEvents("# only a comment\n"), /no events/); assert.throws(() => readTrack("0,0,0.1\n1,0.1,0.2\n2,0.2,0.3\n"), /3 columns/); - assert.throws(() => readTrack("0.0,0.1\n0.0167,\n0.0333,0.3\n"), /not numeric/); + assert.throws(() => readTrack("0.0,0.1\n0.0167,\n0.0333,0.3\n"), /earlier lines had 2/); assert.throws(() => readTrack("0,0.1\n0.5,0.2\n0.4,0.3\n"), /does not increase/); + assert.throws(() => readEvents('[{"t":null}]'), /no numeric time/); assert.throws(() => readEvents('[{"t":""},1]'), /no numeric time/); + assert.throws(() => readEvents('[{"t":true}]'), /no numeric time/); assert.throws(() => readEvents("0x10\n"), /does not start with a time/); + assert.throws(() => readEvents('[{"t":1,"label":{"a":1}}]'), /label must be a string/); + assert.throws(() => readEvents("0:01.5\n"), /does not start with a time/); +}); + +t("valid layouts are accepted: header after blank or comment lines, semicolons, trailing commas, CRLF, BOM", () => { + assert.equal(readTrack("\n\nt,luma\r\n0,0.1,\r\n0.1;0.2\r\n0.2 0.3\r\n").v.length, 3); + assert.equal(readTrack("\uFEFF0,0.1\n0.1,0.2\n0.2,0.3\n").v.length, 3); + assert.deepEqual(readEvents("# list\ntime (s);label\n0.5;kick\n1.0;snare, loud\n").map((e) => [e.t, e.label]), [[0.5, "kick"], [1, "snare, loud"]]); +}); + +t("zero-width windows are expressible: --lag-ms 0 makes any positive lag outside", () => { + assert.equal(run({ offsetMs: 40 }, { lagMs: 0 }).r.summary.pass, false); }); t("silence and a flat track yield no onsets and no pass", () => { @@ -183,7 +204,25 @@ t("readTrack takes header + t,value rows or a single column with fps; readEvents const ff = process.env.FFMPEG ?? "ffmpeg"; let have = true; try { execFileSync(ff, ["-version"], { stdio: "ignore" }); } catch { have = false; } if (!have) console.log("skip video path: no ffmpeg (set FFMPEG)"); - else t("a real browser capture (VFR vp8+opus webm): every logged hit pairs, and the lag is a constant (spread under one frame)", () => { + if (have) for (const [vOff, aOff] of [[0, 0], [0.06, 0], [0, 0.08], [0.25, 0.1], [0.5, 0.5]]) { + const { mkdtempSync, writeFileSync } = await import("node:fs"), { tmpdir } = await import("node:os"), { join } = await import("node:path"); + const dir = mkdtempSync(join(tmpdir(), "sync-clip-")), fx = makeFixture({ offsetMs: 0, fps: 60, seconds: 4, events: [0.5, 1.4, 2.3, 3.2] }); + const wavP = join(dir, "a.wav"), rawP = join(dir, "v.gray"), outP = join(dir, "clip.mkv"); + writeFileSync(wavP, wavFloat32(fx.audio)); + writeFileSync(rawP, Buffer.concat(Array.from(fx.visual.v, (x) => Buffer.alloc(64 * 36, Math.round(255 * x))))); + // encode each stream alone, then mux with -itsoffset and stream copy: that is what puts a start_time on the stream in the file + const vP = join(dir, "v.mkv"), aP = join(dir, "a.mka"); + execFileSync(ff, ["-v", "error", "-y", "-f", "rawvideo", "-pix_fmt", "gray", "-s", "64x36", "-r", "60", "-i", rawP, "-c:v", "ffv1", vP]); + execFileSync(ff, ["-v", "error", "-y", "-i", wavP, "-c:a", "pcm_s16le", aP]); + execFileSync(ff, ["-v", "error", "-y", "-itsoffset", String(vOff), "-i", vP, "-itsoffset", String(aOff), "-i", aP, "-map", "0:v", "-map", "1:a", "-c", "copy", outP]); + const v = readVideo(outP, { ffmpeg: ff }), truth = (aOff - vOff) * 1000; // audio = visual shifted by (aOff - vOff) + // events are on the container clock, which ffmpeg shifts so the earliest stream starts at 0 + const r = syncAnalyze({ audio: v.audio.samples, sampleRate: v.audio.sampleRate, audioStartS: v.audio.startS, visual: v.luma, events: fx.events.map((e) => ({ t: e.t + vOff - Math.min(vOff, aOff), label: e.label })) }); + n++; console.log(`ok ffmpeg clip, video starts ${vOff * 1000} ms, audio ${aOff * 1000} ms: true lag ${truth} ms inside every interval`); + for (const p of r.pairs) assert.ok(p.lagLowMs - 1e-6 <= truth && truth <= p.lagHighMs + 1e-6, `video@${vOff} audio@${aOff}: [${p.lagLowMs}, ${p.lagHighMs}] vs ${truth}`); + assert.equal(r.pairs.length, 4); + } + if (have) t("a real browser capture (VFR vp8+opus webm): every logged hit pairs, and the lag is a constant (spread under one frame)", () => { const v = readVideo(new URL("./fixtures/browser-capture.webm", import.meta.url).pathname, { ffmpeg: ff }); const ev = readEvents(readFileSync(new URL("./fixtures/browser-capture.events.txt", import.meta.url), "utf8")); const r = syncAnalyze({ audio: v.audio.samples, sampleRate: v.audio.sampleRate, audioStartS: v.audio.startS, visual: v.luma, events: ev }); -- 2.47.3 From e4c5d25281328588b3d2780622430b6e48b8e1d9 Mon Sep 17 00:00:00 2001 From: Aqua Date: Tue, 10 Nov 2026 23:42:34 +0000 Subject: [PATCH 4/4] aqua-feel sync: third Soul pass fixes (resampled-video interval, header and CR parsing, docs) - a video resampled to constant frame rate gets an interval half a period wider each side (found at 15/24 fps against ffmpeg-built ground truth); tests at 15 and 24 fps - header lines may contain digits; CR-only line endings split - SYNC.md: wrong-sign reading beyond half the hit spacing, bed block length, test counts Co-Authored-By: Claude Sonnet 5.5 --- tools/aqua-feel/SYNC.md | 12 ++++++------ tools/aqua-feel/feel.mjs | 6 +++--- tools/aqua-feel/sync.mjs | 23 ++++++++++++----------- tools/aqua-feel/sync.test.mjs | 12 +++++++----- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/tools/aqua-feel/SYNC.md b/tools/aqua-feel/SYNC.md index 1248b57..016399a 100644 --- a/tools/aqua-feel/SYNC.md +++ b/tools/aqua-feel/SYNC.md @@ -16,7 +16,7 @@ node feel.mjs sync --wav capture.wav --track frames.csv [--events hits.txt] [--f node feel.mjs sync --video capture.mp4 [--events hits.txt] [--kind luma|flux] # options: --profile default|bt1359|ebu-r37|atsc-is191 --lead-ms N --lag-ms N --pair-window-ms N (default 200) # --min-gap-ms N --allow-unpaired --json out.json --no-gate -node sync.test.mjs # 37 checks on synthetic fixtures; 43 with FFMPEG=/path/to/ffmpeg (also builds clips with known stream offsets and runs a real browser capture) +node sync.test.mjs # 37 checks on synthetic fixtures; 46 with FFMPEG=/path/to/ffmpeg (also builds clips with known stream offsets and runs a real browser capture) node sync-fixture.mjs DIR 80 60 # write a fixture (audio 80 ms late, 60 fps) to try the CLI on ``` @@ -37,15 +37,15 @@ event log, audio onsets pair with visual onsets by proximity (within the pairing another onset of the same kind lies closer than twice the pairing window** (400 ms by default). Nearest-onset pairing is right only if the true lag is under half the spacing of the onsets, and the tool cannot know the lag, so it takes the pairing window as the largest lag you consider plausible. The cost is real: material faster than 2.5 hits per second cannot pass at the default, however good the sync. Narrow `--pair-window-ms` if the lag is known to be small (`50` clears 100 ms -spacing), or space the events out. A lag beyond the window reads as `no-audio-onset`; a lag beyond half the hit spacing but inside the window can read as the -previous hit's (wrong sign), which the ambiguity rule is there to catch. +spacing), or space the events out. A lag beyond the window reads as `no-audio-onset`, or, when the previous hit's sound lands inside the window, as that hit's (wrong sign, e.g. ++250 ms at 0.45 s spacing reads -200 ms); it fails either way, but do not trust the sign of a lag larger than half the hit spacing. Parsers refuse input they cannot read (timecodes, a third column, empty cells, non-increasing times) rather than guess. ## How onsets are found - **Audio:** log-magnitude spectral flux (1024-sample window, 128-sample hop, 60 Hz and up), half-wave rectified; peaks over median + 8 robust sigmas, at least 50 ms apart; each is then refined on the samples to the first one that rises above both 10% of the local peak and - 2.5 times the steady bed (the median block peak of the 0.1 s before, after removing the median as DC). A bed louder than a tenth of the hit still defeats it. + 2.5 times the steady bed (the median block peak of the 85 ms before, after removing the median as DC). A bed louder than a tenth of the hit still defeats it. - **Visual:** the positive step (luma) or the value (flux) of the track; peaks over median + 8 robust sigmas and at least 10% of the track's range, at least 100 ms apart (`--min-gap-ms` changes it, for both trains; flashes closer than that merge into one onset and the events beyond the first go unpaired); the onset is the first frame of the rise reaching 50% of the peak. @@ -75,8 +75,8 @@ lip-sync is more sensitive than a game impact) and on the viewer; they are popul A visual onset can only land on a frame, and a frame is stamped when its picture started to be displayed. A change at time tau first shows in the first frame stamped at or after tau, so the measured visual time is late by 0 to one frame period, and the measured lag (audio minus visual) is **biased low** by that much. The audio onset is good to about half a hop (1.4 ms). Each pair therefore reports `lagLowMs = lag - 1.4` and `lagHighMs = lag + frame period + 1.4`: the truth lies in that interval -(16.7 + 2.7 ms wide at 60 fps; 33.3 + 2.7 ms at 30 fps), and the verdict is read off the interval. This holds for events on and off the frame grid in the fixtures -(`sync.test.mjs`). The display's own latency (the time from stamp to light) is not in it. +(16.7 + 2.7 ms wide at 60 fps; 33.3 + 2.7 ms at 30 fps). A video is resampled to constant frame rate, which snaps each frame to the nearest source frame, so for a video the interval is half a period wider on each side (`[lag - period/2 - hop/2, lag + 1.5 period + hop/2]`), and the verdict is read off the interval. This holds for events on and off the frame grid in the fixtures +(`sync.test.mjs`; the video clips there are built with frame stamps at or after the change, the model's assumption). The display's own latency (the time from stamp to light) is not in it. ## What the metric cannot tell you diff --git a/tools/aqua-feel/feel.mjs b/tools/aqua-feel/feel.mjs index 5c659d6..1e9c370 100644 --- a/tools/aqua-feel/feel.mjs +++ b/tools/aqua-feel/feel.mjs @@ -65,12 +65,12 @@ async function sync() { if (args.length) throw new Error(`unknown argument ${args[0]}`); if (!["luma", "flux"].includes(kind)) throw new Error(`--kind is luma or flux, got "${kind}"`); if (!PROFILES[profile]) throw new Error(`--profile is one of ${Object.keys(PROFILES).join(", ")}, got "${profile}"`); - let audio, sampleRate, audioStartS = 0, visual; - if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; } + let audio, sampleRate, audioStartS = 0, visual, visualResampled = false; + if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; visualResampled = true; } else if (wav && trk) { const w = readWav(readFileSync(wav)); audio = w.samples; sampleRate = w.sampleRate; visual = readTrack(readFileSync(trk, "utf8"), fps); } else throw new Error("give --video FILE, or --wav FILE with --track FILE"); const r = syncAnalyze({ audio, sampleRate, audioStartS, visual, visualKind: kind, profile, events: evs ? readEvents(readFileSync(evs, "utf8")) : undefined, - leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 200, minGapMs: gap, allowUnpaired }); + leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 200, minGapMs: gap, allowUnpaired, visualResampled }); r.caveat = "Nothing here was heard or seen: two onset lists compared. The metric is not the verdict; see SYNC.md."; console.log(formatSync(r)); if (json) writeFileSync(json, JSON.stringify(r, null, 1)); diff --git a/tools/aqua-feel/sync.mjs b/tools/aqua-feel/sync.mjs index d6bcd51..749951e 100644 --- a/tools/aqua-feel/sync.mjs +++ b/tools/aqua-feel/sync.mjs @@ -37,15 +37,15 @@ export function readWav(buf) { } const NUM = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/, num = (x) => (typeof x === "number" ? x : typeof x === "string" && NUM.test(x.trim()) ? Number(x) : NaN); -// A header is a first data line with no digit in it; a line with digits that fails to parse (a timecode, a hex number) is an error. -const isHeader = (line, first) => first && !/\d/.test(line); +// A header is a first data line whose first field does not start like a number; a first field that does (a timecode, a hex number) is an error. +const isHeader = (line, first) => first && !/^[+-]?\.?\d/.test(line); // Frame-diff track: lines of "t,value" (seconds, strictly increasing) or a single "value" column with fps given. Only the first line may be a header. // Anything else that does not parse is an error: a silently dropped or shifted row would give a plausible wrong lag. export function readTrack(text, fps) { const t = [], v = []; let first = true, ncols = 0; - text.replace(/^/, "").split(/\r?\n/).forEach((raw, ln) => { + text.replace(/^/, "").split(/\r\n|\r|\n/).forEach((raw, ln) => { const line = raw.trim().replace(/[\s,;]+$/, ""); if (!line) return; const wasFirst = first; first = false; const tok = line.split(/[\s,;]+/), f = tok.map(num); @@ -75,7 +75,7 @@ export function readEvents(text) { }); } else { out = []; let first = true; - s.split(/\r?\n/).forEach((raw, ln) => { + s.split(/\r\n|\r|\n/).forEach((raw, ln) => { const l = raw.trim(); if (!l || l.startsWith("#")) return; const wasFirst = first; first = false; const m = l.match(/^([^,;\s]+)(?:[,;\s]+(.*))?$/), t = num(m[1]); @@ -112,7 +112,7 @@ export function readVideo(path, { ffmpeg = process.env.FFMPEG ?? "ffmpeg", targe t[f] = f / fps; lum[f] = s / (W * H) / 255; flux[f] = f ? df / (W * H) / 255 : 0; } const a = new Float32Array(audioRaw.buffer.slice(audioRaw.byteOffset, audioRaw.byteOffset + audioRaw.length - (audioRaw.length % 4))); - return { audio: { samples: a, sampleRate: 48000, startS: aStart }, luma: { t, v: lum }, flux: { t, v: flux }, fps, videoStartS: _vStart }; + return { audio: { samples: a, sampleRate: 48000, startS: aStart }, luma: { t, v: lum }, flux: { t, v: flux }, fps, resampled: true, videoStartS: _vStart }; } // ---------- onset detection ---------- @@ -199,7 +199,7 @@ const crowded = (list, o, windowS) => list.some((x) => x !== o && Math.abs(x.t - // Frame quantisation: a visual change at time tau first shows in the frame stamped at or after tau, so visual - tau is in [0, framePeriod) and the // measured lag (audio - visual) is biased LOW by up to a frame period. The truth therefore lies in [lag - audioHop/2, lag + framePeriod + audioHop/2]; // a pair is judged on that interval, not on a point. -export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false }) { +export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false, visualResampled = false }) { if (!PROFILES[profile]) throw new Error(`unknown profile ${profile}`); const prof = { ...PROFILES[profile] }, custom = leadMs !== undefined || lagMs !== undefined; if (leadMs !== undefined) prof.leadMs = leadMs; @@ -208,9 +208,10 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual for (const [n, x] of [["leadMs", prof.leadMs], ["lagMs", prof.lagMs], ["pairWindowMs", pairWindowMs]]) if (!Number.isFinite(x) || x < 0 || (n === "pairWindowMs" && x === 0)) throw new Error(`${n} must be a ${n === "pairWindowMs" ? "positive" : "non-negative"} number`); const gap = minGapMs === undefined ? {} : { minGapS: minGapMs / 1000 }; const a = audioOnsets(audio, sampleRate, { startS: audioStartS, ...gap }), v = visualOnsets(visual, { kind: visualKind, ...gap }); - const hopMs = a.resolutionMs, periodMs = v.framePeriodMs, win = pairWindowMs / 1000; + // A video resampled to constant frame rate by `fps` snaps each frame to the nearest source frame: half a period more slack on each side. + const hopMs = a.resolutionMs, periodMs = v.framePeriodMs, win = pairWindowMs / 1000, snapMs = visualResampled ? periodMs / 2 : 0; const classify = (lag) => { - const lo = lag - hopMs / 2, hi = lag + periodMs + hopMs / 2; // truth interval, ms + const lo = lag - hopMs / 2 - snapMs, hi = lag + periodMs + hopMs / 2 + snapMs; // truth interval, ms if (lo >= -prof.leadMs && hi <= prof.lagMs) return "inside-window"; if (lo > prof.lagMs) return "audio-late-detectable"; if (hi < -prof.leadMs) return "audio-early-detectable"; @@ -219,7 +220,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual const pairs = [], usedA = new Set(), usedV = new Set(); const row = (label, eventS, ao, vo, ambiguous) => { const r = { label, eventS, audioS: ao?.t ?? null, visualS: vo?.t ?? null, audioVsEventMs: ao && eventS !== null ? (ao.t - eventS) * 1e3 : null, visualVsEventMs: vo && eventS !== null ? (vo.t - eventS) * 1e3 : null, lagMs: ao && vo ? (ao.t - vo.t) * 1e3 : null }; - if (r.lagMs !== null) { r.lagLowMs = r.lagMs - hopMs / 2; r.lagHighMs = r.lagMs + periodMs + hopMs / 2; } + if (r.lagMs !== null) { r.lagLowMs = r.lagMs - hopMs / 2 - snapMs; r.lagHighMs = r.lagMs + periodMs + hopMs / 2 + snapMs; } r.verdict = r.lagMs === null ? (ao ? "no-visual-onset" : vo ? "no-audio-onset" : "no-onsets") : ambiguous ? "ambiguous-pairing" : classify(r.lagMs); return r; }; @@ -253,7 +254,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual summary.pass = summary.pairs > 0 && summary.outsideWindow === 0 && summary.missing === 0 && summary.ambiguous === 0 && (allowUnpaired || unpaired === 0); return { schema: "aqua-feel.sync.v1", lagConvention: "lagMs = audio onset - visual onset; positive = audio late, negative = audio early. Truth lies in [lagLowMs, lagHighMs] (visual onsets are quantised up to the next frame)", - profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualKind, audioResolutionMs: hopMs, framePeriodMs: periodMs, + profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualResampled, visualKind, audioResolutionMs: hopMs, framePeriodMs: periodMs, audioOnsets: a.onsets, visualOnsets: v.onsets, pairs, summary, }; } @@ -261,7 +262,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual export function formatSync(r) { const f = (x, d = 1) => (x === null || x === undefined ? " - " : x.toFixed(d).padStart(7)); const L = [`profile ${r.profile.name}: audio may lead by ${r.profile.leadMs} ms, lag by ${r.profile.lagMs} ms (${r.profile.note})`, - `lag = audio - visual (+ = audio late). The visual onset is quantised up to the next frame (period ${r.framePeriodMs.toFixed(1)} ms), so the true lag lies in [lag - ${(r.audioResolutionMs / 2).toFixed(1)}, lag + ${(r.framePeriodMs + r.audioResolutionMs / 2).toFixed(1)}] ms`, + `lag = audio - visual (+ = audio late). The visual onset is quantised up to the next frame (period ${r.framePeriodMs.toFixed(1)} ms), so the true lag lies in [lag - ${(r.audioResolutionMs / 2 + (r.visualResampled ? r.framePeriodMs / 2 : 0)).toFixed(1)}, lag + ${(r.framePeriodMs * (r.visualResampled ? 1.5 : 1) + r.audioResolutionMs / 2).toFixed(1)}] ms${r.visualResampled ? " (video resampled to constant frame rate)" : ""}`, "", "label event s audio s visual s a-ev ms v-ev ms lag ms verdict"]; for (const p of r.pairs) L.push(`${String(p.label).padEnd(10).slice(0, 10)} ${f(p.eventS, 3)} ${f(p.audioS, 3)} ${f(p.visualS, 3)} ${f(p.audioVsEventMs)} ${f(p.visualVsEventMs)} ${f(p.lagMs)} ${p.verdict}`); const s = r.summary, rng = s.minLagMs === null ? "-" : `${s.minLagMs.toFixed(1)}..${s.maxLagMs.toFixed(1)}`; diff --git a/tools/aqua-feel/sync.test.mjs b/tools/aqua-feel/sync.test.mjs index 8091280..f2b4e4d 100644 --- a/tools/aqua-feel/sync.test.mjs +++ b/tools/aqua-feel/sync.test.mjs @@ -155,6 +155,8 @@ t("parsers refuse input that would give a plausible wrong answer", () => { t("valid layouts are accepted: header after blank or comment lines, semicolons, trailing commas, CRLF, BOM", () => { assert.equal(readTrack("\n\nt,luma\r\n0,0.1,\r\n0.1;0.2\r\n0.2 0.3\r\n").v.length, 3); assert.equal(readTrack("\uFEFF0,0.1\n0.1,0.2\n0.2,0.3\n").v.length, 3); + assert.equal(readEvents("0.5,a\r1.0,b\r").length, 2, "CR-only line ends"); + assert.equal(readTrack("t0,luma_0to1\n0,0.1\n0.1,0.2\n0.2,0.3\n").v.length, 3, "a header may contain digits"); assert.deepEqual(readEvents("# list\ntime (s);label\n0.5;kick\n1.0;snare, loud\n").map((e) => [e.t, e.label]), [[0.5, "kick"], [1, "snare, loud"]]); }); @@ -204,7 +206,7 @@ t("readTrack takes header + t,value rows or a single column with fps; readEvents const ff = process.env.FFMPEG ?? "ffmpeg"; let have = true; try { execFileSync(ff, ["-version"], { stdio: "ignore" }); } catch { have = false; } if (!have) console.log("skip video path: no ffmpeg (set FFMPEG)"); - if (have) for (const [vOff, aOff] of [[0, 0], [0.06, 0], [0, 0.08], [0.25, 0.1], [0.5, 0.5]]) { + if (have) for (const [vOff, aOff, vfps] of [[0, 0, 60], [0.06, 0, 60], [0, 0.08, 60], [0.25, 0.1, 60], [0.5, 0.5, 60], [0.1, 0, 15], [0.1, 0, 24], [0, 0.1, 15]]) { const { mkdtempSync, writeFileSync } = await import("node:fs"), { tmpdir } = await import("node:os"), { join } = await import("node:path"); const dir = mkdtempSync(join(tmpdir(), "sync-clip-")), fx = makeFixture({ offsetMs: 0, fps: 60, seconds: 4, events: [0.5, 1.4, 2.3, 3.2] }); const wavP = join(dir, "a.wav"), rawP = join(dir, "v.gray"), outP = join(dir, "clip.mkv"); @@ -212,14 +214,14 @@ t("readTrack takes header + t,value rows or a single column with fps; readEvents writeFileSync(rawP, Buffer.concat(Array.from(fx.visual.v, (x) => Buffer.alloc(64 * 36, Math.round(255 * x))))); // encode each stream alone, then mux with -itsoffset and stream copy: that is what puts a start_time on the stream in the file const vP = join(dir, "v.mkv"), aP = join(dir, "a.mka"); - execFileSync(ff, ["-v", "error", "-y", "-f", "rawvideo", "-pix_fmt", "gray", "-s", "64x36", "-r", "60", "-i", rawP, "-c:v", "ffv1", vP]); + execFileSync(ff, ["-v", "error", "-y", "-f", "rawvideo", "-pix_fmt", "gray", "-s", "64x36", "-r", "60", "-i", rawP, "-vf", `fps=${vfps}:round=up`, "-c:v", "ffv1", vP]); execFileSync(ff, ["-v", "error", "-y", "-i", wavP, "-c:a", "pcm_s16le", aP]); execFileSync(ff, ["-v", "error", "-y", "-itsoffset", String(vOff), "-i", vP, "-itsoffset", String(aOff), "-i", aP, "-map", "0:v", "-map", "1:a", "-c", "copy", outP]); const v = readVideo(outP, { ffmpeg: ff }), truth = (aOff - vOff) * 1000; // audio = visual shifted by (aOff - vOff) // events are on the container clock, which ffmpeg shifts so the earliest stream starts at 0 - const r = syncAnalyze({ audio: v.audio.samples, sampleRate: v.audio.sampleRate, audioStartS: v.audio.startS, visual: v.luma, events: fx.events.map((e) => ({ t: e.t + vOff - Math.min(vOff, aOff), label: e.label })) }); - n++; console.log(`ok ffmpeg clip, video starts ${vOff * 1000} ms, audio ${aOff * 1000} ms: true lag ${truth} ms inside every interval`); - for (const p of r.pairs) assert.ok(p.lagLowMs - 1e-6 <= truth && truth <= p.lagHighMs + 1e-6, `video@${vOff} audio@${aOff}: [${p.lagLowMs}, ${p.lagHighMs}] vs ${truth}`); + const r = syncAnalyze({ audio: v.audio.samples, sampleRate: v.audio.sampleRate, audioStartS: v.audio.startS, visual: v.luma, events: fx.events.map((e) => ({ t: e.t + vOff - Math.min(vOff, aOff), label: e.label })), visualResampled: v.resampled }); + n++; console.log(`ok ffmpeg clip, video (${vfps} fps) starts ${vOff * 1000} ms, audio ${aOff * 1000} ms: true lag ${truth} ms inside every interval`); + for (const p of r.pairs) assert.ok(p.lagLowMs - 1e-6 <= truth && truth <= p.lagHighMs + 1e-6, `video@${vOff}/${vfps}fps audio@${aOff}: [${p.lagLowMs}, ${p.lagHighMs}] vs ${truth}`); assert.equal(r.pairs.length, 4); } if (have) t("a real browser capture (VFR vp8+opus webm): every logged hit pairs, and the lag is a constant (spread under one frame)", () => { -- 2.47.3