diff --git a/tools/aqua-feel/SYNC.md b/tools/aqua-feel/SYNC.md index 016399a..39916e3 100644 --- a/tools/aqua-feel/SYNC.md +++ b/tools/aqua-feel/SYNC.md @@ -89,8 +89,7 @@ onset is good to about half a hop (1.4 ms). Each pair therefore reports `lagLowM that outlives the event) is not modelled. - **Anything about the chain you did not capture.** If the capture is of the app's output, display latency, speaker/Bluetooth latency and a player's own sync are not in it. A capture can be in sync and the playback chain still be 150 ms off. -- **Variable frame rate.** A video is resampled by presentation time to constant frame rate (ffmpeg `fps` filter), so a VFR browser capture works, but the frame period - used in the interval is the resampled one, not the source's real gaps. A CSV track keeps its own timestamps. +- **Variable frame rate.** A video is resampled by presentation time to constant frame rate (ffmpeg `fps` filter). The rate is the stream's declared one; a browser capture declares none (`1k tbr`), so it is the median gap between the frames' own times (24 to 120 fps; before this it silently fell back to 30). The interval's frame period is the larger of the resampled period and the 95th-percentile gap between source frames, so a capture that stalls for 50 ms cannot be reported as tight as a 60 fps one; the report says so when the gap sets the period. A CSV track keeps its own timestamps. - **Stereo is mixed to mono**, so an antiphase stereo track cancels to silence and reads as no onsets. - **Content that is not onset-shaped.** Continuous music against continuous motion has no onsets to compare; the tool returns few or none, and silence in either track is reported as no pass, not as sync. @@ -98,6 +97,6 @@ onset is good to about half a hop (1.4 ms). Each pair therefore reports `lagLowM below the 10%-of-range floor; use a cropped frame-diff track in that case. - **Known stream offsets.** `sync.test.mjs` builds clips with ffmpeg whose video or audio stream starts late (`-itsoffset`, stream copy) and checks that the known lag lies in every pair's interval, so the container start-time bookkeeping has a ground truth; the clips are synthetic flashes and tones. - **A pulsing bed.** The bed estimate is a median of block peaks and cannot follow dynamics: a 4 Hz tremolo bed at 10% depth under the hits pulled some lags off by up to 28 ms in testing. Beds louder than a tenth of the hit and noise gates that cut the sound to digital silence (the noise resuming reads as an onset) can also give a wrong onset that passes when the true lag is well inside the window. -- **One real clip.** `fixtures/browser-capture.webm` is a headless-Chromium MediaRecorder capture (WebAudio beeps and canvas flashes scheduled at the same instants, vp8 + opus, recipe in `fixtures/record-browser-capture.cjs`). It is a test that the video path runs and pairs, not a validation of the lag it reports: nothing independent says what the true lag of that clip is. No Playground UI or Perlines clip has been run. +- **One real clip.** `fixtures/browser-capture.webm` is a headless-Chromium MediaRecorder capture (WebAudio beeps and canvas flashes scheduled at the same instants, vp8 + opus, recipe in `fixtures/record-browser-capture.cjs`). It is a test that the video path runs and pairs, not a validation of the lag it reports: nothing independent says what the true lag of that clip is. `fixtures/perlines-aurora-vfr.webm` is 6 s of Perlines v2 (Aurora disk scene, software-rendered headless Chromium) playing a click-train WAV, with the app's audio output and its canvas recorded; it declares no frame rate and stalls up to 34 ms, and is there to test that. A longer take of all three scenes and what it showed is on the minds.gamecult.org/aqua page named in the PR. No Playground UI clip has been run. - **The tuned constants.** The 8-sigma threshold, window sizes, 200 ms pairing window, 10% floors and 2.5x bed factor were set against synthetic fixtures (`sync-fixture.mjs`: tone-burst hits against 4-frame flashes, known offset, with noise, tone and DC beds) and one browser clip. They have not been tuned on game or music content. diff --git a/tools/aqua-feel/feel.mjs b/tools/aqua-feel/feel.mjs index 1e9c370..f789557 100644 --- a/tools/aqua-feel/feel.mjs +++ b/tools/aqua-feel/feel.mjs @@ -65,12 +65,12 @@ async function sync() { if (args.length) throw new Error(`unknown argument ${args[0]}`); if (!["luma", "flux"].includes(kind)) throw new Error(`--kind is luma or flux, got "${kind}"`); if (!PROFILES[profile]) throw new Error(`--profile is one of ${Object.keys(PROFILES).join(", ")}, got "${profile}"`); - let audio, sampleRate, audioStartS = 0, visual, visualResampled = false; - if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; visualResampled = true; } + let audio, sampleRate, audioStartS = 0, visual, visualResampled = false, visualGapMs = 0; + if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; visualResampled = true; visualGapMs = v.sourceGapMs; } else if (wav && trk) { const w = readWav(readFileSync(wav)); audio = w.samples; sampleRate = w.sampleRate; visual = readTrack(readFileSync(trk, "utf8"), fps); } else throw new Error("give --video FILE, or --wav FILE with --track FILE"); const r = syncAnalyze({ audio, sampleRate, audioStartS, visual, visualKind: kind, profile, events: evs ? readEvents(readFileSync(evs, "utf8")) : undefined, - leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 200, minGapMs: gap, allowUnpaired, visualResampled }); + leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 200, minGapMs: gap, allowUnpaired, visualResampled, visualGapMs }); r.caveat = "Nothing here was heard or seen: two onset lists compared. The metric is not the verdict; see SYNC.md."; console.log(formatSync(r)); if (json) writeFileSync(json, JSON.stringify(r, null, 1)); diff --git a/tools/aqua-feel/fixtures/perlines-aurora-vfr.webm b/tools/aqua-feel/fixtures/perlines-aurora-vfr.webm new file mode 100644 index 0000000..eb6454e Binary files /dev/null and b/tools/aqua-feel/fixtures/perlines-aurora-vfr.webm differ diff --git a/tools/aqua-feel/sync.mjs b/tools/aqua-feel/sync.mjs index 749951e..a1b42d2 100644 --- a/tools/aqua-feel/sync.mjs +++ b/tools/aqua-feel/sync.mjs @@ -87,16 +87,28 @@ export function readEvents(text) { return out; } +// From the presentation times in ffmpeg's showinfo output: the frame rate (median gap, rounded, within [24, 120]; 30 when fewer than two frames) +// and the 95th-percentile gap between source frames, which bounds how stale a displayed frame can be in a variable-frame-rate capture. +export function sourceFrames(showinfo) { + const pts = [...String(showinfo).matchAll(/pts_time:\s*(-?\d+(?:\.\d+)?)/g)].map((m) => Number(m[1])); + const gaps = pts.slice(1).map((x, i) => x - pts[i]).filter((g) => g > 0).sort((a, b) => a - b); + if (!gaps.length) return { fps: 30, gapMs: 0 }; + return { fps: Math.min(120, Math.max(24, Math.round(1 / median(gaps)))), gapMs: 1000 * gaps[Math.min(gaps.length - 1, Math.floor(gaps.length * 0.95))] }; +} + // Video via ffmpeg: audio as mono f32 at 48 kHz, video as 64x36 gray frames; per-frame mean luminance and mean |frame diff| (motion flux). // Stream start_time offsets are applied so both clocks are the container's clock; video is resampled to constant frame rate by pts. export function readVideo(path, { ffmpeg = process.env.FFMPEG ?? "ffmpeg", targetFps } = {}) { // ffmpeg only (no ffprobe: static builds ship without it). `ffmpeg -i` lists the streams; the first decoded frame of each stream says when it starts. - const run = (a) => { const r = spawnSync(ffmpeg, a, { encoding: "utf8", maxBuffer: 1 << 26 }); if (r.error?.code === "ENOENT") throw new Error("ffmpeg is not on PATH (set FFMPEG=/path/to/ffmpeg); give --wav and --track instead"); return String(r.stderr ?? ""); }; + const run = (a) => { const r = spawnSync(ffmpeg, a, { encoding: "utf8", maxBuffer: 1 << 26 }); if (r.error?.code === "ENOENT") throw new Error("ffmpeg is not on PATH (set FFMPEG=/path/to/ffmpeg); give --wav and --track instead"); if (r.error) throw new Error(`ffmpeg failed (${r.error.code}): its frame list may be truncated, so the capture's stalls would be under-measured`); return String(r.stderr ?? ""); }; const lines = run(["-hide_banner", "-i", path]).split("\n"), find = (kind) => lines.find((l) => new RegExp(`Stream #\\d+:\\d+.*: ${kind}:`).test(l)); const vl = find("Video"), al = find("Audio"); if (!vl || !al) throw new Error("file needs one video and one audio stream"); const idx = (l) => Number(l.match(/Stream #\d+:(\d+)/)[1]), fpsm = vl.match(/(\d+(?:\.\d+)?) fps/) ?? vl.match(/(\d+(?:\.\d+)?) tbr/); - const fps = targetFps ?? Number(fpsm?.[1] ?? 30), W = 64, H = 36, vi = idx(vl), ai = idx(al); + const W = 64, H = 36, vi = idx(vl), ai = idx(al); + // A browser capture declares no frame rate (`1k tbr`). The stream's own frame times then say what rate to resample at; guessing 30 fps widened every interval to 51 ms. + const src = sourceFrames(run(["-hide_banner", "-i", path, "-map", `0:${vi}`, "-an", "-vf", "showinfo", "-f", "null", "-"])); + const fps = targetFps ?? (fpsm ? Number(fpsm[1]) : src.fps); // Time of a stream's first frame on the container clock (ffmpeg shifts every stream by the earliest start, so these are relative to that). const first = (flags) => { const m = run(["-hide_banner", "-i", path, ...flags, "-f", "null", "-"]).match(/pts_time:\s*(-?\d+(?:\.\d+)?)/); if (!m) throw new Error("could not read a stream start time from ffmpeg"); return Number(m[1]); }; const _vStart = first(["-map", `0:${vi}`, "-an", "-frames:v", "1", "-vf", "showinfo"]), aStart = first(["-map", `0:${ai}`, "-vn", "-frames:a", "1", "-af", "ashowinfo"]); @@ -112,7 +124,7 @@ export function readVideo(path, { ffmpeg = process.env.FFMPEG ?? "ffmpeg", targe t[f] = f / fps; lum[f] = s / (W * H) / 255; flux[f] = f ? df / (W * H) / 255 : 0; } const a = new Float32Array(audioRaw.buffer.slice(audioRaw.byteOffset, audioRaw.byteOffset + audioRaw.length - (audioRaw.length % 4))); - return { audio: { samples: a, sampleRate: 48000, startS: aStart }, luma: { t, v: lum }, flux: { t, v: flux }, fps, resampled: true, videoStartS: _vStart }; + return { audio: { samples: a, sampleRate: 48000, startS: aStart }, luma: { t, v: lum }, flux: { t, v: flux }, fps, sourceGapMs: src.gapMs, resampled: true, videoStartS: _vStart }; } // ---------- onset detection ---------- @@ -199,7 +211,7 @@ const crowded = (list, o, windowS) => list.some((x) => x !== o && Math.abs(x.t - // Frame quantisation: a visual change at time tau first shows in the frame stamped at or after tau, so visual - tau is in [0, framePeriod) and the // measured lag (audio - visual) is biased LOW by up to a frame period. The truth therefore lies in [lag - audioHop/2, lag + framePeriod + audioHop/2]; // a pair is judged on that interval, not on a point. -export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false, visualResampled = false }) { +export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false, visualResampled = false, visualGapMs = 0 }) { if (!PROFILES[profile]) throw new Error(`unknown profile ${profile}`); const prof = { ...PROFILES[profile] }, custom = leadMs !== undefined || lagMs !== undefined; if (leadMs !== undefined) prof.leadMs = leadMs; @@ -209,7 +221,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual const gap = minGapMs === undefined ? {} : { minGapS: minGapMs / 1000 }; const a = audioOnsets(audio, sampleRate, { startS: audioStartS, ...gap }), v = visualOnsets(visual, { kind: visualKind, ...gap }); // A video resampled to constant frame rate by `fps` snaps each frame to the nearest source frame: half a period more slack on each side. - const hopMs = a.resolutionMs, periodMs = v.framePeriodMs, win = pairWindowMs / 1000, snapMs = visualResampled ? periodMs / 2 : 0; + const hopMs = a.resolutionMs, periodMs = Math.max(v.framePeriodMs, visualGapMs), win = pairWindowMs / 1000, snapMs = visualResampled ? periodMs / 2 : 0; const classify = (lag) => { const lo = lag - hopMs / 2 - snapMs, hi = lag + periodMs + hopMs / 2 + snapMs; // truth interval, ms if (lo >= -prof.leadMs && hi <= prof.lagMs) return "inside-window"; @@ -254,7 +266,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual summary.pass = summary.pairs > 0 && summary.outsideWindow === 0 && summary.missing === 0 && summary.ambiguous === 0 && (allowUnpaired || unpaired === 0); return { schema: "aqua-feel.sync.v1", lagConvention: "lagMs = audio onset - visual onset; positive = audio late, negative = audio early. Truth lies in [lagLowMs, lagHighMs] (visual onsets are quantised up to the next frame)", - profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualResampled, visualKind, audioResolutionMs: hopMs, framePeriodMs: periodMs, + profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualResampled, visualKind, audioResolutionMs: hopMs, framePeriodMs: periodMs, visualGapMs, audioOnsets: a.onsets, visualOnsets: v.onsets, pairs, summary, }; } @@ -262,7 +274,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual export function formatSync(r) { const f = (x, d = 1) => (x === null || x === undefined ? " - " : x.toFixed(d).padStart(7)); const L = [`profile ${r.profile.name}: audio may lead by ${r.profile.leadMs} ms, lag by ${r.profile.lagMs} ms (${r.profile.note})`, - `lag = audio - visual (+ = audio late). The visual onset is quantised up to the next frame (period ${r.framePeriodMs.toFixed(1)} ms), so the true lag lies in [lag - ${(r.audioResolutionMs / 2 + (r.visualResampled ? r.framePeriodMs / 2 : 0)).toFixed(1)}, lag + ${(r.framePeriodMs * (r.visualResampled ? 1.5 : 1) + r.audioResolutionMs / 2).toFixed(1)}] ms${r.visualResampled ? " (video resampled to constant frame rate)" : ""}`, + `lag = audio - visual (+ = audio late). The visual onset is quantised up to the next frame (period ${r.framePeriodMs.toFixed(1)} ms${r.visualGapMs > 0 && r.framePeriodMs === r.visualGapMs ? ": the capture\'s source frames are up to that far apart" : ""}), so the true lag lies in [lag - ${(r.audioResolutionMs / 2 + (r.visualResampled ? r.framePeriodMs / 2 : 0)).toFixed(1)}, lag + ${(r.framePeriodMs * (r.visualResampled ? 1.5 : 1) + r.audioResolutionMs / 2).toFixed(1)}] ms${r.visualResampled ? " (video resampled to constant frame rate)" : ""}`, "", "label event s audio s visual s a-ev ms v-ev ms lag ms verdict"]; for (const p of r.pairs) L.push(`${String(p.label).padEnd(10).slice(0, 10)} ${f(p.eventS, 3)} ${f(p.audioS, 3)} ${f(p.visualS, 3)} ${f(p.audioVsEventMs)} ${f(p.visualVsEventMs)} ${f(p.lagMs)} ${p.verdict}`); const s = r.summary, rng = s.minLagMs === null ? "-" : `${s.minLagMs.toFixed(1)}..${s.maxLagMs.toFixed(1)}`; diff --git a/tools/aqua-feel/sync.test.mjs b/tools/aqua-feel/sync.test.mjs index f2b4e4d..72394da 100644 --- a/tools/aqua-feel/sync.test.mjs +++ b/tools/aqua-feel/sync.test.mjs @@ -1,6 +1,6 @@ // Checks of `aqua-feel sync` on synthetic fixtures with known offsets. No real capture is used. Usage: node sync.test.mjs import assert from "node:assert/strict"; -import { formatSync, syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, PROFILES } from "./sync.mjs"; +import { formatSync, syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, sourceFrames, PROFILES } from "./sync.mjs"; import { readFileSync } from "node:fs"; import { wavFloat32 } from "./render.js"; import { makeFixture, FIXTURE_EVENTS_S } from "./sync-fixture.mjs"; @@ -200,9 +200,19 @@ t("readTrack takes header + t,value rows or a single column with fps; readEvents assert.deepEqual(readEvents('[{"t":2,"label":"x"},3]').map((e) => e.label), ["x", "e1"]); }); +t("sourceFrames: the rate is the median frame gap, the stale-frame bound is the 95th-percentile gap", () => { + const si = (ts) => ts.map((x, i) => `[Parsed_showinfo_0 @ 0x1] n:${i} pts:${x * 1000} pts_time:${x} duration:1`).join("\n"); + const cfr = Array.from({ length: 40 }, (_, i) => i / 50); + { const c = sourceFrames(si(cfr)); assert.equal(c.fps, 50); assert.ok(Math.abs(c.gapMs - 20) < 1e-6); } + const vfr = []; let x = 0; for (let i = 0; i < 100; i++) { vfr.push(x); x += i % 10 === 9 ? 0.1 : 0.0167; } // mostly 60 fps, one 100 ms stall in ten + const r = sourceFrames(si(vfr)); assert.equal(r.fps, 60); assert.ok(Math.abs(r.gapMs - 100) < 1, `gap ${r.gapMs}`); + assert.deepEqual(sourceFrames(si([0])), { fps: 30, gapMs: 0 }); + assert.equal(sourceFrames(si([0, 1, 2])).fps, 24); assert.equal(sourceFrames(si([0, 0.001, 0.002])).fps, 120); +}); + // The video path needs ffmpeg (FFMPEG=/path or on PATH); skipped, and said so, when there is none. { - const { readVideo } = await import("./sync.mjs"), { execFileSync } = await import("node:child_process"); + const { readVideo } = await import("./sync.mjs"), { execFileSync } = await import("node:child_process"), require_fs = await import("node:fs"), { tmpdir } = await import("node:os"), { join } = await import("node:path"); const ff = process.env.FFMPEG ?? "ffmpeg"; let have = true; try { execFileSync(ff, ["-version"], { stdio: "ignore" }); } catch { have = false; } if (!have) console.log("skip video path: no ffmpeg (set FFMPEG)"); @@ -233,6 +243,26 @@ t("readTrack takes header + t,value rows or a single column with fps; readEvents assert.equal(r.summary.unpairedAudio + r.summary.unpairedVisual, 0); for (const p of r.pairs) assert.ok(p.visualVsEventMs >= 0 && p.visualVsEventMs <= 1000 / v.fps, `flash ${p.visualVsEventMs} ms off its logged instant`); }); + if (have) t("a Perlines v2 capture declares no frame rate (1k tbr): the rate comes from its frame times, and the interval is no narrower than its worst frame gaps", () => { + const v = readVideo(new URL("./fixtures/perlines-aurora-vfr.webm", import.meta.url).pathname, { ffmpeg: ff }); + assert.ok(v.fps >= 50, `resampled at ${v.fps} fps, not the 30 fps fallback`); + assert.ok(v.sourceGapMs > 1000 / v.fps * 1.5, `source gap ${v.sourceGapMs} ms is a stalled capture, wider than one frame`); + const r = syncAnalyze({ audio: v.audio.samples, sampleRate: v.audio.sampleRate, audioStartS: v.audio.startS, visual: v.luma, allowUnpaired: true, visualResampled: true, visualGapMs: v.sourceGapMs }); + assert.ok(r.pairs.length >= 3); + for (const p of r.pairs) assert.ok(p.lagHighMs - p.lagLowMs >= v.sourceGapMs * 1.5, `interval ${p.lagHighMs - p.lagLowMs} ms narrower than the capture's own stalls`); + assert.match(formatSync(r), /source frames are up to that far apart/); + }); + if (have) t("the CLI carries the capture's frame gaps into its report (feel.mjs plumbing)", () => { + const { mkdtempSync, readFileSync: rf } = require_fs, out = join(mkdtempSync(join(tmpdir(), "sync-cli-")), "r.json"); + execFileSync("node", [new URL("./feel.mjs", import.meta.url).pathname, "sync", "--video", new URL("./fixtures/perlines-aurora-vfr.webm", import.meta.url).pathname, "--allow-unpaired", "--no-gate", "--json", out], { env: { ...process.env, FFMPEG: ff }, stdio: "ignore" }); + const r = JSON.parse(rf(out, "utf8")); + assert.ok(r.visualGapMs > 1000 / 59 * 1.5, `visualGapMs ${r.visualGapMs}`); assert.equal(r.framePeriodMs, r.visualGapMs); + }); + if (have) t("a failed ffmpeg read (e.g. output buffer overflow) is an error, not a quietly shorter frame list", () => { + const { mkdtempSync, writeFileSync: wf, chmodSync } = require_fs, d = mkdtempSync(join(tmpdir(), "sync-stub-")), stub = join(d, "ffmpeg"); + wf(stub, "#!/bin/sh\nhead -c 70000000 /dev/zero | tr '\\0' 'x' >&2\n"); chmodSync(stub, 0o755); + assert.throws(() => readVideo(new URL("./fixtures/perlines-aurora-vfr.webm", import.meta.url).pathname, { ffmpeg: stub }), /ffmpeg failed \(ENOBUFS\)/); + }); } console.log(`${n} sync checks passed. Fixtures only (synthetic, plus one browser capture when ffmpeg is present); nothing was heard or seen.`);