tools/aqua-feel/SYNC.md

@@ -89,8 +89,7 @@ onset is good to about half a hop (1.4 ms). Each pair therefore reports `lagLowM
   that outlives the event) is not modelled.
 - **Anything about the chain you did not capture.** If the capture is of the app's output, display latency, speaker/Bluetooth
   latency and a player's own sync are not in it. A capture can be in sync and the playback chain still be 150 ms off.
-- **Variable frame rate.** A video is resampled by presentation time to constant frame rate (ffmpeg `fps` filter), so a VFR browser capture works, but the frame period
-  used in the interval is the resampled one, not the source's real gaps. A CSV track keeps its own timestamps.
+- **Variable frame rate.** A video is resampled by presentation time to constant frame rate (ffmpeg `fps` filter). The rate is the stream's declared one; a browser capture declares none (`1k tbr`), so it is the median gap between the frames' own times (24 to 120 fps; before this it silently fell back to 30). The interval's frame period is the larger of the resampled period and the 95th-percentile gap between source frames, so a capture that stalls for 50 ms cannot be reported as tight as a 60 fps one; the report says so when the gap sets the period. A CSV track keeps its own timestamps.
 - **Stereo is mixed to mono**, so an antiphase stereo track cancels to silence and reads as no onsets.
 - **Content that is not onset-shaped.** Continuous music against continuous motion has no onsets to compare; the tool returns
   few or none, and silence in either track is reported as no pass, not as sync.
@@ -98,6 +97,6 @@ onset is good to about half a hop (1.4 ms). Each pair therefore reports `lagLowM
   below the 10%-of-range floor; use a cropped frame-diff track in that case.
 - **Known stream offsets.** `sync.test.mjs` builds clips with ffmpeg whose video or audio stream starts late (`-itsoffset`, stream copy) and checks that the known lag lies in every pair's interval, so the container start-time bookkeeping has a ground truth; the clips are synthetic flashes and tones.
 - **A pulsing bed.** The bed estimate is a median of block peaks and cannot follow dynamics: a 4 Hz tremolo bed at 10% depth under the hits pulled some lags off by up to 28 ms in testing. Beds louder than a tenth of the hit and noise gates that cut the sound to digital silence (the noise resuming reads as an onset) can also give a wrong onset that passes when the true lag is well inside the window.
-- **One real clip.** `fixtures/browser-capture.webm` is a headless-Chromium MediaRecorder capture (WebAudio beeps and canvas flashes scheduled at the same instants, vp8 + opus, recipe in `fixtures/record-browser-capture.cjs`). It is a test that the video path runs and pairs, not a validation of the lag it reports: nothing independent says what the true lag of that clip is. No Playground UI or Perlines clip has been run.
+- **One real clip.** `fixtures/browser-capture.webm` is a headless-Chromium MediaRecorder capture (WebAudio beeps and canvas flashes scheduled at the same instants, vp8 + opus, recipe in `fixtures/record-browser-capture.cjs`). It is a test that the video path runs and pairs, not a validation of the lag it reports: nothing independent says what the true lag of that clip is. `fixtures/perlines-aurora-vfr.webm` is 6 s of Perlines v2 (Aurora disk scene, software-rendered headless Chromium) playing a click-train WAV, with the app's audio output and its canvas recorded; it declares no frame rate and stalls up to 34 ms, and is there to test that. A longer take of all three scenes and what it showed is on the minds.gamecult.org/aqua page named in the PR. No Playground UI clip has been run.
 - **The tuned constants.** The 8-sigma threshold, window sizes, 200 ms pairing window, 10% floors and 2.5x bed factor were set against synthetic
   fixtures (`sync-fixture.mjs`: tone-burst hits against 4-frame flashes, known offset, with noise, tone and DC beds) and one browser clip. They have not been tuned on game or music content.

tools/aqua-feel/feel.mjs

@@ -65,12 +65,12 @@ async function sync() {
     if (args.length) throw new Error(`unknown argument ${args[0]}`);
     if (!["luma", "flux"].includes(kind)) throw new Error(`--kind is luma or flux, got "${kind}"`);
     if (!PROFILES[profile]) throw new Error(`--profile is one of ${Object.keys(PROFILES).join(", ")}, got "${profile}"`);
-    let audio, sampleRate, audioStartS = 0, visual, visualResampled = false;
-    if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; visualResampled = true; }
+    let audio, sampleRate, audioStartS = 0, visual, visualResampled = false, visualGapMs = 0;
+    if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; visualResampled = true; visualGapMs = v.sourceGapMs; }
     else if (wav && trk) { const w = readWav(readFileSync(wav)); audio = w.samples; sampleRate = w.sampleRate; visual = readTrack(readFileSync(trk, "utf8"), fps); }
     else throw new Error("give --video FILE, or --wav FILE with --track FILE");
     const r = syncAnalyze({ audio, sampleRate, audioStartS, visual, visualKind: kind, profile, events: evs ? readEvents(readFileSync(evs, "utf8")) : undefined,
-      leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 200, minGapMs: gap, allowUnpaired, visualResampled });
+      leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 200, minGapMs: gap, allowUnpaired, visualResampled, visualGapMs });
     r.caveat = "Nothing here was heard or seen: two onset lists compared. The metric is not the verdict; see SYNC.md.";
     console.log(formatSync(r));
     if (json) writeFileSync(json, JSON.stringify(r, null, 1));

tools/aqua-feel/sync.mjs

@@ -87,16 +87,28 @@ export function readEvents(text) {
   return out;
 }
 
+// From the presentation times in ffmpeg's showinfo output: the frame rate (median gap, rounded, within [24, 120]; 30 when fewer than two frames)
+// and the 95th-percentile gap between source frames, which bounds how stale a displayed frame can be in a variable-frame-rate capture.
+export function sourceFrames(showinfo) {
+  const pts = [...String(showinfo).matchAll(/pts_time:\s*(-?\d+(?:\.\d+)?)/g)].map((m) => Number(m[1]));
+  const gaps = pts.slice(1).map((x, i) => x - pts[i]).filter((g) => g > 0).sort((a, b) => a - b);
+  if (!gaps.length) return { fps: 30, gapMs: 0 };
+  return { fps: Math.min(120, Math.max(24, Math.round(1 / median(gaps)))), gapMs: 1000 * gaps[Math.min(gaps.length - 1, Math.floor(gaps.length * 0.95))] };
+}
+
 // Video via ffmpeg: audio as mono f32 at 48 kHz, video as 64x36 gray frames; per-frame mean luminance and mean |frame diff| (motion flux).
 // Stream start_time offsets are applied so both clocks are the container's clock; video is resampled to constant frame rate by pts.
 export function readVideo(path, { ffmpeg = process.env.FFMPEG ?? "ffmpeg", targetFps } = {}) {
   // ffmpeg only (no ffprobe: static builds ship without it). `ffmpeg -i` lists the streams; the first decoded frame of each stream says when it starts.
-  const run = (a) => { const r = spawnSync(ffmpeg, a, { encoding: "utf8", maxBuffer: 1 << 26 }); if (r.error?.code === "ENOENT") throw new Error("ffmpeg is not on PATH (set FFMPEG=/path/to/ffmpeg); give --wav and --track instead"); return String(r.stderr ?? ""); };
+  const run = (a) => { const r = spawnSync(ffmpeg, a, { encoding: "utf8", maxBuffer: 1 << 26 }); if (r.error?.code === "ENOENT") throw new Error("ffmpeg is not on PATH (set FFMPEG=/path/to/ffmpeg); give --wav and --track instead"); if (r.error) throw new Error(`ffmpeg failed (${r.error.code}): its frame list may be truncated, so the capture's stalls would be under-measured`); return String(r.stderr ?? ""); };
   const lines = run(["-hide_banner", "-i", path]).split("\n"), find = (kind) => lines.find((l) => new RegExp(`Stream #\\d+:\\d+.*: ${kind}:`).test(l));
   const vl = find("Video"), al = find("Audio");
   if (!vl || !al) throw new Error("file needs one video and one audio stream");
   const idx = (l) => Number(l.match(/Stream #\d+:(\d+)/)[1]), fpsm = vl.match(/(\d+(?:\.\d+)?) fps/) ?? vl.match(/(\d+(?:\.\d+)?) tbr/);
-  const fps = targetFps ?? Number(fpsm?.[1] ?? 30), W = 64, H = 36, vi = idx(vl), ai = idx(al);
+  const W = 64, H = 36, vi = idx(vl), ai = idx(al);
+  // A browser capture declares no frame rate (`1k tbr`). The stream's own frame times then say what rate to resample at; guessing 30 fps widened every interval to 51 ms.
+  const src = sourceFrames(run(["-hide_banner", "-i", path, "-map", `0:${vi}`, "-an", "-vf", "showinfo", "-f", "null", "-"]));
+  const fps = targetFps ?? (fpsm ? Number(fpsm[1]) : src.fps);
   // Time of a stream's first frame on the container clock (ffmpeg shifts every stream by the earliest start, so these are relative to that).
   const first = (flags) => { const m = run(["-hide_banner", "-i", path, ...flags, "-f", "null", "-"]).match(/pts_time:\s*(-?\d+(?:\.\d+)?)/); if (!m) throw new Error("could not read a stream start time from ffmpeg"); return Number(m[1]); };
   const _vStart = first(["-map", `0:${vi}`, "-an", "-frames:v", "1", "-vf", "showinfo"]), aStart = first(["-map", `0:${ai}`, "-vn", "-frames:a", "1", "-af", "ashowinfo"]);
@@ -112,7 +124,7 @@ export function readVideo(path, { ffmpeg = process.env.FFMPEG ?? "ffmpeg", targe
     t[f] = f / fps; lum[f] = s / (W * H) / 255; flux[f] = f ? df / (W * H) / 255 : 0;
   }
   const a = new Float32Array(audioRaw.buffer.slice(audioRaw.byteOffset, audioRaw.byteOffset + audioRaw.length - (audioRaw.length % 4)));
-  return { audio: { samples: a, sampleRate: 48000, startS: aStart }, luma: { t, v: lum }, flux: { t, v: flux }, fps, resampled: true, videoStartS: _vStart };
+  return { audio: { samples: a, sampleRate: 48000, startS: aStart }, luma: { t, v: lum }, flux: { t, v: flux }, fps, sourceGapMs: src.gapMs, resampled: true, videoStartS: _vStart };
 }
 
 // ---------- onset detection ----------
@@ -199,7 +211,7 @@ const crowded = (list, o, windowS) => list.some((x) => x !== o && Math.abs(x.t -
 // Frame quantisation: a visual change at time tau first shows in the frame stamped at or after tau, so visual - tau is in [0, framePeriod) and the
 // measured lag (audio - visual) is biased LOW by up to a frame period. The truth therefore lies in [lag - audioHop/2, lag + framePeriod + audioHop/2];
 // a pair is judged on that interval, not on a point.
-export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false, visualResampled = false }) {
+export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false, visualResampled = false, visualGapMs = 0 }) {
   if (!PROFILES[profile]) throw new Error(`unknown profile ${profile}`);
   const prof = { ...PROFILES[profile] }, custom = leadMs !== undefined || lagMs !== undefined;
   if (leadMs !== undefined) prof.leadMs = leadMs;
@@ -209,7 +221,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual
   const gap = minGapMs === undefined ? {} : { minGapS: minGapMs / 1000 };
   const a = audioOnsets(audio, sampleRate, { startS: audioStartS, ...gap }), v = visualOnsets(visual, { kind: visualKind, ...gap });
   // A video resampled to constant frame rate by `fps` snaps each frame to the nearest source frame: half a period more slack on each side.
-  const hopMs = a.resolutionMs, periodMs = v.framePeriodMs, win = pairWindowMs / 1000, snapMs = visualResampled ? periodMs / 2 : 0;
+  const hopMs = a.resolutionMs, periodMs = Math.max(v.framePeriodMs, visualGapMs), win = pairWindowMs / 1000, snapMs = visualResampled ? periodMs / 2 : 0;
   const classify = (lag) => {
     const lo = lag - hopMs / 2 - snapMs, hi = lag + periodMs + hopMs / 2 + snapMs; // truth interval, ms
     if (lo >= -prof.leadMs && hi <= prof.lagMs) return "inside-window";
@@ -254,7 +266,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual
   summary.pass = summary.pairs > 0 && summary.outsideWindow === 0 && summary.missing === 0 && summary.ambiguous === 0 && (allowUnpaired || unpaired === 0);
   return {
     schema: "aqua-feel.sync.v1", lagConvention: "lagMs = audio onset - visual onset; positive = audio late, negative = audio early. Truth lies in [lagLowMs, lagHighMs] (visual onsets are quantised up to the next frame)",
-    profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualResampled, visualKind, audioResolutionMs: hopMs, framePeriodMs: periodMs,
+    profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualResampled, visualKind, audioResolutionMs: hopMs, framePeriodMs: periodMs, visualGapMs,
     audioOnsets: a.onsets, visualOnsets: v.onsets, pairs, summary,
   };
 }
@@ -262,7 +274,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual
 export function formatSync(r) {
   const f = (x, d = 1) => (x === null || x === undefined ? "   -  " : x.toFixed(d).padStart(7));
   const L = [`profile ${r.profile.name}: audio may lead by ${r.profile.leadMs} ms, lag by ${r.profile.lagMs} ms (${r.profile.note})`,
-    `lag = audio - visual (+ = audio late). The visual onset is quantised up to the next frame (period ${r.framePeriodMs.toFixed(1)} ms), so the true lag lies in [lag - ${(r.audioResolutionMs / 2 + (r.visualResampled ? r.framePeriodMs / 2 : 0)).toFixed(1)}, lag + ${(r.framePeriodMs * (r.visualResampled ? 1.5 : 1) + r.audioResolutionMs / 2).toFixed(1)}] ms${r.visualResampled ? " (video resampled to constant frame rate)" : ""}`,
+    `lag = audio - visual (+ = audio late). The visual onset is quantised up to the next frame (period ${r.framePeriodMs.toFixed(1)} ms${r.visualGapMs > 0 && r.framePeriodMs === r.visualGapMs ? ": the capture\'s source frames are up to that far apart" : ""}), so the true lag lies in [lag - ${(r.audioResolutionMs / 2 + (r.visualResampled ? r.framePeriodMs / 2 : 0)).toFixed(1)}, lag + ${(r.framePeriodMs * (r.visualResampled ? 1.5 : 1) + r.audioResolutionMs / 2).toFixed(1)}] ms${r.visualResampled ? " (video resampled to constant frame rate)" : ""}`,
     "", "label        event s   audio s  visual s  a-ev ms  v-ev ms   lag ms  verdict"];
   for (const p of r.pairs) L.push(`${String(p.label).padEnd(10).slice(0, 10)} ${f(p.eventS, 3)}  ${f(p.audioS, 3)}  ${f(p.visualS, 3)}  ${f(p.audioVsEventMs)}  ${f(p.visualVsEventMs)}  ${f(p.lagMs)}  ${p.verdict}`);
   const s = r.summary, rng = s.minLagMs === null ? "-" : `${s.minLagMs.toFixed(1)}..${s.maxLagMs.toFixed(1)}`;

tools/aqua-feel/sync.test.mjs

@@ -1,6 +1,6 @@
 // Checks of `aqua-feel sync` on synthetic fixtures with known offsets. No real capture is used. Usage: node sync.test.mjs
 import assert from "node:assert/strict";
-import { formatSync, syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, PROFILES } from "./sync.mjs";
+import { formatSync, syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, sourceFrames, PROFILES } from "./sync.mjs";
 import { readFileSync } from "node:fs";
 import { wavFloat32 } from "./render.js";
 import { makeFixture, FIXTURE_EVENTS_S } from "./sync-fixture.mjs";
@@ -200,9 +200,19 @@ t("readTrack takes header + t,value rows or a single column with fps; readEvents
   assert.deepEqual(readEvents('[{"t":2,"label":"x"},3]').map((e) => e.label), ["x", "e1"]);
 });
 
+t("sourceFrames: the rate is the median frame gap, the stale-frame bound is the 95th-percentile gap", () => {
+  const si = (ts) => ts.map((x, i) => `[Parsed_showinfo_0 @ 0x1] n:${i} pts:${x * 1000} pts_time:${x} duration:1`).join("\n");
+  const cfr = Array.from({ length: 40 }, (_, i) => i / 50);
+  { const c = sourceFrames(si(cfr)); assert.equal(c.fps, 50); assert.ok(Math.abs(c.gapMs - 20) < 1e-6); }
+  const vfr = []; let x = 0; for (let i = 0; i < 100; i++) { vfr.push(x); x += i % 10 === 9 ? 0.1 : 0.0167; }  // mostly 60 fps, one 100 ms stall in ten
+  const r = sourceFrames(si(vfr)); assert.equal(r.fps, 60); assert.ok(Math.abs(r.gapMs - 100) < 1, `gap ${r.gapMs}`);
+  assert.deepEqual(sourceFrames(si([0])), { fps: 30, gapMs: 0 });
+  assert.equal(sourceFrames(si([0, 1, 2])).fps, 24); assert.equal(sourceFrames(si([0, 0.001, 0.002])).fps, 120);
+});
+
 // The video path needs ffmpeg (FFMPEG=/path or on PATH); skipped, and said so, when there is none.
 {
-  const { readVideo } = await import("./sync.mjs"), { execFileSync } = await import("node:child_process");
+  const { readVideo } = await import("./sync.mjs"), { execFileSync } = await import("node:child_process"), require_fs = await import("node:fs"), { tmpdir } = await import("node:os"), { join } = await import("node:path");
   const ff = process.env.FFMPEG ?? "ffmpeg";
   let have = true; try { execFileSync(ff, ["-version"], { stdio: "ignore" }); } catch { have = false; }
   if (!have) console.log("skip video path: no ffmpeg (set FFMPEG)");
@@ -233,6 +243,26 @@ t("readTrack takes header + t,value rows or a single column with fps; readEvents
     assert.equal(r.summary.unpairedAudio + r.summary.unpairedVisual, 0);
     for (const p of r.pairs) assert.ok(p.visualVsEventMs >= 0 && p.visualVsEventMs <= 1000 / v.fps, `flash ${p.visualVsEventMs} ms off its logged instant`);
   });
+  if (have) t("a Perlines v2 capture declares no frame rate (1k tbr): the rate comes from its frame times, and the interval is no narrower than its worst frame gaps", () => {
+    const v = readVideo(new URL("./fixtures/perlines-aurora-vfr.webm", import.meta.url).pathname, { ffmpeg: ff });
+    assert.ok(v.fps >= 50, `resampled at ${v.fps} fps, not the 30 fps fallback`);
+    assert.ok(v.sourceGapMs > 1000 / v.fps * 1.5, `source gap ${v.sourceGapMs} ms is a stalled capture, wider than one frame`);
+    const r = syncAnalyze({ audio: v.audio.samples, sampleRate: v.audio.sampleRate, audioStartS: v.audio.startS, visual: v.luma, allowUnpaired: true, visualResampled: true, visualGapMs: v.sourceGapMs });
+    assert.ok(r.pairs.length >= 3);
+    for (const p of r.pairs) assert.ok(p.lagHighMs - p.lagLowMs >= v.sourceGapMs * 1.5, `interval ${p.lagHighMs - p.lagLowMs} ms narrower than the capture's own stalls`);
+    assert.match(formatSync(r), /source frames are up to that far apart/);
+  });
+  if (have) t("the CLI carries the capture's frame gaps into its report (feel.mjs plumbing)", () => {
+    const { mkdtempSync, readFileSync: rf } = require_fs, out = join(mkdtempSync(join(tmpdir(), "sync-cli-")), "r.json");
+    execFileSync("node", [new URL("./feel.mjs", import.meta.url).pathname, "sync", "--video", new URL("./fixtures/perlines-aurora-vfr.webm", import.meta.url).pathname, "--allow-unpaired", "--no-gate", "--json", out], { env: { ...process.env, FFMPEG: ff }, stdio: "ignore" });
+    const r = JSON.parse(rf(out, "utf8"));
+    assert.ok(r.visualGapMs > 1000 / 59 * 1.5, `visualGapMs ${r.visualGapMs}`); assert.equal(r.framePeriodMs, r.visualGapMs);
+  });
+  if (have) t("a failed ffmpeg read (e.g. output buffer overflow) is an error, not a quietly shorter frame list", () => {
+    const { mkdtempSync, writeFileSync: wf, chmodSync } = require_fs, d = mkdtempSync(join(tmpdir(), "sync-stub-")), stub = join(d, "ffmpeg");
+    wf(stub, "#!/bin/sh\nhead -c 70000000 /dev/zero | tr '\\0' 'x' >&2\n"); chmodSync(stub, 0o755);
+    assert.throws(() => readVideo(new URL("./fixtures/perlines-aurora-vfr.webm", import.meta.url).pathname, { ffmpeg: stub }), /ffmpeg failed \(ENOBUFS\)/);
+  });
 }
 
 console.log(`${n} sync checks passed. Fixtures only (synthetic, plus one browser capture when ffmpeg is present); nothing was heard or seen.`);