diff --git a/tools/aqua-feel/SYNC.md b/tools/aqua-feel/SYNC.md index 39916e3..7a99842 100644 --- a/tools/aqua-feel/SYNC.md +++ b/tools/aqua-feel/SYNC.md @@ -15,7 +15,7 @@ node feel.mjs sync --wav capture.wav --track frames.csv [--events hits.txt] [--f # a video with one audio and one video stream (needs ffmpeg on PATH, or FFMPEG=/path/to/ffmpeg) node feel.mjs sync --video capture.mp4 [--events hits.txt] [--kind luma|flux] # options: --profile default|bt1359|ebu-r37|atsc-is191 --lead-ms N --lag-ms N --pair-window-ms N (default 200) -# --min-gap-ms N --allow-unpaired --json out.json --no-gate +# --min-gap-ms N --allow-unpaired --json out.json --no-gate --shape step|ramp (ramp: slow swells, see below) node sync.test.mjs # 37 checks on synthetic fixtures; 46 with FFMPEG=/path/to/ffmpeg (also builds clips with known stream offsets and runs a real browser capture) node sync-fixture.mjs DIR 80 60 # write a fixture (audio 80 ms late, 60 fps) to try the CLI on ``` @@ -70,6 +70,22 @@ fragmentary in the search; figures confirmed against secondary summaries), and t [ATSC recommends AV sync rules](https://tvtechnology.com/opinions/atsc-recommends-av-sync-rules). Windows depend on content (speech lip-sync is more sensitive than a game impact) and on the viewer; they are population thresholds, not a property of your capture. +## Ramp-shaped visuals (`--shape ramp`) + +Some visuals answer a sound with a slow swell (a scene brightening over a few hundred ms), not a step. The default detector looks for a peak in the frame-to-frame step, finds none, and says no visual onset +(`step` found 0 of 5 hits on `fixtures/perlines-trails-ramp.webm`; `ramp` paired 3). `--shape ramp` takes the track's rise over 300 ms as the novelty and picks its peaks (at least 15% of the track's range, 6 noise sigmas, +at least 500 ms apart). For each it finds the quiet level before the rise (the last sample within 2 noise sigmas of the minimum of the 0.6 s before) and the 20% and 80% crossings of the rise to its top. + +**What it claims.** The response started somewhere between the quiet level, backed off by the time from there to the 20% crossing and one frame, and the 20% crossing. The lag interval is that whole range against the audio onset, +so it is far wider than a step's (100 ms and more) and most verdicts are `borderline`. `audio-late-detectable` is sound whatever the ramp's shape (the picture started to move at the latest at the 20% crossing and the sound +still came later than the window); `audio-early-detectable` only if even the widest start estimate lies after the sound. The point value (`visual s`, `lag ms`) is the foot of the line through the 20% and 80% crossings, a +convention that fits a linear ramp and reads early for exponential approach and late for slow-start (smoothstep) shapes; it is not the instant anyone perceives. + +**What it cannot claim.** When the viewer sees the swell begin (that is later than the physical start and depends on contrast, size and attention). That the swell was caused by this sound: pairing is by nearness, and a bed that +swells on its own, a scene change or a second sound inside 500 ms gives a false pair. The back-off for a decaying tail is checked on linear, exponential and smoothstep ramps (250 to 450 ms to peak, decay over about a second), not proved: +a response that starts more than one valley-to-20% time before the quiet minimum would escape the interval. Ramps closer than 500 ms merge; ramps under 15% of the track's range, or on a steady drift that rises over the whole clip, are not found. +The 0.15 floor (0.25 missed 7 of 10 Vector trails hits and 0.10 admitted one false onset) and the other constants were set on that one clip. + ## Uncertainty A visual onset can only land on a frame, and a frame is stamped when its picture started to be displayed. A change at time tau first shows in the first frame diff --git a/tools/aqua-feel/feel.mjs b/tools/aqua-feel/feel.mjs index f789557..7ff3f93 100644 --- a/tools/aqua-feel/feel.mjs +++ b/tools/aqua-feel/feel.mjs @@ -5,7 +5,7 @@ // Usage: node feel.mjs run [--out DIR] [--only SUBSTR] [--no-gate] render, measure, write report + gallery, gate // node feel.mjs gate REPORT.json [THRESHOLDS.json] [--partial] re-gate a report (--partial: a subset; skips id-rule and waiver bookkeeping) // node feel.mjs compare BEFORE.json AFTER.json [--only SUBSTR] before/after table -// node feel.mjs sync --wav A.wav --track T.csv [--events E] | --video V [--fps N] [--kind luma|flux] [--profile P] [--lead-ms N --lag-ms N] [--pair-window-ms N] [--min-gap-ms N] [--allow-unpaired] [--json OUT] A/V synchrony (SYNC.md) +// node feel.mjs sync --wav A.wav --track T.csv [--events E] | --video V [--fps N] [--kind luma|flux] [--shape step|ramp] [--profile P] [--lead-ms N --lag-ms N] [--pair-window-ms N] [--min-gap-ms N] [--allow-unpaired] [--json OUT] A/V synchrony (SYNC.md) // node feel.mjs correlate TRIAL_EXPORT.json ... feel metrics vs listening-trial ratings import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -61,15 +61,16 @@ async function sync() { const num = (n) => { const x = val(n); if (x === undefined) return undefined; if (!Number.isFinite(Number(x)) || x.trim() === "") throw new Error(`${n} needs a number, got "${x}"`); return Number(x); }; const wav = val("--wav"), trk = val("--track"), vid = val("--video"), evs = val("--events"), fps = num("--fps"), json = val("--json"); const kind = val("--kind") ?? "luma", profile = val("--profile") ?? "default", lead = num("--lead-ms"), lag = num("--lag-ms"), pw = num("--pair-window-ms"), gap = num("--min-gap-ms"); - const noGate = bool("--no-gate"), allowUnpaired = bool("--allow-unpaired"); + const shape = val("--shape") ?? "step", noGate = bool("--no-gate"), allowUnpaired = bool("--allow-unpaired"); if (args.length) throw new Error(`unknown argument ${args[0]}`); + if (!["step", "ramp"].includes(shape)) throw new Error(`--shape is step or ramp, got "${shape}"`); if (!["luma", "flux"].includes(kind)) throw new Error(`--kind is luma or flux, got "${kind}"`); if (!PROFILES[profile]) throw new Error(`--profile is one of ${Object.keys(PROFILES).join(", ")}, got "${profile}"`); let audio, sampleRate, audioStartS = 0, visual, visualResampled = false, visualGapMs = 0; if (vid) { const v = readVideo(vid); audio = v.audio.samples; sampleRate = v.audio.sampleRate; audioStartS = v.audio.startS; visual = kind === "flux" ? v.flux : v.luma; visualResampled = true; visualGapMs = v.sourceGapMs; } else if (wav && trk) { const w = readWav(readFileSync(wav)); audio = w.samples; sampleRate = w.sampleRate; visual = readTrack(readFileSync(trk, "utf8"), fps); } else throw new Error("give --video FILE, or --wav FILE with --track FILE"); - const r = syncAnalyze({ audio, sampleRate, audioStartS, visual, visualKind: kind, profile, events: evs ? readEvents(readFileSync(evs, "utf8")) : undefined, + const r = syncAnalyze({ audio, sampleRate, audioStartS, visual, visualKind: kind, visualShape: shape, profile, events: evs ? readEvents(readFileSync(evs, "utf8")) : undefined, leadMs: lead, lagMs: lag, pairWindowMs: pw ?? 200, minGapMs: gap, allowUnpaired, visualResampled, visualGapMs }); r.caveat = "Nothing here was heard or seen: two onset lists compared. The metric is not the verdict; see SYNC.md."; console.log(formatSync(r)); diff --git a/tools/aqua-feel/fixtures/perlines-trails-ramp.webm b/tools/aqua-feel/fixtures/perlines-trails-ramp.webm new file mode 100644 index 0000000..68b8977 Binary files /dev/null and b/tools/aqua-feel/fixtures/perlines-trails-ramp.webm differ diff --git a/tools/aqua-feel/sync.mjs b/tools/aqua-feel/sync.mjs index a1b42d2..afccac4 100644 --- a/tools/aqua-feel/sync.mjs +++ b/tools/aqua-feel/sync.mjs @@ -194,6 +194,31 @@ export function visualOnsets(track, { kind = "luma", k = 8, minGapS = 0.1 } = {} return r; } +// Visual, ramp-shaped response (a slow brightening or swell over hundreds of ms, no step): the track's rise over `riseS` is the novelty; each peak of it +// is one response. From the quiet level before it (the last sample within 2 noise sigmas of the minimum of the 0.6 s before the peak) to the top, the +// onset is the foot of the line through the 20% and 80% crossings, clamped to [valley, 20% crossing]. The start is only known to lie in +// [valley - (valley to 20% time) - one frame, crossing20S]: a ramp has no first frame, so the onset carries that span and the verdict is read off it, not off the foot. +export function visualRampOnsets(track, { minGapS = 0.5, riseS = 0.3, k = 6 } = {}) { + const { t, v } = track, n = v.length, dts = []; for (let i = 1; i < n; i++) dts.push(t[i] - t[i - 1]); + const period = median(dts), w = Math.max(1, Math.round(0.06 / period)); + const sm = new Float64Array(n); for (let i = 0; i < n; i++) { let a = 0, c = 0; for (let q = Math.max(0, i - w); q <= Math.min(n - 1, i + w); q++) { a += v[q]; c++; } sm[i] = a / c; } + const range = Math.max(...v) - Math.min(...v), d1 = Float64Array.from(v.slice(1), (x, i) => Math.abs(x - v[i])), sigma = 1.4826 * median(d1) / Math.SQRT2 + 1e-12; + const at = (x) => { let lo = 0, hi = n - 1; while (lo < hi) { const m = (lo + hi) >> 1; if (t[m] < x) lo = m + 1; else hi = m; } return lo; }; + const nov = new Float64Array(n); for (let i = 0; i < n; i++) nov[i] = Math.max(0, sm[Math.min(n - 1, at(t[i] + riseS))] - sm[i]); + const r = pickOnsets(nov, t, { k, floor: Math.max(0.15 * range, k * sigma), minGapS, riseFrac: 1 }); + r.framePeriodMs = 1000 * period; + r.onsets = r.onsets.map((o) => { + const p = at(o.peakT), b0 = at(o.peakT - 0.6); let mn = Infinity; for (let i = b0; i <= p; i++) mn = Math.min(mn, sm[i]); + let m = p; while (m > b0 && sm[m] > mn + 2 * sigma) m--; + let top = m; for (let i = m; i < Math.min(n, at(o.peakT + riseS + 0.3)); i++) if (sm[i] > sm[top]) top = i; + const A = sm[top] - sm[m], cross = (f) => { let i = m; while (i < top && sm[i] < sm[m] + f * A) i++; return i; }; + const i20 = cross(0.2), i80 = cross(0.8), foot = Math.min(t[i20], Math.max(t[m], t[i20] - (t[i80] - t[i20]) / 3)); + // Over a decaying tail the minimum of the sum lands after the true start (a slow ramp cancels the decay before it turns the curve): back off by the valley-to-20% time. + return { t: foot, loS: t[m] - (t[i20] - t[m]) - period, hiS: t[i20], peakT: t[top], strength: A }; + }); + return r; +} + // ---------- pairing and verdict ---------- // One-to-one greedy matching of onsets to anchors (events, or audio onsets) by distance, within windowS. Returns Map anchorIndex -> onsetIndex. function match(anchors, onsets, windowS) { @@ -211,7 +236,7 @@ const crowded = (list, o, windowS) => list.some((x) => x !== o && Math.abs(x.t - // Frame quantisation: a visual change at time tau first shows in the frame stamped at or after tau, so visual - tau is in [0, framePeriod) and the // measured lag (audio - visual) is biased LOW by up to a frame period. The truth therefore lies in [lag - audioHop/2, lag + framePeriod + audioHop/2]; // a pair is judged on that interval, not on a point. -export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false, visualResampled = false, visualGapMs = 0 }) { +export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visualKind = "luma", visualShape = "step", events, profile = "default", leadMs, lagMs, pairWindowMs = 200, minGapMs, allowUnpaired = false, visualResampled = false, visualGapMs = 0 }) { if (!PROFILES[profile]) throw new Error(`unknown profile ${profile}`); const prof = { ...PROFILES[profile] }, custom = leadMs !== undefined || lagMs !== undefined; if (leadMs !== undefined) prof.leadMs = leadMs; @@ -219,11 +244,10 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual if (custom) { prof.note = `profile ${profile} with the window overridden on the command line`; } for (const [n, x] of [["leadMs", prof.leadMs], ["lagMs", prof.lagMs], ["pairWindowMs", pairWindowMs]]) if (!Number.isFinite(x) || x < 0 || (n === "pairWindowMs" && x === 0)) throw new Error(`${n} must be a ${n === "pairWindowMs" ? "positive" : "non-negative"} number`); const gap = minGapMs === undefined ? {} : { minGapS: minGapMs / 1000 }; - const a = audioOnsets(audio, sampleRate, { startS: audioStartS, ...gap }), v = visualOnsets(visual, { kind: visualKind, ...gap }); + const a = audioOnsets(audio, sampleRate, { startS: audioStartS, ...gap }), v = visualShape === "ramp" ? visualRampOnsets(visual, gap.minGapS === undefined ? {} : { minGapS: gap.minGapS }) : visualOnsets(visual, { kind: visualKind, ...gap }); // A video resampled to constant frame rate by `fps` snaps each frame to the nearest source frame: half a period more slack on each side. const hopMs = a.resolutionMs, periodMs = Math.max(v.framePeriodMs, visualGapMs), win = pairWindowMs / 1000, snapMs = visualResampled ? periodMs / 2 : 0; - const classify = (lag) => { - const lo = lag - hopMs / 2 - snapMs, hi = lag + periodMs + hopMs / 2 + snapMs; // truth interval, ms + const classify = (lo, hi) => { if (lo >= -prof.leadMs && hi <= prof.lagMs) return "inside-window"; if (lo > prof.lagMs) return "audio-late-detectable"; if (hi < -prof.leadMs) return "audio-early-detectable"; @@ -232,8 +256,12 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual const pairs = [], usedA = new Set(), usedV = new Set(); const row = (label, eventS, ao, vo, ambiguous) => { const r = { label, eventS, audioS: ao?.t ?? null, visualS: vo?.t ?? null, audioVsEventMs: ao && eventS !== null ? (ao.t - eventS) * 1e3 : null, visualVsEventMs: vo && eventS !== null ? (vo.t - eventS) * 1e3 : null, lagMs: ao && vo ? (ao.t - vo.t) * 1e3 : null }; - if (r.lagMs !== null) { r.lagLowMs = r.lagMs - hopMs / 2 - snapMs; r.lagHighMs = r.lagMs + periodMs + hopMs / 2 + snapMs; } - r.verdict = r.lagMs === null ? (ao ? "no-visual-onset" : vo ? "no-audio-onset" : "no-onsets") : ambiguous ? "ambiguous-pairing" : classify(r.lagMs); + if (r.lagMs !== null) { + // A step has its change in (visual - period, visual]. A ramp's start lies anywhere in [loS, hiS] (its quiet level to its 20% crossing). + const hiT = vo.hiS ?? vo.t, loT = vo.loS ?? vo.t - periodMs / 1000; + r.lagLowMs = (ao.t - hiT) * 1e3 - hopMs / 2 - snapMs; r.lagHighMs = (ao.t - loT) * 1e3 + hopMs / 2 + snapMs; + } + r.verdict = r.lagMs === null ? (ao ? "no-visual-onset" : vo ? "no-audio-onset" : "no-onsets") : ambiguous ? "ambiguous-pairing" : classify(r.lagLowMs, r.lagHighMs); return r; }; if (events) { @@ -266,7 +294,7 @@ export function syncAnalyze({ audio, sampleRate, audioStartS = 0, visual, visual summary.pass = summary.pairs > 0 && summary.outsideWindow === 0 && summary.missing === 0 && summary.ambiguous === 0 && (allowUnpaired || unpaired === 0); return { schema: "aqua-feel.sync.v1", lagConvention: "lagMs = audio onset - visual onset; positive = audio late, negative = audio early. Truth lies in [lagLowMs, lagHighMs] (visual onsets are quantised up to the next frame)", - profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualResampled, visualKind, audioResolutionMs: hopMs, framePeriodMs: periodMs, visualGapMs, + profile: { name: custom ? `${profile}+override` : profile, ...prof }, allowUnpaired, pairWindowMs, visualResampled, visualKind, visualShape, audioResolutionMs: hopMs, framePeriodMs: periodMs, visualGapMs, audioOnsets: a.onsets, visualOnsets: v.onsets, pairs, summary, }; } @@ -275,6 +303,7 @@ export function formatSync(r) { const f = (x, d = 1) => (x === null || x === undefined ? " - " : x.toFixed(d).padStart(7)); const L = [`profile ${r.profile.name}: audio may lead by ${r.profile.leadMs} ms, lag by ${r.profile.lagMs} ms (${r.profile.note})`, `lag = audio - visual (+ = audio late). The visual onset is quantised up to the next frame (period ${r.framePeriodMs.toFixed(1)} ms${r.visualGapMs > 0 && r.framePeriodMs === r.visualGapMs ? ": the capture\'s source frames are up to that far apart" : ""}), so the true lag lies in [lag - ${(r.audioResolutionMs / 2 + (r.visualResampled ? r.framePeriodMs / 2 : 0)).toFixed(1)}, lag + ${(r.framePeriodMs * (r.visualResampled ? 1.5 : 1) + r.audioResolutionMs / 2).toFixed(1)}] ms${r.visualResampled ? " (video resampled to constant frame rate)" : ""}`, + ...(r.visualShape === "ramp" ? ["--shape ramp: the visual column is the foot of the line through the 20% and 80% crossings of each rise. A ramp has no first frame, so each interval runs from before the quiet minimum to the 20% crossing and is far wider than a step's."] : []), "", "label event s audio s visual s a-ev ms v-ev ms lag ms verdict"]; for (const p of r.pairs) L.push(`${String(p.label).padEnd(10).slice(0, 10)} ${f(p.eventS, 3)} ${f(p.audioS, 3)} ${f(p.visualS, 3)} ${f(p.audioVsEventMs)} ${f(p.visualVsEventMs)} ${f(p.lagMs)} ${p.verdict}`); const s = r.summary, rng = s.minLagMs === null ? "-" : `${s.minLagMs.toFixed(1)}..${s.maxLagMs.toFixed(1)}`; diff --git a/tools/aqua-feel/sync.test.mjs b/tools/aqua-feel/sync.test.mjs index 72394da..63f50ea 100644 --- a/tools/aqua-feel/sync.test.mjs +++ b/tools/aqua-feel/sync.test.mjs @@ -1,6 +1,6 @@ // Checks of `aqua-feel sync` on synthetic fixtures with known offsets. No real capture is used. Usage: node sync.test.mjs import assert from "node:assert/strict"; -import { formatSync, syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, sourceFrames, PROFILES } from "./sync.mjs"; +import { formatSync, syncAnalyze, readWav, readTrack, readEvents, audioOnsets, visualOnsets, visualRampOnsets, sourceFrames, PROFILES } from "./sync.mjs"; import { readFileSync } from "node:fs"; import { wavFloat32 } from "./render.js"; import { makeFixture, FIXTURE_EVENTS_S } from "./sync-fixture.mjs"; @@ -210,6 +210,46 @@ t("sourceFrames: the rate is the median frame gap, the stale-frame bound is the assert.equal(sourceFrames(si([0, 1, 2])).fps, 24); assert.equal(sourceFrames(si([0, 0.001, 0.002])).fps, 120); }); +// Ramp-shaped responses: a brightening that starts at the event and takes hundreds of ms to peak, then decays over about a second; a drifting bed and noise. +// shapes: linear ramp, exponential approach (steepest at the start), smoothstep (slowest at the start). Truth is the instant the change begins. +const SHAPES = { linear: (u) => Math.min(1, u), exponential: (u) => 1 - Math.exp(-4 * Math.min(1, u)), smoothstep: (u) => { const x = Math.min(1, u); return x * x * (3 - 2 * x); } }; +const rampFixture = ({ shape, rampS, fps, offsetMs = 40, events = [0.5042, 1.9117, 3.2209, 4.6033, 5.9161, 7.2087, 8.5], noise = 0.002 }) => { + const fx = makeFixture({ offsetMs, fps, seconds: 10, events }), nf = Math.round(fps * 10), tt = new Float64Array(nf), vv = new Float64Array(nf); let s = 11; + const rand = () => { s = (Math.imul(s, 1664525) + 1013904223) >>> 0; return s / 2 ** 32 - 0.5; }; + for (let f = 0; f < nf; f++) { + const x = f / fps; let y = 0.1 + 0.005 * Math.sin(x * 0.7) + noise * 2 * rand(); + for (const e of events) if (x >= e) { const u = (x - e) / rampS; y += 0.3 * (u < 1 ? SHAPES[shape](u) : Math.exp(-(u - 1) * rampS / 0.8)); } + tt[f] = x; vv[f] = y; + } + return { ...fx, visual: { t: tt, v: vv }, events: events.map((e, i) => ({ t: e, label: `r${i}` })) }; +}; +for (const shape of Object.keys(SHAPES)) for (const [rampS, fps] of [[0.25, 60], [0.45, 60], [0.35, 30]]) { + t(`ramp (${shape}, ${rampS * 1000} ms to peak, ${fps} fps): every response is found, and the true start lies in each pair's interval`, () => { + const fx = rampFixture({ shape, rampS, fps }), r = syncAnalyze({ audio: fx.audio, sampleRate: fx.sampleRate, visual: fx.visual, visualShape: "ramp", events: fx.events }); + assert.equal(r.pairs.length, fx.events.length); assert.equal(r.summary.unpairedVisual, 0); + for (const p of r.pairs) assert.ok(inInterval(p, fx.offsetMs), `${shape}/${rampS}/${fps} ${p.label}: [${p.lagLowMs}, ${p.lagHighMs}] vs ${fx.offsetMs}`); + for (const p of r.pairs) assert.ok(Math.abs(p.visualVsEventMs) <= 0.45 * rampS * 1000 + 1e3 / fps, `${p.label}: foot ${p.visualVsEventMs} ms from the true start`); + }); +} +for (const offsetMs of [-60, 0, 150]) t(`ramp: a known ${offsetMs} ms offset stays inside the interval for every shape (30 and 60 fps)`, () => { + for (const shape of Object.keys(SHAPES)) for (const fps of [30, 60]) { + const fx = rampFixture({ shape, rampS: 0.3, fps, offsetMs }), r = syncAnalyze({ audio: fx.audio, sampleRate: fx.sampleRate, visual: fx.visual, visualShape: "ramp", events: fx.events }); + assert.equal(r.pairs.length, fx.events.length, `${shape}/${fps}`); + for (const p of r.pairs) assert.ok(inInterval(p, offsetMs), `${shape}/${fps}/${offsetMs} ${p.label}: [${p.lagLowMs}, ${p.lagHighMs}]`); + } +}); +t("ramp: the interval is wide and says so (a ramp has no first frame); it is judged on the whole interval", () => { + const fx = rampFixture({ shape: "smoothstep", rampS: 0.45, fps: 60, offsetMs: 0 }), r = syncAnalyze({ audio: fx.audio, sampleRate: fx.sampleRate, visual: fx.visual, visualShape: "ramp", events: fx.events }); + for (const p of r.pairs) { assert.ok(p.lagHighMs - p.lagLowMs > 100, `interval ${p.lagHighMs - p.lagLowMs} ms`); assert.equal(p.verdict, "borderline"); } + const late = rampFixture({ shape: "linear", rampS: 0.3, fps: 60, offsetMs: 400 }), rl = syncAnalyze({ audio: late.audio, sampleRate: late.sampleRate, visual: late.visual, visualShape: "ramp", pairWindowMs: 450, events: late.events }); + assert.ok(rl.pairs.every((p) => p.verdict === "audio-late-detectable"), "audio 400 ms after the picture starts to move is outside whatever the ramp's shape"); +}); +t("ramp: noise, a steady drift and a lone step give no ramp onsets from noise or drift", () => { + const nf = 600, tt = Float64Array.from({ length: nf }, (_, i) => i / 60); let s = 5; const rand = () => { s = (Math.imul(s, 1664525) + 1013904223) >>> 0; return s / 2 ** 32 - 0.5; }; + assert.equal(visualRampOnsets({ t: tt, v: Float64Array.from(tt, () => 0.2 + 0.01 * rand()) }).onsets.length, 0, "noise"); + assert.equal(visualRampOnsets({ t: tt, v: Float64Array.from(tt, (x) => 0.1 + 0.03 * x + 0.004 * rand()) }).onsets.length, 0, "a drift over the whole clip is not a response"); +}); + // The video path needs ffmpeg (FFMPEG=/path or on PATH); skipped, and said so, when there is none. { const { readVideo } = await import("./sync.mjs"), { execFileSync } = await import("node:child_process"), require_fs = await import("node:fs"), { tmpdir } = await import("node:os"), { join } = await import("node:path"); @@ -252,6 +292,13 @@ t("sourceFrames: the rate is the median frame gap, the stale-frame bound is the for (const p of r.pairs) assert.ok(p.lagHighMs - p.lagLowMs >= v.sourceGapMs * 1.5, `interval ${p.lagHighMs - p.lagLowMs} ms narrower than the capture's own stalls`); assert.match(formatSync(r), /source frames are up to that far apart/); }); + if (have) t("a real ramp-shaped response (Perlines v2 Vector trails, off-grid hits): the step detector finds none, the ramp detector pairs at least 3 of 5, each with a wide interval", () => { + const v = readVideo(new URL("./fixtures/perlines-trails-ramp.webm", import.meta.url).pathname, { ffmpeg: ff }); + const go = (visualShape) => syncAnalyze({ audio: v.audio.samples, sampleRate: v.audio.sampleRate, audioStartS: v.audio.startS, visual: v.luma, visualShape, allowUnpaired: true, visualResampled: true, visualGapMs: v.sourceGapMs }); + assert.equal(go("step").pairs.length, 0); + const r = go("ramp"); assert.ok(r.pairs.length >= 3, `${r.pairs.length} paired`); + for (const p of r.pairs) assert.ok(p.lagHighMs - p.lagLowMs > 100, `interval ${p.lagHighMs - p.lagLowMs} ms`); + }); if (have) t("the CLI carries the capture's frame gaps into its report (feel.mjs plumbing)", () => { const { mkdtempSync, readFileSync: rf } = require_fs, out = join(mkdtempSync(join(tmpdir(), "sync-cli-")), "r.json"); execFileSync("node", [new URL("./feel.mjs", import.meta.url).pathname, "sync", "--video", new URL("./fixtures/perlines-aurora-vfr.webm", import.meta.url).pathname, "--allow-unpaired", "--no-gate", "--json", out], { env: { ...process.env, FFMPEG: ff }, stdio: "ignore" });