diff --git a/docs/renders/aqua-feel-native-diff-2026-11-10.md b/docs/renders/aqua-feel-native-diff-2026-11-10.md index b3c6040..e227cb0 100644 --- a/docs/renders/aqua-feel-native-diff-2026-11-10.md +++ b/docs/renders/aqua-feel-native-diff-2026-11-10.md @@ -97,7 +97,7 @@ What the data says: ## Tolerance in `native-diff.mjs` -Added, because the data shows a gap. Across all 13 sources and the 3 conditions the metric deltas stay under |LUFS| 0.017 dB, |true peak| 0.152 dB, centroid 0.3 %. A deliberate +1 dB gain on the native buffer moves LUFS and true peak by 1.0 dB. The limits sit between: 0.1 dB, 0.5 dB, 1 %. Sample-level error has no limit with the library smoother (the waveforms legitimately diverge), but with `--smoother imp3` it has one at -60 dB (the worst row is -75.4 dB; a one-sample shift reads about -4 dB). +The limits (`LIMITS`) and the comparison now live in `tools/aqua-feel/native-compare.js`; `native-diff.test.mjs` trips each one. Added, because the data shows a gap. Across all 13 sources and the 3 conditions the metric deltas stay under |LUFS| 0.017 dB, |true peak| 0.152 dB, centroid 0.3 %. A deliberate +1 dB gain on the native buffer moves LUFS and true peak by 1.0 dB. The limits sit between: 0.1 dB, 0.5 dB, 1 %. Sample-level error has no limit with the library smoother (the waveforms legitimately diverge), but with `--smoother imp3` it has one at -60 dB (the worst row is -75.4 dB; a one-sample shift reads about -4 dB). Mutation check on the tool, `--items smoother --smoother imp3`: unmutated, 12 of 12 clean, exit 0; +1 dB native gain, 12 of 12 flagged (`loudnessLufs`, `truePeakDb`, `relRmsErrDb`), exit 1; one-sample native shift, 12 of 12 flagged (`relRmsErrDb`), exit 1. A one-sample shift is not detectable with the library smoother; `loudnessLufs`, `truePeakDb` and centroid also cannot see it. The limits are tripwires fit to these 13 sources, not taste. diff --git a/tools/aqua-feel/native-compare.js b/tools/aqua-feel/native-compare.js new file mode 100644 index 0000000..c47b0b6 --- /dev/null +++ b/tools/aqua-feel/native-compare.js @@ -0,0 +1,29 @@ +// The comparison native-diff.mjs applies to one source: wasm samples against native samples, and the LIMITS that decide +// whether the difference is acceptable. Pure (no render, no file), so test native-diff.test.mjs can trip each limit with +// known perturbations. Nothing here was heard. +import { measure } from "./metrics.js"; + +// LIMITS sit between the largest deltas seen across 13 sources and three conditions (|LUFS| 0.017, |true peak| 0.152 dB, +// centroid 0.3 %) and a deliberate +1 dB gain (LUFS and true peak 1.0 dB); the sample bound applies only with --smoother imp3. +export const LIMITS = { lufsDb: 0.1, truePeakDb: 0.5, centroidPct: 1, relRmsErrDbImp3: -60 }; + +export const pick = (m) => ({ samplePeak: m.samplePeak, truePeakDb: m.truePeakDb, loudnessLufs: m.loudnessLufs, centroidMedianHz: m.centroidMedianHz, overSamples: m.overSamples, rmsDb: m.rmsDb, crestDb: m.crestDb, noiseToToneDb: m.noiseToToneDb, noiseToToneADb: m.noiseToToneADb, lowMotion: m.lowMotion }); + +// Row for one source, or { error } on a length mismatch (a length change is a failure, not something to trim). +export function compareRenders(id, w, n) { + if (w.length !== n.length) return { id, error: `length mismatch: wasm ${w.length}, native ${n.length}` }; + const len = w.length; let maxErr = 0, at = 0, sq = 0, sqRef = 0; + for (let i = 0; i < len; i++) { const d = Math.abs(w[i] - n[i]); if (d > maxErr) { maxErr = d; at = i; } sq += d * d; sqRef += w[i] * w[i]; } + const mw = measure(w), mn = measure(n); + return { id, lenWasm: w.length, lenNative: n.length, maxErr, maxErrAt: at, rmsErr: Math.sqrt(sq / len), relRmsErrDb: 10 * Math.log10(sq / Math.max(sqRef, 1e-30)), wasm: pick(mw), native: pick(mn), measures: { wasm: mw, native: mn } }; +} + +// Names of the LIMITS this row breaks. `mode` is the --smoother mode; the sample bound applies only to "imp3". +export function violationsOf(r, mode) { + const v = []; + if (Math.abs(r.native.loudnessLufs - r.wasm.loudnessLufs) > LIMITS.lufsDb) v.push("loudnessLufs"); + if (Math.abs(r.native.truePeakDb - r.wasm.truePeakDb) > LIMITS.truePeakDb) v.push("truePeakDb"); + if (100 * Math.abs(r.native.centroidMedianHz - r.wasm.centroidMedianHz) / r.wasm.centroidMedianHz > LIMITS.centroidPct) v.push("centroidMedianHz"); + if (mode === "imp3" && r.relRmsErrDb > LIMITS.relRmsErrDbImp3) v.push("relRmsErrDb"); + return v; +} diff --git a/tools/aqua-feel/native-diff.mjs b/tools/aqua-feel/native-diff.mjs index 3ae628a..8e6525f 100644 --- a/tools/aqua-feel/native-diff.mjs +++ b/tools/aqua-feel/native-diff.mjs @@ -2,8 +2,7 @@ // Faust source, at the same params, 48 kHz and length. Nothing here was heard. // Both sides compile against their own stdlib (faustwasm bundles libfaust's; native reads AQUASYNTH_FAUST_HOME's share/faust), so a // difference here can be library text, not the host: the report records the identities. The script exits 1 -// on a render error, a length mismatch, or a LIMITS violation. LIMITS sit between the largest deltas seen across 13 sources and -// three conditions (|LUFS| 0.017, |true peak| 0.152 dB, centroid 0.3 %) and a deliberate +1 dB gain (LUFS and true peak 1.0 dB); +// on a render error, a length mismatch, or a LIMITS violation. LIMITS (native-compare.js; the evidence for their values is there, and native-diff.test.mjs trips each) bound the differences; // the sample-level bound applies only with --smoother imp3, where the largest error seen is -75.4 dB (dx7-32, unexplained, is -79.5) and a one-sample // shift reads about -4 dB. With the library smoother the waveforms legitimately diverge (thruster -8 to -29 dB), so no sample bound. // usage: node native-diff.mjs [--tract PATH.dsp] [--out DIR] [--items core|smoother] [--smoother library|old|imp3] @@ -16,11 +15,11 @@ import { spawnSync } from "node:child_process"; import { join, dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; -import { SR, measure } from "./metrics.js"; +import { SR } from "./metrics.js"; +import { compareRenders, violationsOf } from "./native-compare.js"; import { gate } from "./gate.js"; import { loadRuntime, compile, renderOnce, writeWav, readText, playground } from "./render.js"; -const LIMITS = { lufsDb: 0.1, truePeakDb: 0.5, centroidPct: 1, relRmsErrDbImp3: -60 }; const here = dirname(fileURLToPath(import.meta.url)), args = process.argv.slice(2); const opt = (k) => { const i = args.indexOf(k); return i < 0 ? null : args[i + 1]; }; const bin = resolve(args[0]), out = resolve(opt("--out") ?? join(tmpdir(), "aqua-feel-native-diff")); @@ -60,22 +59,12 @@ for (const it of items) { const r = spawnSync("dotnet", [bin.endsWith(".dll") ? bin : join(bin, "FaustNativeRender.dll"), dsp, wav, String(it.seconds), ...Object.entries(it.params).map(([k, v]) => `${k}=${v}`)], { encoding: "utf8" }); if (r.status !== 0) { rows.push({ id: it.id, error: (r.stderr || r.stdout).slice(0, 400) }); continue; } const n = readF32Wav(wav); writeWav(join(out, `${it.id.replace(/[^A-Za-z0-9._@-]+/g, "_")}.wasm.wav`), w); - if (w.length !== n.length) { rows.push({ id: it.id, error: `length mismatch: wasm ${w.length}, native ${n.length}` }); continue; } - const len = w.length; let maxErr = 0, at = 0, sq = 0, sqRef = 0; - for (let i = 0; i < len; i++) { const d = Math.abs(w[i] - n[i]); if (d > maxErr) { maxErr = d; at = i; } sq += d * d; sqRef += w[i] * w[i]; } - const mw = measure(w), mn = measure(n); - rows.push({ id: it.id, lenWasm: w.length, lenNative: n.length, maxErr, maxErrAt: at, rmsErr: Math.sqrt(sq / len), relRmsErrDb: 10 * Math.log10(sq / Math.max(sqRef, 1e-30)), - wasm: pick(mw), native: pick(mn), flagsWasm: flagsOf(it.id, it.group, mw), flagsNative: flagsOf(it.id, it.group, mn) }); -} -function pick(m) { return { samplePeak: m.samplePeak, truePeakDb: m.truePeakDb, loudnessLufs: m.loudnessLufs, centroidMedianHz: m.centroidMedianHz, overSamples: m.overSamples, rmsDb: m.rmsDb, crestDb: m.crestDb, noiseToToneDb: m.noiseToToneDb, noiseToToneADb: m.noiseToToneADb, lowMotion: m.lowMotion }; } -for (const r of rows) { - if (r.error) continue; - r.violations = []; - if (Math.abs(r.native.loudnessLufs - r.wasm.loudnessLufs) > LIMITS.lufsDb) r.violations.push("loudnessLufs"); - if (Math.abs(r.native.truePeakDb - r.wasm.truePeakDb) > LIMITS.truePeakDb) r.violations.push("truePeakDb"); - if (100 * Math.abs(r.native.centroidMedianHz - r.wasm.centroidMedianHz) / r.wasm.centroidMedianHz > LIMITS.centroidPct) r.violations.push("centroidMedianHz"); - if (mode === "imp3" && r.relRmsErrDb > LIMITS.relRmsErrDbImp3) r.violations.push("relRmsErrDb"); + const row = compareRenders(it.id, w, n); + if (row.error) { rows.push({ id: it.id, error: row.error }); continue; } + const { measures, ...rest } = row; + rows.push({ ...rest, flagsWasm: flagsOf(it.id, it.group, measures.wasm), flagsNative: flagsOf(it.id, it.group, measures.native) }); } +for (const r of rows) if (!r.error) r.violations = violationsOf(r, mode); writeFileSync(join(out, "native-diff.json"), JSON.stringify({ libfaustWasm: rt.version, sampleRate: SR, items: set, smoother: mode, rows }, null, 1)); if (rows.some((r) => r.error || r.violations.length)) process.exitCode = 1; console.log(JSON.stringify({ libfaustWasm: rt.version, items: set, smoother: mode, rows }, null, 1)); diff --git a/tools/aqua-feel/native-diff.test.mjs b/tools/aqua-feel/native-diff.test.mjs new file mode 100644 index 0000000..98c968d --- /dev/null +++ b/tools/aqua-feel/native-diff.test.mjs @@ -0,0 +1,74 @@ +// Checks that each native-diff LIMITS entry trips on a known perturbation and stays quiet on a clean or in-tolerance pair. +// The pair is synthetic (a seeded tone-and-noise burst against a perturbed copy); no render, no dotnet, nothing heard. +// Usage: node native-diff.test.mjs +import assert from "node:assert/strict"; +import { SR } from "./metrics.js"; +import { LIMITS, compareRenders, violationsOf } from "./native-compare.js"; + +let n = 0; +const t = (name, fn) => { fn(); n++; console.log(`ok ${name}`); }; + +// Tones at 110/330/990 Hz with a slow envelope plus seeded noise: broadband enough that a one-sample shift is far from silent. +const sig = (() => { + let s = 12345; const r = () => { s = (Math.imul(s, 1664525) + 1013904223) >>> 0; return s / 4294967296 - 0.5; }; + const x = new Float32Array(SR * 3); + for (let i = 0; i < x.length; i++) { const t = i / SR; x[i] = 0.25 * (0.6 * Math.sin(2 * Math.PI * 110 * t) + 0.3 * Math.sin(2 * Math.PI * 330 * t) + 0.2 * Math.sin(2 * Math.PI * 990 * t)) * (0.7 + 0.3 * Math.sin(2 * Math.PI * 1.3 * t)) + 0.05 * r(); } + return x; +})(); +const gain = (x, db) => x.map((v) => v * Math.pow(10, db / 20)); +const shift = (x, k) => { const y = new Float32Array(x.length); y.set(x.subarray(0, x.length - k), k); return y; }; +const smooth = (x) => x.map((v, i) => 0.5 * (v + (x[i - 1] ?? 0))); // two-point average: a mild low-pass that moves the spectral centroid +const trips = (native, mode = "library") => violationsOf(compareRenders("probe", sig, native), mode); + +t("identical renders trip nothing in either mode", () => { + assert.deepEqual(trips(sig), []); assert.deepEqual(trips(sig, "imp3"), []); +}); + +t("a +1 dB gain trips loudness and true peak, and neither the centroid nor (library mode) the sample bound", () => { + assert.deepEqual(trips(gain(sig, 1)).sort(), ["loudnessLufs", "truePeakDb"]); +}); + +t("a +1 dB gain also trips the sample bound under imp3 (relative error about -18 dB)", () => { + assert.deepEqual(trips(gain(sig, 1), "imp3").sort(), ["loudnessLufs", "relRmsErrDb", "truePeakDb"]); +}); + +t("loudness trips on its own at +0.2 dB, true peak at +0.6 dB: each limit has its own edge", () => { + assert.deepEqual(trips(gain(sig, 0.2)), ["loudnessLufs"]); + assert.ok(!trips(gain(sig, 0.05)).includes("loudnessLufs"), "+0.05 dB is inside the 0.1 LU limit"); + assert.ok(trips(gain(sig, 0.6)).includes("truePeakDb")); + assert.ok(!trips(gain(sig, 0.4)).includes("truePeakDb"), "+0.4 dB is inside the 0.5 dB true-peak limit"); +}); + +t("a spectral change (a two-point low-pass) trips the centroid limit", () => { + const row = compareRenders("probe", sig, smooth(sig)); + assert.ok(100 * Math.abs(row.native.centroidMedianHz - row.wasm.centroidMedianHz) / row.wasm.centroidMedianHz > LIMITS.centroidPct); + assert.ok(violationsOf(row, "library").includes("centroidMedianHz")); +}); + +t("centroid edge, in literal numbers: 1000 Hz vs 1009 Hz (0.9 %) passes, vs 1011 Hz (1.1 %) trips, in both directions", () => { + const base = compareRenders("probe", sig, sig), at = (w, nat) => ({ ...base, wasm: { ...base.wasm, centroidMedianHz: w }, native: { ...base.native, centroidMedianHz: nat } }); + for (const [w, nat] of [[1000, 1009], [1000, 991], [1000, 1000]]) assert.deepEqual(violationsOf(at(w, nat), "library"), [], `${w} vs ${nat}`); + for (const [w, nat] of [[1000, 1011], [1000, 989]]) assert.deepEqual(violationsOf(at(w, nat), "library"), ["centroidMedianHz"], `${w} vs ${nat}`); +}); + +t("a one-sample shift trips only the sample bound, and only under imp3 (library mode has no sample bound)", () => { + const sh = shift(sig, 1), row = compareRenders("probe", sig, sh); + assert.ok(row.relRmsErrDb > LIMITS.relRmsErrDbImp3, `relRmsErrDb ${row.relRmsErrDb}`); + assert.deepEqual(violationsOf(row, "imp3"), ["relRmsErrDb"]); + assert.deepEqual(violationsOf(row, "library"), []); + assert.deepEqual(violationsOf(row, "old"), []); +}); + +t("an error well inside the sample bound passes under imp3", () => { + const near = sig.map((v, i) => v + (i % 2 ? 1 : -1) * 1e-5); + assert.ok(compareRenders("probe", sig, near).relRmsErrDb < LIMITS.relRmsErrDbImp3); + assert.deepEqual(trips(near, "imp3"), []); +}); + +t("a length mismatch is an error row, not a trimmed comparison", () => { + const row = compareRenders("probe", sig, sig.subarray(0, sig.length - 512)); + assert.match(row.error, /length mismatch: wasm 144000, native 143488/); + assert.equal(row.wasm, undefined); +}); + +console.log(`${n} passed`); diff --git a/tools/aqua-feel/package.json b/tools/aqua-feel/package.json index b0d2271..531b378 100644 --- a/tools/aqua-feel/package.json +++ b/tools/aqua-feel/package.json @@ -3,5 +3,5 @@ "private": true, "type": "module", "description": "Headless feel meter for AquaSynth: renders every shipped patch and Ding offline and measures them. Measures; does not listen.", - "scripts": { "test": "node test.mjs && node sync.test.mjs", "feel": "node feel.mjs", "shelf": "node shelf.mjs build", "shelf:check": "node shelf.mjs check" } + "scripts": { "test": "node test.mjs && node sync.test.mjs && node native-diff.test.mjs", "feel": "node feel.mjs", "shelf": "node shelf.mjs build", "shelf:check": "node shelf.mjs check" } } diff --git a/tools/aqua-trial/README.md b/tools/aqua-trial/README.md index 81890fd..bd4e3bc 100644 --- a/tools/aqua-trial/README.md +++ b/tools/aqua-trial/README.md @@ -4,7 +4,7 @@ Blinded, randomized A/B ratings of a synthesized thruster, to test Deru's "dead thruster's noise layers (rasp, rattle) are what makes it sound dead, and that a tone-forward mix lands better. **Status: the page has never been run in a browser, and nothing has been heard.** Node covers the logic and -the patch through the same faustwasm runtime the page loads (`node test.mjs`, 15 checks). The DOM glue +the patch through the same faustwasm runtime the page loads (`node test.mjs`, 21 checks). The DOM glue (`web/trial.js`), AudioContext playback and the download are untested. Follow-up: browser-verify. ## Entry points @@ -27,7 +27,7 @@ the patch through the same faustwasm runtime the page loads (`node test.mjs`, 15 ## Privacy Answers live in the page's memory only. The page makes no network request after loading its own assets and -writes nothing to storage. **Export** downloads a JSON file (patch, params, peak, rms, rating, word, time, +writes nothing to storage. **Export** downloads a JSON file (patch, the sha-256 of the DSP source it compiled, params, peak, rms, rating, word, time, user agent, sample rate); sending it is the listener's act. ## Measured finding before any listening @@ -40,5 +40,5 @@ already small, so noise 100 vs 0 differs far less than it did in the original mi 0.189 / 0.161 / 0.148 / 0.140). The v5 mix is now a variant (`v5noise100`), so the noise layers can be compared against the current mix and the old one. The v5 variant measured at 48 kHz, rpm 600: rms 0.0734, peak 0.189, against 0.0725 and 0.160 for the current mix at noise 100: a level difference of about 0.1 dB rms, so the variants again differ mostly in texture, not level. Adding it makes ten comparisons -rather than six. `analyze.mjs` prints `mix` beside `noise%`; its `(context) noise%` correlation now pools the v5 variant (noise 100) -with the current noise-100 variant, so read it with the table. Nothing here was heard, before or after the change. +rather than six. `analyze.mjs` prints `mix` beside `noise%`; its `(context) noise%` correlation uses the current mix only (the v5 variant shares noise 100 and is left out). +`analyze.mjs` rejects an export without `dspSha256` (a record from a page build that did not record its DSP could not be told from mix 0) and warns when exports differ from each other or from this checkout's `thruster-trial.dsp`. Nothing here was heard, before or after the change. diff --git a/tools/aqua-trial/analyze.mjs b/tools/aqua-trial/analyze.mjs index 2b5fa40..ba23943 100644 --- a/tools/aqua-trial/analyze.mjs +++ b/tools/aqua-trial/analyze.mjs @@ -3,7 +3,9 @@ // and whether render peak / rms predicted the ratings. Usage: node analyze.mjs export1.json [export2.json ...] // Pure functions are exported for test.mjs; the CLI runs only when this file is the entry point. import { readFileSync } from "node:fs"; -import { pathToFileURL } from "node:url"; +import { createHash } from "node:crypto"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { rng, shuffle, SCHEMA } from "./logic.js"; export const mean = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length; @@ -48,7 +50,13 @@ export function predictor(records, key) { return { key, n: records.length, rho, p, predicts }; } -export function analyze(exports) { +// An export names the sha-256 of the DSP source the page compiled. One without it predates the field (a record from the +// page build before 2026-11-05 would otherwise read as mix 0), so it is rejected, not guessed. `expectedDsp` is the sha-256 +// of the thruster-trial.dsp this checkout holds; exports from a different DSP are kept but flagged in the report. +export function analyze(exports, { expectedDsp } = {}) { + exports.forEach((e, i) => { if (!/^[0-9a-f]{64}$/.test(e.dspSha256 ?? "")) throw new Error(`export ${i + 1} has no dspSha256: it comes from a page build that did not record which DSP it rendered`); }); + const dspHashes = [...new Set(exports.map((e) => e.dspSha256))]; + const dspMismatch = expectedDsp ? dspHashes.filter((h) => h !== expectedDsp) : []; const records = exports.flatMap((e, i) => e.records.map((r) => ({ ...r, listener: i }))); const byVariant = new Map(); for (const r of records) (byVariant.get(r.variant) ?? byVariant.set(r.variant, []).get(r.variant)).push(r); @@ -64,8 +72,9 @@ export function analyze(exports) { }).sort((a, b) => a.mix - b.mix || b.noise - a.noise); const predictors = ["peak", "rms"].map((k) => predictor(records, k)); // Context: the manipulated variable itself, so a peak/rms "prediction" can be told apart from noise-amount doing the work. - const noise = predictor(records.map((r) => ({ ...r, noise: r.params.noise })), "noise"); - return { listeners: exports.length, records: records.length, variants, predictors, noise }; + // Only the current mix (mix 0): v5noise100 shares noise 100 with noise100 and would pool two mixes into one noise level. + const noise = predictor(records.filter((r) => (r.params.mix ?? 0) === 0).map((r) => ({ ...r, noise: r.params.noise })), "noise"); + return { listeners: exports.length, records: records.length, variants, predictors, noise, dspHashes, dspMismatch }; } const f = (x, d = 3) => (Number.isFinite(x) ? x.toFixed(d) : "n/a"); @@ -78,7 +87,9 @@ export function format(rep) { } out.push("", "Did render level predict the rating? (Spearman rho over individual ratings, permutation p; predicts = |rho|>=0.5 and p<0.05)"); for (const p of rep.predictors) out.push(` ${p.key.padEnd(5)} rho ${f(p.rho, 2).padStart(6)} p ${f(p.p, 4)} n ${p.n} -> ${p.predicts ? "PREDICTS" : "no evidence it predicts"}`); - out.push(` (context) noise% rho ${f(rep.noise.rho, 2)} p ${f(rep.noise.p, 4)}`); + out.push(` (context) noise% rho ${f(rep.noise.rho, 2)} p ${f(rep.noise.p, 4)} n ${rep.noise.n} (mix 0 only)`); + if (rep.dspHashes.length > 1) out.push("", `WARNING: exports come from ${rep.dspHashes.length} different DSP sources (${rep.dspHashes.map((h) => h.slice(0, 7)).join(", ")}); do not read them as one trial.`); + if (rep.dspMismatch.length) out.push("", `WARNING: ${rep.dspMismatch.map((h) => h.slice(0, 7)).join(", ")} is not the sha-256 of this checkout's thruster-trial.dsp; the export was rendered from different DSP text.`); out.push("", "Peak and rms fall with noise amount here (noise is not loudness-compensated), so a level effect and a noise effect are confounded, and this script does not separate them."); return out.join("\n"); } @@ -91,5 +102,6 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) if (e.schema !== SCHEMA) throw new Error(`${p}: schema ${e.schema}, expected ${SCHEMA}`); return e; }); - console.log(format(analyze(exports))); + const expectedDsp = createHash("sha256").update(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "thruster-trial.dsp"))).digest("hex"); + console.log(format(analyze(exports, { expectedDsp }))); } diff --git a/tools/aqua-trial/logic.js b/tools/aqua-trial/logic.js index fcce6c6..95ddcb6 100644 --- a/tools/aqua-trial/logic.js +++ b/tools/aqua-trial/logic.js @@ -67,8 +67,15 @@ export function cleanWord(text) { export const validRating = (n) => Number.isInteger(n) && n >= 1 && n <= 5; +// sha-256 of a string as lowercase hex (Web Crypto: browsers and node 20+). The page hashes the DSP text it compiles. +export async function sha256Hex(text) { + const d = new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(text))); + return [...d].map((b) => b.toString(16).padStart(2, "0")).join(""); +} + // answers: [{ A: {rating, word, time, plays}, B: {...} }] aligned with design. -export function buildExport({ seed, design, renders, answers, sampleRate, userAgent, exportedAt }) { +export function buildExport({ seed, design, renders, answers, dspSha256, sampleRate, userAgent, exportedAt }) { + if (!/^[0-9a-f]{64}$/.test(dspSha256 ?? "")) throw new Error("export needs the sha-256 of the DSP source that was rendered"); const records = []; design.forEach((trial, t) => { for (const pos of ["A", "B"]) { @@ -82,5 +89,5 @@ export function buildExport({ seed, design, renders, answers, sampleRate, userAg }); } }); - return { schema: SCHEMA, patch: PATCH, seed, sampleRate, userAgent, exportedAt, seconds: SECONDS, records }; + return { schema: SCHEMA, patch: PATCH, dspSha256, seed, sampleRate, userAgent, exportedAt, seconds: SECONDS, records }; } diff --git a/tools/aqua-trial/test.mjs b/tools/aqua-trial/test.mjs index ac15813..36167a7 100644 --- a/tools/aqua-trial/test.mjs +++ b/tools/aqua-trial/test.mjs @@ -5,11 +5,12 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { makeDesign, variants, stats, cleanWord, validRating, buildExport, renderVariant, rng, shuffle, SCHEMA, PATCH } from "./logic.js"; +import { makeDesign, variants, stats, cleanWord, validRating, buildExport, sha256Hex, renderVariant, rng, shuffle, SCHEMA, PATCH } from "./logic.js"; import { analyze, format, mean, sd, spearman, ranks, pearson } from "./analyze.mjs"; const here = dirname(fileURLToPath(import.meta.url)); const dist = resolve(process.argv[2] ?? join(here, "dist")); +const DSP_SHA = await sha256Hex(readFileSync(join(here, "thruster-trial.dsp"), "utf8")); let n = 0; const t = async (name, fn) => { await fn(); n++; console.log(`ok ${name}`); }; @@ -50,7 +51,7 @@ const fakeExport = (seed, rate) => { const design = makeDesign(seed), rand = rng(seed); const renders = { noise100: { peak: 0.19, rms: 0.073 }, noise50: { peak: 0.15, rms: 0.05 }, noise20: { peak: 0.12, rms: 0.04 }, noise0: { peak: 0.1, rms: 0.035 }, v5noise100: { peak: 0.19, rms: 0.073 } }; const answers = design.map(() => Object.fromEntries(["A", "B"].map((p) => [p, { rating: 1 + Math.floor(rand() * 5), word: "w", time: "2026-11-04T00:00:00Z", plays: 1 }]))); - return { design, renders, answers, doc: buildExport({ seed, design, renders, answers, sampleRate: rate, userAgent: "test-ua", exportedAt: "2026-11-04T00:00:00Z" }) }; + return { design, renders, answers, doc: buildExport({ seed, design, renders, answers, dspSha256: DSP_SHA, sampleRate: rate, userAgent: "test-ua", exportedAt: "2026-11-04T00:00:00Z" }) }; }; await t("export: one record per heard sound with the required fields", () => { @@ -65,10 +66,17 @@ await t("export: one record per heard sound with the required fields", () => { await t("export: unanswered trials are omitted, not invented", () => { const { design, renders, answers } = fakeExport(3, 48000); - const doc = buildExport({ seed: 3, design, renders, answers: answers.slice(0, 2), sampleRate: 48000, userAgent: "u", exportedAt: "x" }); + const doc = buildExport({ seed: 3, design, renders, answers: answers.slice(0, 2), dspSha256: DSP_SHA, sampleRate: 48000, userAgent: "u", exportedAt: "x" }); assert.equal(doc.records.length, 4); }); +await t("export: carries the sha-256 of the DSP source and refuses to be built without one", async () => { + const { doc, design, renders, answers } = fakeExport(3, 48000); + assert.equal(doc.dspSha256, DSP_SHA); assert.match(doc.dspSha256, /^[0-9a-f]{64}$/); + assert.equal(await sha256Hex("abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + for (const bad of [undefined, "", "abc", DSP_SHA.toUpperCase()]) assert.throws(() => buildExport({ seed: 3, design, renders, answers, dspSha256: bad, sampleRate: 48000, userAgent: "u", exportedAt: "x" }), /sha-256/); +}); + await t("analyze maths: ranks with ties, spearman, pearson, sd", () => { assert.deepEqual(ranks([10, 20, 20, 30]), [1, 2.5, 2.5, 4]); assert.equal(spearman([1, 2, 3, 4], [10, 20, 30, 40]), 1); @@ -105,6 +113,27 @@ await t("analyze: ratings unrelated to level are not reported as predicted", () assert.match(format(rep), /no evidence/); }); +await t("analyze: the context noise% correlation does not pool the v5 mix into noise 100", () => { + // current-mix ratings rise with noise; the v5 variant (noise 100 too) is rated worst, which would drag a pooled correlation down + const exports = synth((r) => (r.params.mix === 1 ? 1 : r.params.noise === 100 ? 5 : r.params.noise === 50 ? 4 : r.params.noise === 20 ? 3 : 2)); + const rep = analyze(exports); + assert.equal(rep.noise.n, 12 * 16, "each listener rates a variant in 4 trials, so 4 current-mix variants give 16 records and the v5 variant 4, which are left out"); + assert.ok(rep.noise.rho > 0.99, `rho ${rep.noise.rho}`); + assert.match(format(rep), /mix 0 only/); +}); + +await t("analyze: an export without dspSha256 is rejected; mixed or foreign DSP hashes are flagged", () => { + const ok = fakeExport(5, 48000).doc, old = { ...ok }; delete old.dspSha256; + assert.throws(() => analyze([ok, old]), /no dspSha256/); + assert.throws(() => analyze([{ ...ok, dspSha256: "nothex" }]), /no dspSha256/); + const clean = analyze([ok], { expectedDsp: DSP_SHA }); + assert.deepEqual(clean.dspMismatch, []); assert.doesNotMatch(format(clean), /WARNING/); + const other = { ...ok, dspSha256: "0".repeat(64) }; + const mixed = analyze([ok, other], { expectedDsp: DSP_SHA }); + assert.deepEqual(mixed.dspMismatch, ["0".repeat(64)]); + assert.match(format(mixed), /2 different DSP sources/); assert.match(format(mixed), /not the sha-256 of this checkout/); +}); + await t("analyze: per-variant mean and spread are computed per variant", () => { const rep = analyze(synth((r) => (r.params.noise === 100 ? 5 : r.params.noise === 50 ? 3 : 1 + (r.trial % 2) * 2))); const by = Object.fromEntries(rep.variants.map((v) => [v.id, v])); @@ -123,6 +152,10 @@ await t("analyze CLI: runs on export files and rejects a foreign schema", async writeFileSync(b, JSON.stringify({ ...fakeExport(5, 48000).doc, schema: "other" })); assert.match(execFileSync("node", [join(here, "analyze.mjs"), a], { encoding: "utf8" }), /1 listener export/); assert.notEqual(spawnSync("node", [join(here, "analyze.mjs"), b]).status, 0); + const c = join(d, "c.json"), nohash = fakeExport(5, 48000).doc; delete nohash.dspSha256; writeFileSync(c, JSON.stringify(nohash)); + const rc = spawnSync("node", [join(here, "analyze.mjs"), c], { encoding: "utf8" }); + assert.notEqual(rc.status, 0); assert.match(rc.stderr, /no dspSha256/); + assert.doesNotMatch(execFileSync("node", [join(here, "analyze.mjs"), a], { encoding: "utf8" }), /WARNING/, "the checkout's own DSP hash matches"); assert.equal(spawnSync("node", [join(here, "analyze.mjs")]).status, 2); }); @@ -148,21 +181,30 @@ await t("patch: exposes rpm, noise and mix controls", () => { assert.ok(trial.controls.some((c) => c.label === "rpm")); }); -await t("patch: noise=100 is sample-identical to patches/thruster/thruster.dsp at rpm 600", async () => { - const a = await renderVariant(trial.g, trial.controls, { params: { rpm: 600, noise: 100 } }, SR, 2); - const b = await renderVariant(base.g, base.controls, { params: { rpm: 600 } }, SR, 2); - assert.equal(a.samples.length, b.samples.length); - let maxDiff = 0; for (let i = 0; i < a.samples.length; i++) maxDiff = Math.max(maxDiff, Math.abs(a.samples[i] - b.samples[i])); - assert.ok(maxDiff < 1e-6, `max diff ${maxDiff}`); +const IDENTITY_RPMS = [300, 600, 1200, 4800]; + +await t("patch: noise=100 is sample-identical to patches/thruster/thruster.dsp at rpm 300/600/1200/4800", async () => { + for (const rpm of IDENTITY_RPMS) { + const a = await renderVariant(trial.g, trial.controls, { params: { rpm, noise: 100 } }, SR, 1); + const b = await renderVariant(base.g, base.controls, { params: { rpm } }, SR, 1); + assert.ok(maxDiffOf(a, b) < 1e-6, `rpm ${rpm}: max diff ${maxDiffOf(a, b)}`); + } }); -await t("patch: mix=1 noise=100 is sample-identical to the v5 thruster as cut at c6561da, at rpm 600, and differs from mix=0", async () => { +await t("patch: mix=1 noise=100 is sample-identical to the v5 thruster as cut at c6561da, at rpm 300/600/1200/4800, and differs from mix=0", async () => { const a = await renderVariant(trial.g, trial.controls, { params: { rpm: 600, noise: 100, mix: 1 } }, SR, 2); const b = await renderVariant(v5.g, v5.controls, { params: { rpm: 600 } }, SR, 2); const cur = await renderVariant(trial.g, trial.controls, { params: { rpm: 600, noise: 100, mix: 0 } }, SR, 2); assert.ok(maxDiffOf(a, b) < 1e-6, `max diff ${maxDiffOf(a, b)}`); assert.ok(maxDiffOf(a, cur) > 1e-2, "the two mixes must be audibly different signals, not equal"); assert.ok(a.rms > cur.rms, `v5 rms ${a.rms} should exceed current ${cur.rms}`); + for (const rpm of IDENTITY_RPMS.filter((r) => r !== 600)) { + const m1 = await renderVariant(trial.g, trial.controls, { params: { rpm, noise: 100, mix: 1 } }, SR, 1); + const ref = await renderVariant(v5.g, v5.controls, { params: { rpm } }, SR, 1); + const m0 = await renderVariant(trial.g, trial.controls, { params: { rpm, noise: 100, mix: 0 } }, SR, 1); + assert.ok(maxDiffOf(m1, ref) < 1e-6, `rpm ${rpm}: mix 1 vs v5 max diff ${maxDiffOf(m1, ref)}`); + assert.ok(maxDiffOf(m1, m0) > 1e-3, `rpm ${rpm}: mix 1 must differ from mix 0`); + } }); await t("patch: all variants finite and non-silent, rendered at 44100 and 48000, noise 0 is deterministic", async () => { @@ -187,4 +229,36 @@ await t("patch: noise genuinely removes the noise layers (rms falls monotonicall assert.ok(rms[3] > 1e-3); }); +// Energy of x in 2.5-5 kHz (Goertzel at 40 bins). At rpm 4800 the rasp band sits at 700 + 4800*0.6 = 3580 Hz, above every +// rattle mode (173..1130 Hz) and the rumble, so energy up there is the rasp and nothing else. +const highBandEnergy = (x) => { + let e = 0; + for (let k = 0; k < 40; k++) { + const w = 2 * Math.PI * (2500 + k * 64) / SR, c = 2 * Math.cos(w); + let s1 = 0, s2 = 0; + for (let i = 0; i < x.length; i++) { const s0 = x[i] + c * s1 - s2; s2 = s1; s1 = s0; } + e += s1 * s1 + s2 * s2 - c * s1 * s2; + } + return e; +}; +const diffOf = (a, b) => a.samples.map((v, i) => v - b.samples[i]); + +await t("patch: the noise knob scales the rasp: its band energy is 0 at noise 0 and goes as (noise/100)^2, at rpm 4800", async () => { + const at = (noise) => renderVariant(trial.g, trial.controls, { params: { rpm: 4800, noise, mix: 0 } }, SR, 1); + const [x0, x50, x100] = [await at(0), await at(50), await at(100)]; + const e0 = highBandEnergy(x0.samples), e50 = highBandEnergy(x50.samples), e100 = highBandEnergy(x100.samples); + assert.ok(e100 > 100 * e0, `rasp band energy at noise 100 (${e100}) must dwarf noise 0 (${e0})`); + // tanh at the output bends the scaling a little, hence the tolerance + assert.ok(Math.abs(Math.sqrt(e50 / e100) - 0.5) < 0.05, `amplitude ratio ${Math.sqrt(e50 / e100)}`); +}); + +await t("patch: the noise knob scales rasp and rattle together at rpm 600 (what changes with noise is proportional to noise)", async () => { + const at = (noise) => renderVariant(trial.g, trial.controls, { params: { rpm: 600, noise, mix: 0 } }, SR, 1); + const [x0, x50, x100] = [await at(0), await at(50), await at(100)]; + const rms = (d) => Math.sqrt(d.reduce((s, v) => s + v * v, 0) / d.length); + const full = rms(diffOf(x100, x0)), half = rms(diffOf(x50, x0)); + assert.ok(full > 1e-3, `noise layers contribute ${full}`); + assert.ok(Math.abs(half / full - 0.5) < 0.05, `half/full ${half / full}`); +}); + console.log(`${n} passed`); diff --git a/tools/aqua-trial/web/trial.js b/tools/aqua-trial/web/trial.js index 9c3f4f8..dec8677 100644 --- a/tools/aqua-trial/web/trial.js +++ b/tools/aqua-trial/web/trial.js @@ -3,7 +3,7 @@ // Nothing here makes a network request after the page assets load, and nothing is written to storage. import { instantiateFaustModuleFromFile, LibFaust, FaustCompiler, FaustMonoDspGenerator } from "./vendor/faustwasm/index.js"; import { controlsOf } from "./ui.js"; -import { variants, makeDesign, renderVariant, cleanWord, validRating, buildExport, PATCH } from "./logic.js"; +import { variants, makeDesign, renderVariant, cleanWord, validRating, buildExport, sha256Hex, PATCH } from "./logic.js"; const $ = (s, root = document) => root.querySelector(s); const status = (t) => { $("#status").textContent = t; }; @@ -11,7 +11,7 @@ const seed = crypto.getRandomValues(new Uint32Array(1))[0]; const design = makeDesign(seed); const answers = []; const renders = {}; // variant id -> { buffer, peak, rms } -let ctx, source, current = 0, plays = { A: 0, B: 0 }; +let ctx, source, dspSha256, current = 0, plays = { A: 0, B: 0 }; $("#agree").onchange = () => { $("#start").disabled = !$("#agree").checked; }; $("#start").onclick = start; @@ -38,7 +38,9 @@ async function start() { status("Loading the Faust compiler…"); const compiler = new FaustCompiler(new LibFaust(await instantiateFaustModuleFromFile("./vendor/libfaust-wasm/libfaust-wasm.js"))); const g = new FaustMonoDspGenerator(); - if (!await g.compile(compiler, "thruster_trial", await (await fetch(`./${PATCH}.dsp`)).text(), "-ftz 2")) throw new Error(compiler.getErrorMessage()); + const dspText = await (await fetch(`./${PATCH}.dsp`)).text(); + dspSha256 = await sha256Hex(dspText); // the exact source compiled, so an export names the DSP that produced it + if (!await g.compile(compiler, "thruster_trial", dspText, "-ftz 2")) throw new Error(compiler.getErrorMessage()); const controls = controlsOf(JSON.parse(g.factory.json)); for (const v of variants()) { status(`Rendering ${Object.keys(renders).length + 1} of ${variants().length}…`); @@ -100,7 +102,7 @@ function next() { function exportJson() { const doc = buildExport({ - seed, design, answers, sampleRate: ctx.sampleRate, userAgent: navigator.userAgent, exportedAt: new Date().toISOString(), + seed, design, answers, dspSha256, sampleRate: ctx.sampleRate, userAgent: navigator.userAgent, exportedAt: new Date().toISOString(), renders: Object.fromEntries(Object.entries(renders).map(([id, r]) => [id, { peak: r.peak, rms: r.rms }])), }); const url = URL.createObjectURL(new Blob([JSON.stringify(doc, null, 2)], { type: "application/json" }));