From da902619de5ad9f3b6201d8bdbd924083d2b8dbe Mon Sep 17 00:00:00 2001 From: Aqua Date: Thu, 5 Nov 2026 18:52:29 +0000 Subject: [PATCH 1/7] web: onset detector and key estimator with synthetic-signal tests Spectral-flux onset detector (max-filtered reference, median threshold, refractory) and peak-based chroma with Krumhansl profile key estimate. Own FFT; no external dependencies. Co-Authored-By: Claude Sonnet 5.5 --- web/package.json | 12 +++ web/src/dsp.js | 180 ++++++++++++++++++++++++++++++++++++++++++ web/test/dsp.test.mjs | 107 +++++++++++++++++++++++++ 3 files changed, 299 insertions(+) create mode 100644 web/package.json create mode 100644 web/src/dsp.js create mode 100644 web/test/dsp.test.mjs diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..ffa5137 --- /dev/null +++ b/web/package.json @@ -0,0 +1,12 @@ +{ + "name": "perlines-web", + "version": "2.0.0", + "description": "Perlines v2: zero-install browser audio visualizer (WebAudio + WebGL2)", + "type": "module", + "license": "MIT", + "private": true, + "scripts": { + "test": "node --test test/dsp.test.mjs", + "serve": "node tools/serve.mjs" + } +} diff --git a/web/src/dsp.js b/web/src/dsp.js new file mode 100644 index 0000000..2b21430 --- /dev/null +++ b/web/src/dsp.js @@ -0,0 +1,180 @@ +// Perlines v2 analysis: FFT, onset detector, chroma + key estimate. +// Pure ES module, no DOM, no dependencies: the browser and the node tests load the same file. +// Written from the published ideas (spectral flux with a max-filtered reference, adaptive +// median threshold, Krumhansl-Schmuckler key profiles), not from any GPL implementation. + +/** In-place radix-2 complex FFT. re/im are Float32Array|Float64Array of power-of-two length. */ +export function fft(re, im) { + const n = re.length; + for (let i = 1, j = 0; i < n; i++) { + let bit = n >> 1; + for (; j & bit; bit >>= 1) j ^= bit; + j ^= bit; + if (i < j) { + let t = re[i]; re[i] = re[j]; re[j] = t; + t = im[i]; im[i] = im[j]; im[j] = t; + } + } + for (let len = 2; len <= n; len <<= 1) { + const ang = -2 * Math.PI / len, wr = Math.cos(ang), wi = Math.sin(ang); + for (let i = 0; i < n; i += len) { + let cr = 1, ci = 0; + for (let k = 0; k < len >> 1; k++) { + const a = i + k, b = a + (len >> 1); + const xr = re[b] * cr - im[b] * ci, xi = re[b] * ci + im[b] * cr; + re[b] = re[a] - xr; im[b] = im[a] - xi; + re[a] += xr; im[a] += xi; + const t = cr * wr - ci * wi; ci = cr * wi + ci * wr; cr = t; + } + } + } +} + +/** Hann-windowed magnitude spectrum (n/2 bins, scaled so a full-scale sine reads ~0.5). */ +export function magnitudeSpectrum(samples, offset, n, out) { + const re = new Float64Array(n), im = new Float64Array(n); + for (let i = 0; i < n; i++) { + const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / (n - 1)); + re[i] = (samples[offset + i] || 0) * w; + } + fft(re, im); + const m = out || new Float32Array(n >> 1); + for (let k = 0; k < n >> 1; k++) m[k] = Math.hypot(re[k], im[k]) * 2 / n; + return m; +} + +/** Decibel array (as WebAudio getFloatFrequencyData gives) to linear magnitude, in place into out. */ +export function dbToLinear(db, out) { + for (let i = 0; i < db.length; i++) out[i] = Math.pow(10, Math.max(db[i], -160) / 20); + return out; +} + +/** + * Onset detector: log-compressed spectral flux against a max-filtered previous frame + * (the max filter ignores vibrato wobble), a moving-median adaptive threshold, local-peak picking + * and a refractory period. Causal: an onset on frame n-1 is confirmed when frame n arrives. + * + * push(mags, t) -> null | { time, strength } (time = the onset frame's own timestamp, seconds) + */ +export class OnsetDetector { + constructor(opts = {}) { + this.compress = opts.compress ?? 200; // log(1 + compress * mag) + this.window = opts.window ?? 43; // median history, frames + this.delta = opts.delta ?? 0.35; // absolute floor above the median, in flux units + this.ratio = opts.ratio ?? 1.6; // and the flux must exceed ratio * median + this.refractory = opts.refractory ?? 0.06; // seconds + this.minBin = opts.minBin ?? 2; + this.maxBin = opts.maxBin ?? 0; // 0 = all bins + this.prev = null; + this.hist = []; // recent flux values + this.f1 = 0; this.f2 = 0; this.t1 = 0; // flux[n-1], flux[n-2], time[n-1] + this.last = -1e9; + this.flux = 0; // latest raw flux, for display + } + + push(mags, t) { + const n = this.maxBin || mags.length; + const cur = new Float32Array(n); + for (let k = 0; k < n; k++) cur[k] = Math.log1p(this.compress * mags[k]); + let flux = 0; + if (this.prev) { + const p = this.prev; + for (let k = this.minBin; k < n; k++) { + const ref = Math.max(p[k - 1] ?? 0, p[k], p[k + 1] ?? 0); + const d = cur[k] - ref; + if (d > 0) flux += d; + } + flux /= Math.sqrt(n); + } + this.prev = cur; + this.flux = flux; + + // candidate is the previous frame: f1 must be a local maximum over f2 and the new flux + let out = null; + const cand = this.f1; + if (this.hist.length >= 8 && cand > this.f2 && cand >= flux) { + const sorted = this.hist.slice().sort((a, b) => a - b); + const med = sorted[sorted.length >> 1]; + if (cand > med * this.ratio + this.delta && this.t1 - this.last >= this.refractory) { + out = { time: this.t1, strength: Math.min(1, (cand - med) / (cand + 1e-6) * 1.5) }; + this.last = this.t1; + } + } + this.hist.push(flux); + if (this.hist.length > this.window) this.hist.shift(); + this.f2 = this.f1; this.f1 = flux; this.t1 = t; + return out; + } +} + +const NOTE_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']; +export { NOTE_NAMES }; +// Krumhansl-Kessler probe-tone profiles (published psychology data). +const MAJOR = [6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88]; +const MINOR = [6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17]; + +function corr(a, b, rot) { + let ma = 0, mb = 0; + for (let i = 0; i < 12; i++) { ma += a[(i + rot) % 12]; mb += b[i]; } + ma /= 12; mb /= 12; + let num = 0, da = 0, db = 0; + for (let i = 0; i < 12; i++) { + const x = a[(i + rot) % 12] - ma, y = b[i] - mb; + num += x * y; da += x * x; db += y * y; + } + return da > 1e-12 && db > 1e-12 ? num / Math.sqrt(da * db) : 0; +} + +/** + * Chroma accumulator + key estimate. Each frame folds spectral peaks between lowHz and highHz into + * 12 pitch classes, a slow exponential average (halfLife seconds) smooths them, and the key is the + * best of 24 rotations of the Krumhansl profiles by correlation. + */ +export class KeyEstimator { + constructor(sampleRate, binCount, opts = {}) { + this.halfLife = opts.halfLife ?? 4; + this.lowHz = opts.lowHz ?? 80; + this.highHz = opts.highHz ?? 2000; + this.chroma = new Float32Array(12); + this.key = { tonic: 0, mode: 'major', confidence: 0, name: 'C' }; + this.hzPerBin = sampleRate / 2 / binCount; + this.lo = Math.max(2, Math.floor(this.lowHz / this.hzPerBin)); + this.hi = Math.min(binCount - 2, Math.ceil(this.highHz / this.hzPerBin)); + } + + // Peak-based: bins are wider than a semitone below ~250 Hz, so each spectral peak is refined by + // parabolic interpolation and its energy goes to the pitch class of the refined frequency. + push(mags, dt) { + const frame = new Float32Array(12); + let top = 0; + for (let k = this.lo; k <= this.hi; k++) if (mags[k] > top) top = mags[k]; + const floor = top * 0.05; + for (let k = this.lo; k <= this.hi; k++) { + const m = mags[k]; + if (m < floor || m < 1e-5 || m < mags[k - 1] || m <= mags[k + 1]) continue; + const a = Math.log(mags[k - 1] + 1e-9), b = Math.log(m + 1e-9), c = Math.log(mags[k + 1] + 1e-9); + const d = 0.5 * (a - c) / (a - 2 * b + c); + const midi = 69 + 12 * Math.log2((k + d) * this.hzPerBin / 440); + const near = Math.round(midi); + frame[((near % 12) + 12) % 12] += Math.sqrt(m) * (1 - Math.min(0.5, Math.abs(midi - near))); // sqrt tames loud-bass dominance + } + const a = Math.pow(0.5, dt / this.halfLife); + for (let i = 0; i < 12; i++) this.chroma[i] = this.chroma[i] * a + frame[i] * (1 - a); + this.key = this.estimate(); + return this.key; + } + + estimate() { + let best = { tonic: 0, mode: 'major', score: -2 }, second = -2; + for (let r = 0; r < 12; r++) { + for (const [mode, prof] of [['major', MAJOR], ['minor', MINOR]]) { + const s = corr(this.chroma, prof, r); + if (s > best.score) { second = best.score; best = { tonic: r, mode, score: s }; } + else if (s > second) second = s; + } + } + const energy = this.chroma.reduce((x, y) => x + y, 0); + const confidence = energy < 1e-4 ? 0 : Math.max(0, Math.min(1, (best.score - second) * 4 + best.score * 0.3)); + return { tonic: best.tonic, mode: best.mode, confidence, name: NOTE_NAMES[best.tonic] + (best.mode === 'minor' ? 'm' : '') }; + } +} diff --git a/web/test/dsp.test.mjs b/web/test/dsp.test.mjs new file mode 100644 index 0000000..b8c5771 --- /dev/null +++ b/web/test/dsp.test.mjs @@ -0,0 +1,107 @@ +// node --test web/test (node >= 20). Synthetic-signal checks of the onset detector and key estimate. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { magnitudeSpectrum, OnsetDetector, KeyEstimator } from '../src/dsp.js'; + +const SR = 44100, FFT = 2048; + +function rng(seed) { let s = seed >>> 0; return () => ((s = (s * 1664525 + 1013904223) >>> 0) / 4294967296) * 2 - 1; } + +function clickTrack(times, seconds, noiseAmp = 0.003) { + const x = new Float32Array(Math.ceil(seconds * SR)), r = rng(7); + for (let i = 0; i < x.length; i++) x[i] = r() * noiseAmp; + for (const t of times) { + const s = Math.round(t * SR); + for (let i = 0; i < 900 && s + i < x.length; i++) x[s + i] += r() * 0.8 * Math.exp(-i / 180); + } + return x; +} + +function detect(x, hopSec, opts) { + const det = new OnsetDetector(opts), hop = Math.round(hopSec * SR), found = []; + const mags = new Float32Array(FFT >> 1); + for (let off = 0; off + FFT <= x.length; off += hop) { + magnitudeSpectrum(x, off, FFT, mags); + // the analyser window ends at "now", so a frame's time is its window end, like AnalyserNode + const o = det.push(mags, (off + FFT) / SR); + if (o) found.push(o.time); + } + return found; +} + +function score(truth, found, tol) { + // an onset in the window ending at time T is visible from T-FFT/SR .. T; allow that window plus tol + const used = new Set(); let hit = 0; + for (const t of truth) { + const i = found.findIndex((f, j) => !used.has(j) && f >= t - tol && f <= t + FFT / SR + tol); + if (i >= 0) { used.add(i); hit++; } + } + return { recall: hit / truth.length, falsePositives: found.length - used.size }; +} + +const TRUTH = [0.5, 1.0, 1.4, 2.1, 2.6, 3.3, 3.8, 4.6, 5.0, 5.7]; + +for (const hop of [512 / SR, 1 / 60]) { + test(`clicks found at hop ${(hop * 1000).toFixed(1)} ms`, () => { + const x = clickTrack(TRUTH, 6.5); + const found = detect(x, hop); + const s = score(TRUTH, found, 0.03); + console.log(` hop ${(hop * 1000).toFixed(1)}ms: recall ${s.recall}, false positives ${s.falsePositives}, found ${found.length}/${TRUTH.length}`); + assert.ok(s.recall >= 0.9, `recall ${s.recall}`); + assert.ok(s.falsePositives <= 1, `false positives ${s.falsePositives}`); + }); +} + +test('silence and steady noise give no onsets', () => { + assert.equal(detect(new Float32Array(SR * 4), 1 / 60).length, 0); + const r = rng(3), x = new Float32Array(SR * 4).map(() => r() * 0.1); + assert.equal(detect(x, 1 / 60).length, 0); +}); + +test('a sustained tone does not retrigger', () => { + const x = new Float32Array(SR * 4).map((_, i) => 0.3 * Math.sin(2 * Math.PI * 440 * i / SR)); + assert.ok(detect(x, 1 / 60).length <= 1); // the very first rise may count once +}); + +test('refractory period merges a doublet 20 ms apart', () => { + const x = clickTrack([0.5, 1.5, 2.5, 3.0, 3.02], 4.5); + const found = detect(x, 512 / SR).filter(t => t > 2.9 && t < 3.4); + assert.equal(found.length, 1); +}); + +function chord(midis, seconds) { + const x = new Float32Array(Math.ceil(seconds * SR)); + for (const m of midis) { + const f = 440 * 2 ** ((m - 69) / 12); + for (let h = 1; h <= 4; h++) for (let i = 0; i < x.length; i++) x[i] += Math.sin(2 * Math.PI * f * h * i / SR) * 0.1 / h; + } + return x; +} + +function keyOf(x) { + const k = new KeyEstimator(SR, FFT >> 1), mags = new Float32Array(FFT >> 1), hop = 735; + for (let off = 0; off + FFT <= x.length; off += hop) { magnitudeSpectrum(x, off, FFT, mags); k.push(mags, hop / SR); } + return k.key; +} + +test('key: C major scale-ish progression reads C major', () => { + // C-F-G-C triads, then lingering C major + const parts = [[60, 64, 67], [65, 69, 72], [67, 71, 74], [60, 64, 67], [60, 64, 67]]; + const x = new Float32Array(SR * 15); + parts.forEach((p, i) => x.set(chord(p, 3), i * 3 * SR)); + const k = keyOf(x); + assert.equal(k.name, 'C'); + assert.ok(k.confidence > 0.2, `confidence ${k.confidence}`); +}); + +test('key: A minor progression reads A minor, and key tracks a transposition', () => { + // i - iv - i - V - i - i: the E major chord's G# is the leading tone that separates A minor from C major + const am = [[57, 60, 64], [62, 65, 69], [57, 60, 64], [64, 68, 71], [57, 60, 64], [57, 60, 64]]; + const x = new Float32Array(SR * 18); + am.forEach((p, i) => x.set(chord(p, 3), i * 3 * SR)); + assert.equal(keyOf(x).name, 'Am'); + const up = am.map(p => p.map(m => m + 2)); // B minor + const y = new Float32Array(SR * 18); + up.forEach((p, i) => y.set(chord(p, 3), i * 3 * SR)); + assert.equal(keyOf(y).name, 'Bm'); +}); From 700908975861463580663e1a909336536bbfdbc7 Mon Sep 17 00:00:00 2001 From: Aqua Date: Thu, 5 Nov 2026 18:55:36 +0000 Subject: [PATCH 2/7] web: WebGL2 scenes, audio inputs, analysis state, page Aurora disk (spectrum-to-mesh extrusion), burning-water spray (stateless onset-driven particles), vector trails (Perlin curl paths). File, mic and synthetic demo inputs; frame-time HUD and ?bench mode. Co-Authored-By: Claude Sonnet 5.5 --- web/index.html | 42 +++++++++++ web/src/analysis.js | 88 ++++++++++++++++++++++ web/src/audio.js | 50 +++++++++++++ web/src/gl.js | 119 +++++++++++++++++++++++++++++ web/src/main.js | 156 +++++++++++++++++++++++++++++++++++++++ web/src/noise.js | 43 +++++++++++ web/src/scenes/aurora.js | 76 +++++++++++++++++++ web/src/scenes/trails.js | 90 ++++++++++++++++++++++ web/src/scenes/water.js | 120 ++++++++++++++++++++++++++++++ web/tools/serve.mjs | 17 +++++ 10 files changed, 801 insertions(+) create mode 100644 web/index.html create mode 100644 web/src/analysis.js create mode 100644 web/src/audio.js create mode 100644 web/src/gl.js create mode 100644 web/src/main.js create mode 100644 web/src/noise.js create mode 100644 web/src/scenes/aurora.js create mode 100644 web/src/scenes/trails.js create mode 100644 web/src/scenes/water.js create mode 100644 web/tools/serve.mjs diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..24d3960 --- /dev/null +++ b/web/index.html @@ -0,0 +1,42 @@ + + + + + +Perlines + + + + +
+ + + + + + + + +
+
Perlines
Drop an audio file anywhere, or use the microphone.
Keys: 1 2 3 scenes, F fullscreen, H HUD.
+Nothing leaves your browser: audio is analysed locally.
+
+ +

+
+
+
diff --git a/web/src/analysis.js b/web/src/analysis.js
new file mode 100644
index 0000000..4b52e47
--- /dev/null
+++ b/web/src/analysis.js
@@ -0,0 +1,88 @@
+// Turns a frame of dB spectrum into the shared state every scene reads: log-spaced smoothed bands,
+// band levels, onset events, chroma, key and the tint colour derived from the key.
+import { dbToLinear, OnsetDetector, KeyEstimator } from './dsp.js';
+
+export const NB = 128;          // log-spaced display bands
+export const HIST = 128;        // history rows (spectrum texture height)
+
+function hslToRgb(h, s, l) {
+  const f = n => { const k = (n + h * 12) % 12, a = s * Math.min(l, 1 - l); return l - a * Math.max(-1, Math.min(k - 3, 9 - k, 1)); };
+  return [f(0), f(8), f(4)];
+}
+
+export class Analysis {
+  constructor(sampleRate, binCount) {
+    this.sampleRate = sampleRate; this.binCount = binCount;
+    this.lin = new Float32Array(binCount);
+    this.onsets = new OnsetDetector({ maxBin: Math.min(binCount, Math.round(8000 / (sampleRate / 2 / binCount))) });
+    this.keyEst = new KeyEstimator(sampleRate, binCount);
+    this.bands = new Float32Array(NB);
+    this.row = new Uint8Array(NB);            // newest row, as texture bytes
+    this.bass = 0; this.mid = 0; this.treble = 0; this.level = 0;
+    this.onset = { count: 0, strength: 0, time: -1 };   // count increments per onset; scenes diff it
+    this.flux = 0;
+    this.key = this.keyEst.key;
+    this.hue = 0.6; this.tint = hslToRgb(0.6, 0.7, 0.55);
+    this.time = 0; this.dt = 1 / 60;
+    // band -> bin edges, log spaced 40 Hz .. 16 kHz
+    const hz = sampleRate / 2 / binCount, lo = 40, hi = Math.min(16000, sampleRate / 2 - hz);
+    this.edges = Array.from({ length: NB + 1 }, (_, i) => Math.min(binCount - 1, Math.round(lo * Math.pow(hi / lo, i / NB) / hz)));
+  }
+
+  /** db: Float32Array in decibels (AnalyserNode.getFloatFrequencyData); t: seconds. */
+  update(db, t) {
+    const dt = Math.min(0.1, Math.max(1e-3, t - this.time || 1 / 60));
+    this.time = t; this.dt = dt;
+    const lin = dbToLinear(db, this.lin);
+
+    const o = this.onsets.push(lin, t);
+    this.flux = this.onsets.flux;
+    if (o) { this.onset.count++; this.onset.strength = o.strength; this.onset.time = t; }
+
+    // bands: peak of dB inside each band, mapped -90..-25 dB to 0..1, fast attack / slow release
+    let b = 0, m = 0, tr = 0, nb = 0, nm = 0, nt = 0;
+    for (let i = 0; i < NB; i++) {
+      const a = this.edges[i], z = Math.max(a + 1, this.edges[i + 1]);
+      let pk = -160;
+      for (let k = a; k < z; k++) if (db[k] > pk) pk = db[k];
+      const v = Math.max(0, Math.min(1, (pk + 90) / 65));
+      const cur = this.bands[i];
+      this.bands[i] = v > cur ? cur + (v - cur) * 0.8 : cur + (v - cur) * (1 - Math.pow(0.02, dt));
+      this.row[i] = Math.round(this.bands[i] * 255);
+      if (i < NB * 0.2) { b += this.bands[i]; nb++; } else if (i < NB * 0.6) { m += this.bands[i]; nm++; } else { tr += this.bands[i]; nt++; }
+    }
+    this.bass = b / nb; this.mid = m / nm; this.treble = tr / nt;
+    this.level = (this.bass + this.mid + this.treble) / 3;
+
+    const key = this.keyEst.push(lin, dt);
+    this.key = key;
+    // tint: hue walks the circle of fifths so related keys sit near each other; confidence gates the move
+    const target = ((key.tonic * 7) % 12) / 12;
+    let d = target - this.hue; d -= Math.round(d);
+    this.hue = (this.hue + d * Math.min(1, dt * (0.15 + key.confidence * 0.8)) + 1) % 1;
+    const major = key.mode === 'major';
+    this.tint = hslToRgb(this.hue, major ? 0.8 : 0.6, major ? 0.58 : 0.5);
+    return this;
+  }
+}
+
+/** Synthetic spectrum for demo mode and benchmarks: a 120 bpm kick/hat pattern over an A-minor pad. */
+export class DemoFeed {
+  constructor(sampleRate, binCount) { this.sr = sampleRate; this.n = binCount; this.db = new Float32Array(binCount); }
+  frame(t) {
+    const { db, n, sr } = this, hz = sr / 2 / n, beat = 0.5;
+    const ph = t % beat, kick = Math.exp(-ph * 9), hat = Math.exp(-((t + beat / 2) % beat) * 40) * ((Math.floor(t / (beat / 2)) % 2) ? 1 : 0.4);
+    const snare = Math.floor(t / beat) % 2 ? Math.exp(-ph * 14) : 0;
+    const chord = [220, 261.63, 329.63, 440, 523.25, 659.25];
+    for (let k = 0; k < n; k++) {
+      const f = k * hz + 1;
+      let v = -105 + 12 * Math.sin(k * 12.9898 + t * 3) * 0.3;
+      v += 62 * kick * Math.exp(-Math.pow(Math.log2(f / 55), 2) * 1.5);
+      v += 52 * snare * Math.exp(-Math.pow(Math.log2(f / 1800), 2) * 0.35);
+      v += 45 * hat * Math.exp(-Math.pow(Math.log2(f / 9000), 2) * 0.5);
+      for (const c of chord) v += 55 * Math.exp(-Math.pow((f - c) / (hz * 0.8 + c * 0.01), 2));
+      db[k] = v;
+    }
+    return db;
+  }
+}
diff --git a/web/src/audio.js b/web/src/audio.js
new file mode 100644
index 0000000..aeaacc2
--- /dev/null
+++ b/web/src/audio.js
@@ -0,0 +1,50 @@
+// Audio inputs. All go through one AnalyserNode; the analysis code reads dB frames from it.
+// File playback uses an