Spectrogram and waveform PNG beside each clip, burst times in a separate note
clip_images.py (numpy + matplotlib Agg, STFT by numpy.fft) writes a deterministic spec and wave PNG per clip and, from the with/without --no-burst renders, burst-notes.md/json. Both kits' render.sh end with it; build.mjs --images embeds the pictures in a collapsed block and widens the page policy to data: images only then. The notes never enter listen.html or kit.json. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Aqua
parent
185eecafdc
commit
813ae0b432
9 changed files with 272 additions and 13 deletions
|
|
@ -24,7 +24,7 @@ Privacy: the page has no network access at all (the browser is told to refuse it
|
|||
|
||||
```
|
||||
node build.mjs --manifest kit.json --clips ./clips --out ./dist # writes dist/listen.html
|
||||
node test.mjs # 16 checks, no dependencies, no browser
|
||||
node test.mjs # 17 checks, no dependencies, no browser
|
||||
node smoke.mjs dist/listen.html # optional browser smoke, see below
|
||||
```
|
||||
|
||||
|
|
@ -59,6 +59,12 @@ The blank verdict files this was meant to match were not found under `/opt/commo
|
|||
`kits/tract-weak-spots` and `kits/tract-stops` hold a `kit.json`, the `phones/*.json` that WordRender reads, and a `render.sh` that renders the clips (`render.sh <repo-root> <out-root>`; needs CULTLIB_ROOT and AQUASYNTH_FAUST_HOME, see the script header). Build a page from the rendered clips:
|
||||
`node tools/aqua-listen/build.mjs --manifest tools/aqua-listen/kits/tract-stops/kit.json --clips <out-root>/clips --out <out-root>/dist`. The titles in those kit.json files carry measured figures, a prediction and a falsifier; the page hides them from the listener.
|
||||
|
||||
### Pictures and burst notes (for a reader who cannot listen)
|
||||
|
||||
`clip_images.py <clips> <images> [--kit kit.json --renders <out-root>]` writes `<stem>.spec.png` (spectrogram, Hann 512 / hop 128, 0-8 kHz, -90..0 dB re the clip peak) and `<stem>.wave.png` (fixed +-0.5) for every wav, byte-identical between runs, and with `--kit --renders` also `burst-notes.md` and `burst-notes.json` next to the images: per plosive closure start, release (start of the with-minus-without `--no-burst` difference above 1/20 of its peak), burst end and the window to look in. Both kits' `render.sh` end with it. Requirement: `python3` with numpy and matplotlib (Agg), set `PYTHON=` to one that has them; no scipy, nothing for the dotnet build. Test: `python3 -I test_clip_images.py`.
|
||||
|
||||
`build.mjs --images <images>` embeds a clip's two pictures in a collapsed "Pictures of this sound" block (matched by the wav's stem) and lets the page's policy admit `data:` images, only then. The burst notes are never read by the build and never enter `listen.html` or `kit.json`: give the page to the listener and the notes to whoever reads the pictures.
|
||||
|
||||
## Files
|
||||
|
||||
`logic.cjs` validation, ordering, export shape (shared by build, page, tests) · `build.mjs` assembles the page · `web/` page template, glue and styles · `test.mjs` · `smoke.mjs`
|
||||
`clip_images.py` + `test_clip_images.py` pictures and burst notes · `kits/` · `logic.cjs` validation, ordering, export shape (shared by build, page, tests) · `build.mjs` assembles the page · `web/` page template, glue and styles · `test.mjs` · `smoke.mjs`
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
#!/usr/bin/env node
|
||||
// Builds one self-contained listening page. Usage:
|
||||
// node build.mjs --manifest kit.json --clips DIR --out DIR (writes DIR/listen.html; --clips defaults to the manifest's folder)
|
||||
// node build.mjs --manifest kit.json --clips DIR --out DIR [--images DIR] (writes DIR/listen.html; --clips defaults to the manifest's folder)
|
||||
// --images: clip_images.py's output; a clip whose wav stem has <stem>.spec.png and <stem>.wave.png there gets both in a collapsed block on its card.
|
||||
import { readFileSync, writeFileSync, mkdirSync, existsSync, statSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createRequire } from "node:module";
|
||||
import { dirname, extname, join, resolve } from "node:path";
|
||||
import { basename, dirname, extname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -12,7 +13,7 @@ const L = createRequire(import.meta.url)("./logic.cjs");
|
|||
const MIME = { ".wav": "audio/wav", ".mp3": "audio/mpeg", ".ogg": "audio/ogg", ".flac": "audio/flac" };
|
||||
|
||||
// Throws one Error listing every manifest problem and every missing file. Returns {html, hash, bytes}.
|
||||
export function buildKit(manifest, clipsDir) {
|
||||
export function buildKit(manifest, clipsDir, imagesDir) {
|
||||
const errs = L.validateManifest(manifest);
|
||||
if (errs.length === 0) for (const c of manifest.clips) {
|
||||
const f = resolve(clipsDir, c.file);
|
||||
|
|
@ -27,13 +28,20 @@ export function buildKit(manifest, clipsDir) {
|
|||
h.update(buf);
|
||||
audio[c.id] = { mime: MIME[extname(c.file).toLowerCase()], b64: buf.toString("base64") };
|
||||
}
|
||||
const images = {};
|
||||
if (imagesDir) for (const c of manifest.clips) {
|
||||
const stem = basename(c.file, extname(c.file)), pics = [["spectrogram", "spec"], ["waveform", "wave"]].map(([label, k]) => [label, resolve(imagesDir, `${stem}.${k}.png`)]);
|
||||
if (!pics.every(([, f]) => existsSync(f))) continue;
|
||||
images[c.id] = pics.map(([label, f]) => { const buf = readFileSync(f); h.update(buf); return { label, b64: buf.toString("base64") }; });
|
||||
}
|
||||
const hash = h.digest("hex").slice(0, 8);
|
||||
const data = JSON.stringify({ manifest, hash, audio }).replace(/</g, "\\u003c");
|
||||
const data = JSON.stringify({ manifest, hash, audio, images }).replace(/</g, "\\u003c");
|
||||
const logic = `globalThis.AquaListen=(()=>{const module={exports:{}};${readFileSync(join(here, "logic.cjs"), "utf8")}\nreturn module.exports})();`;
|
||||
const part = (n) => readFileSync(join(here, "web", n), "utf8");
|
||||
const esc = (s) => s.replace(/&/g, "&").replace(/</g, "<");
|
||||
// data: images only when the page carries pictures; the page without them keeps the tighter policy
|
||||
// function replacers: no `$` patterns in the substituted text are interpreted
|
||||
const html = part("index.html")
|
||||
const html = part("index.html").replace("media-src blob:", () => Object.keys(images).length ? "media-src blob:; img-src data:" : "media-src blob:")
|
||||
.replace("/*TITLE*/", () => esc(manifest.kit.title)).replace("/*STYLES*/", () => part("styles.css"))
|
||||
.replace("/*DATA*/", () => data).replace("/*LOGIC*/", () => logic.replace(/<\/script/gi, "<\\/script")).replace("/*PAGE*/", () => part("page.js").replace(/<\/script/gi, "<\\/script"));
|
||||
return { html, hash, bytes: Buffer.byteLength(html) };
|
||||
|
|
@ -42,10 +50,10 @@ export function buildKit(manifest, clipsDir) {
|
|||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const a = process.argv.slice(2), opt = (n) => { const i = a.indexOf(n); return i < 0 ? undefined : a[i + 1]; };
|
||||
const mf = opt("--manifest"), out = opt("--out");
|
||||
if (!mf || !out) { console.error("usage: node build.mjs --manifest kit.json --clips DIR --out DIR"); process.exit(2); }
|
||||
if (!mf || !out) { console.error("usage: node build.mjs --manifest kit.json --clips DIR --out DIR [--images DIR]"); process.exit(2); }
|
||||
try {
|
||||
const manifest = JSON.parse(readFileSync(mf, "utf8"));
|
||||
const r = buildKit(manifest, resolve(opt("--clips") ?? dirname(resolve(mf))));
|
||||
const r = buildKit(manifest, resolve(opt("--clips") ?? dirname(resolve(mf))), opt("--images") && resolve(opt("--images")));
|
||||
mkdirSync(out, { recursive: true });
|
||||
writeFileSync(join(out, "listen.html"), r.html);
|
||||
console.log(`built ${join(out, "listen.html")}\nclips ${manifest.clips.length} bytes ${r.bytes} hash ${r.hash}`);
|
||||
|
|
|
|||
161
tools/aqua-listen/clip_images.py
Normal file
161
tools/aqua-listen/clip_images.py
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
#!/usr/bin/env python3
|
||||
"""A spectrogram and a waveform PNG for every clip of a kit, and (with --kit and --renders) the burst notes.
|
||||
|
||||
python3 -I clip_images.py <clips-dir> <images-dir> [--kit kit.json --renders <out-root>]
|
||||
|
||||
Needs numpy and matplotlib (Agg); no scipy. Writes <images-dir>/<wav stem>.spec.png and .wave.png for each *.wav in <clips-dir> (sorted).
|
||||
The images are deterministic: fixed figure size, dpi and STFT (Hann 512, hop 128, 0-8 kHz, -90..0 dB re the clip's loudest bin), no PNG
|
||||
metadata, so two runs give identical bytes. Nothing in the pictures marks a time; the notes do, and they are written next to <images-dir>
|
||||
(burst-notes.md and burst-notes.json) and are not read by build.mjs.
|
||||
|
||||
--renders is the out-root of a kit's render.sh (out/<w>/script.aqua and out/<w>/<w>.wav, outnb/<w>/<w>.wav). For each clip of --kit the
|
||||
note gives, for each plosive gesture of the word's script: closure start (the gesture's start), release = start of the with-minus-without
|
||||
(--no-burst) waveform difference above 1/20 of its peak in the gesture's window (the definition PlosiveBurstExistsOnlyAroundItsOwnRelease
|
||||
uses), burst end = last such sample of the same burst, and the window to look at. A --no-burst clip says 'no burst expected'.
|
||||
"""
|
||||
import argparse, json, re, sys, wave
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
WIN, HOP, FMAX = 512, 128, 8000.0
|
||||
DB_LO = -90.0
|
||||
|
||||
|
||||
def read_wav(path):
|
||||
with wave.open(str(path), "rb") as w:
|
||||
assert w.getsampwidth() == 2 and w.getnchannels() == 1, f"{path}: expected 16-bit mono"
|
||||
return np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float64) / 32768.0, w.getframerate()
|
||||
|
||||
|
||||
def stft_db(x, sr):
|
||||
"""dB re the loudest bin of the clip; rows are frequency 0..FMAX, columns are frames (Hann window, numpy.fft)."""
|
||||
x = np.concatenate([x, np.zeros(max(0, WIN - len(x)))])
|
||||
n = 1 + (len(x) - WIN) // HOP
|
||||
idx = np.arange(WIN)[None, :] + HOP * np.arange(n)[:, None]
|
||||
mag = np.abs(np.fft.rfft(x[idx] * np.hanning(WIN)[None, :], axis=1)).T
|
||||
f = np.fft.rfftfreq(WIN, 1.0 / sr)
|
||||
mag = mag[f <= FMAX]
|
||||
return 20 * np.log10(np.maximum(mag, 1e-12) / max(mag.max(), 1e-12)), f[f <= FMAX], (np.arange(n) * HOP + WIN / 2) / sr
|
||||
|
||||
|
||||
def save(fig, path):
|
||||
fig.savefig(path, dpi=100, metadata={"Software": None})
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def images(wav, out):
|
||||
x, sr = read_wav(wav)
|
||||
db, f, t = stft_db(x, sr)
|
||||
fig, ax = plt.subplots(figsize=(8, 3.2))
|
||||
im = ax.imshow(np.clip(db, DB_LO, 0), origin="lower", aspect="auto", cmap="magma", vmin=DB_LO, vmax=0,
|
||||
extent=[t[0] - HOP / 2 / sr, t[-1] + HOP / 2 / sr, f[0], f[-1]])
|
||||
ax.set_xlabel("time (s)"); ax.set_ylabel("Hz"); ax.set_title(f"spectrogram, Hann {WIN} hop {HOP}, 0-8 kHz", fontsize=9)
|
||||
fig.colorbar(im, ax=ax, label="dB re clip peak")
|
||||
fig.tight_layout(); save(fig, out / f"{wav.stem}.spec.png")
|
||||
fig, ax = plt.subplots(figsize=(8, 2.4))
|
||||
ax.plot(np.arange(len(x)) / sr, x, lw=0.5, color="black")
|
||||
ax.set_ylim(-0.5, 0.5); ax.set_xlim(0, len(x) / sr); ax.set_xlabel("time (s)"); ax.set_ylabel("amplitude (full scale 1.0)")
|
||||
ax.set_title("waveform, fixed range +-0.5", fontsize=9); ax.grid(alpha=0.3)
|
||||
fig.tight_layout(); save(fig, out / f"{wav.stem}.wave.png")
|
||||
|
||||
|
||||
PLOSIVE_IPA = set("pbtdkgɡcɟqɢʔ")
|
||||
|
||||
|
||||
def plosives(script):
|
||||
"""(start, dur) of every plosive phoneme_gesture in a render's script.aqua (WordRender writes ipa=, no descriptor)."""
|
||||
out = []
|
||||
for line in script.read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("phoneme_gesture"):
|
||||
kv = dict(p.split("=", 1) for p in line.split()[1:] if "=" in p)
|
||||
if kv.get("ipa") in PLOSIVE_IPA: out.append((float(kv["start"]), float(kv["dur"])))
|
||||
return out
|
||||
|
||||
|
||||
def burst_note(root, stem):
|
||||
"""Notes for one render stem: the with-minus-without difference of out/<stem> and outnb/<stem>, per plosive."""
|
||||
script = root / "out" / stem / "script.aqua"
|
||||
if not script.exists():
|
||||
return {"burst": "not measured: no render of this word under out/"}
|
||||
ps = plosives(script)
|
||||
if not ps:
|
||||
return {"burst": "none: the word has no plosive"}
|
||||
twin = root / "outnb" / stem / f"{stem}.wav"
|
||||
if not twin.exists():
|
||||
return {"burst": "not measured: no --no-burst twin render of this word"}
|
||||
a, sr = read_wav(root / "out" / stem / f"{stem}.wav")
|
||||
b, _ = read_wav(twin)
|
||||
n = min(len(a), len(b)); d = np.abs(a[:n] - b[:n])
|
||||
rows = []
|
||||
for start, dur in ps:
|
||||
lo, hi = int(start * sr), min(n, int((start + dur + 0.12) * sr))
|
||||
peak = d[lo:hi].max() if hi > lo else 0.0
|
||||
hot = np.flatnonzero(d[lo:hi] > peak / 20) + lo if peak > 0 else np.array([], dtype=int)
|
||||
if not len(hot):
|
||||
rows.append({"closure_start_s": start, "release_s": None, "burst_end_s": None, "look_s": None, "note": "no difference between the renders"})
|
||||
continue
|
||||
end = hot[0]
|
||||
for i in hot[1:]: # the first burst: stop at a gap over 5 ms
|
||||
if i - end > 0.005 * sr: break
|
||||
end = i
|
||||
rel, fin = hot[0] / sr, end / sr
|
||||
rows.append({"closure_start_s": round(start, 4), "release_s": round(rel, 4), "burst_end_s": round(fin, 4),
|
||||
"difference_peak": round(float(peak), 4), "look_s": [round(max(0.0, rel - 0.03), 4), round(fin + 0.05, 4)]})
|
||||
return {"burst": rows}
|
||||
|
||||
|
||||
def render_of(root, clip):
|
||||
"""The out/<w> render a clip file was copied from (byte-identical wav), or None: two clip ids can be one render."""
|
||||
data = clip.read_bytes() if clip.exists() else None
|
||||
for d in sorted((root / "out").glob("*/")):
|
||||
w = d / f"{d.name}.wav"
|
||||
if data is not None and w.exists() and w.read_bytes() == data: return d.name
|
||||
return None
|
||||
|
||||
|
||||
def notes(kit, root, clips):
|
||||
out = []
|
||||
for c in kit["clips"]:
|
||||
stem = Path(c["file"]).stem
|
||||
if stem.endswith("_noburst"):
|
||||
out.append({"id": c["id"], "file": c["file"], "word": stem[:-8], "burst": "no burst expected"}); continue
|
||||
word = render_of(root, clips / c["file"]) or stem
|
||||
out.append({"id": c["id"], "file": c["file"], "word": word, **burst_note(root, word)})
|
||||
return out
|
||||
|
||||
|
||||
def notes_md(rows):
|
||||
lines = ["# Burst notes", "", "Times in seconds from the clip start. Release = start of the with-minus-without difference above 1/20 of its peak. Look in the spectrogram at 2-8 kHz inside the window.", ""]
|
||||
lines += ["| clip id | file | closure start | release | burst end | look in |", "|---|---|---|---|---|---|"]
|
||||
for r in rows:
|
||||
if isinstance(r["burst"], str):
|
||||
lines.append(f"| {r['id']} | {r['file']} | {r['burst']} | | | |"); continue
|
||||
for p in r["burst"]:
|
||||
look = f"{p['look_s'][0]} to {p['look_s'][1]}" if p.get("look_s") else p.get("note", "")
|
||||
lines.append(f"| {r['id']} | {r['file']} | {p['closure_start_s']} | {p['release_s']} | {p['burst_end_s']} | {look} |")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("clips"); ap.add_argument("images"); ap.add_argument("--kit"); ap.add_argument("--renders")
|
||||
a = ap.parse_args()
|
||||
clips, out = Path(a.clips), Path(a.images)
|
||||
if bool(a.kit) != bool(a.renders): ap.error("--kit and --renders go together")
|
||||
wavs = sorted(clips.glob("*.wav"))
|
||||
if not wavs: sys.exit(f"no .wav files in {clips}")
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for w in wavs: images(w, out)
|
||||
print(f"{len(wavs)} clips, {2 * len(wavs)} images in {out}")
|
||||
if a.kit:
|
||||
rows = notes(json.loads(Path(a.kit).read_text(encoding="utf-8")), Path(a.renders), clips)
|
||||
(out.parent / "burst-notes.json").write_text(json.dumps(rows, indent=1) + "\n", encoding="utf-8")
|
||||
(out.parent / "burst-notes.md").write_text(notes_md(rows), encoding="utf-8")
|
||||
print(f"burst notes for {len(rows)} clips in {out.parent}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -4,7 +4,8 @@
|
|||
# render.sh <repo-root> <out-root> [jobs]
|
||||
# Needs CULTLIB_ROOT at the admitted CultLib revision and AQUASYNTH_FAUST_HOME. Under load each render is minutes (a Faust compile).
|
||||
# <out-root>/out/<w>/<w>.wav <out-root>/outnb/<w>/<w>.wav (--no-burst) <out-root>/clips/*.wav the files kit.json names
|
||||
# Build the page: node tools/aqua-listen/build.mjs --manifest tools/aqua-listen/kits/tract-stops/kit.json --clips <out-root>/clips --out <out-root>/dist
|
||||
# <out-root>/images, burst-notes.* clip_images.py: PNGs per clip and the burst times (needs numpy + matplotlib: PYTHON=<python with both>)
|
||||
# Build the page: node tools/aqua-listen/build.mjs --manifest tools/aqua-listen/kits/tract-stops/kit.json --clips <out-root>/clips --out <out-root>/dist [--images <out-root>/images]
|
||||
set -eu
|
||||
REPO=$(cd "$1" && pwd); OUT=$2; JOBS=${3:-4}; HERE=$REPO/tools/aqua-listen/kits/tract-stops
|
||||
dotnet build "$REPO/tools/WordRender" -c Release -v q -o "$OUT/wr" | grep -E "rror\(s\)"
|
||||
|
|
@ -13,8 +14,9 @@ job() { n=$1; o=$2; f=$3; shift 3; dotnet "$OUT/wr/WordRender.dll" "$OUT/$o/$n"
|
|||
export -f job; export OUT HERE
|
||||
{
|
||||
for n in ata ada aka ala apa aba; do echo "$n out 140"; done
|
||||
for n in ata ada aka; do echo "$n outnb 140 --no-burst"; done
|
||||
for n in ata ada aka apa aba; do echo "$n outnb 140 --no-burst"; done # apa aba twins feed the burst notes only, no clip
|
||||
} | xargs -P "$JOBS" -L1 bash -c 'job "$@"' _
|
||||
C=$OUT/clips
|
||||
for n in ata ada aka ala apa aba; do cp "$OUT/out/$n/$n.wav" "$C/$n.wav"; done
|
||||
for n in ata ada aka; do cp "$OUT/outnb/$n/$n.wav" "$C/${n}_noburst.wav"; done
|
||||
${PYTHON:-python3} -I "$REPO/tools/aqua-listen/clip_images.py" "$C" "$OUT/images" --kit "$HERE/kit.json" --renders "$OUT"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
# <out-root>/out/<w>/<w>.wav 140 Hz renders; v_<vowel>_s is the vowel with primary stress, nibu has it on /i/
|
||||
# <out-root>/out148/v_<vowel> the unstressed vowel at 148.4 Hz (140 x 1.06: the f0 rise of primary stress alone)
|
||||
# <out-root>/outnb/<w> the same word with --no-burst (plosive release_flow splines dropped)
|
||||
# <out-root>/images, burst-notes.* clip_images.py: PNGs per clip and the burst times (needs numpy + matplotlib: PYTHON=<python with both>)
|
||||
# <out-root>/clips/*.wav the files kit.json names; weak_spots.py <out-root> measures the rest
|
||||
set -eu
|
||||
REPO=$(cd "$1" && pwd); OUT=$2; PRE=${3:-}; JOBS=${4:-4}; HERE=$REPO/tools/aqua-listen/kits/tract-weak-spots
|
||||
|
|
@ -15,7 +16,7 @@ job() { n=$1; o=$2; f=$3; shift 3; dotnet "$OUT/wr/WordRender.dll" "$OUT/$o/$n"
|
|||
export -f job; export OUT HERE
|
||||
{
|
||||
for n in nibu nibu_flat v_i v_i_s bin bun ba ama ana v_ih v_oo_short v_ah v_ah_s v_e v_e_s v_o v_o_s v_u v_u_s; do echo "$n out 140"; done
|
||||
for n in bin ba bun; do echo "$n outnb 140 --no-burst"; done
|
||||
for n in bin ba bun nibu nibu_flat; do echo "$n outnb 140 --no-burst"; done
|
||||
for v in i e ah o u ih oo_short; do echo "v_$v out148 148.4"; done
|
||||
} | xargs -P "$JOBS" -L1 bash -c 'job "$@"' _
|
||||
if [ -n "$PRE" ]; then
|
||||
|
|
@ -27,3 +28,4 @@ C=$OUT/clips
|
|||
cp "$OUT/out/nibu_flat/nibu_flat.wav" "$C/nibu_flat.wav"; cp "$OUT/out/nibu_flat/nibu_flat.wav" "$C/nibu_flat_after.wav" # one render, two ids: the before/after pair is one-to-one
|
||||
for n in nibu v_i v_i_s v_ih v_oo_short ama ana ba bun; do cp "$OUT/out/$n/$n.wav" "$C/$n.wav"; done
|
||||
for n in ba bun; do cp "$OUT/outnb/$n/$n.wav" "$C/${n}_noburst.wav"; done
|
||||
${PYTHON:-python3} -I "$REPO/tools/aqua-listen/clip_images.py" "$C" "$OUT/images" --kit "$HERE/kit.json" --renders "$OUT"
|
||||
|
|
|
|||
|
|
@ -101,6 +101,17 @@ test("build refuses a missing file and a manifest escape, naming them", () => {
|
|||
assert.throws(() => buildKit(mf([clip("a", { file: "../x.wav" })]), dir), /relative path/);
|
||||
assert.throws(() => buildKit(mf([clip("a", { title: "" })]), dir), /title is required/);
|
||||
});
|
||||
test("pictures ride in a collapsed block only for clips that have both, and only then does the policy admit data: images", () => {
|
||||
const pics = mkdtempSync(join(tmpdir(), "aqua-listen-img-")), png = Buffer.from("89504e470d0a1a0a", "hex");
|
||||
writeFileSync(join(pics, "a.spec.png"), png); writeFileSync(join(pics, "a.wave.png"), png); writeFileSync(join(pics, "b.spec.png"), png);
|
||||
const withPics = buildKit(manifest, dir, pics), data = JSON.parse(/<script id="kit-data" type="application\/json">(.*?)<\/script>/s.exec(withPics.html)[1]);
|
||||
assert.deepEqual(Object.keys(data.images), ["a"]);
|
||||
assert.deepEqual(data.images.a.map((p) => p.label), ["spectrogram", "waveform"]);
|
||||
assert.match(withPics.html, /media-src blob:; img-src data:/);
|
||||
const bare = buildKit(manifest, dir);
|
||||
assert.doesNotMatch(bare.html, /img-src/); assert.deepEqual(JSON.parse(/<script id="kit-data" type="application\/json">(.*?)<\/script>/s.exec(bare.html)[1]).images, {});
|
||||
assert.notEqual(withPics.hash, bare.hash);
|
||||
});
|
||||
test("markup in a title cannot break the page", () => {
|
||||
const m = mf([clip("a", { title: "</script><b>" , feel: "</script>"})]);
|
||||
const h = buildKit(m, dir).html;
|
||||
|
|
|
|||
67
tools/aqua-listen/test_clip_images.py
Normal file
67
tools/aqua-listen/test_clip_images.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
"""python3 -I test_clip_images.py (numpy + matplotlib; synthetic clips, no render needed)"""
|
||||
import hashlib, json, sys, tempfile, unittest, wave
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import clip_images as ci
|
||||
|
||||
SR, BURST_AT = 48000, 0.30
|
||||
|
||||
|
||||
def write(path, x):
|
||||
with wave.open(str(path), "wb") as w:
|
||||
w.setnchannels(1); w.setsampwidth(2); w.setframerate(SR)
|
||||
w.writeframes((np.clip(x, -1, 1) * 32767).astype("<i2").tobytes())
|
||||
|
||||
|
||||
def word(with_burst):
|
||||
t = np.arange(int(0.6 * SR)) / SR
|
||||
x = 0.2 * np.sin(2 * np.pi * 300 * t) * (np.abs(t - 0.3) > 0.05) # a vowel with a gap round the release
|
||||
if with_burst:
|
||||
i = int(BURST_AT * SR); n = int(0.012 * SR)
|
||||
x[i:i + n] += np.random.default_rng(1).normal(0, 0.1, n) * np.hanning(n)
|
||||
return x
|
||||
|
||||
|
||||
class Fixture(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp()); self.clips = self.tmp / "clips"; self.clips.mkdir()
|
||||
write(self.clips / "ba.wav", word(True)); write(self.clips / "ba_noburst.wav", word(False))
|
||||
self.root = self.tmp / "r"
|
||||
for sub, wb in (("out", True), ("outnb", False)):
|
||||
(self.root / sub / "ba").mkdir(parents=True); write(self.root / sub / "ba" / "ba.wav", word(wb))
|
||||
(self.root / "out" / "ba" / "script.aqua").write_text("phoneme_gesture name=ba_0_ɑ ipa=ɑ start=0 dur=0.2 intensity=1\nphoneme_gesture name=ba_1_b ipa=b start=0.2 dur=0.095 intensity=1\n")
|
||||
self.kit = self.tmp / "kit.json"
|
||||
self.kit.write_text(json.dumps({"clips": [{"id": "x-ba", "file": "ba.wav"}, {"id": "x-ba-nb", "file": "ba_noburst.wav"}]}))
|
||||
|
||||
def run_tool(self, out, *extra):
|
||||
sys.argv = ["clip_images", str(self.clips), str(out), *extra]; ci.main()
|
||||
|
||||
def test_images_are_deterministic_and_complete(self):
|
||||
a, b = self.tmp / "a" / "images", self.tmp / "b" / "images"
|
||||
self.run_tool(a); self.run_tool(b)
|
||||
names = sorted(p.name for p in a.iterdir())
|
||||
self.assertEqual(names, ["ba.spec.png", "ba.wave.png", "ba_noburst.spec.png", "ba_noburst.wave.png"])
|
||||
for n in names:
|
||||
self.assertEqual(hashlib.sha256((a / n).read_bytes()).hexdigest(), hashlib.sha256((b / n).read_bytes()).hexdigest(), n)
|
||||
|
||||
def test_notes_find_the_release_and_the_twin_says_none(self):
|
||||
self.run_tool(self.tmp / "o" / "images", "--kit", str(self.kit), "--renders", str(self.root))
|
||||
rows = json.loads((self.tmp / "o" / "burst-notes.json").read_text())
|
||||
p = rows[0]["burst"][0]
|
||||
self.assertAlmostEqual(p["release_s"], BURST_AT, delta=0.003)
|
||||
self.assertGreater(p["burst_end_s"], BURST_AT + 0.004); self.assertLess(p["burst_end_s"], BURST_AT + 0.02)
|
||||
self.assertEqual(rows[1]["burst"], "no burst expected")
|
||||
self.assertIn("no burst expected", (self.tmp / "o" / "burst-notes.md").read_text())
|
||||
|
||||
def test_burst_shows_in_the_spectrogram_at_the_noted_time_only(self):
|
||||
def hi_db(x, at):
|
||||
db, f, t = ci.stft_db(x, SR); col = int(np.argmin(np.abs(t - at)))
|
||||
lin = 10 ** (db[f > 2000, col] / 20); return 20 * np.log10(np.sqrt((lin ** 2).sum()) + 1e-12)
|
||||
w, nb = word(True), word(False)
|
||||
self.assertGreaterEqual(hi_db(w, BURST_AT + 0.006) - hi_db(nb, BURST_AT + 0.006), 10)
|
||||
self.assertLess(hi_db(w, BURST_AT + 0.106) - hi_db(nb, BURST_AT + 0.106), 10) # a note 100 ms late would find nothing
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
// Page glue. The kit (manifest, content hash, base64 audio) is in #kit-data; logic is AquaListen (logic.cjs, inlined).
|
||||
// Page glue. The kit (manifest, content hash, base64 audio, optional base64 PNGs per clip id) is in #kit-data; logic is AquaListen (logic.cjs, inlined).
|
||||
(() => {
|
||||
const L = globalThis.AquaListen, kit = JSON.parse(document.getElementById("kit-data").textContent);
|
||||
const app = document.getElementById("app"), KEY = `aqua-listen:${kit.manifest.kit.id}:${kit.hash}`;
|
||||
|
|
@ -62,6 +62,7 @@
|
|||
h("h2", {}, o.slot ? `Sound ${o.slot}` : `Sound ${n}`),
|
||||
h("p", { class: "feel" }, `It should feel: ${c.feel}`),
|
||||
audio,
|
||||
(kit.images[id] || []).length ? h("details", {}, h("summary", {}, "Pictures of this sound (optional)"), kit.images[id].map((p) => h("img", { src: "data:image/png;base64," + p.b64, alt: p.label, class: "pic" }))) : [],
|
||||
h("div", { class: "row" }, h("span", {}, "Keep, fix or drop?"), radios("v-" + id, L.VERDICTS, a.verdict, (v) => touch(id, { verdict: v }))),
|
||||
h("div", { class: "row" }, h("span", {}, "Did it land? 1 = not at all, 5 = completely"), radios("l-" + id, [1, 2, 3, 4, 5], a.landed, (v) => touch(id, { landed: Number(v) }))),
|
||||
reason);
|
||||
|
|
|
|||
|
|
@ -16,3 +16,4 @@ button:disabled { opacity: 0.4; cursor: default; }
|
|||
.bar { position: sticky; top: 0; background: var(--bg); padding: 0.5rem 0; border-bottom: 1px solid var(--line); z-index: 1; }
|
||||
.pair { border: 2px dashed var(--line); border-radius: 10px; padding: 0 0.75rem; margin: 1rem 0; }
|
||||
small, .hint { opacity: 0.75; }
|
||||
img.pic { max-width: 100%; height: auto; display: block; margin: 0.5rem 0; }
|
||||
|
|
|
|||
Loading…
Reference in a new issue