Test: Soul s2 findings as behavioural tests (id suffix burst, FIFO listed as BVH, work slot and sweep)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/fixtures/stub-worker.mjs

@@ -1,11 +1,11 @@
 // Stands in for kimodo-resident-worker.py in the daemon's tests. It binds port 0, prints the ready line the
 // daemon waits for, and answers /health and /generate with what the daemon gave it. The --model value picks a
 // misbehaviour: crash-before-ready, hang-before-ready, sick, slow-health, stubborn, fail-generate,
-// refuse-generate, drop-generate, slow-generate, writer (appends to <output>.writes every 50 ms, never answers, ignores SIGTERM), slow-to-stop (no answer; prints a line and exits 1.5 s after SIGTERM), chatty (prints a line the moment it is healthy), encoder (a grandchild, then no answer), graceful (exits 0.5 s after SIGTERM), die-after-ready, bvh-outside (lists a BVH outside the job's directory), bvh-link (lists a symlink as its BVH), plant-link (a normal answer with a file symlink and a directory symlink planted beside the BVH). A request with bvh writes <output>.bvh (motion/motion_NN.bvh for several samples) and lists it. Any other value is just the model the worker serves.
+// refuse-generate, drop-generate, slow-generate, writer (appends to <output>.writes every 50 ms, never answers, ignores SIGTERM), slow-to-stop (no answer; prints a line and exits 1.5 s after SIGTERM), chatty (prints a line the moment it is healthy), encoder (a grandchild, then no answer), graceful (exits 0.5 s after SIGTERM), die-after-ready, bvh-outside (lists a BVH outside the job's directory), bvh-link (lists a symlink as its BVH), bvh-fifo (lists a FIFO as its BVH), plant-link (a normal answer with a file symlink and a directory symlink planted beside the BVH). A request with bvh writes <output>.bvh (motion/motion_NN.bvh for several samples) and lists it. Any other value is just the model the worker serves.
 import http from "node:http";
 import os from "node:os";
 import path from "node:path";
-import { spawn } from "node:child_process";
+import { execFileSync, spawn } from "node:child_process";
 import { appendFileSync, mkdirSync, symlinkSync, writeFileSync } from "node:fs";
 
 const argv = process.argv.slice(2);
@@ -87,6 +87,9 @@ if (model === "hang-before-ready") {
           if (model === "bvh-outside") {
             writeFileSync(outside, "BVH-OUTSIDE\n");
             files = [outside];
+          } else if (model === "bvh-fifo") {
+            execFileSync("mkfifo", [`${payload.output}.bvh`]);
+            files = [`${payload.output}.bvh`];
           } else if (model === "bvh-link") {
             writeFileSync(outside, "BVH-OUTSIDE\n");
             symlinkSync(outside, `${payload.output}.bvh`);

scripts/test-vili-daemon.mjs

@@ -1869,3 +1869,110 @@ test("the job record lists regular files only: a planted symlink, to a file or t
     await daemon.stop();
   }
 });
+
+// Soul s2: the job id's random suffix is what carries a burst. The clock alone repeats inside one millisecond, and
+// the exclusive mkdir's few retries are spent there, so a constant suffix fails a burst and passes every small test.
+test("four hundred simultaneous generates each get their own job id, directory and record", async () => {
+  const daemon = await startDaemon();
+  try {
+    const answers = await Promise.all(Array.from({ length: 400 }, () => generate(daemon)));
+    assert.deepEqual([...new Set(answers.map((answer) => answer.status))], [200]);
+    const ids = answers.map((answer) => answer.body.jobId);
+    for (const id of ids) assert.match(id, /^vili-\d{13}-[0-9a-f]{8}$/);
+    assert.equal(new Set(ids).size, 400, "no two jobs share an id");
+    assert.equal(readdirSync(path.join(daemon.stateRoot, "artifacts")).length, 400);
+    assert.equal(jobFiles(daemon).length, 400);
+  } finally {
+    await daemon.stop();
+  }
+});
+
+// Soul s2: opening a path that blocks (a FIFO with no writer) must not wait. The open is non-blocking and the check
+// is made on the open descriptor, so the listed node is refused at once and the filesystem threads stay free.
+test("a listed BVH that is a FIFO is refused promptly and other file work proceeds", async () => {
+  const daemon = await startDaemon();
+  const within = (promise, ms, what) => Promise.race([
+    promise,
+    new Promise((_, reject) => setTimeout(() => reject(new Error(`${what} did not settle in ${ms} ms`)), ms)),
+  ]);
+  try {
+    // more than libuv's four filesystem threads: a blocking open per request would starve all of them
+    const answers = await within(Promise.all(Array.from({ length: 6 }, () =>
+      generate(daemon, exportRequest({ model: "bvh-fifo", clipName: "canary-fifo-clip" })))), 8000, "the FIFO jobs");
+    for (const answer of answers) {
+      assert.equal(answer.status, 502);
+      assert.equal(answer.body.ok, false);
+      assert.ok(!answer.raw.includes("canary-fifo-clip"), "the refusal does not echo the clip name");
+    }
+    assert.equal(exportStarts(daemon).length, 0, "no Blender for a node that is not a regular file");
+    const other = await within(generate(daemon, exportRequest({ clipName: "after" })), 8000, "an ordinary job");
+    assert.equal(other.status, 200);
+    assert.ok(!daemon.stdout().includes("canary-fifo-clip") && !daemon.stderr().includes("canary-fifo-clip"));
+  } finally {
+    await daemon.stop();
+  }
+});
+
+// Soul s2: work/ is a declared state slot, and a daemon that holds the state root sweeps the export directories a
+// killed daemon left, those and nothing else.
+test("the recipe declares the work directory as a state slot", () => {
+  const recipe = readFileSync(path.join(here, "..", "deployment", "idunn", "recipe.toml"), "utf8");
+  assert.match(recipe, /\[\[state\.slots\]\]\nid = "work"\nrelative_path = "work"\nkind = "external-directory"/);
+});
+
+test("a restarted daemon sweeps the export directories a killed one left, and touches nothing else", async () => {
+  const first = await startDaemon();
+  let second = null;
+  try {
+    const pending = generate(first, exportRequest({ clipName: "slow" })).catch(() => null);
+    await until(() => exportStarts(first).length === 1, "the export to start");
+    const work = path.join(first.stateRoot, "work");
+    const [stale] = readdirSync(work);
+    assert.match(stale, /^export-[A-Za-z0-9]{6}$/);
+    first.proc.kill("SIGKILL");
+    await until(() => first.proc.signalCode !== null, "the daemon to die");
+    await pending;
+    assert.deepEqual(readdirSync(work), [stale], "the killed export left its directory");
+    // things under the state root that are not stale export directories
+    const decoys = {
+      "work/keep": "K",
+      "work/export-short/in.bvh": "S",
+      "work/exports-keep/in.bvh": "E",
+      "jobs/decoy.json": "J",
+      "artifacts/decoy/motion.bvh": "A",
+      "logs/decoy.log": "L",
+      "decoy-root-file": "R",
+    };
+    for (const [relative, text] of Object.entries(decoys)) {
+      mkdirSync(path.dirname(path.join(first.stateRoot, relative)), { recursive: true });
+      writeFileSync(path.join(first.stateRoot, relative), text);
+    }
+    second = await startDaemon({ root: first.root });
+    assert.deepEqual(readdirSync(work).sort(), ["export-short", "exports-keep", "keep"], "the stale export is gone, the rest stays");
+    for (const [relative, text] of Object.entries(decoys)) assert.equal(readFileSync(path.join(second.stateRoot, relative), "utf8"), text, relative);
+    assert.equal((await generate(second, exportRequest({ clipName: "after" }))).status, 200);
+  } finally {
+    if (first.proc.signalCode === null) first.proc.kill("SIGKILL");
+    await (second ?? first).stop();
+  }
+});
+
+test("a daemon that does not hold the state root sweeps nothing", async () => {
+  const first = await startDaemon();
+  let second = null;
+  try {
+    const pending = generate(first, exportRequest({ clipName: "slow" }));
+    await until(() => exportStarts(first).length === 1, "the export to start");
+    const work = path.join(first.stateRoot, "work");
+    const live = readdirSync(work);
+    assert.equal(live.length, 1);
+    second = await startDaemon({ root: first.root, hold: false });
+    await until(() => second.proc.exitCode !== null, "the second daemon to refuse", 20);
+    assert.equal(second.proc.exitCode, 2);
+    assert.deepEqual(readdirSync(work), live, "the live export's directory is still there");
+    assert.equal((await pending).status, 200, "the live export finished");
+  } finally {
+    if (second && second.proc.exitCode === null && second.proc.signalCode === null) second.proc.kill("SIGKILL");
+    await first.stop();
+  }
+});