From 16e585d0b9c4af5816bd87ecafcfb41b8c2ca7c6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 18:32:26 +0100 Subject: [PATCH] Constraints given twice are a 400, so the record names the one input the worker read The worker reads one path; with inline constraints and constraints_path both sent, the daemon preferred the path but the record kept the inline list. Two inputs are now refused before anything is written, in the daemon's fixed-text 400 shape, echoing nothing. Co-Authored-By: Claude Opus 5.5 --- scripts/test-vili-daemon.mjs | 4140 +++++++++++++++++----------------- scripts/vili-daemon.mjs | 2479 ++++++++++---------- 2 files changed, 3325 insertions(+), 3294 deletions(-) diff --git a/scripts/test-vili-daemon.mjs b/scripts/test-vili-daemon.mjs index 0e827c9..7c9af59 100644 --- a/scripts/test-vili-daemon.mjs +++ b/scripts/test-vili-daemon.mjs @@ -1,2058 +1,2082 @@ -// node --test scripts/test-vili-daemon.mjs -// Runs the real daemon against scripts/fixtures/stub-worker.mjs as its --python/--worker-entry. -import assert from "node:assert/strict"; -import { spawn } from "node:child_process"; -import { createHash } from "node:crypto"; -import { chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; -import http from "node:http"; -import net from "node:net"; -import os from "node:os"; -import path from "node:path"; -import test from "node:test"; -import { fileURLToPath } from "node:url"; -import { decode, encode } from "@msgpack/msgpack"; - -const here = path.dirname(fileURLToPath(import.meta.url)); -const daemonPath = path.join(here, "vili-daemon.mjs"); -const stubPath = path.join(here, "fixtures", "stub-worker.mjs"); -const stubBlenderPath = path.join(here, "fixtures", "stub-blender.mjs"); -const exportScriptPath = path.join(here, "bvh-to-fbx.py"); -const fakeIdunnEntry = path.join(here, "fixtures", "fake-idunn-entry.mjs"); -const THREADS = 5; -// The stub Blender is executed, and the recipe's runner mounts /tmp noexec (and the root read-only), so it cannot live -// there. It lives under the checkout's gitignored .vili, which the runner's workspace leaves writable. -const execScratch = path.join(here, "..", ".vili", "test-scratch"); - -async function until(condition, what, seconds = 15) { - const deadline = Date.now() + seconds * 1000; - for (;;) { - const value = await condition(); - if (value) return value; - assert.ok(Date.now() < deadline, `timed out waiting for ${what}`); - await new Promise((resolve) => setTimeout(resolve, 25)); - } -} - -// A zombie is dead for our purposes: its parent (or init) has simply not reaped it yet. -function isAlive(pid) { - try { - process.kill(pid, 0); - } catch { - return false; - } - try { - return !/^\d+ \(.*\) Z/.test(readFileSync(`/proc/${pid}/stat`, "utf8")); - } catch { - return false; - } -} - -async function startDaemon({ - omit = [], viaEnv = false, threads = String(THREADS), python = process.execPath, nested = false, host = "127.0.0.1", - relativeModels = false, extraArgs = [], root: givenRoot = null, entry = daemonPath, extraEnv = {}, hold = entry === daemonPath, - blender = "stub", -} = {}) { - const root = givenRoot ?? mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const stateRoot = nested ? path.join(root, "a", "b", "state") : root; - const modelsRoot = path.join(root, "models"); - const settings = { - host, - port: "0", - "state-root": stateRoot, - python, - "worker-entry": stubPath, - "models-root": relativeModels ? "models-rel" : modelsRoot, - "kimodo-commit": "abc1234", - "python-env": "test-env", - }; - if (threads !== null) settings.threads = threads; - // Blender is a copy of the stub in a directory of its own (the stub logs its starts beside itself): "stub" is - // ready, "broken" fails its version probe, "missing" is a path with nothing there, null is unconfigured. - mkdirSync(execScratch, { recursive: true }); - const blenderDir = mkdtempSync(path.join(execScratch, "blender-")); - const blenderPath = blender === null ? null : path.join(blenderDir, { stub: "blender", broken: "blender-broken", missing: "absent" }[blender]); - if (blender === "stub" || blender === "broken") { - copyFileSync(stubBlenderPath, blenderPath); - chmodSync(blenderPath, 0o755); - } - if (blenderPath) settings.blender = blenderPath; - for (const name of omit) delete settings[name]; - const env = { ...process.env, LEAK_CHECK: "leaked", LANG: "xx_TEST.UTF-8", ...extraEnv }; - for (const name of Object.keys(env)) if (name.startsWith("VILI_")) delete env[name]; - const argv = [entry, ...extraArgs]; - for (const [name, value] of Object.entries(settings)) { - if (viaEnv) env[`VILI_${{ host: "DAEMON_HOST", port: "DAEMON_PORT", "state-root": "STATE_ROOT", "worker-entry": "WORKER_ENTRY" }[name] ?? name.toUpperCase().replace("-", "_")}`] = value; - else argv.push(`--${name}`, value); - } - const proc = spawn(process.execPath, argv, { env, cwd: root, stdio: ["ignore", "pipe", "pipe"] }); - let output = ""; - let errors = ""; - proc.stdout.on("data", (chunk) => { output += chunk; }); - proc.stderr.on("data", (chunk) => { errors += chunk; }); - const listening = await until(() => /listening on http:\/\/([^:]+):(\d+)/.exec(output), "the daemon to listen"); - const port = listening[2]; - // the hand-run daemon takes its lock after it listens; a test that wants a served state root waits for it - if (hold) await until(() => /holds the state root/.test(output), "the daemon to hold the state root"); - const spawnLog = () => { - const log = path.join(stateRoot, "embeddings", "stub-spawns.log"); - return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean) : []; - }; - const startsLog = path.join(blenderDir, "starts.log"); - const startLines = () => (existsSync(startsLog) ? readFileSync(startsLog, "utf8").split("\n").filter(Boolean) : []); - return { - root, - stateRoot, - modelsRoot, - blenderDir, - blenderPath, - // every start of the stub Blender, the version probe included - starts: () => startLines().filter((line) => line.startsWith("{")).map((line) => JSON.parse(line)), - blenderGrandchildren: () => startLines().filter((line) => line.startsWith("grandchild ")).map((line) => Number(line.split(" ")[1])), - proc, - boundHost: listening[1], - port: Number(port), - stderr: () => errors, - stdout: () => output, - spawns: () => spawnLog().filter((line) => /^\d+$/.test(line)).map(Number), - grandchildren: () => spawnLog().filter((line) => line.startsWith("grandchild ")).map((line) => Number(line.split(" ")[1])), - async stop() { - try { - if (proc.exitCode === null && proc.signalCode === null) { - proc.kill("SIGTERM"); - await until(() => proc.exitCode !== null || proc.signalCode !== null, "the daemon to exit", 30); - } - } finally { - // a daemon that did not stop on SIGTERM must not outlive the test and hold the run open - if (proc.exitCode === null && proc.signalCode === null) proc.kill("SIGKILL"); - rmSync(root, { recursive: true, force: true }); - rmSync(blenderDir, { recursive: true, force: true }); - } - }, - }; -} - -function request(port, method, urlPath, { body, headers = {} } = {}) { - return new Promise((resolve, reject) => { - const req = http.request({ host: "127.0.0.1", port, method, path: urlPath, headers }, (res) => { - const chunks = []; - res.on("data", (chunk) => chunks.push(chunk)); - res.on("end", () => { - const raw = Buffer.concat(chunks).toString("utf8"); - let parsed = null; - try { - parsed = JSON.parse(raw); - } catch { - // not JSON - } - resolve({ status: res.statusCode, headers: res.headers, raw, body: parsed }); - }); - }); - req.on("error", reject); - req.end(body); - }); -} - -const generate = (daemon, payload = { prompt: "a person waves", duration: 2 }, headers = {}) => - request(daemon.port, "POST", "/motion/generate", { - body: typeof payload === "string" ? payload : JSON.stringify(payload), - headers: { "content-type": "application/json", ...headers }, - }); -const operatorState = async (daemon) => (await request(daemon.port, "GET", "/operator-state")).body; -const backendOf = async (daemon) => (await operatorState(daemon)).backend; - -test("the worker's port comes from its ready line", async () => { - const daemon = await startDaemon(); - try { - const answer = await generate(daemon); - assert.equal(answer.status, 200); - const backend = await backendOf(daemon); - assert.equal(backend.state, "ready"); - assert.notEqual(daemon.port, 8824, "--port 0 binds a free port"); - assert.ok(Number.isInteger(backend.port) && backend.port > 1 && backend.port !== 8825); - assert.equal(answer.body.worker.pid, backend.pid); - assert.deepEqual(answer.body.worker.argv.slice(-4), ["--host", "127.0.0.1", "--port", "0"]); - // the request reached the stub on the port the stub printed, not on a decoy line's port - assert.equal((await request(backend.port, "GET", "/health")).body.requests, 1); - } finally { - await daemon.stop(); - } -}); - -test("one child, respawned after exit", async () => { - const daemon = await startDaemon(); - try { - const [first, second] = await Promise.all([generate(daemon), generate(daemon)]); - assert.equal(first.status, 200); - assert.equal(second.status, 200); - assert.equal(daemon.spawns().length, 1); - const [pid] = daemon.spawns(); - process.kill(pid, "SIGKILL"); - await until(async () => (await backendOf(daemon)).state === "stopped", "the daemon to see the exit"); - assert.match((await backendOf(daemon)).lastError, /^the worker exited \(SIGKILL\)$/); - const third = await generate(daemon); - assert.equal(third.status, 200); - const spawns = daemon.spawns(); - assert.equal(spawns.length, 2); - assert.notEqual(spawns[1], pid); - assert.equal(third.body.worker.pid, spawns[1]); - assert.equal((await backendOf(daemon)).lastError, null); - } finally { - await daemon.stop(); - } -}); - -test("environment and nice", async () => { - const daemon = await startDaemon(); - try { - const { env, priority } = (await generate(daemon)).body.worker; - assert.equal(env.OMP_NUM_THREADS, String(THREADS)); - assert.equal(env.MKL_NUM_THREADS, String(THREADS)); - assert.equal(env.VILI_THREADS, String(THREADS)); - assert.equal(env.HF_HUB_OFFLINE, "1"); - assert.equal(env.TRANSFORMERS_OFFLINE, "1"); - assert.equal(env.HF_HOME, daemon.modelsRoot); - assert.equal(env.VILI_KIMODO_COMMIT, "abc1234"); - assert.equal(env.VILI_PYTHON_ENV, "test-env"); - assert.equal(env.PYTHONDONTWRITEBYTECODE, "1"); - assert.equal(env.PYTHONUNBUFFERED, "1"); - assert.equal(env.PATH, process.env.PATH); - assert.equal(env.HOME, process.env.HOME); - assert.equal(env.LANG, "xx_TEST.UTF-8"); - assert.ok(env.PYTHONPATH.endsWith(path.join("vendor", "CultLib", "packages", "cultcache-py", "src"))); - assert.equal(env.LEAK_CHECK, undefined, "the daemon's own environment does not reach the worker"); - assert.equal(priority, 10); - } finally { - await daemon.stop(); - } -}); - -test("every flag has an environment twin, and the thread cap defaults to 8", async () => { - const viaEnv = await startDaemon({ viaEnv: true, threads: "16" }); - try { - const answer = await generate(viaEnv); - assert.equal(answer.status, 200); - assert.equal(answer.body.worker.env.OMP_NUM_THREADS, "16"); - assert.deepEqual(await backendOf(viaEnv), { - state: "ready", pid: answer.body.worker.pid, port: (await backendOf(viaEnv)).port, threads: 16, pythonEnv: "test-env", - kimodoCommit: "abc1234", modelsRoot: viaEnv.modelsRoot, lastError: null, blender: "Blender 0.0.0 (stub)", - }); - } finally { - await viaEnv.stop(); - } - const byDefault = await startDaemon({ threads: null }); - try { - const answer = await generate(byDefault); - assert.equal(answer.body.worker.env.OMP_NUM_THREADS, "8"); - assert.equal((await backendOf(byDefault)).threads, 8); - } finally { - await byDefault.stop(); - } - const one = await startDaemon({ threads: "1", nested: true }); - try { - assert.equal((await generate(one)).body.worker.env.OMP_NUM_THREADS, "1"); - } finally { - await one.stop(); - } -}); - -test("the worker's output lands in its log, and the ready line does not", async () => { - const daemon = await startDaemon(); - try { - assert.equal((await generate(daemon)).status, 200); - const log = path.join(daemon.stateRoot, "logs", "worker.log"); - const text = await until(() => { - const content = existsSync(log) ? readFileSync(log, "utf8") : ""; - return content.includes("stub-stdout-line") && content.includes("stub-stderr-line") ? content : null; - }, "the worker log"); - assert.ok(!text.includes(`"port":${(await backendOf(daemon)).port}`), "the real ready line is not logged"); - } finally { - await daemon.stop(); - } -}); - -test("a job is a record on disk", async () => { - const daemon = await startDaemon(); - try { - const answer = await generate(daemon, { prompt: "a person waves", duration: 2, constraints: [{ type: "root2d" }] }); - assert.equal(answer.status, 200); - const job = answer.body; - assert.equal(job.schema, "gamecult.vili.motion_job.v1"); - assert.equal(job.request.schema, "gamecult.vili.motion_request.v0"); - assert.equal(job.prompt, "a person waves"); - assert.equal(job.model, "stub-model"); - assert.equal(job.ok, true); - assert.equal(job.error, null); - assert.equal(job.artifactDir, path.join(daemon.stateRoot, "artifacts", job.jobId)); - assert.equal(job.worker.output, path.join(job.artifactDir, "motion")); - assert.equal(job.worker.contentType, "application/json"); - assert.equal(job.worker.prompt, "a person waves"); - assert.equal(job.worker.constraints, path.join(job.artifactDir, "constraints.json")); - assert.deepEqual(JSON.parse(readFileSync(job.worker.constraints, "utf8")), [{ type: "root2d" }]); - assert.deepEqual(JSON.parse(readFileSync(path.join(daemon.stateRoot, "jobs", `${job.jobId}.json`), "utf8")), job); - assert.ok(Number.isInteger(job.seed) && job.seed >= 0 && job.seed < 2 ** 31, "no seed was sent, so one was minted"); - assert.deepEqual(job.files.map((file) => file.name), ["constraints.json", "meta.json", "request.json"]); - for (const file of job.files) { - const bytes = readFileSync(path.join(job.artifactDir, file.name)); - assert.equal(file.bytes, bytes.length); - assert.equal(file.sha256, createHash("sha256").update(bytes).digest("hex")); - } - assert.equal(job.export, undefined, "a job that asked for no FBX has no export block"); - const batch = await generate(daemon, { texts: ["a", "b"], durations: [1, 1] }); - assert.equal(batch.status, 200); - assert.equal(batch.body.prompt, null); - assert.equal(batch.body.worker.prompt, undefined); - } finally { - await daemon.stop(); - } -}); - -test("a requested model restarts the worker only when it is not the one serving", async () => { - const daemon = await startDaemon(); - try { - assert.equal((await generate(daemon)).status, 200); - assert.equal((await generate(daemon, { prompt: "x", model: "stub-model" })).status, 200); - assert.equal((await generate(daemon, { prompt: "x", model: "Stub Model" })).status, 200); - assert.equal(daemon.spawns().length, 1); - const [first] = daemon.spawns(); - const other = await generate(daemon, { prompt: "x", model: "other-model" }); - assert.equal(other.status, 200); - assert.equal(other.body.model, "other-model"); - assert.deepEqual(other.body.worker.argv.slice(-2), ["--model", "other-model"]); - assert.equal(daemon.spawns().length, 2); - await until(() => !isAlive(first), "the first worker to be gone"); - assert.equal((await generate(daemon)).status, 200); - assert.equal(daemon.spawns().length, 2, "a request that names no model keeps the one serving"); - } finally { - await daemon.stop(); - } -}); - -for (const signal of ["SIGTERM", "SIGINT"]) { - test(`${signal} leaves no child`, async () => { - const daemon = await startDaemon(); - try { - assert.equal((await generate(daemon)).status, 200); - const [pid] = daemon.spawns(); - assert.ok(isAlive(pid)); - daemon.proc.kill(signal); - await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 8); - assert.equal(isAlive(pid), false); - } finally { - await daemon.stop(); - } - }); -} - -test("a worker that is stopping gets its grace, not a SIGKILL", async () => { - const daemon = await startDaemon(); - try { - assert.equal((await generate(daemon, { prompt: "x", model: "graceful" })).status, 200); - daemon.proc.kill("SIGTERM"); - await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 8); - assert.ok(readFileSync(path.join(daemon.stateRoot, "embeddings", "stub-spawns.log"), "utf8").includes("graceful-exit")); - } finally { - await daemon.stop(); - } -}); - -test("a worker that dies while the daemon waits for its health fails the job at once", async () => { - const daemon = await startDaemon(); - try { - const began = Date.now(); - const failed = await generate(daemon, { prompt: "x", model: "die-after-ready" }); - assert.equal(failed.status, 502); - assert.ok(Date.now() - began < 20000, "the default deadline is minutes; a dead worker does not wait for it"); - const backend = await backendOf(daemon); - assert.equal(backend.state, "stopped"); - assert.equal(backend.lastError, "the worker exited (code 4)"); - } finally { - await daemon.stop(); - } -}); - -test("the daemon binds loopback unless told otherwise, and ignores flags it does not know", async () => { - const byDefault = await startDaemon({ omit: ["host"], extraArgs: ["--bogus"] }); - try { - assert.equal(byDefault.boundHost, "127.0.0.1"); - assert.equal((await request(byDefault.port, "GET", "/health")).status, 200); - } finally { - await byDefault.stop(); - } - const named = await startDaemon({ host: "127.0.0.2" }); - try { - assert.equal(named.boundHost, "127.0.0.2"); - assert.equal((await fetch(`http://127.0.0.2:${named.port}/health`)).status, 200); - } finally { - await named.stop(); - } -}); - -test("a relative models root reaches the worker as an absolute path", async () => { - const daemon = await startDaemon({ relativeModels: true }); - try { - const { env } = (await generate(daemon)).body.worker; - assert.equal(env.HF_HOME, path.join(path.dirname(daemon.modelsRoot), "models-rel")); - } finally { - await daemon.stop(); - } -}); - -test("a worker that ignores SIGTERM is killed with everything it started", async () => { - const daemon = await startDaemon(); - try { - assert.equal((await generate(daemon, { prompt: "x", model: "stubborn" })).status, 200); - const [pid] = daemon.spawns(); - const [grandchild] = daemon.grandchildren(); - assert.ok(isAlive(pid) && isAlive(grandchild)); - daemon.proc.kill("SIGTERM"); - await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 30); - assert.equal(isAlive(pid), false); - assert.equal(isAlive(grandchild), false); - } finally { - await daemon.stop(); - } -}); - -// The daemon owns the worker's whole tree (motion-pipeline:finding:cut-vili-cpu-body.s1.orphaned-descendants). -const aliveWorkers = (daemon) => daemon.spawns().filter(isAlive); - -test("a worker killed mid-request leaves no encoder child, and the next job has exactly one worker", async () => { - const daemon = await startDaemon(); - try { - const pending = generate(daemon, { prompt: "x", model: "encoder" }); - const [grandchild] = await until(() => { const found = daemon.grandchildren(); return found.length ? found : null; }, "the encoder child"); - const [worker] = daemon.spawns(); - assert.ok(isAlive(worker) && isAlive(grandchild)); - process.kill(worker, "SIGKILL"); - assert.equal(readFileSync(path.join(daemon.stateRoot, "worker.pid"), "utf8"), `${worker}\n`); - assert.equal((await pending).status, 502); - await until(() => !isAlive(grandchild), "the encoder child to die with its worker"); - await until(() => !existsSync(path.join(daemon.stateRoot, "worker.pid")), "the dead worker's pid file to go"); - assert.equal((await generate(daemon, { prompt: "y" })).status, 200); - assert.equal(aliveWorkers(daemon).length, 1); - assert.equal(readFileSync(path.join(daemon.stateRoot, "worker.pid"), "utf8"), `${aliveWorkers(daemon)[0]}\n`); - assert.equal(isAlive(grandchild), false); - } finally { - await daemon.stop(); - } -}); - -// "encoder" leaves a SIGTERM-proof grandchild beside the worker; "stubborn" is a worker that ignores SIGTERM; the plain stub is just a worker. -for (const model of ["encoder", "stubborn", "stub-model"]) { - test(`a restarted daemon kills the ${model} worker its crashed predecessor left, and runs exactly one`, async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const first = await startDaemon({ root }); - const stray = generate(first, { prompt: "x", model }).catch(() => null); - await until(() => first.spawns().length, "the worker"); - if (model === "encoder") await until(() => first.grandchildren().length, "the encoder child"); - await until(() => request(first.port, "GET", "/health").then((r) => r.body?.state?.backend?.state === "ready"), "the worker to be ready"); - const [worker] = first.spawns(); - const strays = [worker, ...first.grandchildren()]; - first.proc.kill("SIGKILL"); - await until(() => first.proc.exitCode !== null || first.proc.signalCode !== null, "the first daemon to die"); - await stray; - assert.ok(strays.every(isAlive), "the probe must reproduce the orphan before the fix can be measured"); - const second = await startDaemon({ root }); - try { - await until(() => !strays.some(isAlive), "the orphans to be killed"); - assert.equal(existsSync(path.join(root, "worker.pid")), false); - assert.equal((await generate(second, { prompt: "y" })).status, 200); - assert.equal(aliveWorkers(second).length, 1); - second.proc.kill("SIGTERM"); - await until(() => second.proc.exitCode !== null, "the second daemon to exit", 30); - assert.equal(aliveWorkers(second).length, 0); - } finally { - await second.stop(); - } - }); -} - -test("a live worker outside the pid file's group is left alone", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const first = await startDaemon({ root }); - const bystander = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); - try { - assert.equal((await generate(first)).status, 200); - const [worker] = first.spawns(); - first.proc.kill("SIGKILL"); - await until(() => first.proc.exitCode !== null || first.proc.signalCode !== null, "the first daemon to die"); - writeFileSync(path.join(root, "worker.pid"), `${bystander.pid}\n`); - const second = await startDaemon({ root }); - assert.ok(isAlive(worker) && isAlive(bystander.pid), "only the group the pid file names, and only if it is ours, is killed"); - second.proc.kill("SIGTERM"); - await until(() => second.proc.exitCode !== null, "the second daemon to exit", 30); - process.kill(-worker, "SIGKILL"); - } finally { - bystander.kill("SIGKILL"); - await first.stop(); - } -}); - -// The marker that proves a group is ours is this state root's own embeddings directory. A live worker of another -// state root, named by a stale pid file, carries another root's marker and is left alone. -test("a pid file naming another state root's live worker kills nothing", async () => { - const other = await startDaemon(); - const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - try { - assert.equal((await generate(other)).status, 200); - const [foreign] = other.spawns(); - writeFileSync(path.join(root, "worker.pid"), `${foreign}\n`); - const daemon = await startDaemon({ root }); - try { - assert.ok(isAlive(foreign), "a worker carrying another root's marker must survive the reclaim"); - assert.equal(existsSync(path.join(root, "worker.pid")), false, "the stale pid file is still forgotten"); - } finally { - await daemon.stop(); - } - } finally { - await other.stop(); - } -}); - -test("a pid file naming a process group that is not the worker's kills nothing", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const bystander = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); - writeFileSync(path.join(root, "worker.pid"), `${bystander.pid}\n`); - const daemon = await startDaemon({ root }); - try { - assert.equal(isAlive(bystander.pid), true); - assert.equal(existsSync(path.join(root, "worker.pid")), false); - } finally { - bystander.kill("SIGKILL"); - await daemon.stop(); - } -}); - -test("cross-site is refused before the worker", async () => { - const daemon = await startDaemon(); - try { - assert.equal((await generate(daemon)).status, 200); - const withOrigin = await generate(daemon, undefined, { origin: "https://example.test" }); - assert.equal(withOrigin.status, 403); - assert.deepEqual(withOrigin.body, { ok: false, code: "cross-site" }); - const plain = await generate(daemon, undefined, { "content-type": "text/plain" }); - assert.equal(plain.status, 403); - assert.deepEqual(plain.body, { ok: false, code: "cross-site" }); - const untyped = await request(daemon.port, "POST", "/motion/generate", { body: JSON.stringify({ prompt: "x" }) }); - assert.equal(untyped.status, 403); - const options = await request(daemon.port, "OPTIONS", "/motion/generate", { headers: { origin: "https://example.test" } }); - assert.equal(options.status, 404); - const health = await request(daemon.port, "GET", "/health"); - for (const reply of [withOrigin, plain, untyped, options, health]) { - assert.equal(Object.keys(reply.headers).filter((name) => name.startsWith("access-control-")).length, 0); - } - const backend = await backendOf(daemon); - assert.equal((await request(backend.port, "GET", "/health")).body.requests, 1, "only the same-site request reached the worker"); - // the media type is matched as a type, not as a string - assert.equal((await generate(daemon, undefined, { "content-type": "Application/JSON; charset=utf-8" })).status, 200); - assert.equal((await generate(daemon, undefined, { "content-type": "application/json ; charset=utf-8" })).status, 200); - // and only the generate route answers a POST - const stray = await request(daemon.port, "POST", "/motion/other", { - body: JSON.stringify({ prompt: "x" }), - headers: { "content-type": "application/json" }, - }); - assert.equal(stray.status, 404); - } finally { - await daemon.stop(); - } -}); - -for (const omitted of ["models-root", "kimodo-commit", "python-env"]) { - test(`unconfigured (no ${omitted}) refuses before spawning`, async () => { - const daemon = await startDaemon({ omit: [omitted] }); - try { - const answer = await generate(daemon); - assert.equal(answer.status, 503); - assert.deepEqual(answer.body, { ok: false, code: "worker-unconfigured" }); - assert.deepEqual(daemon.spawns(), []); - assert.equal((await backendOf(daemon)).state, "stopped"); - } finally { - await daemon.stop(); - } - }); -} - -test("an error never echoes the request", async () => { - const daemon = await startDaemon(); - try { - const malformed = await generate(daemon, "{\"prompt\": \"the launch code is swordfish\", "); - assert.equal(malformed.status, 400); - assert.deepEqual(malformed.body, { ok: false, error: "Invalid JSON." }); - const missing = await generate(daemon, { texts: ["the launch code is swordfish"] }); - assert.equal(missing.status, 400); - assert.ok(!missing.raw.includes("swordfish")); - // a body the handler cannot use at all: the answer names nothing, and neither does the log - const unusable = await generate(daemon, "null"); - assert.equal(unusable.status, 500); - assert.deepEqual(unusable.body, { ok: false, code: "internal" }); - await until(() => daemon.stderr().includes("request failed: TypeError"), "the daemon to log the class"); - assert.ok(!daemon.stderr().includes("swordfish")); - } finally { - await daemon.stop(); - } -}); - -test("a worker that dies before it is ready fails the job, and the next job tries again", async () => { - const daemon = await startDaemon(); - try { - const failed = await generate(daemon, { prompt: "x", model: "crash-before-ready" }); - assert.equal(failed.status, 502); - assert.equal(failed.body.ok, false); - assert.equal(failed.body.error, "the worker request failed (Error)"); - const backend = await backendOf(daemon); - assert.equal(backend.state, "failed"); - assert.equal(backend.lastError, "the worker exited (code 3)"); - assert.equal(backend.pid, null); - assert.equal(backend.port, null); - assert.equal((await generate(daemon)).status, 200); - assert.equal((await backendOf(daemon)).state, "ready"); - assert.equal(daemon.spawns().length, 2); - } finally { - await daemon.stop(); - } -}); - -test("a worker that never says it is ready is killed at the deadline", async () => { - const daemon = await startDaemon(); - try { - const pending = generate(daemon, { prompt: "x", model: "hang-before-ready", workerTimeoutMs: 1500 }); - const starting = await until(async () => { - const backend = await backendOf(daemon); - return backend.state === "starting" ? backend : null; - }, "the starting state"); - assert.ok(Number.isInteger(starting.pid) && starting.port === null); - const failed = await pending; - assert.equal(failed.status, 502); - const backend = await backendOf(daemon); - assert.equal(backend.state, "failed"); - assert.equal(backend.lastError, "the worker was not ready in time"); - assert.equal(isAlive(daemon.spawns()[0]), false); - } finally { - await daemon.stop(); - } -}); - -test("a worker that is ready but not healthy fails at the deadline; one that gets healthy in time serves", async () => { - const daemon = await startDaemon(); - try { - const failed = await generate(daemon, { prompt: "x", model: "sick", workerTimeoutMs: 1500 }); - assert.equal(failed.status, 502); - const backend = await backendOf(daemon); - assert.equal(backend.state, "failed"); - assert.equal(backend.lastError, "the worker did not become healthy in time"); - assert.equal(isAlive(daemon.spawns()[0]), false); - const slow = await generate(daemon, { prompt: "x", model: "slow-health" }); - assert.equal(slow.status, 200); - assert.equal((await backendOf(daemon)).state, "ready"); - } finally { - await daemon.stop(); - } -}); - -test("a missing interpreter fails the job at once", async () => { - const daemon = await startDaemon({ python: "/no/such/python" }); - try { - const began = Date.now(); - const failed = await generate(daemon); - assert.equal(failed.status, 502); - assert.ok(Date.now() - began < 5000, "nothing waits on a process that never existed"); - const backend = await backendOf(daemon); - assert.equal(backend.state, "failed"); - assert.equal(backend.lastError, "the worker could not be spawned"); - assert.equal(existsSync(path.join(daemon.stateRoot, "worker.pid")), false); - } finally { - await daemon.stop(); - } -}); - -test("a worker that refuses, errors or drops the request fails the job without echoing it", async () => { - const daemon = await startDaemon(); - try { - const refused = await generate(daemon, { prompt: "swordfish", model: "refuse-generate" }); - assert.equal(refused.status, 502); - assert.equal(refused.body.ok, false); - assert.equal(refused.body.error, null); - const errored = await generate(daemon, { prompt: "swordfish", model: "fail-generate" }); - assert.equal(errored.status, 502); - assert.equal(errored.body.error, "the worker answered 500"); - const dropped = await generate(daemon, { prompt: "swordfish", model: "drop-generate" }); - assert.equal(dropped.status, 502); - assert.equal(dropped.body.error, "the worker request failed (TypeError)"); - const stalled = await generate(daemon, { prompt: "swordfish", model: "slow-generate", timeoutMs: 700 }); - assert.equal(stalled.status, 502); - assert.equal(stalled.body.error, "the worker request failed (TimeoutError)"); - for (const answer of [refused, errored, dropped, stalled]) assert.ok(!answer.body.error?.includes("swordfish")); - } finally { - await daemon.stop(); - } -}); - -test("health, operator state and the unknown", async () => { - const daemon = await startDaemon(); - try { - for (const route of ["/", "/health"]) { - const answer = await request(daemon.port, "GET", route); - assert.equal(answer.status, 200); - assert.equal(answer.body.ok, true); - assert.equal(answer.body.service, "vili"); - assert.equal(answer.body.state.backend.threads, THREADS); - } - assert.equal((await request(daemon.port, "GET", "/motion/generate")).status, 404); - // the retired operator route, spelled in parts so the retirement grep over scripts/ stays empty - assert.equal((await request(daemon.port, "GET", ["", "eve", "operator"].join("/"))).status, 404); - const unknown = await request(daemon.port, "GET", "/provider-advertisement"); - assert.equal(unknown.status, 404); - assert.equal(unknown.raw, "not found\n"); - const state = await operatorState(daemon); - assert.equal(state.host, os.hostname()); - assert.deepEqual(Object.keys(state.routes), ["health", "operatorState", "generate", "artifacts", "restFbx"]); - } finally { - await daemon.stop(); - } -}); - -test("the typed service state is refreshed while the daemon runs", async () => { - const daemon = await startDaemon({ nested: true }); - try { - const store = path.join(daemon.stateRoot, "vili.service.cc"); - assert.ok(existsSync(store)); - assert.ok(!readFileSync(store).includes("ready")); - assert.equal((await generate(daemon)).status, 200); - await until(() => readFileSync(store).includes("ready"), "the store to show the worker ready", 20); - } finally { - await daemon.stop(); - } -}); - -test("a thread count outside 1 to 16 stops the daemon without echoing it", async () => { - for (const threads of ["0", "17", "99", "100", "8x", "x8", "-1", "1.5"]) { - const proc = spawn(process.execPath, [daemonPath, "--threads", threads], { stdio: ["ignore", "pipe", "pipe"] }); - let output = ""; - proc.stdout.on("data", (chunk) => { output += chunk; }); - proc.stderr.on("data", (chunk) => { output += chunk; }); - const code = await new Promise((resolve) => proc.on("exit", resolve)); - assert.equal(code, 2); - assert.match(output, /threads must be an integer from 1 to 16/); - assert.ok(!output.replace("from 1 to 16", "").includes(threads), `the refusal repeated ${threads}`); - } -}); - -const msgpackPost = (daemon, body, headers = {}) => - new Promise((resolve, reject) => { - const req = http.request({ host: "127.0.0.1", port: daemon.port, method: "POST", path: "/cultnet/snapshot", headers: { "content-type": "application/msgpack", ...headers } }, (res) => { - const chunks = []; - res.on("data", (chunk) => chunks.push(chunk)); - res.on("end", () => resolve({ status: res.statusCode, headers: res.headers, raw: Buffer.concat(chunks) })); - }); - req.on("error", reject); - req.end(body); - }); -const SECRET = "canary-9f3c-never-echoed"; -const snapshotRequest = (messageId = "m1") => Buffer.from(encode({ schemaVersion: "cultnet.snapshot_request.v0", messageId })); - -// A second daemon on a state root a live daemon holds must refuse, and must not touch the first one's worker. -test("a second daemon on a live state root exits 2 and kills nothing", async () => { - const first = await startDaemon(); - let second = null; - try { - assert.equal((await generate(first)).status, 200); - const [worker] = first.spawns(); - assert.ok(isAlive(worker)); - second = await startDaemon({ root: first.root, hold: false }); - await until(() => second.proc.exitCode !== null, "the second daemon to refuse", 20); - assert.equal(second.proc.exitCode, 2); - assert.match(second.stderr(), /held by a running daemon/); - assert.ok(isAlive(worker), "the first daemon's worker must survive the refusal"); - assert.equal((await generate(first)).status, 200); - assert.equal(aliveWorkers(first).length, 1); - } finally { - if (second && second.proc.exitCode === null && second.proc.signalCode === null) second.proc.kill("SIGKILL"); // a daemon that failed to refuse must not outlive the test - await first.stop(); - } -}); - -// Two daemons start together over the lock a crashed daemon left. Exactly one owns the root, every time: the -// other exits 2, and one generate leaves one worker on the root. -for (let trial = 1; trial <= 40; trial += 1) { - test(`two daemons racing a crashed daemon's lock leave one owner and one worker (trial ${trial})`, async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const crashed = await startDaemon({ root }); - crashed.proc.kill("SIGKILL"); - await until(() => crashed.proc.exitCode !== null || crashed.proc.signalCode !== null, "the first daemon to die"); - const racers = await Promise.all([startDaemon({ root, hold: false }), startDaemon({ root, hold: false })]); - try { - await until(() => racers.every((racer) => racer.proc.exitCode !== null || /holds the state root/.test(racer.stdout())), "both racers to decide", 20); - await new Promise((resolve) => setTimeout(resolve, 300)); - const owners = racers.filter((racer) => racer.proc.exitCode === null); - assert.equal(owners.length, 1, "exactly one daemon stays up"); - assert.match(owners[0].stdout(), /holds the state root/); - const loser = racers.find((racer) => racer !== owners[0]); - assert.equal(loser.proc.exitCode, 2); - assert.match(loser.stderr(), /held by a running daemon/); - for (const racer of owners) assert.equal((await generate(racer)).status, 200); - assert.equal(aliveWorkers(owners[0]).length, 1, "one worker on the root"); - } finally { - for (const racer of racers) if (racer.proc.exitCode === null && racer.proc.signalCode === null) racer.proc.kill("SIGKILL"); - for (const pid of racers[0].spawns()) { try { process.kill(-pid, "SIGKILL"); } catch { /* gone */ } } - rmSync(root, { recursive: true, force: true }); - } - }); -} - -// canonicalPath's fallback: a state root that does not exist yet is resolved through its longest existing prefix, so a -// daemon naming it through a symlink and one naming it directly still contend for one lock. -for (let trial = 1; trial <= 10; trial += 1) { - test(`two daemons naming a not-yet-existing root, one through a symlink, leave one owner (trial ${trial})`, async () => { - const real = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const link = `${real}-link`; - symlinkSync(real, link); - const racers = await Promise.all([startDaemon({ root: real, nested: true, hold: false }), startDaemon({ root: link, nested: true, hold: false })]); - try { - await until(() => racers.every((racer) => racer.proc.exitCode !== null || /holds the state root/.test(racer.stdout())), "both racers to decide", 20); - await new Promise((resolve) => setTimeout(resolve, 300)); - const owners = racers.filter((racer) => racer.proc.exitCode === null); - assert.equal(owners.length, 1, "exactly one daemon stays up"); - const loser = racers.find((racer) => racer !== owners[0]); - assert.equal(loser.proc.exitCode, 2); - assert.match(loser.stderr(), /held by a running daemon/); - } finally { - for (const racer of racers) if (racer.proc.exitCode === null && racer.proc.signalCode === null) racer.proc.kill("SIGKILL"); - rmSync(link, { force: true }); - rmSync(real, { recursive: true, force: true }); - } - }); -} -const idunnDaemon = async ({ leased = false, extraEnv = {}, loseWhenExists = null, python = undefined } = {}) => { - const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const lease = path.join(root, "lease"); - const healthLog = path.join(root, "health.log"); - if (leased) writeFileSync(lease, ""); - const daemon = await startDaemon({ - root, nested: true, entry: fakeIdunnEntry, ...(python ? { python } : {}), extraEnv: { - FAKE_IDUNN_LEASE: lease, FAKE_IDUNN_HEALTH_LOG: healthLog, ...extraEnv, - ...(loseWhenExists ? { FAKE_IDUNN_STATE_ROOT: path.join(root, "a", "b", "state"), FAKE_IDUNN_LOSE_WHEN_EXISTS: loseWhenExists } : {}), - }, - }); - return Object.assign(daemon, { lease, health: () => (existsSync(healthLog) ? readFileSync(healthLog, "utf8").split("\n").filter(Boolean) : []) }); -}; - -test("before the lease nothing is written, spawned or accepted", async () => { - const daemon = await idunnDaemon(); - try { - const answer = await generate(daemon); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-pending"); - assert.deepEqual(daemon.spawns(), []); - assert.deepEqual(daemon.starts(), [], "no Blender before the lease, the version probe included"); - assert.equal(existsSync(daemon.stateRoot), false, "the state root must not exist before the lease"); - assert.deepEqual(daemon.health(), []); - } finally { - await daemon.stop(); - } -}); - -test("with the lease the daemon writes and serves, and no pid file exists", async () => { - const daemon = await idunnDaemon(); - try { - writeFileSync(daemon.lease, ""); - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - const answer = await generate(daemon); - assert.equal(answer.status, 200); - assert.equal(existsSync(path.join(daemon.stateRoot, "worker.pid")), false); - assert.equal(existsSync(path.join(daemon.stateRoot, "daemon.lock")), false); - assert.deepEqual(daemon.health(), ["active"]); - } finally { - await daemon.stop(); - } -}); - -test("losing the lease kills the worker tree, refuses, and writes nothing more", async () => { - const daemon = await idunnDaemon({ leased: true }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - assert.equal((await generate(daemon, { prompt: "x", model: "stubborn" })).status, 200); - const [worker] = daemon.spawns(); - const [grandchild] = await until(() => { const found = daemon.grandchildren(); return found.length ? found : null; }, "the armed grandchild"); - assert.ok(isAlive(worker) && isAlive(grandchild)); - rmSync(daemon.lease); - await until(() => !isAlive(worker) && !isAlive(grandchild), "the worker tree to die", 20); - const answer = await generate(daemon); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - await until(() => daemon.health().join() === "active,degraded", "the health to say degraded"); - const store = path.join(daemon.stateRoot, "vili.service.cc"); - const before = statSync(store).mtimeMs; - const jobs = readdirSync(path.join(daemon.stateRoot, "jobs")).length; - await new Promise((resolve) => setTimeout(resolve, 6500)); - assert.equal(statSync(store).mtimeMs, before); - assert.equal(readdirSync(path.join(daemon.stateRoot, "jobs")).length, jobs); - } finally { - await daemon.stop(); - } -}); - -// A generate whose headers are in and whose body is still arriving when the lease goes. The lease is gone, so -// nothing is created and nothing is spawned when the body completes, whether the daemon has noticed the loss yet -// (its 5 s tick) or not. -for (const noticed of [false, true]) { - test(`a request in flight when the lease is lost writes and spawns nothing (loss ${noticed ? "noticed" : "not yet noticed"})`, async () => { - const daemon = await idunnDaemon({ leased: true }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - const body = JSON.stringify({ prompt: "x", duration: 2 }); - const socket = net.connect(daemon.port, "127.0.0.1"); - let raw = ""; - socket.on("data", (chunk) => { raw += chunk; }); - const closed = new Promise((resolve) => socket.on("close", resolve)); - socket.write(`POST /motion/generate HTTP/1.1\r\nhost: vili\r\ncontent-type: application/json\r\ncontent-length: ${body.length}\r\nconnection: close\r\n\r\n${body.slice(0, 5)}`); - await new Promise((resolve) => setTimeout(resolve, 200)); - rmSync(daemon.lease); - if (noticed) await until(() => daemon.health().join() === "active,degraded", "the daemon to notice the loss", 15); - socket.write(body.slice(5)); - await Promise.race([closed, new Promise((resolve) => setTimeout(resolve, 15000))]); - socket.destroy(); - assert.match(raw, /^HTTP\/1\.1 503 /); - assert.match(raw, /write-lease-lost/); - assert.deepEqual(daemon.spawns(), [], "no worker is spawned after the loss"); - const artifacts = path.join(daemon.stateRoot, "artifacts"); - assert.deepEqual(existsSync(artifacts) ? readdirSync(artifacts) : [], [], "no artifact directory is created"); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), []); - } finally { - await daemon.stop(); - } - }); -} - -// The lease goes between two writes of one request. Each case names the path whose appearance takes the lease, and -// what must not exist afterwards: every write and the spawn are gated where they happen, not by the request's -// earlier checks. -const losses = [ - { when: "artifacts/*", absent: ["artifacts/*/meta.json", "logs", "embeddings"] }, - { when: "artifacts/*", payload: { prompt: "a person waves", duration: 2, constraints: [{ type: "root2d" }] }, absent: ["artifacts/*/constraints.json", "artifacts/*/meta.json"] }, - { when: "artifacts/*/meta.json", absent: ["logs", "embeddings"] }, - { when: "logs", absent: ["embeddings"] }, - { when: "embeddings", absent: ["logs/worker.log"] }, - { when: "embeddings/stub-healthy", absent: ["artifacts/*/request.json", "jobs/*"] }, -]; -for (const { when, absent, payload } of losses) { - test(`a request is stopped at its next write when the lease goes at ${when}${payload ? " (constraints in the body)" : ""}`, async () => { - const daemon = await idunnDaemon({ leased: true, loseWhenExists: when }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - const answer = await generate(daemon, payload); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - for (const gone of absent) { - const parts = gone.split("/"); - const found = (base, rest) => (rest.length === 0 ? existsSync(base) - : rest[0] === "*" ? existsSync(base) && readdirSync(base).some((entry) => found(path.join(base, entry), rest.slice(1))) - : found(path.join(base, rest[0]), rest.slice(1))); - assert.equal(found(daemon.stateRoot, parts), false, `${gone} must not exist`); - } - if (when === "embeddings") assert.deepEqual(daemon.spawns(), []); - } finally { - await daemon.stop(); - } - }); -} - -// The daemon's own copy of a worker's stdout into worker.log is a write like any other. -test("a worker's output line is not logged once the lease is gone", async () => { - const daemon = await idunnDaemon({ leased: true, loseWhenExists: "embeddings/stub-spawns.log" }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - const answer = await generate(daemon); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - const log = readFileSync(path.join(daemon.stateRoot, "logs", "worker.log"), "utf8"); - assert.ok(!log.includes("stub-stdout-line"), "the daemon must not copy the worker's stdout after the loss"); - } finally { - await daemon.stop(); - } -}); - -// The lease goes while the first persist is under way: the daemon survives it, stays degraded, and the rest of the -// startup writes are not made. -test("a lease lost while the state root is opened leaves the daemon up, degraded and writing nothing more", async () => { - const daemon = await idunnDaemon({ leased: true, loseWhenExists: "vili.service.cc" }); - try { - await until(() => daemon.health().includes("degraded") || daemon.proc.exitCode !== null, "the loss to be seen", 15); - assert.equal(daemon.proc.exitCode, null, "the daemon must not die of it"); - assert.deepEqual(daemon.health(), ["degraded"]); - const store = readFileSync(path.join(daemon.stateRoot, "vili.service.cc")); - assert.ok(store.includes("operator_state") && !store.includes("command_boundary"), "the second document is not written"); - const answer = await generate(daemon); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - } finally { - await daemon.stop(); - } -}); - -test("a request arriving right after the lease is lost writes and spawns nothing", async () => { - const daemon = await idunnDaemon({ leased: true }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - rmSync(daemon.lease); - const answer = await generate(daemon); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - assert.deepEqual(daemon.spawns(), []); - const artifacts = path.join(daemon.stateRoot, "artifacts"); - assert.deepEqual(existsSync(artifacts) ? readdirSync(artifacts) : [], []); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), []); - } finally { - await daemon.stop(); - } -}); - -test("a worker respawned under the lease is killed when the lease is lost", async () => { - const daemon = await idunnDaemon({ leased: true }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - assert.equal((await generate(daemon)).status, 200); - const [first] = daemon.spawns(); - process.kill(-first, "SIGKILL"); - await until(async () => (await backendOf(daemon)).state === "stopped", "the daemon to see the worker gone"); - assert.equal((await generate(daemon)).status, 200); - const second = daemon.spawns().find((pid) => pid !== first); - assert.ok(second && isAlive(second)); - rmSync(daemon.lease); - await until(() => !isAlive(second), "the respawned worker to die", 20); - } finally { - await daemon.stop(); - } -}); - -// Idunn stops an incumbent (SIGTERM to the unit) before it revokes the lease. What Vili owns in that sequence: the -// worker tree is dead when the process ends, it exits 0, and nothing is written from the signal on. -test("on SIGTERM the daemon stops its worker tree, writes nothing more, and exits 0", async () => { - const daemon = await idunnDaemon({ leased: true }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - assert.equal((await generate(daemon, { prompt: "x", model: "stubborn" })).status, 200); - const [worker] = daemon.spawns(); - const [grandchild] = await until(() => { const found = daemon.grandchildren(); return found.length ? found : null; }, "the armed grandchild"); - const store = path.join(daemon.stateRoot, "vili.service.cc"); - const before = statSync(store).mtimeMs; - const jobs = readdirSync(path.join(daemon.stateRoot, "jobs")); - daemon.proc.kill("SIGTERM"); - await until(() => daemon.proc.exitCode !== null || daemon.proc.signalCode !== null, "the daemon to exit", 30); - assert.equal(daemon.proc.exitCode, 0); - assert.ok(!isAlive(worker) && !isAlive(grandchild), "the worker tree is dead when the daemon is gone"); - assert.equal(statSync(store).mtimeMs, before, "the 5 s surface tick must not write during the stop"); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), jobs); - assert.ok(existsSync(daemon.lease), "the daemon never touches the lease Idunn revokes"); - } finally { - await daemon.stop(); - } -}); - -test("a job that finishes after the lease is lost is not recorded", async () => { - const daemon = await idunnDaemon({ leased: true }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - const slow = generate(daemon, { prompt: "x", model: "slow-generate" }); - slow.catch(() => {}); // a failing assertion below must not leave this request pending - await until(() => daemon.spawns().length, "the worker"); - await until(() => request(daemon.port, "GET", "/health").then((r) => r.body?.state?.backend?.state === "ready"), "the worker to be ready"); - rmSync(daemon.lease); - const answer = await Promise.race([slow, new Promise((resolve) => setTimeout(() => resolve({ status: "no answer in 30 s" }), 30000))]); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), []); - } finally { - await daemon.stop(); - } -}); - -test("the route challenge is answered ahead of the cross-site refusal, and an Origin is still refused", async () => { - const daemon = await idunnDaemon(); - try { - // the challenge is how Idunn learns the incarnation is warming, so it is answered before the lease - const answered = await msgpackPost(daemon, snapshotRequest("m-42")); - assert.equal(answered.status, 200); - assert.equal(answered.headers["content-type"], "application/msgpack"); - const message = decode(answered.raw); - assert.equal(message.schemaVersion, "cultnet.snapshot_response_raw.v0"); - assert.equal(message.messageId, "m-42"); - assert.equal(message.documents.length, 1); - assert.equal((await msgpackPost(daemon, snapshotRequest(), { origin: "https://evil.example" })).status, 403); - assert.equal((await msgpackPost(daemon, Buffer.from("not msgpack at all"))).status, 400); - assert.equal((await msgpackPost(daemon, Buffer.from(encode({ schemaVersion: "cultnet.snapshot_response_raw.v0", messageId: "x", documents: [] })))).status, 400); - assert.equal((await msgpackPost(daemon, Buffer.alloc(70 * 1024))).status, 400); - const text = await request(daemon.port, "POST", "/motion/generate", { body: "{}", headers: { "content-type": "text/plain" } }); - assert.equal(text.status, 403); - } finally { - await daemon.stop(); - } -}); - -test("a signer that cannot answer gives 503 and echoes nothing", async () => { - const daemon = await idunnDaemon({ extraEnv: { FAKE_IDUNN_ANSWER_THROWS: "1" } }); - try { - const answered = await msgpackPost(daemon, snapshotRequest(SECRET)); - assert.equal(answered.status, 503); - assert.ok(!answered.raw.toString("utf8").includes(SECRET)); - assert.ok(!daemon.stderr().includes(SECRET)); - } finally { - await daemon.stop(); - } -}); - -test("the hand-run daemon has no route challenge", async () => { - const daemon = await startDaemon(); - try { - assert.equal((await msgpackPost(daemon, snapshotRequest())).status, 404); - assert.equal((await msgpackPost(daemon, snapshotRequest(), { origin: "https://evil.example" })).status, 404); - } finally { - await daemon.stop(); - } -}); - -// ---- the Idunn entry's own rules: its argument contract, and what it derives for the daemon - -const entryPath = path.join(here, "idunn-entry.mjs"); -const runEntry = (args, env = {}) => new Promise((resolve) => { - const proc = spawn(process.execPath, [entryPath, ...args], { env: { PATH: process.env.PATH, ...env }, stdio: ["ignore", "pipe", "pipe"] }); - let output = ""; - proc.stdout.on("data", (chunk) => { output += chunk; }); - proc.stderr.on("data", (chunk) => { output += chunk; }); - proc.on("exit", (code) => resolve({ code, output })); -}); - -test("the entry takes exactly --state-root --kimodo-commit <40 hex>, and echoes none of a bad value", async () => { - const commit = "6bb58488037dd65360ff0c5d1692b403a23309f7"; - const good = ["--state-root", "/var/lib/gamecult/vili", "--kimodo-commit", commit]; - const bads = [ - [], - [...good, "--extra", "x"], - ["--kimodo-commit", commit, "--state-root", "/var/lib/gamecult/vili"], - ["--state-root", "canary-relative-path", "--kimodo-commit", commit], - ["--state-root", "/var/lib/gamecult/vili", "--kimodo-commit", "canary-not-a-commit"], - ["--state-root", "/var/lib/gamecult/vili", "--kimodo-commit", commit.toUpperCase()], - ]; - for (const args of bads) { - const answer = await runEntry(args, { GAMECULT_IDUNN_CANDIDATE_BIND: "127.0.0.1:18861" }); - assert.equal(answer.code, 2, JSON.stringify(args)); - assert.ok(!answer.output.includes("canary"), `the refusal repeated a value of ${JSON.stringify(args)}`); - } - for (const badBind of ["canary-not-a-bind", "127.0.0.1:65536", "127.0.0.1:99999", "127.0.0.1:123456", "127.0.0.1:"]) { - const bind = await runEntry(good, { GAMECULT_IDUNN_CANDIDATE_BIND: badBind }); - assert.equal(bind.code, 2, badBind); - assert.match(bind.output, /CANDIDATE_BIND is not host:port/); - assert.ok(!bind.output.includes("canary")); - assert.ok(!bind.output.includes("65536") && !bind.output.includes("99999")); - } -}); - -// A startup failure that kills the unit names its cause from what the failing code itself says (its error code, or -// the fixed text of a plain Error), and never repeats a value: an fs error's own message carries the path. -test("a failed startup names the cause and repeats no value", async () => { - const good = ["--state-root", "/var/lib/gamecult/vili", "--kimodo-commit", "6bb58488037dd65360ff0c5d1692b403a23309f7"]; - const bind = { GAMECULT_IDUNN_CANDIDATE_BIND: "127.0.0.1:18861" }; - const unset = await runEntry(good, bind); - assert.equal(unset.code, 1); - assert.match(unset.output, /GAMECULT_IDUNN_RUNTIME_BUNDLE is required/); - const missing = await runEntry(good, { ...bind, GAMECULT_IDUNN_RUNTIME_BUNDLE: "/canary-bundle-dir" }); - assert.equal(missing.code, 1); - assert.match(missing.output, /ENOENT/); - assert.ok(!missing.output.includes("canary"), "the failure repeated a value"); -}); - -test("the entry derives the daemon's host, port, interpreter and environment identity from the release", async () => { - const { createHash } = await import("node:crypto"); - const { daemonEnvironment } = await import("./idunn-entry.mjs"); - const release = mkdtempSync(path.join(os.tmpdir(), "vili-entry-")); - try { - mkdirSync(path.join(release, "vili", "scripts"), { recursive: true }); - mkdirSync(path.join(release, "vili", "deployment", "idunn"), { recursive: true }); - writeFileSync(path.join(release, "vili", "deployment", "idunn", "requirements-build.lock"), "build lock bytes\n"); - writeFileSync(path.join(release, "vili", "deployment", "idunn", "requirements.lock"), "runtime lock bytes\n"); - const entryDir = path.join(release, "vili", "scripts"); - assert.equal(daemonEnvironment("127.0.0.1:65535", entryDir).VILI_DAEMON_PORT, "65535", "the top of the port range is accepted"); - for (const bind of ["127.0.0.1:18861", "http://127.0.0.1:18861"]) { - const derived = daemonEnvironment(bind, entryDir); - assert.equal(derived.VILI_DAEMON_HOST, "127.0.0.1"); - assert.equal(derived.VILI_DAEMON_PORT, "18861"); - assert.equal(derived.GAMECULT_IDUNN_CANDIDATE_BIND, "http://127.0.0.1:18861", "cultnet-ts compares the bind to the Expected route's endpoint, which carries the scheme Idunn's bare host:port lacks"); - assert.equal(derived.VILI_PYTHON, path.join(release, "python", "bin", "python3.12")); - assert.equal(derived.VILI_BLENDER, path.join(release, "blender", "blender")); - assert.deepEqual(Object.keys(derived).sort(), ["GAMECULT_IDUNN_CANDIDATE_BIND", "VILI_BLENDER", "VILI_DAEMON_HOST", "VILI_DAEMON_PORT", "VILI_PYTHON", "VILI_PYTHON_ENV"], - "the entry sets exactly these: no LD_LIBRARY_PATH for the whole unit"); - assert.equal(derived.VILI_PYTHON_ENV, `locks-sha256:${createHash("sha256").update("build lock bytes\nruntime lock bytes\n").digest("hex")}`); - } - } finally { - rmSync(release, { recursive: true, force: true }); - } -}); - -// The lease is asked while a generate awaits its worker, not only when the service documents are rewritten. The -// bound is one WATCH_MS ask plus the SIGKILL; the test allows a second. The loss is made just after a rewrite of the -// service documents (whose own gates have asked), the longest wait for an implementation that asks only then. -test("a worker still writing when the lease is lost is dead within a second", async () => { - const daemon = await idunnDaemon({ leased: true }); - try { - const store = path.join(daemon.stateRoot, "vili.service.cc"); - await until(() => existsSync(store), "the service store", 10); - const pending = generate(daemon, { prompt: "x", model: "writer" }); - pending.catch(() => {}); - const writes = () => { - const dir = path.join(daemon.stateRoot, "artifacts"); - for (const entry of existsSync(dir) ? readdirSync(dir) : []) { - const file = path.join(dir, entry, "motion.writes"); - if (existsSync(file)) return file; - } - return null; - }; - const file = await until(writes, "the worker to write"); - const [worker] = daemon.spawns(); - const rewritten = statSync(store).mtimeMs; - await until(() => statSync(store).mtimeMs !== rewritten, "the service documents to be rewritten", 8); - await new Promise((resolve) => setTimeout(resolve, 150)); // the rewrite is done: its own gates have asked - const lostAt = Date.now(); - rmSync(daemon.lease); - await until(() => !isAlive(worker), "the worker to die", 10); - assert.ok(Date.now() - lostAt < 1000, `the worker lived ${Date.now() - lostAt} ms after the loss`); - const stamps = readFileSync(file, "utf8").trim().split("\n").map(Number); - assert.ok(Math.max(...stamps) - lostAt < 1000, "the worker wrote more than a second after the loss"); - const answer = await Promise.race([pending, new Promise((resolve) => setTimeout(() => resolve({ status: "no answer" }), 10000))]); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), []); - } finally { - await daemon.stop(); - } -}); - -// A request in flight and one arriving during the stop. The worker takes 1.5 s to stop, so the stop is long enough -// for the second request to arrive in it. From the signal on nothing is written and nothing is spawned. -for (const kind of ["fake Idunn entry", "hand-run"]) { - test(`SIGTERM (${kind}): nothing is written or spawned from the signal on, and the worker tree is dead at exit 0`, async () => { - const daemon = kind === "hand-run" ? await startDaemon() : await idunnDaemon({ leased: true }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - const inFlight = generate(daemon, { prompt: "x", model: "slow-to-stop" }); - inFlight.catch(() => {}); - await until(() => daemon.spawns().length, "the worker"); - await until(() => request(daemon.port, "GET", "/health").then((r) => r.body?.state?.backend?.state === "ready"), "the worker to be ready"); - const store = path.join(daemon.stateRoot, "vili.service.cc"); - const storeBefore = statSync(store).mtimeMs; - const artifacts = readdirSync(path.join(daemon.stateRoot, "artifacts")); - daemon.proc.kill("SIGTERM"); - await new Promise((resolve) => setTimeout(resolve, 300)); - const second = await generate(daemon).catch((error) => ({ status: error.code })); - assert.notEqual(second.status, 200, "a request arriving during the stop is not served"); - await until(() => daemon.proc.exitCode !== null || daemon.proc.signalCode !== null, "the daemon to exit", 30); - assert.equal(daemon.proc.exitCode, 0); - assert.equal(daemon.spawns().length, 1, "no worker is spawned after the signal"); - assert.ok(daemon.spawns().every((pid) => !isAlive(pid)), "the worker tree is dead at exit"); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "artifacts")), artifacts, "no artifact is created after the signal"); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), [], "no job is recorded after the signal"); - assert.equal(statSync(store).mtimeMs, storeBefore); - assert.ok(!readFileSync(path.join(daemon.stateRoot, "logs", "worker.log"), "utf8").includes("stopping-line"), "the worker's output while it stops is not copied"); - } finally { - await daemon.stop(); - } - }); -} - -// One notion of root identity: a root is the same root under either spelling, for the lock and for reclaim. -for (const [crashedOn, restartedOn] of [["link", "real"], ["real", "link"]]) { - test(`a daemon restarted on the ${restartedOn} spelling of its root reclaims the worker a crash on the ${crashedOn} spelling left`, async () => { - const real = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const link = `${real}-link`; - symlinkSync(real, link); - const spelling = { real, link }; - const first = await startDaemon({ root: spelling[crashedOn] }); - let second = null; - let stray = null; - try { - assert.equal((await generate(first)).status, 200); - [stray] = first.spawns(); - first.proc.kill("SIGKILL"); - await until(() => first.proc.exitCode !== null || first.proc.signalCode !== null, "the first daemon to die"); - assert.ok(isAlive(stray), "the probe must reproduce the orphan before the fix can be measured"); - second = await startDaemon({ root: spelling[restartedOn] }); - await until(() => !isAlive(stray), "the predecessor's worker to be reclaimed"); - assert.equal((await generate(second)).status, 200); - assert.equal(aliveWorkers(second).length, 1, "one live worker on the one root"); - } finally { - if (stray) { try { process.kill(-stray, "SIGKILL"); } catch { /* gone */ } } - if (second) await second.stop(); - rmSync(link, { force: true }); - rmSync(real, { recursive: true, force: true }); - } - }); -} - -// The group is the worker's, and its leader may be the first to go. Any member carrying the root's marker proves it. -test("a restarted daemon kills a worker group whose leader is gone and a member lives", async () => { - const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const first = await startDaemon({ root }); - const stray = generate(first, { prompt: "x", model: "encoder" }).catch(() => null); - const [grandchild] = await until(() => { const found = first.grandchildren(); return found.length ? found : null; }, "the encoder child"); - const [worker] = first.spawns(); - first.proc.kill("SIGKILL"); - await until(() => first.proc.exitCode !== null || first.proc.signalCode !== null, "the first daemon to die"); - process.kill(worker, "SIGKILL"); // the leader only - await until(() => !isAlive(worker), "the leader to die"); - await stray; - assert.ok(isAlive(grandchild), "the probe must leave a member alive in the leaderless group"); - const second = await startDaemon({ root }); - try { - await until(() => !isAlive(grandchild), "the leaderless group's member to be killed"); - assert.equal(existsSync(path.join(root, "worker.pid")), false); - } finally { - try { process.kill(-worker, "SIGKILL"); } catch { /* gone */ } - await second.stop(); - } -}); - -// The copy of worker output is gated at every line, not at the first: here the lease goes after the first line -// was logged, by the worker's first healthy answer, and the line it prints then must not reach the log. -test("a worker's output line after the first is not logged once the lease is gone", async () => { - const daemon = await idunnDaemon({ leased: true, loseWhenExists: "embeddings/stub-healthy" }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - const answer = await generate(daemon, { prompt: "x", model: "chatty" }); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - await new Promise((resolve) => setTimeout(resolve, 500)); - const log = readFileSync(path.join(daemon.stateRoot, "logs", "worker.log"), "utf8"); - assert.ok(log.includes("stub-stdout-line"), "the first line was logged under the lease"); - assert.ok(!log.includes("after-healthy-line"), "a line after the loss must not be copied"); - } finally { - await daemon.stop(); - } -}); - -// ---- FBX export: the daemon converts each BVH the worker lists, through the sealed Blender, under the write gate. - -const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); -const exportRequest = (extra = {}) => ({ prompt: "a person waves", duration: 2, fbx: true, ...extra }); -const exportStarts = (daemon) => daemon.starts().filter((start) => start.argv.includes("--")); -const jobFiles = (daemon) => readdirSync(path.join(daemon.stateRoot, "jobs")); -const getArtifact = (daemon, urlPath) => request(daemon.port, "GET", urlPath); - -test("an fbx job exports every listed BVH", async () => { - const daemon = await startDaemon(); - try { - assert.equal((await backendOf(daemon)).blender, "Blender 0.0.0 (stub)"); - const answer = await generate(daemon, exportRequest({ clipName: "wave_01", seed: 7 })); - assert.equal(answer.status, 200); - const job = answer.body; - assert.equal(job.schema, "gamecult.vili.motion_job.v1"); - assert.equal(job.ok, true); - assert.equal(job.worker.bvh, true, "fbx implies the BVH"); - assert.equal(job.worker.bvhStandardTpose, true, "fbx implies the standard T-pose"); - assert.equal(job.request.bvh, true); - assert.equal(job.request.bvhStandardTpose, true); - assert.deepEqual(job.export.clips, [{ file: "motion.fbx", clipName: "wave_01", bones: 78, frames: [1, 60], fps: 30 }]); - assert.equal(job.export.blender, "Blender 0.0.0 (stub)"); - assert.equal(job.export.script, `sha256:${sha256(readFileSync(exportScriptPath))}`); - assert.equal(job.export.settings, "standard-tpose soma77 metres hashseed0 time1970"); - const fbx = readFileSync(path.join(job.artifactDir, "motion.fbx")); - assert.equal(fbx.toString(), "FBX-STUB:BVH-STUB\n"); - assert.deepEqual(job.files.map((file) => file.name), ["meta.json", "motion.bvh", "motion.fbx", "motion.npz", "request.json"]); - assert.equal(job.files.find((file) => file.name === "motion.fbx").sha256, sha256(fbx)); - assert.equal(job.files.find((file) => file.name === "motion.fbx").bytes, fbx.length); - assert.deepEqual(JSON.parse(readFileSync(path.join(daemon.stateRoot, "jobs", `${job.jobId}.json`), "utf8")), job); - const request_ = JSON.parse(readFileSync(path.join(job.artifactDir, "request.json"), "utf8")); - assert.equal(request_.fbx, undefined, "the worker is told nothing of FBX"); - assert.equal(request_.clipName, undefined); - - const [probe, run] = daemon.starts(); - assert.deepEqual(probe.argv, ["-b", "--factory-startup", "--version"]); - assert.deepEqual(run.argv.slice(0, 8), [ - "-b", "--factory-startup", "--python-use-system-env", "--python-exit-code", "1", "--python", exportScriptPath, "--", - ]); - // Blender reads a private copy and writes a private FBX, in a directory made for this export alone: it is never - // handed a path in the job's directory, and the directory is gone once the export is over. - const [privateBvh, privateFbx, claimed] = run.argv.slice(8); - assert.match(path.relative(daemon.stateRoot, privateBvh), /^work\/export-[A-Za-z0-9]{6}\/in\.bvh$/); - assert.equal(privateFbx, path.join(path.dirname(privateBvh), "out.fbx")); - assert.equal(claimed, "wave_01"); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "work")), []); - assert.deepEqual(run.env, { - PATH: process.env.PATH, - LANG: "xx_TEST.UTF-8", - HOME: "/nonexistent", - PYTHONHASHSEED: "0", - PYTHONNOUSERSITE: "1", - LD_LIBRARY_PATH: path.join(daemon.blenderDir, "lib"), - }, "the export environment is these six and nothing else (no VILI_*, no LEAK_CHECK)"); - assert.equal(run.priority, 10); - assert.equal(run.pgid, run.pid, "Blender leads its own process group"); - const log = readFileSync(path.join(daemon.stateRoot, "logs", "export.log"), "utf8"); - assert.match(log, /stub-blender-stderr wave_01/); - - const named = await generate(daemon, exportRequest()); - assert.equal(named.body.export.clips[0].clipName, named.body.jobId, "the clip is named after the job by default"); - const several = await generate(daemon, exportRequest({ clipName: "pair", numSamples: 2 })); - assert.equal(several.status, 200); - assert.deepEqual(several.body.export.clips.map((clip) => [clip.file, clip.clipName]), [ - ["motion/motion_00.fbx", "pair_00"], - ["motion/motion_01.fbx", "pair_01"], - ]); - } finally { - await daemon.stop(); - } -}); - -test("fbx refusals happen before anything is written or spawned", async () => { - const daemon = await startDaemon(); - try { - for (const bad of [ - { bvhStandardTpose: false }, - { bvh_standard_tpose: "false" }, - { clipName: "a b canary" }, - { clipName: "x".repeat(65) }, - { clipName: "" }, - { clipName: 5 }, - ]) { - const answer = await generate(daemon, exportRequest(bad)); - assert.equal(answer.status, 400, JSON.stringify(bad)); - assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); - } - assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made"); - assert.deepEqual(jobFiles(daemon), []); - assert.deepEqual(daemon.spawns(), [], "no worker was started"); - assert.equal(exportStarts(daemon).length, 0); - assert.equal((await generate(daemon, exportRequest({ clipName: "x".repeat(64) }))).status, 200, "64 characters is the limit"); - } finally { - await daemon.stop(); - } - for (const blender of [null, "broken", "missing"]) { - const unready = await startDaemon({ blender }); - try { - assert.equal((await backendOf(unready)).blender, blender === null ? "unconfigured" : "unavailable"); - const answer = await generate(unready, exportRequest()); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "blender-unavailable"); - assert.deepEqual(unready.spawns(), [], "no worker was started"); - assert.equal(existsSync(path.join(unready.stateRoot, "artifacts")), false, "no artifact directory was made"); - assert.deepEqual(jobFiles(unready), []); - assert.equal(exportStarts(unready).length, 0); - assert.equal((await generate(unready)).status, 200, "a job that asks for no FBX needs no Blender"); - } finally { - await unready.stop(); - } - } -}); - -test("loop asks for the seamless window, and the window is the FBX's frame range", async () => { - const daemon = await startDaemon(); - try { - const constraints = [{ type: "root2d", frame_indices: [0, 10], smooth_root_2d: [[0, 0], [0, 0.5]] }]; - const answer = await generate(daemon, exportRequest({ clipName: "walk", seed: 7, loop: true, constraints })); - assert.equal(answer.status, 200); - const job = answer.body; - assert.equal(job.request.loop, true); - assert.deepEqual(job.worker.loops, [{ first: 3, last: 223, poseCost: 0.03, velocityCost: 0.2 }]); - assert.deepEqual(job.request.constraints, constraints, "the record keeps inline constraints as sent"); - assert.deepEqual(JSON.parse(readFileSync(path.join(job.artifactDir, "constraints.json"), "utf8")), constraints); - assert.equal(JSON.parse(readFileSync(path.join(job.artifactDir, "request.json"), "utf8")).loop, true, "the worker is asked for the loop"); - assert.equal(job.worker.constraints, path.join(job.artifactDir, "constraints.json"), "the worker still gets the path"); - const [run] = exportStarts(daemon); - assert.deepEqual(run.argv.slice(run.argv.indexOf("--") + 3), ["walk", "--frames", "4", "224"], "Blender's frames count from 1"); - assert.deepEqual(job.export.clips.map((clip) => clip.frames), [[1, 60]], "the clip's frames are what Blender reports"); - - const several = await generate(daemon, exportRequest({ clipName: "pair", numSamples: 2, loop: true })); - assert.equal(several.status, 200); - assert.deepEqual(several.body.worker.loops.map((loop) => [loop.first, loop.last]), [[3, 223], [4, 224]]); - const frames = Object.fromEntries(exportStarts(daemon).slice(1).map((start) => { - const [, , clip, ...rest] = start.argv.slice(start.argv.indexOf("--") + 1); - return [clip, rest]; - })); - assert.deepEqual(frames, { pair_00: ["--frames", "4", "224"], pair_01: ["--frames", "5", "225"] }, "each sample is exported over its own window"); - - const plain = await generate(daemon, exportRequest({ clipName: "plain" })); - assert.equal(plain.status, 200); - assert.equal(plain.body.worker.loops, undefined); - assert.equal(plain.body.request.loop, undefined); - assert.equal(JSON.parse(readFileSync(path.join(plain.body.artifactDir, "request.json"), "utf8")).loop, undefined); - assert.equal(exportStarts(daemon).at(-1).argv.at(-1), "plain", "no window, no --frames"); - const kept = await generate(daemon, exportRequest({ constraints_path: "/some/where.json", constraints: undefined })); - assert.equal(kept.body.request.constraints, "/some/where.json", "a path stays a path"); - } finally { - await daemon.stop(); - } -}); - -test("loop is a boolean and needs fbx, refused before anything is written or spawned", async () => { - const daemon = await startDaemon(); - try { - for (const bad of [{ fbx: false, loop: true }, { fbx: undefined, loop: true }, { loop: "yes canary" }, { loop: 1 }, { loop: null }, { loop: "true" }]) { - const answer = await generate(daemon, exportRequest(bad)); - assert.equal(answer.status, 400, JSON.stringify(bad)); - assert.deepEqual(answer.body, { ok: false, error: "Invalid fbx request." }); - assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); - } - assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made"); - assert.deepEqual(jobFiles(daemon), []); - assert.deepEqual(daemon.spawns(), [], "no worker was started"); - assert.equal(exportStarts(daemon).length, 0); - assert.equal((await generate(daemon, exportRequest({ loop: false }))).status, 200, "loop false is no loop"); - assert.equal((await generate(daemon, { prompt: "a person waves", duration: 2, loop: false })).status, 200); - } finally { - await daemon.stop(); - } -}); - -test("a loop request whose answer has no usable window for a sample fails without exporting that sample", async () => { - for (const [model, exported] of [["no-loops", 0], ["bad-loops", 0], ["short-loops", 1]]) { - const daemon = await startDaemon(); - try { - const answer = await generate(daemon, exportRequest({ model, loop: true, numSamples: 2 })); - assert.equal(answer.status, 502, model); - assert.equal(answer.body.ok, false); - assert.equal(answer.body.error, "fbx-export-failed (no loop window)"); - assert.equal(exportStarts(daemon).length, exported, `${model}: only a sample with a window is exported`); - assert.equal(answer.body.export.clips.length, exported); - } finally { - await daemon.stop(); - } - } -}); - -test("the BVH is never given to Blender unless it is a regular file inside the job's directory", async () => { - const daemon = await startDaemon(); - try { - for (const model of ["bvh-outside", "bvh-link"]) { - const answer = await generate(daemon, exportRequest({ model, clipName: "canary_clip" })); - assert.equal(answer.status, 502, model); - assert.equal(answer.body.ok, false); - assert.equal(answer.body.error, "fbx-export-refused"); - assert.equal(answer.body.export.clips.length, 0); - assert.ok(existsSync(path.join(daemon.stateRoot, "jobs", `${answer.body.jobId}.json`)), "the failed job is still a record"); - } - assert.equal(exportStarts(daemon).length, 0, "Blender was never started for either"); - assert.ok(!(daemon.stdout() + daemon.stderr()).includes("canary")); - } finally { - await daemon.stop(); - } -}); - -test("a failing or hung Blender fails the job, leaks nothing, and leaves nothing running", async () => { - const daemon = await startDaemon(); - try { - const failed = await generate(daemon, exportRequest({ clipName: "fail_canary" })); - assert.equal(failed.status, 502); - assert.equal(failed.body.ok, false); - assert.equal(failed.body.error, "fbx-export-failed (exit 1)"); - assert.ok(!failed.raw.includes("stub-blender-stderr"), "Blender's output is not in the answer"); - assert.ok(!failed.body.error.includes("canary") && !(daemon.stdout() + daemon.stderr()).includes("canary")); - assert.match(readFileSync(path.join(daemon.stateRoot, "logs", "export.log"), "utf8"), /stub-blender-stderr fail_canary/); - assert.ok(existsSync(path.join(daemon.stateRoot, "jobs", `${failed.body.jobId}.json`))); - - for (const clip of ["silent", "garbage", "nofile"]) { - const odd = await generate(daemon, exportRequest({ clipName: clip })); - assert.equal(odd.status, 502, clip); - assert.equal(odd.body.error, "fbx-export-failed (no result)", clip); - } - const lied = await generate(daemon, exportRequest({ clipName: "lies" })); - assert.equal(lied.status, 502, "an export that exits non-zero is no export, whatever it printed"); - assert.equal(lied.body.error, "fbx-export-failed (exit 3)"); - - const known = new Set(daemon.blenderGrandchildren()); - const orphaned = await generate(daemon, exportRequest({ clipName: "orphan" })); - assert.equal(orphaned.status, 200, "an export that exits 0 is a good export"); - const left = await until(() => daemon.blenderGrandchildren().filter((pid) => !known.has(pid))[0], "the orphan's grandchild"); - await until(() => !isAlive(left), "what Blender left running to be killed with its group"); - - const before = daemon.starts().length; - known.add(left); - const hung = await Promise.race([ - generate(daemon, exportRequest({ clipName: "hang", exportTimeoutMs: 2500 })), - new Promise((_, reject) => setTimeout(() => reject(new Error("the hung export was never stopped")), 20000)), - ]); - assert.equal(hung.status, 502); - assert.equal(hung.body.error, "fbx-export-failed (signal SIGKILL)"); - const [run] = daemon.starts().slice(before); - const stuck = await until(() => daemon.blenderGrandchildren().filter((pid) => !known.has(pid))[0], "the hung Blender's grandchild"); - await until(() => !isAlive(run.pid) && !isAlive(stuck), "the hung export's group to be dead"); - assert.equal(hung.body.files.some((file) => file.name.endsWith(".fbx")), false); - } finally { - await daemon.stop(); - } -}); - -test("SIGTERM during an export stops its group, writes nothing more, and exits 0", async () => { - const daemon = await startDaemon(); - try { - const pending = generate(daemon, exportRequest({ clipName: "slow" })); - pending.catch(() => {}); - const run = await until(() => exportStarts(daemon)[0], "the export to start"); - daemon.proc.kill("SIGTERM"); - await until(() => !isAlive(run.pid), "the export to be stopped", 5); - await until(() => daemon.proc.exitCode !== null, "the daemon to exit"); - assert.equal(daemon.proc.exitCode, 0); - assert.deepEqual(jobFiles(daemon), [], "no record after the signal"); - const [artifacts] = [path.join(daemon.stateRoot, "artifacts")]; - for (const job of readdirSync(artifacts)) assert.equal(existsSync(path.join(artifacts, job, "motion.fbx")), false); - } finally { - await daemon.stop(); - } -}); - -test("what a stopping export still prints is not logged, and it is killed when it ignores SIGTERM", async () => { - const daemon = await startDaemon(); - try { - const pending = generate(daemon, exportRequest({ clipName: "hang", exportTimeoutMs: 60000 })); - pending.catch(() => {}); - const run = await until(() => exportStarts(daemon)[0], "the export to start"); - const logFile = path.join(daemon.stateRoot, "logs", "export.log"); - await until(() => existsSync(logFile) && readFileSync(logFile, "utf8").includes("stub-blender-stderr hang"), "the export's first line to be logged"); - daemon.proc.kill("SIGTERM"); - await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 30); - assert.equal(daemon.proc.exitCode, 0); - await until(() => !isAlive(run.pid), "the export that ignored SIGTERM to be killed"); - const log = readFileSync(path.join(daemon.stateRoot, "logs", "export.log"), "utf8"); - assert.match(log, /stub-blender-stderr hang/); - assert.ok(!log.includes("after-sigterm-line"), "a line printed after the signal reached the log"); - } finally { - await daemon.stop(); - } -}); - -test("losing the lease kills a running export within a second and records nothing", async () => { - const daemon = await idunnDaemon({ leased: true }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - await until(async () => (await backendOf(daemon)).blender === "Blender 0.0.0 (stub)", "the version probe, run once the lease was held"); - assert.equal(daemon.starts().length, 1); - assert.equal(existsSync(path.join(daemon.stateRoot, "logs")), false, "the probe writes nothing to the state root"); - const pending = generate(daemon, exportRequest({ clipName: "slow" })); - const ended = pending.then((answer) => { throw new Error(`the job ended before its export started: ${answer.status} ${answer.raw.slice(0, 300)}`); }); - ended.catch(() => {}); - const run = await Promise.race([until(() => exportStarts(daemon)[0], "the export to start"), ended]); - rmSync(daemon.lease); - const lost = Date.now(); - await until(() => !isAlive(run.pid), "the export to be dead", 2); - assert.ok(Date.now() - lost < 1500, "the export outlived the lease by more than a second"); - assert.equal((await pending).status, 503); - assert.deepEqual(jobFiles(daemon), []); - assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts", readdirSync(path.join(daemon.stateRoot, "artifacts"))[0], "motion.fbx")), false); - } finally { - await daemon.stop(); - } -}); - -test("the seed is minted when absent and recorded for every job", async () => { - const daemon = await startDaemon(); - try { - for (const extra of [{}, { fbx: false }]) { - const job = (await generate(daemon, exportRequest(extra))).body; - assert.ok(Number.isInteger(job.seed) && job.seed >= 0 && job.seed < 2 ** 31); - assert.equal(job.request.seed, job.seed); - assert.equal(job.worker.seed, job.seed, "the worker was given the minted seed"); - assert.equal(job.worker.metaSeed, job.seed); - assert.equal(JSON.parse(readFileSync(path.join(job.artifactDir, "meta.json"), "utf8")).seed, job.seed); - } - const given = (await generate(daemon, exportRequest({ seed: 7 }))).body; - assert.equal(given.seed, 7); - assert.equal(given.worker.seed, 7); - assert.equal(given.worker.metaSeed, 7); - assert.equal(JSON.parse(readFileSync(path.join(given.artifactDir, "meta.json"), "utf8")).seed, 7); - const minted = new Set(); - for (let count = 0; count < 3; count += 1) minted.add((await generate(daemon, exportRequest())).body.seed); - assert.ok(minted.size > 1, "minted seeds differ from job to job"); - const zero = (await generate(daemon, exportRequest({ seed: 0 }))).body; - assert.equal(zero.seed, 0, "0 is a seed"); - } finally { - await daemon.stop(); - } -}); - -test("the artifact read route serves only a job's own FBX and BVH files", async () => { - const daemon = await startDaemon(); - try { - const job = (await generate(daemon, exportRequest({ clipName: "walk", numSamples: 1 }))).body; - const base = `/motion/artifacts/${job.jobId}`; - const served = await getArtifact(daemon, `${base}/motion.fbx`); - assert.equal(served.status, 200); - assert.equal(served.headers["content-type"], "application/octet-stream"); - assert.equal(served.raw, "FBX-STUB:BVH-STUB\n"); - assert.equal((await getArtifact(daemon, `${base}/motion.bvh`)).raw, "BVH-STUB\n"); - - // a directory that is not a job's own (its id is not vili-<13 digits>) serves nothing even with a file in it - mkdirSync(path.join(daemon.stateRoot, "artifacts", "vili-123"), { recursive: true }); - writeFileSync(path.join(daemon.stateRoot, "artifacts", "vili-123", "motion.fbx"), "PLANTED"); - const refused = [ - `${base}/meta.json`, - `${base}/request.json`, - `${base}/motion.npz`, - `${base}/..%2fmeta.json`, - `${base}/%2e%2e/${job.jobId}/meta.json`, - `${base}/../../jobs/${job.jobId}.json`, - `${base}/motion/motion_00.fbx`, - `/motion/artifacts/vili-123/motion.fbx`, - `/motion/artifacts/vili-canary-canary/motion.fbx`, - `/motion/artifacts/vili-9999999999999/motion.fbx`, - `/motion/artifacts/${job.jobId}`, - `/motion/artifacts/${job.jobId}/`, - ]; - for (const url of refused) { - const answer = await getArtifact(daemon, url); - assert.equal(answer.status, 404, url); - assert.equal(answer.raw, "not found\n", "a refusal echoes nothing"); - } - assert.equal((await request(daemon.port, "POST", `${base}/motion.fbx`, { body: "{}", headers: { "content-type": "application/json" } })).status, 404); - - // a symlink planted in the artifact directory, as the file or as a directory above it, is never followed - const outside = path.join(daemon.root, "outside.bin"); - writeFileSync(outside, "SECRET"); - rmSync(path.join(job.artifactDir, "motion.bvh")); - symlinkSync(outside, path.join(job.artifactDir, "motion.bvh")); - assert.equal((await getArtifact(daemon, `${base}/motion.bvh`)).status, 404); - mkdirSync(path.join(daemon.root, "elsewhere")); - writeFileSync(path.join(daemon.root, "elsewhere", "motion_00.fbx"), "SECRET"); - symlinkSync(path.join(daemon.root, "elsewhere"), path.join(job.artifactDir, "motion")); - const through = await getArtifact(daemon, `${base}/motion/motion_00.fbx`); - assert.equal(through.status, 404); - assert.ok(!through.raw.includes("SECRET")); - } finally { - await daemon.stop(); - } -}); - -// A worker interpreter whose site-packages carry Kimodo's rest skeleton, as the release's does: the stub worker runs on node. -function restPython() { - const root = mkdtempSync(path.join(os.tmpdir(), "vili-py-")); - const python = path.join(root, "bin", "python3.12"); - mkdirSync(path.dirname(python), { recursive: true }); - symlinkSync(process.execPath, python); - const skeletons = path.join(root, "lib", "python3.12", "site-packages", "kimodo", "assets", "skeletons", "somaskel77"); - mkdirSync(skeletons, { recursive: true }); - writeFileSync(path.join(skeletons, "somaskel77_standard_tpose.bvh"), "REST-BVH\n"); - return { root, python }; -} - -test("rest-fbx is a job whose FBX is readable", async () => { - const { root: pyRoot, python } = restPython(); - const daemon = await startDaemon({ python }); - try { - const answer = await request(daemon.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); - assert.equal(answer.status, 200); - const job = answer.body; - assert.match(job.jobId, /^rest-[0-9]{13}-[0-9a-f]{8}$/); - assert.equal(job.schema, "gamecult.vili.motion_job.v1"); - assert.deepEqual(job.export.clips, [{ file: "SOMA77_Rest.fbx", clipName: "SOMA77_Rest", bones: 78, frames: [1, 60], fps: 30 }]); - assert.deepEqual(job.files.map((file) => file.name), ["SOMA77_Rest.bvh", "SOMA77_Rest.fbx"]); - assert.deepEqual(JSON.parse(readFileSync(path.join(daemon.stateRoot, "jobs", `${job.jobId}.json`), "utf8")), job); - const served = await getArtifact(daemon, `/motion/artifacts/${job.jobId}/SOMA77_Rest.fbx`); - assert.equal(served.status, 200); - assert.equal(served.raw, "FBX-STUB:REST-BVH\n"); - assert.deepEqual(daemon.spawns(), [], "no worker is needed for a rest pose"); - const cross = await request(daemon.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "text/plain" } }); - assert.equal(cross.status, 403); - } finally { - await daemon.stop(); - rmSync(pyRoot, { recursive: true, force: true }); - } - - const bare = await startDaemon(); - try { - const missing = await request(bare.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); - assert.equal(missing.status, 503); - assert.equal(missing.body.code, "rest-skeleton-missing"); - } finally { - await bare.stop(); - } - const unready = await startDaemon({ blender: null }); - try { - const refused = await request(unready.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); - assert.equal(refused.status, 503); - assert.equal(refused.body.code, "blender-unavailable"); - assert.equal(existsSync(path.join(unready.stateRoot, "artifacts")), false); - } finally { - await unready.stop(); - } - const waiting = await idunnDaemon(); - try { - const early = await request(waiting.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); - assert.equal(early.status, 503); - assert.equal(early.body.code, "write-lease-pending"); - assert.deepEqual(waiting.starts(), []); - } finally { - await waiting.stop(); - } -}); - -test("operator state and the command boundary name the export", async () => { - const daemon = await startDaemon(); - try { - const state = await operatorState(daemon); - assert.equal(state.routes.artifacts, "/motion/artifacts/{jobId}/{name}"); - assert.equal(state.routes.restFbx, "/motion/rest-fbx"); - assert.ok(state.authority.owns.includes("clip export to FBX")); - } finally { - await daemon.stop(); - } -}); - -// ---- Findings of Soul's pass s1 on the FBX export (motion-pipeline:verdict:cut-vili-fbx-export.s1). - -const readJson = (file) => JSON.parse(readFileSync(file, "utf8")); -const restRequest = (daemon) => request(daemon.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); - -test("concurrent generate and rest-fbx requests never share a job id, a directory, a record or a file", async () => { - const rest = restPython(); - const daemon = await startDaemon({ python: rest.python }); - try { - const answers = []; - for (let round = 0; round < 10; round += 1) { - answers.push(...await Promise.all([ - generate(daemon, exportRequest({ clipName: `a${round}` })), - generate(daemon, exportRequest({ clipName: `b${round}` })), - restRequest(daemon), - restRequest(daemon), - ])); - } - assert.equal(answers.length, 40); - for (const answer of answers) { - assert.equal(answer.status, 200, answer.raw.slice(0, 300)); - assert.equal(answer.body.ok, true); - } - const ids = answers.map((answer) => answer.body.jobId); - assert.equal(new Set(ids).size, 40, "every job has its own id"); - assert.equal(new Set(answers.map((answer) => answer.body.artifactDir)).size, 40, "and its own directory"); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "artifacts")).sort(), [...ids].sort()); - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")).sort(), ids.map((id) => `${id}.json`).sort(), "and its own record"); - for (const { body } of answers) { - assert.deepEqual(readJson(path.join(daemon.stateRoot, "jobs", `${body.jobId}.json`)), body, "the record is the job's own answer"); - assert.equal(body.export.clips.length, 1); - } - const names = answers.filter((answer) => answer.body.jobId.startsWith("vili-")).map((answer) => answer.body.export.clips[0].clipName).sort(); - assert.deepEqual(names, Array.from({ length: 10 }, (_, round) => [`a${round}`, `b${round}`]).flat().sort(), "each generate job exported its own clip"); - const first = answers[0].body.jobId; - assert.equal((await getArtifact(daemon, `/motion/artifacts/${first}/motion.fbx`)).status, 200, "the read route serves the minted ids"); - } finally { - await daemon.stop(); - rmSync(rest.root, { recursive: true, force: true }); - } -}); - -test("a seed that is not an integer from 0 to 2^31 is refused before anything is written, without echoing it", async () => { - const daemon = await startDaemon(); - try { - const bad = [7.9, "7", "canary", [], [3], true, -1, 2 ** 31, 1e21, { canary: 1 }]; - for (const seed of bad) { - for (const payload of [exportRequest({ seed }), exportRequest({ meta: { seed } })]) { - const answer = await generate(daemon, payload); - assert.equal(answer.status, 400, JSON.stringify(payload)); - assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); - assert.ok(!answer.raw.includes("7.9"), "the refusal repeated the input"); - } - } - assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "nothing was written"); - assert.deepEqual(daemon.spawns(), [], "no worker was started"); - assert.equal(exportStarts(daemon).length, 0); - for (const seed of [0, 2 ** 31 - 1]) { - const job = (await generate(daemon, exportRequest({ seed }))).body; - assert.equal(job.seed, seed); - assert.equal(job.worker.seed, seed, "the seed recorded is the seed the worker was given"); - } - } finally { - await daemon.stop(); - } -}); - -test("a BVH or FBX path swapped for a symlink after the job was checked is never followed", async () => { - for (const swap of ["bvh", "fbx", "both", "directory"]) { - const daemon = await startDaemon(); - try { - const outside = path.join(daemon.root, "outside-secret"); - const victim = path.join(daemon.root, "victim"); - writeFileSync(outside, "SECRET"); - writeFileSync(victim, "VICTIM"); - // sample 0 ("slow_00") exports for six seconds; the swap lands while it does, before sample 1 is touched - const pending = generate(daemon, exportRequest({ clipName: "slow", numSamples: 2 })); - await until(() => exportStarts(daemon).length === 1, "the first export to start"); - const [job] = readdirSync(path.join(daemon.stateRoot, "artifacts")); - const motion = path.join(daemon.stateRoot, "artifacts", job, "motion"); - const elsewhere = path.join(daemon.root, "elsewhere"); - if (swap === "directory") { - // the directory itself is replaced by a link to one that holds a regular file of the right name - mkdirSync(elsewhere); - writeFileSync(path.join(elsewhere, "motion_01.bvh"), "SECRET"); - renameSync(motion, `${motion}-moved`); - symlinkSync(elsewhere, motion); - } - if (swap === "bvh" || swap === "both") { - rmSync(path.join(motion, "motion_01.bvh")); - symlinkSync(outside, path.join(motion, "motion_01.bvh")); - } - if (swap === "fbx" || swap === "both") symlinkSync(victim, path.join(motion, "motion_01.fbx")); - const answer = await pending; - assert.equal(answer.status, 502, swap); - assert.equal(answer.body.ok, false); - assert.equal(readFileSync(victim, "utf8"), "VICTIM", `${swap}: nothing was written through the planted link`); - assert.equal(readFileSync(outside, "utf8"), "SECRET"); - for (const name of readdirSync(swap === "directory" ? elsewhere : motion)) { - if (name.endsWith(".fbx") && !lstatSync(path.join(motion, name)).isSymbolicLink()) { - assert.ok(!readFileSync(path.join(motion, name), "utf8").includes("SECRET"), `${swap}: ${name} holds bytes read through the planted link`); - } - } - assert.ok(!answer.raw.includes("SECRET") && !answer.raw.includes("VICTIM")); - } finally { - await daemon.stop(); - } - } -}); - -test("the worker's environment is its named keys, and LD_LIBRARY_PATH reaches Blender only", async () => { - const daemon = await startDaemon({ extraEnv: { LD_LIBRARY_PATH: "/canary-lib" } }); - try { - const job = (await generate(daemon, exportRequest({ clipName: "wave" }))).body; - assert.equal(job.ok, true); - const named = ["HF_HOME", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "VILI_EMBEDDINGS_DIR", "VILI_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS", - "PYTHONPATH", "PYTHONDONTWRITEBYTECODE", "PYTHONUNBUFFERED", "VILI_KIMODO_COMMIT", "VILI_PYTHON_ENV"]; - const forwarded = ["PATH", "HOME", "LANG"].filter((name) => process.env[name] !== undefined || name === "LANG"); - assert.deepEqual(job.worker.envKeys, [...named, ...forwarded].sort(), "the worker gets exactly these keys, LD_LIBRARY_PATH not among them"); - const run = daemon.starts().find((start) => start.argv.includes("--")); - assert.equal(run.env.LD_LIBRARY_PATH, path.join(daemon.blenderDir, "lib"), "Blender gets the release's libraries, not the host's"); - } finally { - await daemon.stop(); - } -}); - -test("the lease goes between rest-fbx's directory and its copy: nothing is copied, nothing is exported", async () => { - const rest = restPython(); - const daemon = await idunnDaemon({ leased: true, loseWhenExists: "artifacts/*", python: rest.python }); - try { - await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); - await until(async () => /^Blender /.test((await backendOf(daemon)).blender), "the Blender probe"); - const answer = await restRequest(daemon); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "write-lease-lost"); - for (const job of readdirSync(path.join(daemon.stateRoot, "artifacts"))) { - assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "artifacts", job)), [], "the job's directory is empty"); - } - assert.equal(exportStarts(daemon).length, 0); - } finally { - await daemon.stop(); - rmSync(rest.root, { recursive: true, force: true }); - } -}); - -test("the job record lists regular files only: a planted symlink, to a file or to a directory, is not listed", async () => { - const daemon = await startDaemon(); - try { - const job = (await generate(daemon, exportRequest({ model: "plant-link", clipName: "plant" }))).body; - assert.equal(job.ok, true); - assert.equal(readdirSync(job.artifactDir).filter((name) => name.includes("link")).length, 2, "both symlinks are in the job directory"); - assert.deepEqual(job.files.map((file) => file.name), ["meta.json", "motion.bvh", "motion.fbx", "motion.npz", "request.json"]); - } finally { - await daemon.stop(); - } -}); - -// Soul s2: the job id's random suffix is what carries a burst. The clock alone repeats inside one millisecond, and -// the exclusive mkdir's few retries are spent there, so a constant suffix fails a burst and passes every small test. -test("four hundred simultaneous generates each get their own job id, directory and record", async () => { - const daemon = await startDaemon(); - try { - const answers = await Promise.all(Array.from({ length: 400 }, () => generate(daemon))); - assert.deepEqual([...new Set(answers.map((answer) => answer.status))], [200]); - const ids = answers.map((answer) => answer.body.jobId); - for (const id of ids) assert.match(id, /^vili-\d{13}-[0-9a-f]{8}$/); - assert.equal(new Set(ids).size, 400, "no two jobs share an id"); - assert.equal(readdirSync(path.join(daemon.stateRoot, "artifacts")).length, 400); - assert.equal(jobFiles(daemon).length, 400); - } finally { - await daemon.stop(); - } -}); - -// Soul s2: opening a path that blocks (a FIFO with no writer) must not wait. The open is non-blocking and the check -// is made on the open descriptor, so the listed node is refused at once and the filesystem threads stay free. -test("a listed BVH that is a FIFO is refused promptly and other file work proceeds", async () => { - const daemon = await startDaemon(); - const within = (promise, ms, what) => Promise.race([ - promise, - new Promise((_, reject) => setTimeout(() => reject(new Error(`${what} did not settle in ${ms} ms`)), ms)), - ]); - try { - // more than libuv's four filesystem threads: a blocking open per request would starve all of them - const answers = await within(Promise.all(Array.from({ length: 6 }, () => - generate(daemon, exportRequest({ model: "bvh-fifo", clipName: "canary-fifo-clip" })))), 8000, "the FIFO jobs"); - for (const answer of answers) { - assert.equal(answer.status, 502); - assert.equal(answer.body.ok, false); - const { request: recorded, ...outcome } = answer.body; // the record keeps the request; the refusal itself must not echo it - assert.ok(recorded && !JSON.stringify(outcome).includes("canary-fifo-clip"), "the refusal does not echo the clip name"); - } - assert.equal(exportStarts(daemon).length, 0, "no Blender for a node that is not a regular file"); - const other = await within(generate(daemon, exportRequest({ model: "stub-model", clipName: "after" })), 8000, "an ordinary job"); - assert.equal(other.status, 200); - assert.ok(!daemon.stdout().includes("canary-fifo-clip") && !daemon.stderr().includes("canary-fifo-clip")); - } finally { - await daemon.stop(); - } -}); - -// Soul s2: work/ is a declared state slot, and a daemon that holds the state root sweeps the export directories a -// killed daemon left, those and nothing else. -test("the recipe declares the work directory as a state slot", () => { - const recipe = readFileSync(path.join(here, "..", "deployment", "idunn", "recipe.toml"), "utf8"); - assert.match(recipe, /\[\[state\.slots\]\]\nid = "work"\nrelative_path = "work"\nkind = "external-directory"/); -}); - -test("a restarted daemon sweeps the export directories a killed one left, and touches nothing else", async () => { - const first = await startDaemon(); - let second = null; - try { - const pending = generate(first, exportRequest({ clipName: "slow" })).catch(() => null); - await until(() => exportStarts(first).length === 1, "the export to start"); - const work = path.join(first.stateRoot, "work"); - const [stale] = readdirSync(work); - assert.match(stale, /^export-[A-Za-z0-9]{6}$/); - first.proc.kill("SIGKILL"); - await until(() => first.proc.signalCode !== null, "the daemon to die"); - await pending; - assert.deepEqual(readdirSync(work), [stale], "the killed export left its directory"); - // things under the state root that are not stale export directories - const decoys = { - "work/keep": "K", - "work/export-short/in.bvh": "S", - "work/exports-keep/in.bvh": "E", - "jobs/decoy.json": "J", - "artifacts/decoy/motion.bvh": "A", - "logs/decoy.log": "L", - "decoy-root-file": "R", - }; - for (const [relative, text] of Object.entries(decoys)) { - mkdirSync(path.dirname(path.join(first.stateRoot, relative)), { recursive: true }); - writeFileSync(path.join(first.stateRoot, relative), text); - } - second = await startDaemon({ root: first.root }); - assert.deepEqual(readdirSync(work).sort(), ["export-short", "exports-keep", "keep"], "the stale export is gone, the rest stays"); - for (const [relative, text] of Object.entries(decoys)) assert.equal(readFileSync(path.join(second.stateRoot, relative), "utf8"), text, relative); - assert.equal((await generate(second, exportRequest({ clipName: "after" }))).status, 200); - } finally { - if (first.proc.signalCode === null) first.proc.kill("SIGKILL"); - await (second ?? first).stop(); - } -}); - -test("a daemon that does not hold the state root sweeps nothing", async () => { - const first = await startDaemon(); - let second = null; - try { - const pending = generate(first, exportRequest({ clipName: "slow" })); - await until(() => exportStarts(first).length === 1, "the export to start"); - const work = path.join(first.stateRoot, "work"); - const live = readdirSync(work); - assert.equal(live.length, 1); - second = await startDaemon({ root: first.root, hold: false }); - await until(() => second.proc.exitCode !== null, "the second daemon to refuse", 20); - assert.equal(second.proc.exitCode, 2); - assert.deepEqual(readdirSync(work), live, "the live export's directory is still there"); - assert.equal((await pending).status, 200, "the live export finished"); - } finally { - if (second && second.proc.exitCode === null && second.proc.signalCode === null) second.proc.kill("SIGKILL"); - await first.stop(); - } -}); +// node --test scripts/test-vili-daemon.mjs +// Runs the real daemon against scripts/fixtures/stub-worker.mjs as its --python/--worker-entry. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import http from "node:http"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { decode, encode } from "@msgpack/msgpack"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const daemonPath = path.join(here, "vili-daemon.mjs"); +const stubPath = path.join(here, "fixtures", "stub-worker.mjs"); +const stubBlenderPath = path.join(here, "fixtures", "stub-blender.mjs"); +const exportScriptPath = path.join(here, "bvh-to-fbx.py"); +const fakeIdunnEntry = path.join(here, "fixtures", "fake-idunn-entry.mjs"); +const THREADS = 5; +// The stub Blender is executed, and the recipe's runner mounts /tmp noexec (and the root read-only), so it cannot live +// there. It lives under the checkout's gitignored .vili, which the runner's workspace leaves writable. +const execScratch = path.join(here, "..", ".vili", "test-scratch"); + +async function until(condition, what, seconds = 15) { + const deadline = Date.now() + seconds * 1000; + for (;;) { + const value = await condition(); + if (value) return value; + assert.ok(Date.now() < deadline, `timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, 25)); + } +} + +// A zombie is dead for our purposes: its parent (or init) has simply not reaped it yet. +function isAlive(pid) { + try { + process.kill(pid, 0); + } catch { + return false; + } + try { + return !/^\d+ \(.*\) Z/.test(readFileSync(`/proc/${pid}/stat`, "utf8")); + } catch { + return false; + } +} + +async function startDaemon({ + omit = [], viaEnv = false, threads = String(THREADS), python = process.execPath, nested = false, host = "127.0.0.1", + relativeModels = false, extraArgs = [], root: givenRoot = null, entry = daemonPath, extraEnv = {}, hold = entry === daemonPath, + blender = "stub", +} = {}) { + const root = givenRoot ?? mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const stateRoot = nested ? path.join(root, "a", "b", "state") : root; + const modelsRoot = path.join(root, "models"); + const settings = { + host, + port: "0", + "state-root": stateRoot, + python, + "worker-entry": stubPath, + "models-root": relativeModels ? "models-rel" : modelsRoot, + "kimodo-commit": "abc1234", + "python-env": "test-env", + }; + if (threads !== null) settings.threads = threads; + // Blender is a copy of the stub in a directory of its own (the stub logs its starts beside itself): "stub" is + // ready, "broken" fails its version probe, "missing" is a path with nothing there, null is unconfigured. + mkdirSync(execScratch, { recursive: true }); + const blenderDir = mkdtempSync(path.join(execScratch, "blender-")); + const blenderPath = blender === null ? null : path.join(blenderDir, { stub: "blender", broken: "blender-broken", missing: "absent" }[blender]); + if (blender === "stub" || blender === "broken") { + copyFileSync(stubBlenderPath, blenderPath); + chmodSync(blenderPath, 0o755); + } + if (blenderPath) settings.blender = blenderPath; + for (const name of omit) delete settings[name]; + const env = { ...process.env, LEAK_CHECK: "leaked", LANG: "xx_TEST.UTF-8", ...extraEnv }; + for (const name of Object.keys(env)) if (name.startsWith("VILI_")) delete env[name]; + const argv = [entry, ...extraArgs]; + for (const [name, value] of Object.entries(settings)) { + if (viaEnv) env[`VILI_${{ host: "DAEMON_HOST", port: "DAEMON_PORT", "state-root": "STATE_ROOT", "worker-entry": "WORKER_ENTRY" }[name] ?? name.toUpperCase().replace("-", "_")}`] = value; + else argv.push(`--${name}`, value); + } + const proc = spawn(process.execPath, argv, { env, cwd: root, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + let errors = ""; + proc.stdout.on("data", (chunk) => { output += chunk; }); + proc.stderr.on("data", (chunk) => { errors += chunk; }); + const listening = await until(() => /listening on http:\/\/([^:]+):(\d+)/.exec(output), "the daemon to listen"); + const port = listening[2]; + // the hand-run daemon takes its lock after it listens; a test that wants a served state root waits for it + if (hold) await until(() => /holds the state root/.test(output), "the daemon to hold the state root"); + const spawnLog = () => { + const log = path.join(stateRoot, "embeddings", "stub-spawns.log"); + return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean) : []; + }; + const startsLog = path.join(blenderDir, "starts.log"); + const startLines = () => (existsSync(startsLog) ? readFileSync(startsLog, "utf8").split("\n").filter(Boolean) : []); + return { + root, + stateRoot, + modelsRoot, + blenderDir, + blenderPath, + // every start of the stub Blender, the version probe included + starts: () => startLines().filter((line) => line.startsWith("{")).map((line) => JSON.parse(line)), + blenderGrandchildren: () => startLines().filter((line) => line.startsWith("grandchild ")).map((line) => Number(line.split(" ")[1])), + proc, + boundHost: listening[1], + port: Number(port), + stderr: () => errors, + stdout: () => output, + spawns: () => spawnLog().filter((line) => /^\d+$/.test(line)).map(Number), + grandchildren: () => spawnLog().filter((line) => line.startsWith("grandchild ")).map((line) => Number(line.split(" ")[1])), + async stop() { + try { + if (proc.exitCode === null && proc.signalCode === null) { + proc.kill("SIGTERM"); + await until(() => proc.exitCode !== null || proc.signalCode !== null, "the daemon to exit", 30); + } + } finally { + // a daemon that did not stop on SIGTERM must not outlive the test and hold the run open + if (proc.exitCode === null && proc.signalCode === null) proc.kill("SIGKILL"); + rmSync(root, { recursive: true, force: true }); + rmSync(blenderDir, { recursive: true, force: true }); + } + }, + }; +} + +function request(port, method, urlPath, { body, headers = {} } = {}) { + return new Promise((resolve, reject) => { + const req = http.request({ host: "127.0.0.1", port, method, path: urlPath, headers }, (res) => { + const chunks = []; + res.on("data", (chunk) => chunks.push(chunk)); + res.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed = null; + try { + parsed = JSON.parse(raw); + } catch { + // not JSON + } + resolve({ status: res.statusCode, headers: res.headers, raw, body: parsed }); + }); + }); + req.on("error", reject); + req.end(body); + }); +} + +const generate = (daemon, payload = { prompt: "a person waves", duration: 2 }, headers = {}) => + request(daemon.port, "POST", "/motion/generate", { + body: typeof payload === "string" ? payload : JSON.stringify(payload), + headers: { "content-type": "application/json", ...headers }, + }); +const operatorState = async (daemon) => (await request(daemon.port, "GET", "/operator-state")).body; +const backendOf = async (daemon) => (await operatorState(daemon)).backend; + +test("the worker's port comes from its ready line", async () => { + const daemon = await startDaemon(); + try { + const answer = await generate(daemon); + assert.equal(answer.status, 200); + const backend = await backendOf(daemon); + assert.equal(backend.state, "ready"); + assert.notEqual(daemon.port, 8824, "--port 0 binds a free port"); + assert.ok(Number.isInteger(backend.port) && backend.port > 1 && backend.port !== 8825); + assert.equal(answer.body.worker.pid, backend.pid); + assert.deepEqual(answer.body.worker.argv.slice(-4), ["--host", "127.0.0.1", "--port", "0"]); + // the request reached the stub on the port the stub printed, not on a decoy line's port + assert.equal((await request(backend.port, "GET", "/health")).body.requests, 1); + } finally { + await daemon.stop(); + } +}); + +test("one child, respawned after exit", async () => { + const daemon = await startDaemon(); + try { + const [first, second] = await Promise.all([generate(daemon), generate(daemon)]); + assert.equal(first.status, 200); + assert.equal(second.status, 200); + assert.equal(daemon.spawns().length, 1); + const [pid] = daemon.spawns(); + process.kill(pid, "SIGKILL"); + await until(async () => (await backendOf(daemon)).state === "stopped", "the daemon to see the exit"); + assert.match((await backendOf(daemon)).lastError, /^the worker exited \(SIGKILL\)$/); + const third = await generate(daemon); + assert.equal(third.status, 200); + const spawns = daemon.spawns(); + assert.equal(spawns.length, 2); + assert.notEqual(spawns[1], pid); + assert.equal(third.body.worker.pid, spawns[1]); + assert.equal((await backendOf(daemon)).lastError, null); + } finally { + await daemon.stop(); + } +}); + +test("environment and nice", async () => { + const daemon = await startDaemon(); + try { + const { env, priority } = (await generate(daemon)).body.worker; + assert.equal(env.OMP_NUM_THREADS, String(THREADS)); + assert.equal(env.MKL_NUM_THREADS, String(THREADS)); + assert.equal(env.VILI_THREADS, String(THREADS)); + assert.equal(env.HF_HUB_OFFLINE, "1"); + assert.equal(env.TRANSFORMERS_OFFLINE, "1"); + assert.equal(env.HF_HOME, daemon.modelsRoot); + assert.equal(env.VILI_KIMODO_COMMIT, "abc1234"); + assert.equal(env.VILI_PYTHON_ENV, "test-env"); + assert.equal(env.PYTHONDONTWRITEBYTECODE, "1"); + assert.equal(env.PYTHONUNBUFFERED, "1"); + assert.equal(env.PATH, process.env.PATH); + assert.equal(env.HOME, process.env.HOME); + assert.equal(env.LANG, "xx_TEST.UTF-8"); + assert.ok(env.PYTHONPATH.endsWith(path.join("vendor", "CultLib", "packages", "cultcache-py", "src"))); + assert.equal(env.LEAK_CHECK, undefined, "the daemon's own environment does not reach the worker"); + assert.equal(priority, 10); + } finally { + await daemon.stop(); + } +}); + +test("every flag has an environment twin, and the thread cap defaults to 8", async () => { + const viaEnv = await startDaemon({ viaEnv: true, threads: "16" }); + try { + const answer = await generate(viaEnv); + assert.equal(answer.status, 200); + assert.equal(answer.body.worker.env.OMP_NUM_THREADS, "16"); + assert.deepEqual(await backendOf(viaEnv), { + state: "ready", pid: answer.body.worker.pid, port: (await backendOf(viaEnv)).port, threads: 16, pythonEnv: "test-env", + kimodoCommit: "abc1234", modelsRoot: viaEnv.modelsRoot, lastError: null, blender: "Blender 0.0.0 (stub)", + }); + } finally { + await viaEnv.stop(); + } + const byDefault = await startDaemon({ threads: null }); + try { + const answer = await generate(byDefault); + assert.equal(answer.body.worker.env.OMP_NUM_THREADS, "8"); + assert.equal((await backendOf(byDefault)).threads, 8); + } finally { + await byDefault.stop(); + } + const one = await startDaemon({ threads: "1", nested: true }); + try { + assert.equal((await generate(one)).body.worker.env.OMP_NUM_THREADS, "1"); + } finally { + await one.stop(); + } +}); + +test("the worker's output lands in its log, and the ready line does not", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon)).status, 200); + const log = path.join(daemon.stateRoot, "logs", "worker.log"); + const text = await until(() => { + const content = existsSync(log) ? readFileSync(log, "utf8") : ""; + return content.includes("stub-stdout-line") && content.includes("stub-stderr-line") ? content : null; + }, "the worker log"); + assert.ok(!text.includes(`"port":${(await backendOf(daemon)).port}`), "the real ready line is not logged"); + } finally { + await daemon.stop(); + } +}); + +test("a job is a record on disk", async () => { + const daemon = await startDaemon(); + try { + const answer = await generate(daemon, { prompt: "a person waves", duration: 2, constraints: [{ type: "root2d" }] }); + assert.equal(answer.status, 200); + const job = answer.body; + assert.equal(job.schema, "gamecult.vili.motion_job.v1"); + assert.equal(job.request.schema, "gamecult.vili.motion_request.v0"); + assert.equal(job.prompt, "a person waves"); + assert.equal(job.model, "stub-model"); + assert.equal(job.ok, true); + assert.equal(job.error, null); + assert.equal(job.artifactDir, path.join(daemon.stateRoot, "artifacts", job.jobId)); + assert.equal(job.worker.output, path.join(job.artifactDir, "motion")); + assert.equal(job.worker.contentType, "application/json"); + assert.equal(job.worker.prompt, "a person waves"); + assert.equal(job.worker.constraints, path.join(job.artifactDir, "constraints.json")); + assert.deepEqual(JSON.parse(readFileSync(job.worker.constraints, "utf8")), [{ type: "root2d" }]); + assert.deepEqual(JSON.parse(readFileSync(path.join(daemon.stateRoot, "jobs", `${job.jobId}.json`), "utf8")), job); + assert.ok(Number.isInteger(job.seed) && job.seed >= 0 && job.seed < 2 ** 31, "no seed was sent, so one was minted"); + assert.deepEqual(job.files.map((file) => file.name), ["constraints.json", "meta.json", "request.json"]); + for (const file of job.files) { + const bytes = readFileSync(path.join(job.artifactDir, file.name)); + assert.equal(file.bytes, bytes.length); + assert.equal(file.sha256, createHash("sha256").update(bytes).digest("hex")); + } + assert.equal(job.export, undefined, "a job that asked for no FBX has no export block"); + const batch = await generate(daemon, { texts: ["a", "b"], durations: [1, 1] }); + assert.equal(batch.status, 200); + assert.equal(batch.body.prompt, null); + assert.equal(batch.body.worker.prompt, undefined); + } finally { + await daemon.stop(); + } +}); + +test("a requested model restarts the worker only when it is not the one serving", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon)).status, 200); + assert.equal((await generate(daemon, { prompt: "x", model: "stub-model" })).status, 200); + assert.equal((await generate(daemon, { prompt: "x", model: "Stub Model" })).status, 200); + assert.equal(daemon.spawns().length, 1); + const [first] = daemon.spawns(); + const other = await generate(daemon, { prompt: "x", model: "other-model" }); + assert.equal(other.status, 200); + assert.equal(other.body.model, "other-model"); + assert.deepEqual(other.body.worker.argv.slice(-2), ["--model", "other-model"]); + assert.equal(daemon.spawns().length, 2); + await until(() => !isAlive(first), "the first worker to be gone"); + assert.equal((await generate(daemon)).status, 200); + assert.equal(daemon.spawns().length, 2, "a request that names no model keeps the one serving"); + } finally { + await daemon.stop(); + } +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + test(`${signal} leaves no child`, async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon)).status, 200); + const [pid] = daemon.spawns(); + assert.ok(isAlive(pid)); + daemon.proc.kill(signal); + await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 8); + assert.equal(isAlive(pid), false); + } finally { + await daemon.stop(); + } + }); +} + +test("a worker that is stopping gets its grace, not a SIGKILL", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon, { prompt: "x", model: "graceful" })).status, 200); + daemon.proc.kill("SIGTERM"); + await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 8); + assert.ok(readFileSync(path.join(daemon.stateRoot, "embeddings", "stub-spawns.log"), "utf8").includes("graceful-exit")); + } finally { + await daemon.stop(); + } +}); + +test("a worker that dies while the daemon waits for its health fails the job at once", async () => { + const daemon = await startDaemon(); + try { + const began = Date.now(); + const failed = await generate(daemon, { prompt: "x", model: "die-after-ready" }); + assert.equal(failed.status, 502); + assert.ok(Date.now() - began < 20000, "the default deadline is minutes; a dead worker does not wait for it"); + const backend = await backendOf(daemon); + assert.equal(backend.state, "stopped"); + assert.equal(backend.lastError, "the worker exited (code 4)"); + } finally { + await daemon.stop(); + } +}); + +test("the daemon binds loopback unless told otherwise, and ignores flags it does not know", async () => { + const byDefault = await startDaemon({ omit: ["host"], extraArgs: ["--bogus"] }); + try { + assert.equal(byDefault.boundHost, "127.0.0.1"); + assert.equal((await request(byDefault.port, "GET", "/health")).status, 200); + } finally { + await byDefault.stop(); + } + const named = await startDaemon({ host: "127.0.0.2" }); + try { + assert.equal(named.boundHost, "127.0.0.2"); + assert.equal((await fetch(`http://127.0.0.2:${named.port}/health`)).status, 200); + } finally { + await named.stop(); + } +}); + +test("a relative models root reaches the worker as an absolute path", async () => { + const daemon = await startDaemon({ relativeModels: true }); + try { + const { env } = (await generate(daemon)).body.worker; + assert.equal(env.HF_HOME, path.join(path.dirname(daemon.modelsRoot), "models-rel")); + } finally { + await daemon.stop(); + } +}); + +test("a worker that ignores SIGTERM is killed with everything it started", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon, { prompt: "x", model: "stubborn" })).status, 200); + const [pid] = daemon.spawns(); + const [grandchild] = daemon.grandchildren(); + assert.ok(isAlive(pid) && isAlive(grandchild)); + daemon.proc.kill("SIGTERM"); + await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 30); + assert.equal(isAlive(pid), false); + assert.equal(isAlive(grandchild), false); + } finally { + await daemon.stop(); + } +}); + +// The daemon owns the worker's whole tree (motion-pipeline:finding:cut-vili-cpu-body.s1.orphaned-descendants). +const aliveWorkers = (daemon) => daemon.spawns().filter(isAlive); + +test("a worker killed mid-request leaves no encoder child, and the next job has exactly one worker", async () => { + const daemon = await startDaemon(); + try { + const pending = generate(daemon, { prompt: "x", model: "encoder" }); + const [grandchild] = await until(() => { const found = daemon.grandchildren(); return found.length ? found : null; }, "the encoder child"); + const [worker] = daemon.spawns(); + assert.ok(isAlive(worker) && isAlive(grandchild)); + process.kill(worker, "SIGKILL"); + assert.equal(readFileSync(path.join(daemon.stateRoot, "worker.pid"), "utf8"), `${worker}\n`); + assert.equal((await pending).status, 502); + await until(() => !isAlive(grandchild), "the encoder child to die with its worker"); + await until(() => !existsSync(path.join(daemon.stateRoot, "worker.pid")), "the dead worker's pid file to go"); + assert.equal((await generate(daemon, { prompt: "y" })).status, 200); + assert.equal(aliveWorkers(daemon).length, 1); + assert.equal(readFileSync(path.join(daemon.stateRoot, "worker.pid"), "utf8"), `${aliveWorkers(daemon)[0]}\n`); + assert.equal(isAlive(grandchild), false); + } finally { + await daemon.stop(); + } +}); + +// "encoder" leaves a SIGTERM-proof grandchild beside the worker; "stubborn" is a worker that ignores SIGTERM; the plain stub is just a worker. +for (const model of ["encoder", "stubborn", "stub-model"]) { + test(`a restarted daemon kills the ${model} worker its crashed predecessor left, and runs exactly one`, async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const first = await startDaemon({ root }); + const stray = generate(first, { prompt: "x", model }).catch(() => null); + await until(() => first.spawns().length, "the worker"); + if (model === "encoder") await until(() => first.grandchildren().length, "the encoder child"); + await until(() => request(first.port, "GET", "/health").then((r) => r.body?.state?.backend?.state === "ready"), "the worker to be ready"); + const [worker] = first.spawns(); + const strays = [worker, ...first.grandchildren()]; + first.proc.kill("SIGKILL"); + await until(() => first.proc.exitCode !== null || first.proc.signalCode !== null, "the first daemon to die"); + await stray; + assert.ok(strays.every(isAlive), "the probe must reproduce the orphan before the fix can be measured"); + const second = await startDaemon({ root }); + try { + await until(() => !strays.some(isAlive), "the orphans to be killed"); + assert.equal(existsSync(path.join(root, "worker.pid")), false); + assert.equal((await generate(second, { prompt: "y" })).status, 200); + assert.equal(aliveWorkers(second).length, 1); + second.proc.kill("SIGTERM"); + await until(() => second.proc.exitCode !== null, "the second daemon to exit", 30); + assert.equal(aliveWorkers(second).length, 0); + } finally { + await second.stop(); + } + }); +} + +test("a live worker outside the pid file's group is left alone", async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const first = await startDaemon({ root }); + const bystander = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + try { + assert.equal((await generate(first)).status, 200); + const [worker] = first.spawns(); + first.proc.kill("SIGKILL"); + await until(() => first.proc.exitCode !== null || first.proc.signalCode !== null, "the first daemon to die"); + writeFileSync(path.join(root, "worker.pid"), `${bystander.pid}\n`); + const second = await startDaemon({ root }); + assert.ok(isAlive(worker) && isAlive(bystander.pid), "only the group the pid file names, and only if it is ours, is killed"); + second.proc.kill("SIGTERM"); + await until(() => second.proc.exitCode !== null, "the second daemon to exit", 30); + process.kill(-worker, "SIGKILL"); + } finally { + bystander.kill("SIGKILL"); + await first.stop(); + } +}); + +// The marker that proves a group is ours is this state root's own embeddings directory. A live worker of another +// state root, named by a stale pid file, carries another root's marker and is left alone. +test("a pid file naming another state root's live worker kills nothing", async () => { + const other = await startDaemon(); + const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + try { + assert.equal((await generate(other)).status, 200); + const [foreign] = other.spawns(); + writeFileSync(path.join(root, "worker.pid"), `${foreign}\n`); + const daemon = await startDaemon({ root }); + try { + assert.ok(isAlive(foreign), "a worker carrying another root's marker must survive the reclaim"); + assert.equal(existsSync(path.join(root, "worker.pid")), false, "the stale pid file is still forgotten"); + } finally { + await daemon.stop(); + } + } finally { + await other.stop(); + } +}); + +test("a pid file naming a process group that is not the worker's kills nothing", async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const bystander = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + writeFileSync(path.join(root, "worker.pid"), `${bystander.pid}\n`); + const daemon = await startDaemon({ root }); + try { + assert.equal(isAlive(bystander.pid), true); + assert.equal(existsSync(path.join(root, "worker.pid")), false); + } finally { + bystander.kill("SIGKILL"); + await daemon.stop(); + } +}); + +test("cross-site is refused before the worker", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon)).status, 200); + const withOrigin = await generate(daemon, undefined, { origin: "https://example.test" }); + assert.equal(withOrigin.status, 403); + assert.deepEqual(withOrigin.body, { ok: false, code: "cross-site" }); + const plain = await generate(daemon, undefined, { "content-type": "text/plain" }); + assert.equal(plain.status, 403); + assert.deepEqual(plain.body, { ok: false, code: "cross-site" }); + const untyped = await request(daemon.port, "POST", "/motion/generate", { body: JSON.stringify({ prompt: "x" }) }); + assert.equal(untyped.status, 403); + const options = await request(daemon.port, "OPTIONS", "/motion/generate", { headers: { origin: "https://example.test" } }); + assert.equal(options.status, 404); + const health = await request(daemon.port, "GET", "/health"); + for (const reply of [withOrigin, plain, untyped, options, health]) { + assert.equal(Object.keys(reply.headers).filter((name) => name.startsWith("access-control-")).length, 0); + } + const backend = await backendOf(daemon); + assert.equal((await request(backend.port, "GET", "/health")).body.requests, 1, "only the same-site request reached the worker"); + // the media type is matched as a type, not as a string + assert.equal((await generate(daemon, undefined, { "content-type": "Application/JSON; charset=utf-8" })).status, 200); + assert.equal((await generate(daemon, undefined, { "content-type": "application/json ; charset=utf-8" })).status, 200); + // and only the generate route answers a POST + const stray = await request(daemon.port, "POST", "/motion/other", { + body: JSON.stringify({ prompt: "x" }), + headers: { "content-type": "application/json" }, + }); + assert.equal(stray.status, 404); + } finally { + await daemon.stop(); + } +}); + +for (const omitted of ["models-root", "kimodo-commit", "python-env"]) { + test(`unconfigured (no ${omitted}) refuses before spawning`, async () => { + const daemon = await startDaemon({ omit: [omitted] }); + try { + const answer = await generate(daemon); + assert.equal(answer.status, 503); + assert.deepEqual(answer.body, { ok: false, code: "worker-unconfigured" }); + assert.deepEqual(daemon.spawns(), []); + assert.equal((await backendOf(daemon)).state, "stopped"); + } finally { + await daemon.stop(); + } + }); +} + +test("an error never echoes the request", async () => { + const daemon = await startDaemon(); + try { + const malformed = await generate(daemon, "{\"prompt\": \"the launch code is swordfish\", "); + assert.equal(malformed.status, 400); + assert.deepEqual(malformed.body, { ok: false, error: "Invalid JSON." }); + const missing = await generate(daemon, { texts: ["the launch code is swordfish"] }); + assert.equal(missing.status, 400); + assert.ok(!missing.raw.includes("swordfish")); + // a body the handler cannot use at all: the answer names nothing, and neither does the log + const unusable = await generate(daemon, "null"); + assert.equal(unusable.status, 500); + assert.deepEqual(unusable.body, { ok: false, code: "internal" }); + await until(() => daemon.stderr().includes("request failed: TypeError"), "the daemon to log the class"); + assert.ok(!daemon.stderr().includes("swordfish")); + } finally { + await daemon.stop(); + } +}); + +test("a worker that dies before it is ready fails the job, and the next job tries again", async () => { + const daemon = await startDaemon(); + try { + const failed = await generate(daemon, { prompt: "x", model: "crash-before-ready" }); + assert.equal(failed.status, 502); + assert.equal(failed.body.ok, false); + assert.equal(failed.body.error, "the worker request failed (Error)"); + const backend = await backendOf(daemon); + assert.equal(backend.state, "failed"); + assert.equal(backend.lastError, "the worker exited (code 3)"); + assert.equal(backend.pid, null); + assert.equal(backend.port, null); + assert.equal((await generate(daemon)).status, 200); + assert.equal((await backendOf(daemon)).state, "ready"); + assert.equal(daemon.spawns().length, 2); + } finally { + await daemon.stop(); + } +}); + +test("a worker that never says it is ready is killed at the deadline", async () => { + const daemon = await startDaemon(); + try { + const pending = generate(daemon, { prompt: "x", model: "hang-before-ready", workerTimeoutMs: 1500 }); + const starting = await until(async () => { + const backend = await backendOf(daemon); + return backend.state === "starting" ? backend : null; + }, "the starting state"); + assert.ok(Number.isInteger(starting.pid) && starting.port === null); + const failed = await pending; + assert.equal(failed.status, 502); + const backend = await backendOf(daemon); + assert.equal(backend.state, "failed"); + assert.equal(backend.lastError, "the worker was not ready in time"); + assert.equal(isAlive(daemon.spawns()[0]), false); + } finally { + await daemon.stop(); + } +}); + +test("a worker that is ready but not healthy fails at the deadline; one that gets healthy in time serves", async () => { + const daemon = await startDaemon(); + try { + const failed = await generate(daemon, { prompt: "x", model: "sick", workerTimeoutMs: 1500 }); + assert.equal(failed.status, 502); + const backend = await backendOf(daemon); + assert.equal(backend.state, "failed"); + assert.equal(backend.lastError, "the worker did not become healthy in time"); + assert.equal(isAlive(daemon.spawns()[0]), false); + const slow = await generate(daemon, { prompt: "x", model: "slow-health" }); + assert.equal(slow.status, 200); + assert.equal((await backendOf(daemon)).state, "ready"); + } finally { + await daemon.stop(); + } +}); + +test("a missing interpreter fails the job at once", async () => { + const daemon = await startDaemon({ python: "/no/such/python" }); + try { + const began = Date.now(); + const failed = await generate(daemon); + assert.equal(failed.status, 502); + assert.ok(Date.now() - began < 5000, "nothing waits on a process that never existed"); + const backend = await backendOf(daemon); + assert.equal(backend.state, "failed"); + assert.equal(backend.lastError, "the worker could not be spawned"); + assert.equal(existsSync(path.join(daemon.stateRoot, "worker.pid")), false); + } finally { + await daemon.stop(); + } +}); + +test("a worker that refuses, errors or drops the request fails the job without echoing it", async () => { + const daemon = await startDaemon(); + try { + const refused = await generate(daemon, { prompt: "swordfish", model: "refuse-generate" }); + assert.equal(refused.status, 502); + assert.equal(refused.body.ok, false); + assert.equal(refused.body.error, null); + const errored = await generate(daemon, { prompt: "swordfish", model: "fail-generate" }); + assert.equal(errored.status, 502); + assert.equal(errored.body.error, "the worker answered 500"); + const dropped = await generate(daemon, { prompt: "swordfish", model: "drop-generate" }); + assert.equal(dropped.status, 502); + assert.equal(dropped.body.error, "the worker request failed (TypeError)"); + const stalled = await generate(daemon, { prompt: "swordfish", model: "slow-generate", timeoutMs: 700 }); + assert.equal(stalled.status, 502); + assert.equal(stalled.body.error, "the worker request failed (TimeoutError)"); + for (const answer of [refused, errored, dropped, stalled]) assert.ok(!answer.body.error?.includes("swordfish")); + } finally { + await daemon.stop(); + } +}); + +test("health, operator state and the unknown", async () => { + const daemon = await startDaemon(); + try { + for (const route of ["/", "/health"]) { + const answer = await request(daemon.port, "GET", route); + assert.equal(answer.status, 200); + assert.equal(answer.body.ok, true); + assert.equal(answer.body.service, "vili"); + assert.equal(answer.body.state.backend.threads, THREADS); + } + assert.equal((await request(daemon.port, "GET", "/motion/generate")).status, 404); + // the retired operator route, spelled in parts so the retirement grep over scripts/ stays empty + assert.equal((await request(daemon.port, "GET", ["", "eve", "operator"].join("/"))).status, 404); + const unknown = await request(daemon.port, "GET", "/provider-advertisement"); + assert.equal(unknown.status, 404); + assert.equal(unknown.raw, "not found\n"); + const state = await operatorState(daemon); + assert.equal(state.host, os.hostname()); + assert.deepEqual(Object.keys(state.routes), ["health", "operatorState", "generate", "artifacts", "restFbx"]); + } finally { + await daemon.stop(); + } +}); + +test("the typed service state is refreshed while the daemon runs", async () => { + const daemon = await startDaemon({ nested: true }); + try { + const store = path.join(daemon.stateRoot, "vili.service.cc"); + assert.ok(existsSync(store)); + assert.ok(!readFileSync(store).includes("ready")); + assert.equal((await generate(daemon)).status, 200); + await until(() => readFileSync(store).includes("ready"), "the store to show the worker ready", 20); + } finally { + await daemon.stop(); + } +}); + +test("a thread count outside 1 to 16 stops the daemon without echoing it", async () => { + for (const threads of ["0", "17", "99", "100", "8x", "x8", "-1", "1.5"]) { + const proc = spawn(process.execPath, [daemonPath, "--threads", threads], { stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + proc.stdout.on("data", (chunk) => { output += chunk; }); + proc.stderr.on("data", (chunk) => { output += chunk; }); + const code = await new Promise((resolve) => proc.on("exit", resolve)); + assert.equal(code, 2); + assert.match(output, /threads must be an integer from 1 to 16/); + assert.ok(!output.replace("from 1 to 16", "").includes(threads), `the refusal repeated ${threads}`); + } +}); + +const msgpackPost = (daemon, body, headers = {}) => + new Promise((resolve, reject) => { + const req = http.request({ host: "127.0.0.1", port: daemon.port, method: "POST", path: "/cultnet/snapshot", headers: { "content-type": "application/msgpack", ...headers } }, (res) => { + const chunks = []; + res.on("data", (chunk) => chunks.push(chunk)); + res.on("end", () => resolve({ status: res.statusCode, headers: res.headers, raw: Buffer.concat(chunks) })); + }); + req.on("error", reject); + req.end(body); + }); +const SECRET = "canary-9f3c-never-echoed"; +const snapshotRequest = (messageId = "m1") => Buffer.from(encode({ schemaVersion: "cultnet.snapshot_request.v0", messageId })); + +// A second daemon on a state root a live daemon holds must refuse, and must not touch the first one's worker. +test("a second daemon on a live state root exits 2 and kills nothing", async () => { + const first = await startDaemon(); + let second = null; + try { + assert.equal((await generate(first)).status, 200); + const [worker] = first.spawns(); + assert.ok(isAlive(worker)); + second = await startDaemon({ root: first.root, hold: false }); + await until(() => second.proc.exitCode !== null, "the second daemon to refuse", 20); + assert.equal(second.proc.exitCode, 2); + assert.match(second.stderr(), /held by a running daemon/); + assert.ok(isAlive(worker), "the first daemon's worker must survive the refusal"); + assert.equal((await generate(first)).status, 200); + assert.equal(aliveWorkers(first).length, 1); + } finally { + if (second && second.proc.exitCode === null && second.proc.signalCode === null) second.proc.kill("SIGKILL"); // a daemon that failed to refuse must not outlive the test + await first.stop(); + } +}); + +// Two daemons start together over the lock a crashed daemon left. Exactly one owns the root, every time: the +// other exits 2, and one generate leaves one worker on the root. +for (let trial = 1; trial <= 40; trial += 1) { + test(`two daemons racing a crashed daemon's lock leave one owner and one worker (trial ${trial})`, async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const crashed = await startDaemon({ root }); + crashed.proc.kill("SIGKILL"); + await until(() => crashed.proc.exitCode !== null || crashed.proc.signalCode !== null, "the first daemon to die"); + const racers = await Promise.all([startDaemon({ root, hold: false }), startDaemon({ root, hold: false })]); + try { + await until(() => racers.every((racer) => racer.proc.exitCode !== null || /holds the state root/.test(racer.stdout())), "both racers to decide", 20); + await new Promise((resolve) => setTimeout(resolve, 300)); + const owners = racers.filter((racer) => racer.proc.exitCode === null); + assert.equal(owners.length, 1, "exactly one daemon stays up"); + assert.match(owners[0].stdout(), /holds the state root/); + const loser = racers.find((racer) => racer !== owners[0]); + assert.equal(loser.proc.exitCode, 2); + assert.match(loser.stderr(), /held by a running daemon/); + for (const racer of owners) assert.equal((await generate(racer)).status, 200); + assert.equal(aliveWorkers(owners[0]).length, 1, "one worker on the root"); + } finally { + for (const racer of racers) if (racer.proc.exitCode === null && racer.proc.signalCode === null) racer.proc.kill("SIGKILL"); + for (const pid of racers[0].spawns()) { try { process.kill(-pid, "SIGKILL"); } catch { /* gone */ } } + rmSync(root, { recursive: true, force: true }); + } + }); +} + +// canonicalPath's fallback: a state root that does not exist yet is resolved through its longest existing prefix, so a +// daemon naming it through a symlink and one naming it directly still contend for one lock. +for (let trial = 1; trial <= 10; trial += 1) { + test(`two daemons naming a not-yet-existing root, one through a symlink, leave one owner (trial ${trial})`, async () => { + const real = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const link = `${real}-link`; + symlinkSync(real, link); + const racers = await Promise.all([startDaemon({ root: real, nested: true, hold: false }), startDaemon({ root: link, nested: true, hold: false })]); + try { + await until(() => racers.every((racer) => racer.proc.exitCode !== null || /holds the state root/.test(racer.stdout())), "both racers to decide", 20); + await new Promise((resolve) => setTimeout(resolve, 300)); + const owners = racers.filter((racer) => racer.proc.exitCode === null); + assert.equal(owners.length, 1, "exactly one daemon stays up"); + const loser = racers.find((racer) => racer !== owners[0]); + assert.equal(loser.proc.exitCode, 2); + assert.match(loser.stderr(), /held by a running daemon/); + } finally { + for (const racer of racers) if (racer.proc.exitCode === null && racer.proc.signalCode === null) racer.proc.kill("SIGKILL"); + rmSync(link, { force: true }); + rmSync(real, { recursive: true, force: true }); + } + }); +} +const idunnDaemon = async ({ leased = false, extraEnv = {}, loseWhenExists = null, python = undefined } = {}) => { + const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const lease = path.join(root, "lease"); + const healthLog = path.join(root, "health.log"); + if (leased) writeFileSync(lease, ""); + const daemon = await startDaemon({ + root, nested: true, entry: fakeIdunnEntry, ...(python ? { python } : {}), extraEnv: { + FAKE_IDUNN_LEASE: lease, FAKE_IDUNN_HEALTH_LOG: healthLog, ...extraEnv, + ...(loseWhenExists ? { FAKE_IDUNN_STATE_ROOT: path.join(root, "a", "b", "state"), FAKE_IDUNN_LOSE_WHEN_EXISTS: loseWhenExists } : {}), + }, + }); + return Object.assign(daemon, { lease, health: () => (existsSync(healthLog) ? readFileSync(healthLog, "utf8").split("\n").filter(Boolean) : []) }); +}; + +test("before the lease nothing is written, spawned or accepted", async () => { + const daemon = await idunnDaemon(); + try { + const answer = await generate(daemon); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-pending"); + assert.deepEqual(daemon.spawns(), []); + assert.deepEqual(daemon.starts(), [], "no Blender before the lease, the version probe included"); + assert.equal(existsSync(daemon.stateRoot), false, "the state root must not exist before the lease"); + assert.deepEqual(daemon.health(), []); + } finally { + await daemon.stop(); + } +}); + +test("with the lease the daemon writes and serves, and no pid file exists", async () => { + const daemon = await idunnDaemon(); + try { + writeFileSync(daemon.lease, ""); + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + const answer = await generate(daemon); + assert.equal(answer.status, 200); + assert.equal(existsSync(path.join(daemon.stateRoot, "worker.pid")), false); + assert.equal(existsSync(path.join(daemon.stateRoot, "daemon.lock")), false); + assert.deepEqual(daemon.health(), ["active"]); + } finally { + await daemon.stop(); + } +}); + +test("losing the lease kills the worker tree, refuses, and writes nothing more", async () => { + const daemon = await idunnDaemon({ leased: true }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + assert.equal((await generate(daemon, { prompt: "x", model: "stubborn" })).status, 200); + const [worker] = daemon.spawns(); + const [grandchild] = await until(() => { const found = daemon.grandchildren(); return found.length ? found : null; }, "the armed grandchild"); + assert.ok(isAlive(worker) && isAlive(grandchild)); + rmSync(daemon.lease); + await until(() => !isAlive(worker) && !isAlive(grandchild), "the worker tree to die", 20); + const answer = await generate(daemon); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + await until(() => daemon.health().join() === "active,degraded", "the health to say degraded"); + const store = path.join(daemon.stateRoot, "vili.service.cc"); + const before = statSync(store).mtimeMs; + const jobs = readdirSync(path.join(daemon.stateRoot, "jobs")).length; + await new Promise((resolve) => setTimeout(resolve, 6500)); + assert.equal(statSync(store).mtimeMs, before); + assert.equal(readdirSync(path.join(daemon.stateRoot, "jobs")).length, jobs); + } finally { + await daemon.stop(); + } +}); + +// A generate whose headers are in and whose body is still arriving when the lease goes. The lease is gone, so +// nothing is created and nothing is spawned when the body completes, whether the daemon has noticed the loss yet +// (its 5 s tick) or not. +for (const noticed of [false, true]) { + test(`a request in flight when the lease is lost writes and spawns nothing (loss ${noticed ? "noticed" : "not yet noticed"})`, async () => { + const daemon = await idunnDaemon({ leased: true }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + const body = JSON.stringify({ prompt: "x", duration: 2 }); + const socket = net.connect(daemon.port, "127.0.0.1"); + let raw = ""; + socket.on("data", (chunk) => { raw += chunk; }); + const closed = new Promise((resolve) => socket.on("close", resolve)); + socket.write(`POST /motion/generate HTTP/1.1\r\nhost: vili\r\ncontent-type: application/json\r\ncontent-length: ${body.length}\r\nconnection: close\r\n\r\n${body.slice(0, 5)}`); + await new Promise((resolve) => setTimeout(resolve, 200)); + rmSync(daemon.lease); + if (noticed) await until(() => daemon.health().join() === "active,degraded", "the daemon to notice the loss", 15); + socket.write(body.slice(5)); + await Promise.race([closed, new Promise((resolve) => setTimeout(resolve, 15000))]); + socket.destroy(); + assert.match(raw, /^HTTP\/1\.1 503 /); + assert.match(raw, /write-lease-lost/); + assert.deepEqual(daemon.spawns(), [], "no worker is spawned after the loss"); + const artifacts = path.join(daemon.stateRoot, "artifacts"); + assert.deepEqual(existsSync(artifacts) ? readdirSync(artifacts) : [], [], "no artifact directory is created"); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), []); + } finally { + await daemon.stop(); + } + }); +} + +// The lease goes between two writes of one request. Each case names the path whose appearance takes the lease, and +// what must not exist afterwards: every write and the spawn are gated where they happen, not by the request's +// earlier checks. +const losses = [ + { when: "artifacts/*", absent: ["artifacts/*/meta.json", "logs", "embeddings"] }, + { when: "artifacts/*", payload: { prompt: "a person waves", duration: 2, constraints: [{ type: "root2d" }] }, absent: ["artifacts/*/constraints.json", "artifacts/*/meta.json"] }, + { when: "artifacts/*/meta.json", absent: ["logs", "embeddings"] }, + { when: "logs", absent: ["embeddings"] }, + { when: "embeddings", absent: ["logs/worker.log"] }, + { when: "embeddings/stub-healthy", absent: ["artifacts/*/request.json", "jobs/*"] }, +]; +for (const { when, absent, payload } of losses) { + test(`a request is stopped at its next write when the lease goes at ${when}${payload ? " (constraints in the body)" : ""}`, async () => { + const daemon = await idunnDaemon({ leased: true, loseWhenExists: when }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + const answer = await generate(daemon, payload); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + for (const gone of absent) { + const parts = gone.split("/"); + const found = (base, rest) => (rest.length === 0 ? existsSync(base) + : rest[0] === "*" ? existsSync(base) && readdirSync(base).some((entry) => found(path.join(base, entry), rest.slice(1))) + : found(path.join(base, rest[0]), rest.slice(1))); + assert.equal(found(daemon.stateRoot, parts), false, `${gone} must not exist`); + } + if (when === "embeddings") assert.deepEqual(daemon.spawns(), []); + } finally { + await daemon.stop(); + } + }); +} + +// The daemon's own copy of a worker's stdout into worker.log is a write like any other. +test("a worker's output line is not logged once the lease is gone", async () => { + const daemon = await idunnDaemon({ leased: true, loseWhenExists: "embeddings/stub-spawns.log" }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + const answer = await generate(daemon); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + const log = readFileSync(path.join(daemon.stateRoot, "logs", "worker.log"), "utf8"); + assert.ok(!log.includes("stub-stdout-line"), "the daemon must not copy the worker's stdout after the loss"); + } finally { + await daemon.stop(); + } +}); + +// The lease goes while the first persist is under way: the daemon survives it, stays degraded, and the rest of the +// startup writes are not made. +test("a lease lost while the state root is opened leaves the daemon up, degraded and writing nothing more", async () => { + const daemon = await idunnDaemon({ leased: true, loseWhenExists: "vili.service.cc" }); + try { + await until(() => daemon.health().includes("degraded") || daemon.proc.exitCode !== null, "the loss to be seen", 15); + assert.equal(daemon.proc.exitCode, null, "the daemon must not die of it"); + assert.deepEqual(daemon.health(), ["degraded"]); + const store = readFileSync(path.join(daemon.stateRoot, "vili.service.cc")); + assert.ok(store.includes("operator_state") && !store.includes("command_boundary"), "the second document is not written"); + const answer = await generate(daemon); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + } finally { + await daemon.stop(); + } +}); + +test("a request arriving right after the lease is lost writes and spawns nothing", async () => { + const daemon = await idunnDaemon({ leased: true }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + rmSync(daemon.lease); + const answer = await generate(daemon); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + assert.deepEqual(daemon.spawns(), []); + const artifacts = path.join(daemon.stateRoot, "artifacts"); + assert.deepEqual(existsSync(artifacts) ? readdirSync(artifacts) : [], []); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), []); + } finally { + await daemon.stop(); + } +}); + +test("a worker respawned under the lease is killed when the lease is lost", async () => { + const daemon = await idunnDaemon({ leased: true }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + assert.equal((await generate(daemon)).status, 200); + const [first] = daemon.spawns(); + process.kill(-first, "SIGKILL"); + await until(async () => (await backendOf(daemon)).state === "stopped", "the daemon to see the worker gone"); + assert.equal((await generate(daemon)).status, 200); + const second = daemon.spawns().find((pid) => pid !== first); + assert.ok(second && isAlive(second)); + rmSync(daemon.lease); + await until(() => !isAlive(second), "the respawned worker to die", 20); + } finally { + await daemon.stop(); + } +}); + +// Idunn stops an incumbent (SIGTERM to the unit) before it revokes the lease. What Vili owns in that sequence: the +// worker tree is dead when the process ends, it exits 0, and nothing is written from the signal on. +test("on SIGTERM the daemon stops its worker tree, writes nothing more, and exits 0", async () => { + const daemon = await idunnDaemon({ leased: true }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + assert.equal((await generate(daemon, { prompt: "x", model: "stubborn" })).status, 200); + const [worker] = daemon.spawns(); + const [grandchild] = await until(() => { const found = daemon.grandchildren(); return found.length ? found : null; }, "the armed grandchild"); + const store = path.join(daemon.stateRoot, "vili.service.cc"); + const before = statSync(store).mtimeMs; + const jobs = readdirSync(path.join(daemon.stateRoot, "jobs")); + daemon.proc.kill("SIGTERM"); + await until(() => daemon.proc.exitCode !== null || daemon.proc.signalCode !== null, "the daemon to exit", 30); + assert.equal(daemon.proc.exitCode, 0); + assert.ok(!isAlive(worker) && !isAlive(grandchild), "the worker tree is dead when the daemon is gone"); + assert.equal(statSync(store).mtimeMs, before, "the 5 s surface tick must not write during the stop"); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), jobs); + assert.ok(existsSync(daemon.lease), "the daemon never touches the lease Idunn revokes"); + } finally { + await daemon.stop(); + } +}); + +test("a job that finishes after the lease is lost is not recorded", async () => { + const daemon = await idunnDaemon({ leased: true }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + const slow = generate(daemon, { prompt: "x", model: "slow-generate" }); + slow.catch(() => {}); // a failing assertion below must not leave this request pending + await until(() => daemon.spawns().length, "the worker"); + await until(() => request(daemon.port, "GET", "/health").then((r) => r.body?.state?.backend?.state === "ready"), "the worker to be ready"); + rmSync(daemon.lease); + const answer = await Promise.race([slow, new Promise((resolve) => setTimeout(() => resolve({ status: "no answer in 30 s" }), 30000))]); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), []); + } finally { + await daemon.stop(); + } +}); + +test("the route challenge is answered ahead of the cross-site refusal, and an Origin is still refused", async () => { + const daemon = await idunnDaemon(); + try { + // the challenge is how Idunn learns the incarnation is warming, so it is answered before the lease + const answered = await msgpackPost(daemon, snapshotRequest("m-42")); + assert.equal(answered.status, 200); + assert.equal(answered.headers["content-type"], "application/msgpack"); + const message = decode(answered.raw); + assert.equal(message.schemaVersion, "cultnet.snapshot_response_raw.v0"); + assert.equal(message.messageId, "m-42"); + assert.equal(message.documents.length, 1); + assert.equal((await msgpackPost(daemon, snapshotRequest(), { origin: "https://evil.example" })).status, 403); + assert.equal((await msgpackPost(daemon, Buffer.from("not msgpack at all"))).status, 400); + assert.equal((await msgpackPost(daemon, Buffer.from(encode({ schemaVersion: "cultnet.snapshot_response_raw.v0", messageId: "x", documents: [] })))).status, 400); + assert.equal((await msgpackPost(daemon, Buffer.alloc(70 * 1024))).status, 400); + const text = await request(daemon.port, "POST", "/motion/generate", { body: "{}", headers: { "content-type": "text/plain" } }); + assert.equal(text.status, 403); + } finally { + await daemon.stop(); + } +}); + +test("a signer that cannot answer gives 503 and echoes nothing", async () => { + const daemon = await idunnDaemon({ extraEnv: { FAKE_IDUNN_ANSWER_THROWS: "1" } }); + try { + const answered = await msgpackPost(daemon, snapshotRequest(SECRET)); + assert.equal(answered.status, 503); + assert.ok(!answered.raw.toString("utf8").includes(SECRET)); + assert.ok(!daemon.stderr().includes(SECRET)); + } finally { + await daemon.stop(); + } +}); + +test("the hand-run daemon has no route challenge", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await msgpackPost(daemon, snapshotRequest())).status, 404); + assert.equal((await msgpackPost(daemon, snapshotRequest(), { origin: "https://evil.example" })).status, 404); + } finally { + await daemon.stop(); + } +}); + +// ---- the Idunn entry's own rules: its argument contract, and what it derives for the daemon + +const entryPath = path.join(here, "idunn-entry.mjs"); +const runEntry = (args, env = {}) => new Promise((resolve) => { + const proc = spawn(process.execPath, [entryPath, ...args], { env: { PATH: process.env.PATH, ...env }, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + proc.stdout.on("data", (chunk) => { output += chunk; }); + proc.stderr.on("data", (chunk) => { output += chunk; }); + proc.on("exit", (code) => resolve({ code, output })); +}); + +test("the entry takes exactly --state-root --kimodo-commit <40 hex>, and echoes none of a bad value", async () => { + const commit = "6bb58488037dd65360ff0c5d1692b403a23309f7"; + const good = ["--state-root", "/var/lib/gamecult/vili", "--kimodo-commit", commit]; + const bads = [ + [], + [...good, "--extra", "x"], + ["--kimodo-commit", commit, "--state-root", "/var/lib/gamecult/vili"], + ["--state-root", "canary-relative-path", "--kimodo-commit", commit], + ["--state-root", "/var/lib/gamecult/vili", "--kimodo-commit", "canary-not-a-commit"], + ["--state-root", "/var/lib/gamecult/vili", "--kimodo-commit", commit.toUpperCase()], + ]; + for (const args of bads) { + const answer = await runEntry(args, { GAMECULT_IDUNN_CANDIDATE_BIND: "127.0.0.1:18861" }); + assert.equal(answer.code, 2, JSON.stringify(args)); + assert.ok(!answer.output.includes("canary"), `the refusal repeated a value of ${JSON.stringify(args)}`); + } + for (const badBind of ["canary-not-a-bind", "127.0.0.1:65536", "127.0.0.1:99999", "127.0.0.1:123456", "127.0.0.1:"]) { + const bind = await runEntry(good, { GAMECULT_IDUNN_CANDIDATE_BIND: badBind }); + assert.equal(bind.code, 2, badBind); + assert.match(bind.output, /CANDIDATE_BIND is not host:port/); + assert.ok(!bind.output.includes("canary")); + assert.ok(!bind.output.includes("65536") && !bind.output.includes("99999")); + } +}); + +// A startup failure that kills the unit names its cause from what the failing code itself says (its error code, or +// the fixed text of a plain Error), and never repeats a value: an fs error's own message carries the path. +test("a failed startup names the cause and repeats no value", async () => { + const good = ["--state-root", "/var/lib/gamecult/vili", "--kimodo-commit", "6bb58488037dd65360ff0c5d1692b403a23309f7"]; + const bind = { GAMECULT_IDUNN_CANDIDATE_BIND: "127.0.0.1:18861" }; + const unset = await runEntry(good, bind); + assert.equal(unset.code, 1); + assert.match(unset.output, /GAMECULT_IDUNN_RUNTIME_BUNDLE is required/); + const missing = await runEntry(good, { ...bind, GAMECULT_IDUNN_RUNTIME_BUNDLE: "/canary-bundle-dir" }); + assert.equal(missing.code, 1); + assert.match(missing.output, /ENOENT/); + assert.ok(!missing.output.includes("canary"), "the failure repeated a value"); +}); + +test("the entry derives the daemon's host, port, interpreter and environment identity from the release", async () => { + const { createHash } = await import("node:crypto"); + const { daemonEnvironment } = await import("./idunn-entry.mjs"); + const release = mkdtempSync(path.join(os.tmpdir(), "vili-entry-")); + try { + mkdirSync(path.join(release, "vili", "scripts"), { recursive: true }); + mkdirSync(path.join(release, "vili", "deployment", "idunn"), { recursive: true }); + writeFileSync(path.join(release, "vili", "deployment", "idunn", "requirements-build.lock"), "build lock bytes\n"); + writeFileSync(path.join(release, "vili", "deployment", "idunn", "requirements.lock"), "runtime lock bytes\n"); + const entryDir = path.join(release, "vili", "scripts"); + assert.equal(daemonEnvironment("127.0.0.1:65535", entryDir).VILI_DAEMON_PORT, "65535", "the top of the port range is accepted"); + for (const bind of ["127.0.0.1:18861", "http://127.0.0.1:18861"]) { + const derived = daemonEnvironment(bind, entryDir); + assert.equal(derived.VILI_DAEMON_HOST, "127.0.0.1"); + assert.equal(derived.VILI_DAEMON_PORT, "18861"); + assert.equal(derived.GAMECULT_IDUNN_CANDIDATE_BIND, "http://127.0.0.1:18861", "cultnet-ts compares the bind to the Expected route's endpoint, which carries the scheme Idunn's bare host:port lacks"); + assert.equal(derived.VILI_PYTHON, path.join(release, "python", "bin", "python3.12")); + assert.equal(derived.VILI_BLENDER, path.join(release, "blender", "blender")); + assert.deepEqual(Object.keys(derived).sort(), ["GAMECULT_IDUNN_CANDIDATE_BIND", "VILI_BLENDER", "VILI_DAEMON_HOST", "VILI_DAEMON_PORT", "VILI_PYTHON", "VILI_PYTHON_ENV"], + "the entry sets exactly these: no LD_LIBRARY_PATH for the whole unit"); + assert.equal(derived.VILI_PYTHON_ENV, `locks-sha256:${createHash("sha256").update("build lock bytes\nruntime lock bytes\n").digest("hex")}`); + } + } finally { + rmSync(release, { recursive: true, force: true }); + } +}); + +// The lease is asked while a generate awaits its worker, not only when the service documents are rewritten. The +// bound is one WATCH_MS ask plus the SIGKILL; the test allows a second. The loss is made just after a rewrite of the +// service documents (whose own gates have asked), the longest wait for an implementation that asks only then. +test("a worker still writing when the lease is lost is dead within a second", async () => { + const daemon = await idunnDaemon({ leased: true }); + try { + const store = path.join(daemon.stateRoot, "vili.service.cc"); + await until(() => existsSync(store), "the service store", 10); + const pending = generate(daemon, { prompt: "x", model: "writer" }); + pending.catch(() => {}); + const writes = () => { + const dir = path.join(daemon.stateRoot, "artifacts"); + for (const entry of existsSync(dir) ? readdirSync(dir) : []) { + const file = path.join(dir, entry, "motion.writes"); + if (existsSync(file)) return file; + } + return null; + }; + const file = await until(writes, "the worker to write"); + const [worker] = daemon.spawns(); + const rewritten = statSync(store).mtimeMs; + await until(() => statSync(store).mtimeMs !== rewritten, "the service documents to be rewritten", 8); + await new Promise((resolve) => setTimeout(resolve, 150)); // the rewrite is done: its own gates have asked + const lostAt = Date.now(); + rmSync(daemon.lease); + await until(() => !isAlive(worker), "the worker to die", 10); + assert.ok(Date.now() - lostAt < 1000, `the worker lived ${Date.now() - lostAt} ms after the loss`); + const stamps = readFileSync(file, "utf8").trim().split("\n").map(Number); + assert.ok(Math.max(...stamps) - lostAt < 1000, "the worker wrote more than a second after the loss"); + const answer = await Promise.race([pending, new Promise((resolve) => setTimeout(() => resolve({ status: "no answer" }), 10000))]); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), []); + } finally { + await daemon.stop(); + } +}); + +// A request in flight and one arriving during the stop. The worker takes 1.5 s to stop, so the stop is long enough +// for the second request to arrive in it. From the signal on nothing is written and nothing is spawned. +for (const kind of ["fake Idunn entry", "hand-run"]) { + test(`SIGTERM (${kind}): nothing is written or spawned from the signal on, and the worker tree is dead at exit 0`, async () => { + const daemon = kind === "hand-run" ? await startDaemon() : await idunnDaemon({ leased: true }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + const inFlight = generate(daemon, { prompt: "x", model: "slow-to-stop" }); + inFlight.catch(() => {}); + await until(() => daemon.spawns().length, "the worker"); + await until(() => request(daemon.port, "GET", "/health").then((r) => r.body?.state?.backend?.state === "ready"), "the worker to be ready"); + const store = path.join(daemon.stateRoot, "vili.service.cc"); + const storeBefore = statSync(store).mtimeMs; + const artifacts = readdirSync(path.join(daemon.stateRoot, "artifacts")); + daemon.proc.kill("SIGTERM"); + await new Promise((resolve) => setTimeout(resolve, 300)); + const second = await generate(daemon).catch((error) => ({ status: error.code })); + assert.notEqual(second.status, 200, "a request arriving during the stop is not served"); + await until(() => daemon.proc.exitCode !== null || daemon.proc.signalCode !== null, "the daemon to exit", 30); + assert.equal(daemon.proc.exitCode, 0); + assert.equal(daemon.spawns().length, 1, "no worker is spawned after the signal"); + assert.ok(daemon.spawns().every((pid) => !isAlive(pid)), "the worker tree is dead at exit"); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "artifacts")), artifacts, "no artifact is created after the signal"); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")), [], "no job is recorded after the signal"); + assert.equal(statSync(store).mtimeMs, storeBefore); + assert.ok(!readFileSync(path.join(daemon.stateRoot, "logs", "worker.log"), "utf8").includes("stopping-line"), "the worker's output while it stops is not copied"); + } finally { + await daemon.stop(); + } + }); +} + +// One notion of root identity: a root is the same root under either spelling, for the lock and for reclaim. +for (const [crashedOn, restartedOn] of [["link", "real"], ["real", "link"]]) { + test(`a daemon restarted on the ${restartedOn} spelling of its root reclaims the worker a crash on the ${crashedOn} spelling left`, async () => { + const real = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const link = `${real}-link`; + symlinkSync(real, link); + const spelling = { real, link }; + const first = await startDaemon({ root: spelling[crashedOn] }); + let second = null; + let stray = null; + try { + assert.equal((await generate(first)).status, 200); + [stray] = first.spawns(); + first.proc.kill("SIGKILL"); + await until(() => first.proc.exitCode !== null || first.proc.signalCode !== null, "the first daemon to die"); + assert.ok(isAlive(stray), "the probe must reproduce the orphan before the fix can be measured"); + second = await startDaemon({ root: spelling[restartedOn] }); + await until(() => !isAlive(stray), "the predecessor's worker to be reclaimed"); + assert.equal((await generate(second)).status, 200); + assert.equal(aliveWorkers(second).length, 1, "one live worker on the one root"); + } finally { + if (stray) { try { process.kill(-stray, "SIGKILL"); } catch { /* gone */ } } + if (second) await second.stop(); + rmSync(link, { force: true }); + rmSync(real, { recursive: true, force: true }); + } + }); +} + +// The group is the worker's, and its leader may be the first to go. Any member carrying the root's marker proves it. +test("a restarted daemon kills a worker group whose leader is gone and a member lives", async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const first = await startDaemon({ root }); + const stray = generate(first, { prompt: "x", model: "encoder" }).catch(() => null); + const [grandchild] = await until(() => { const found = first.grandchildren(); return found.length ? found : null; }, "the encoder child"); + const [worker] = first.spawns(); + first.proc.kill("SIGKILL"); + await until(() => first.proc.exitCode !== null || first.proc.signalCode !== null, "the first daemon to die"); + process.kill(worker, "SIGKILL"); // the leader only + await until(() => !isAlive(worker), "the leader to die"); + await stray; + assert.ok(isAlive(grandchild), "the probe must leave a member alive in the leaderless group"); + const second = await startDaemon({ root }); + try { + await until(() => !isAlive(grandchild), "the leaderless group's member to be killed"); + assert.equal(existsSync(path.join(root, "worker.pid")), false); + } finally { + try { process.kill(-worker, "SIGKILL"); } catch { /* gone */ } + await second.stop(); + } +}); + +// The copy of worker output is gated at every line, not at the first: here the lease goes after the first line +// was logged, by the worker's first healthy answer, and the line it prints then must not reach the log. +test("a worker's output line after the first is not logged once the lease is gone", async () => { + const daemon = await idunnDaemon({ leased: true, loseWhenExists: "embeddings/stub-healthy" }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + const answer = await generate(daemon, { prompt: "x", model: "chatty" }); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + await new Promise((resolve) => setTimeout(resolve, 500)); + const log = readFileSync(path.join(daemon.stateRoot, "logs", "worker.log"), "utf8"); + assert.ok(log.includes("stub-stdout-line"), "the first line was logged under the lease"); + assert.ok(!log.includes("after-healthy-line"), "a line after the loss must not be copied"); + } finally { + await daemon.stop(); + } +}); + +// ---- FBX export: the daemon converts each BVH the worker lists, through the sealed Blender, under the write gate. + +const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const exportRequest = (extra = {}) => ({ prompt: "a person waves", duration: 2, fbx: true, ...extra }); +const exportStarts = (daemon) => daemon.starts().filter((start) => start.argv.includes("--")); +const jobFiles = (daemon) => readdirSync(path.join(daemon.stateRoot, "jobs")); +const getArtifact = (daemon, urlPath) => request(daemon.port, "GET", urlPath); + +test("an fbx job exports every listed BVH", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await backendOf(daemon)).blender, "Blender 0.0.0 (stub)"); + const answer = await generate(daemon, exportRequest({ clipName: "wave_01", seed: 7 })); + assert.equal(answer.status, 200); + const job = answer.body; + assert.equal(job.schema, "gamecult.vili.motion_job.v1"); + assert.equal(job.ok, true); + assert.equal(job.worker.bvh, true, "fbx implies the BVH"); + assert.equal(job.worker.bvhStandardTpose, true, "fbx implies the standard T-pose"); + assert.equal(job.request.bvh, true); + assert.equal(job.request.bvhStandardTpose, true); + assert.deepEqual(job.export.clips, [{ file: "motion.fbx", clipName: "wave_01", bones: 78, frames: [1, 60], fps: 30 }]); + assert.equal(job.export.blender, "Blender 0.0.0 (stub)"); + assert.equal(job.export.script, `sha256:${sha256(readFileSync(exportScriptPath))}`); + assert.equal(job.export.settings, "standard-tpose soma77 metres hashseed0 time1970"); + const fbx = readFileSync(path.join(job.artifactDir, "motion.fbx")); + assert.equal(fbx.toString(), "FBX-STUB:BVH-STUB\n"); + assert.deepEqual(job.files.map((file) => file.name), ["meta.json", "motion.bvh", "motion.fbx", "motion.npz", "request.json"]); + assert.equal(job.files.find((file) => file.name === "motion.fbx").sha256, sha256(fbx)); + assert.equal(job.files.find((file) => file.name === "motion.fbx").bytes, fbx.length); + assert.deepEqual(JSON.parse(readFileSync(path.join(daemon.stateRoot, "jobs", `${job.jobId}.json`), "utf8")), job); + const request_ = JSON.parse(readFileSync(path.join(job.artifactDir, "request.json"), "utf8")); + assert.equal(request_.fbx, undefined, "the worker is told nothing of FBX"); + assert.equal(request_.clipName, undefined); + + const [probe, run] = daemon.starts(); + assert.deepEqual(probe.argv, ["-b", "--factory-startup", "--version"]); + assert.deepEqual(run.argv.slice(0, 8), [ + "-b", "--factory-startup", "--python-use-system-env", "--python-exit-code", "1", "--python", exportScriptPath, "--", + ]); + // Blender reads a private copy and writes a private FBX, in a directory made for this export alone: it is never + // handed a path in the job's directory, and the directory is gone once the export is over. + const [privateBvh, privateFbx, claimed] = run.argv.slice(8); + assert.match(path.relative(daemon.stateRoot, privateBvh), /^work\/export-[A-Za-z0-9]{6}\/in\.bvh$/); + assert.equal(privateFbx, path.join(path.dirname(privateBvh), "out.fbx")); + assert.equal(claimed, "wave_01"); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "work")), []); + assert.deepEqual(run.env, { + PATH: process.env.PATH, + LANG: "xx_TEST.UTF-8", + HOME: "/nonexistent", + PYTHONHASHSEED: "0", + PYTHONNOUSERSITE: "1", + LD_LIBRARY_PATH: path.join(daemon.blenderDir, "lib"), + }, "the export environment is these six and nothing else (no VILI_*, no LEAK_CHECK)"); + assert.equal(run.priority, 10); + assert.equal(run.pgid, run.pid, "Blender leads its own process group"); + const log = readFileSync(path.join(daemon.stateRoot, "logs", "export.log"), "utf8"); + assert.match(log, /stub-blender-stderr wave_01/); + + const named = await generate(daemon, exportRequest()); + assert.equal(named.body.export.clips[0].clipName, named.body.jobId, "the clip is named after the job by default"); + const several = await generate(daemon, exportRequest({ clipName: "pair", numSamples: 2 })); + assert.equal(several.status, 200); + assert.deepEqual(several.body.export.clips.map((clip) => [clip.file, clip.clipName]), [ + ["motion/motion_00.fbx", "pair_00"], + ["motion/motion_01.fbx", "pair_01"], + ]); + } finally { + await daemon.stop(); + } +}); + +test("fbx refusals happen before anything is written or spawned", async () => { + const daemon = await startDaemon(); + try { + for (const bad of [ + { bvhStandardTpose: false }, + { bvh_standard_tpose: "false" }, + { clipName: "a b canary" }, + { clipName: "x".repeat(65) }, + { clipName: "" }, + { clipName: 5 }, + ]) { + const answer = await generate(daemon, exportRequest(bad)); + assert.equal(answer.status, 400, JSON.stringify(bad)); + assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); + } + assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made"); + assert.deepEqual(jobFiles(daemon), []); + assert.deepEqual(daemon.spawns(), [], "no worker was started"); + assert.equal(exportStarts(daemon).length, 0); + assert.equal((await generate(daemon, exportRequest({ clipName: "x".repeat(64) }))).status, 200, "64 characters is the limit"); + } finally { + await daemon.stop(); + } + for (const blender of [null, "broken", "missing"]) { + const unready = await startDaemon({ blender }); + try { + assert.equal((await backendOf(unready)).blender, blender === null ? "unconfigured" : "unavailable"); + const answer = await generate(unready, exportRequest()); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "blender-unavailable"); + assert.deepEqual(unready.spawns(), [], "no worker was started"); + assert.equal(existsSync(path.join(unready.stateRoot, "artifacts")), false, "no artifact directory was made"); + assert.deepEqual(jobFiles(unready), []); + assert.equal(exportStarts(unready).length, 0); + assert.equal((await generate(unready)).status, 200, "a job that asks for no FBX needs no Blender"); + } finally { + await unready.stop(); + } + } +}); + +test("loop asks for the seamless window, and the window is the FBX's frame range", async () => { + const daemon = await startDaemon(); + try { + const constraints = [{ type: "root2d", frame_indices: [0, 10], smooth_root_2d: [[0, 0], [0, 0.5]] }]; + const answer = await generate(daemon, exportRequest({ clipName: "walk", seed: 7, loop: true, constraints })); + assert.equal(answer.status, 200); + const job = answer.body; + assert.equal(job.request.loop, true); + assert.deepEqual(job.worker.loops, [{ first: 3, last: 223, poseCost: 0.03, velocityCost: 0.2 }]); + assert.deepEqual(job.request.constraints, constraints, "the record keeps inline constraints as sent"); + assert.deepEqual(JSON.parse(readFileSync(path.join(job.artifactDir, "constraints.json"), "utf8")), constraints); + assert.equal(JSON.parse(readFileSync(path.join(job.artifactDir, "request.json"), "utf8")).loop, true, "the worker is asked for the loop"); + assert.equal(job.worker.constraints, path.join(job.artifactDir, "constraints.json"), "the worker still gets the path"); + const [run] = exportStarts(daemon); + assert.deepEqual(run.argv.slice(run.argv.indexOf("--") + 3), ["walk", "--frames", "4", "224"], "Blender's frames count from 1"); + assert.deepEqual(job.export.clips.map((clip) => clip.frames), [[1, 60]], "the clip's frames are what Blender reports"); + + const several = await generate(daemon, exportRequest({ clipName: "pair", numSamples: 2, loop: true })); + assert.equal(several.status, 200); + assert.deepEqual(several.body.worker.loops.map((loop) => [loop.first, loop.last]), [[3, 223], [4, 224]]); + const frames = Object.fromEntries(exportStarts(daemon).slice(1).map((start) => { + const [, , clip, ...rest] = start.argv.slice(start.argv.indexOf("--") + 1); + return [clip, rest]; + })); + assert.deepEqual(frames, { pair_00: ["--frames", "4", "224"], pair_01: ["--frames", "5", "225"] }, "each sample is exported over its own window"); + + const plain = await generate(daemon, exportRequest({ clipName: "plain" })); + assert.equal(plain.status, 200); + assert.equal(plain.body.worker.loops, undefined); + assert.equal(plain.body.request.loop, undefined); + assert.equal(JSON.parse(readFileSync(path.join(plain.body.artifactDir, "request.json"), "utf8")).loop, undefined); + assert.equal(exportStarts(daemon).at(-1).argv.at(-1), "plain", "no window, no --frames"); + const kept = await generate(daemon, exportRequest({ constraints_path: "/some/where.json", constraints: undefined })); + assert.equal(kept.body.request.constraints, "/some/where.json", "a path stays a path"); + } finally { + await daemon.stop(); + } +}); + +test("constraints given twice are refused before anything is written or spawned", async () => { + const daemon = await startDaemon(); + try { + const inline = [{ type: "root2d" }]; + for (const bad of [ + { constraints: inline, constraints_path: "/canary/a.json" }, + { constraints: inline, constraintsPath: "/canary/a.json" }, + { constraints: "/canary/a.json", constraints_path: "/canary/b.json" }, + { constraints_path: "/canary/a.json", constraintsPath: "/canary/b.json" }, + ]) { + const answer = await generate(daemon, exportRequest(bad)); + assert.equal(answer.status, 400, JSON.stringify(bad)); + assert.deepEqual(answer.body, { ok: false, error: "Invalid constraints request." }); + assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); + } + assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made"); + assert.deepEqual(jobFiles(daemon), []); + assert.deepEqual(daemon.spawns(), [], "no worker was started"); + assert.equal((await generate(daemon, exportRequest({ constraints_path: "/some/where.json" }))).status, 200, "one input is fine"); + } finally { + await daemon.stop(); + } +}); + +test("loop is a boolean and needs fbx, refused before anything is written or spawned", async () => { + const daemon = await startDaemon(); + try { + for (const bad of [{ fbx: false, loop: true }, { fbx: undefined, loop: true }, { loop: "yes canary" }, { loop: 1 }, { loop: null }, { loop: "true" }]) { + const answer = await generate(daemon, exportRequest(bad)); + assert.equal(answer.status, 400, JSON.stringify(bad)); + assert.deepEqual(answer.body, { ok: false, error: "Invalid fbx request." }); + assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); + } + assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made"); + assert.deepEqual(jobFiles(daemon), []); + assert.deepEqual(daemon.spawns(), [], "no worker was started"); + assert.equal(exportStarts(daemon).length, 0); + assert.equal((await generate(daemon, exportRequest({ loop: false }))).status, 200, "loop false is no loop"); + assert.equal((await generate(daemon, { prompt: "a person waves", duration: 2, loop: false })).status, 200); + } finally { + await daemon.stop(); + } +}); + +test("a loop request whose answer has no usable window for a sample fails without exporting that sample", async () => { + for (const [model, exported] of [["no-loops", 0], ["bad-loops", 0], ["short-loops", 1]]) { + const daemon = await startDaemon(); + try { + const answer = await generate(daemon, exportRequest({ model, loop: true, numSamples: 2 })); + assert.equal(answer.status, 502, model); + assert.equal(answer.body.ok, false); + assert.equal(answer.body.error, "fbx-export-failed (no loop window)"); + assert.equal(exportStarts(daemon).length, exported, `${model}: only a sample with a window is exported`); + assert.equal(answer.body.export.clips.length, exported); + } finally { + await daemon.stop(); + } + } +}); + +test("the BVH is never given to Blender unless it is a regular file inside the job's directory", async () => { + const daemon = await startDaemon(); + try { + for (const model of ["bvh-outside", "bvh-link"]) { + const answer = await generate(daemon, exportRequest({ model, clipName: "canary_clip" })); + assert.equal(answer.status, 502, model); + assert.equal(answer.body.ok, false); + assert.equal(answer.body.error, "fbx-export-refused"); + assert.equal(answer.body.export.clips.length, 0); + assert.ok(existsSync(path.join(daemon.stateRoot, "jobs", `${answer.body.jobId}.json`)), "the failed job is still a record"); + } + assert.equal(exportStarts(daemon).length, 0, "Blender was never started for either"); + assert.ok(!(daemon.stdout() + daemon.stderr()).includes("canary")); + } finally { + await daemon.stop(); + } +}); + +test("a failing or hung Blender fails the job, leaks nothing, and leaves nothing running", async () => { + const daemon = await startDaemon(); + try { + const failed = await generate(daemon, exportRequest({ clipName: "fail_canary" })); + assert.equal(failed.status, 502); + assert.equal(failed.body.ok, false); + assert.equal(failed.body.error, "fbx-export-failed (exit 1)"); + assert.ok(!failed.raw.includes("stub-blender-stderr"), "Blender's output is not in the answer"); + assert.ok(!failed.body.error.includes("canary") && !(daemon.stdout() + daemon.stderr()).includes("canary")); + assert.match(readFileSync(path.join(daemon.stateRoot, "logs", "export.log"), "utf8"), /stub-blender-stderr fail_canary/); + assert.ok(existsSync(path.join(daemon.stateRoot, "jobs", `${failed.body.jobId}.json`))); + + for (const clip of ["silent", "garbage", "nofile"]) { + const odd = await generate(daemon, exportRequest({ clipName: clip })); + assert.equal(odd.status, 502, clip); + assert.equal(odd.body.error, "fbx-export-failed (no result)", clip); + } + const lied = await generate(daemon, exportRequest({ clipName: "lies" })); + assert.equal(lied.status, 502, "an export that exits non-zero is no export, whatever it printed"); + assert.equal(lied.body.error, "fbx-export-failed (exit 3)"); + + const known = new Set(daemon.blenderGrandchildren()); + const orphaned = await generate(daemon, exportRequest({ clipName: "orphan" })); + assert.equal(orphaned.status, 200, "an export that exits 0 is a good export"); + const left = await until(() => daemon.blenderGrandchildren().filter((pid) => !known.has(pid))[0], "the orphan's grandchild"); + await until(() => !isAlive(left), "what Blender left running to be killed with its group"); + + const before = daemon.starts().length; + known.add(left); + const hung = await Promise.race([ + generate(daemon, exportRequest({ clipName: "hang", exportTimeoutMs: 2500 })), + new Promise((_, reject) => setTimeout(() => reject(new Error("the hung export was never stopped")), 20000)), + ]); + assert.equal(hung.status, 502); + assert.equal(hung.body.error, "fbx-export-failed (signal SIGKILL)"); + const [run] = daemon.starts().slice(before); + const stuck = await until(() => daemon.blenderGrandchildren().filter((pid) => !known.has(pid))[0], "the hung Blender's grandchild"); + await until(() => !isAlive(run.pid) && !isAlive(stuck), "the hung export's group to be dead"); + assert.equal(hung.body.files.some((file) => file.name.endsWith(".fbx")), false); + } finally { + await daemon.stop(); + } +}); + +test("SIGTERM during an export stops its group, writes nothing more, and exits 0", async () => { + const daemon = await startDaemon(); + try { + const pending = generate(daemon, exportRequest({ clipName: "slow" })); + pending.catch(() => {}); + const run = await until(() => exportStarts(daemon)[0], "the export to start"); + daemon.proc.kill("SIGTERM"); + await until(() => !isAlive(run.pid), "the export to be stopped", 5); + await until(() => daemon.proc.exitCode !== null, "the daemon to exit"); + assert.equal(daemon.proc.exitCode, 0); + assert.deepEqual(jobFiles(daemon), [], "no record after the signal"); + const [artifacts] = [path.join(daemon.stateRoot, "artifacts")]; + for (const job of readdirSync(artifacts)) assert.equal(existsSync(path.join(artifacts, job, "motion.fbx")), false); + } finally { + await daemon.stop(); + } +}); + +test("what a stopping export still prints is not logged, and it is killed when it ignores SIGTERM", async () => { + const daemon = await startDaemon(); + try { + const pending = generate(daemon, exportRequest({ clipName: "hang", exportTimeoutMs: 60000 })); + pending.catch(() => {}); + const run = await until(() => exportStarts(daemon)[0], "the export to start"); + const logFile = path.join(daemon.stateRoot, "logs", "export.log"); + await until(() => existsSync(logFile) && readFileSync(logFile, "utf8").includes("stub-blender-stderr hang"), "the export's first line to be logged"); + daemon.proc.kill("SIGTERM"); + await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 30); + assert.equal(daemon.proc.exitCode, 0); + await until(() => !isAlive(run.pid), "the export that ignored SIGTERM to be killed"); + const log = readFileSync(path.join(daemon.stateRoot, "logs", "export.log"), "utf8"); + assert.match(log, /stub-blender-stderr hang/); + assert.ok(!log.includes("after-sigterm-line"), "a line printed after the signal reached the log"); + } finally { + await daemon.stop(); + } +}); + +test("losing the lease kills a running export within a second and records nothing", async () => { + const daemon = await idunnDaemon({ leased: true }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + await until(async () => (await backendOf(daemon)).blender === "Blender 0.0.0 (stub)", "the version probe, run once the lease was held"); + assert.equal(daemon.starts().length, 1); + assert.equal(existsSync(path.join(daemon.stateRoot, "logs")), false, "the probe writes nothing to the state root"); + const pending = generate(daemon, exportRequest({ clipName: "slow" })); + const ended = pending.then((answer) => { throw new Error(`the job ended before its export started: ${answer.status} ${answer.raw.slice(0, 300)}`); }); + ended.catch(() => {}); + const run = await Promise.race([until(() => exportStarts(daemon)[0], "the export to start"), ended]); + rmSync(daemon.lease); + const lost = Date.now(); + await until(() => !isAlive(run.pid), "the export to be dead", 2); + assert.ok(Date.now() - lost < 1500, "the export outlived the lease by more than a second"); + assert.equal((await pending).status, 503); + assert.deepEqual(jobFiles(daemon), []); + assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts", readdirSync(path.join(daemon.stateRoot, "artifacts"))[0], "motion.fbx")), false); + } finally { + await daemon.stop(); + } +}); + +test("the seed is minted when absent and recorded for every job", async () => { + const daemon = await startDaemon(); + try { + for (const extra of [{}, { fbx: false }]) { + const job = (await generate(daemon, exportRequest(extra))).body; + assert.ok(Number.isInteger(job.seed) && job.seed >= 0 && job.seed < 2 ** 31); + assert.equal(job.request.seed, job.seed); + assert.equal(job.worker.seed, job.seed, "the worker was given the minted seed"); + assert.equal(job.worker.metaSeed, job.seed); + assert.equal(JSON.parse(readFileSync(path.join(job.artifactDir, "meta.json"), "utf8")).seed, job.seed); + } + const given = (await generate(daemon, exportRequest({ seed: 7 }))).body; + assert.equal(given.seed, 7); + assert.equal(given.worker.seed, 7); + assert.equal(given.worker.metaSeed, 7); + assert.equal(JSON.parse(readFileSync(path.join(given.artifactDir, "meta.json"), "utf8")).seed, 7); + const minted = new Set(); + for (let count = 0; count < 3; count += 1) minted.add((await generate(daemon, exportRequest())).body.seed); + assert.ok(minted.size > 1, "minted seeds differ from job to job"); + const zero = (await generate(daemon, exportRequest({ seed: 0 }))).body; + assert.equal(zero.seed, 0, "0 is a seed"); + } finally { + await daemon.stop(); + } +}); + +test("the artifact read route serves only a job's own FBX and BVH files", async () => { + const daemon = await startDaemon(); + try { + const job = (await generate(daemon, exportRequest({ clipName: "walk", numSamples: 1 }))).body; + const base = `/motion/artifacts/${job.jobId}`; + const served = await getArtifact(daemon, `${base}/motion.fbx`); + assert.equal(served.status, 200); + assert.equal(served.headers["content-type"], "application/octet-stream"); + assert.equal(served.raw, "FBX-STUB:BVH-STUB\n"); + assert.equal((await getArtifact(daemon, `${base}/motion.bvh`)).raw, "BVH-STUB\n"); + + // a directory that is not a job's own (its id is not vili-<13 digits>) serves nothing even with a file in it + mkdirSync(path.join(daemon.stateRoot, "artifacts", "vili-123"), { recursive: true }); + writeFileSync(path.join(daemon.stateRoot, "artifacts", "vili-123", "motion.fbx"), "PLANTED"); + const refused = [ + `${base}/meta.json`, + `${base}/request.json`, + `${base}/motion.npz`, + `${base}/..%2fmeta.json`, + `${base}/%2e%2e/${job.jobId}/meta.json`, + `${base}/../../jobs/${job.jobId}.json`, + `${base}/motion/motion_00.fbx`, + `/motion/artifacts/vili-123/motion.fbx`, + `/motion/artifacts/vili-canary-canary/motion.fbx`, + `/motion/artifacts/vili-9999999999999/motion.fbx`, + `/motion/artifacts/${job.jobId}`, + `/motion/artifacts/${job.jobId}/`, + ]; + for (const url of refused) { + const answer = await getArtifact(daemon, url); + assert.equal(answer.status, 404, url); + assert.equal(answer.raw, "not found\n", "a refusal echoes nothing"); + } + assert.equal((await request(daemon.port, "POST", `${base}/motion.fbx`, { body: "{}", headers: { "content-type": "application/json" } })).status, 404); + + // a symlink planted in the artifact directory, as the file or as a directory above it, is never followed + const outside = path.join(daemon.root, "outside.bin"); + writeFileSync(outside, "SECRET"); + rmSync(path.join(job.artifactDir, "motion.bvh")); + symlinkSync(outside, path.join(job.artifactDir, "motion.bvh")); + assert.equal((await getArtifact(daemon, `${base}/motion.bvh`)).status, 404); + mkdirSync(path.join(daemon.root, "elsewhere")); + writeFileSync(path.join(daemon.root, "elsewhere", "motion_00.fbx"), "SECRET"); + symlinkSync(path.join(daemon.root, "elsewhere"), path.join(job.artifactDir, "motion")); + const through = await getArtifact(daemon, `${base}/motion/motion_00.fbx`); + assert.equal(through.status, 404); + assert.ok(!through.raw.includes("SECRET")); + } finally { + await daemon.stop(); + } +}); + +// A worker interpreter whose site-packages carry Kimodo's rest skeleton, as the release's does: the stub worker runs on node. +function restPython() { + const root = mkdtempSync(path.join(os.tmpdir(), "vili-py-")); + const python = path.join(root, "bin", "python3.12"); + mkdirSync(path.dirname(python), { recursive: true }); + symlinkSync(process.execPath, python); + const skeletons = path.join(root, "lib", "python3.12", "site-packages", "kimodo", "assets", "skeletons", "somaskel77"); + mkdirSync(skeletons, { recursive: true }); + writeFileSync(path.join(skeletons, "somaskel77_standard_tpose.bvh"), "REST-BVH\n"); + return { root, python }; +} + +test("rest-fbx is a job whose FBX is readable", async () => { + const { root: pyRoot, python } = restPython(); + const daemon = await startDaemon({ python }); + try { + const answer = await request(daemon.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); + assert.equal(answer.status, 200); + const job = answer.body; + assert.match(job.jobId, /^rest-[0-9]{13}-[0-9a-f]{8}$/); + assert.equal(job.schema, "gamecult.vili.motion_job.v1"); + assert.deepEqual(job.export.clips, [{ file: "SOMA77_Rest.fbx", clipName: "SOMA77_Rest", bones: 78, frames: [1, 60], fps: 30 }]); + assert.deepEqual(job.files.map((file) => file.name), ["SOMA77_Rest.bvh", "SOMA77_Rest.fbx"]); + assert.deepEqual(JSON.parse(readFileSync(path.join(daemon.stateRoot, "jobs", `${job.jobId}.json`), "utf8")), job); + const served = await getArtifact(daemon, `/motion/artifacts/${job.jobId}/SOMA77_Rest.fbx`); + assert.equal(served.status, 200); + assert.equal(served.raw, "FBX-STUB:REST-BVH\n"); + assert.deepEqual(daemon.spawns(), [], "no worker is needed for a rest pose"); + const cross = await request(daemon.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "text/plain" } }); + assert.equal(cross.status, 403); + } finally { + await daemon.stop(); + rmSync(pyRoot, { recursive: true, force: true }); + } + + const bare = await startDaemon(); + try { + const missing = await request(bare.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); + assert.equal(missing.status, 503); + assert.equal(missing.body.code, "rest-skeleton-missing"); + } finally { + await bare.stop(); + } + const unready = await startDaemon({ blender: null }); + try { + const refused = await request(unready.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); + assert.equal(refused.status, 503); + assert.equal(refused.body.code, "blender-unavailable"); + assert.equal(existsSync(path.join(unready.stateRoot, "artifacts")), false); + } finally { + await unready.stop(); + } + const waiting = await idunnDaemon(); + try { + const early = await request(waiting.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); + assert.equal(early.status, 503); + assert.equal(early.body.code, "write-lease-pending"); + assert.deepEqual(waiting.starts(), []); + } finally { + await waiting.stop(); + } +}); + +test("operator state and the command boundary name the export", async () => { + const daemon = await startDaemon(); + try { + const state = await operatorState(daemon); + assert.equal(state.routes.artifacts, "/motion/artifacts/{jobId}/{name}"); + assert.equal(state.routes.restFbx, "/motion/rest-fbx"); + assert.ok(state.authority.owns.includes("clip export to FBX")); + } finally { + await daemon.stop(); + } +}); + +// ---- Findings of Soul's pass s1 on the FBX export (motion-pipeline:verdict:cut-vili-fbx-export.s1). + +const readJson = (file) => JSON.parse(readFileSync(file, "utf8")); +const restRequest = (daemon) => request(daemon.port, "POST", "/motion/rest-fbx", { body: "{}", headers: { "content-type": "application/json" } }); + +test("concurrent generate and rest-fbx requests never share a job id, a directory, a record or a file", async () => { + const rest = restPython(); + const daemon = await startDaemon({ python: rest.python }); + try { + const answers = []; + for (let round = 0; round < 10; round += 1) { + answers.push(...await Promise.all([ + generate(daemon, exportRequest({ clipName: `a${round}` })), + generate(daemon, exportRequest({ clipName: `b${round}` })), + restRequest(daemon), + restRequest(daemon), + ])); + } + assert.equal(answers.length, 40); + for (const answer of answers) { + assert.equal(answer.status, 200, answer.raw.slice(0, 300)); + assert.equal(answer.body.ok, true); + } + const ids = answers.map((answer) => answer.body.jobId); + assert.equal(new Set(ids).size, 40, "every job has its own id"); + assert.equal(new Set(answers.map((answer) => answer.body.artifactDir)).size, 40, "and its own directory"); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "artifacts")).sort(), [...ids].sort()); + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "jobs")).sort(), ids.map((id) => `${id}.json`).sort(), "and its own record"); + for (const { body } of answers) { + assert.deepEqual(readJson(path.join(daemon.stateRoot, "jobs", `${body.jobId}.json`)), body, "the record is the job's own answer"); + assert.equal(body.export.clips.length, 1); + } + const names = answers.filter((answer) => answer.body.jobId.startsWith("vili-")).map((answer) => answer.body.export.clips[0].clipName).sort(); + assert.deepEqual(names, Array.from({ length: 10 }, (_, round) => [`a${round}`, `b${round}`]).flat().sort(), "each generate job exported its own clip"); + const first = answers[0].body.jobId; + assert.equal((await getArtifact(daemon, `/motion/artifacts/${first}/motion.fbx`)).status, 200, "the read route serves the minted ids"); + } finally { + await daemon.stop(); + rmSync(rest.root, { recursive: true, force: true }); + } +}); + +test("a seed that is not an integer from 0 to 2^31 is refused before anything is written, without echoing it", async () => { + const daemon = await startDaemon(); + try { + const bad = [7.9, "7", "canary", [], [3], true, -1, 2 ** 31, 1e21, { canary: 1 }]; + for (const seed of bad) { + for (const payload of [exportRequest({ seed }), exportRequest({ meta: { seed } })]) { + const answer = await generate(daemon, payload); + assert.equal(answer.status, 400, JSON.stringify(payload)); + assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); + assert.ok(!answer.raw.includes("7.9"), "the refusal repeated the input"); + } + } + assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "nothing was written"); + assert.deepEqual(daemon.spawns(), [], "no worker was started"); + assert.equal(exportStarts(daemon).length, 0); + for (const seed of [0, 2 ** 31 - 1]) { + const job = (await generate(daemon, exportRequest({ seed }))).body; + assert.equal(job.seed, seed); + assert.equal(job.worker.seed, seed, "the seed recorded is the seed the worker was given"); + } + } finally { + await daemon.stop(); + } +}); + +test("a BVH or FBX path swapped for a symlink after the job was checked is never followed", async () => { + for (const swap of ["bvh", "fbx", "both", "directory"]) { + const daemon = await startDaemon(); + try { + const outside = path.join(daemon.root, "outside-secret"); + const victim = path.join(daemon.root, "victim"); + writeFileSync(outside, "SECRET"); + writeFileSync(victim, "VICTIM"); + // sample 0 ("slow_00") exports for six seconds; the swap lands while it does, before sample 1 is touched + const pending = generate(daemon, exportRequest({ clipName: "slow", numSamples: 2 })); + await until(() => exportStarts(daemon).length === 1, "the first export to start"); + const [job] = readdirSync(path.join(daemon.stateRoot, "artifacts")); + const motion = path.join(daemon.stateRoot, "artifacts", job, "motion"); + const elsewhere = path.join(daemon.root, "elsewhere"); + if (swap === "directory") { + // the directory itself is replaced by a link to one that holds a regular file of the right name + mkdirSync(elsewhere); + writeFileSync(path.join(elsewhere, "motion_01.bvh"), "SECRET"); + renameSync(motion, `${motion}-moved`); + symlinkSync(elsewhere, motion); + } + if (swap === "bvh" || swap === "both") { + rmSync(path.join(motion, "motion_01.bvh")); + symlinkSync(outside, path.join(motion, "motion_01.bvh")); + } + if (swap === "fbx" || swap === "both") symlinkSync(victim, path.join(motion, "motion_01.fbx")); + const answer = await pending; + assert.equal(answer.status, 502, swap); + assert.equal(answer.body.ok, false); + assert.equal(readFileSync(victim, "utf8"), "VICTIM", `${swap}: nothing was written through the planted link`); + assert.equal(readFileSync(outside, "utf8"), "SECRET"); + for (const name of readdirSync(swap === "directory" ? elsewhere : motion)) { + if (name.endsWith(".fbx") && !lstatSync(path.join(motion, name)).isSymbolicLink()) { + assert.ok(!readFileSync(path.join(motion, name), "utf8").includes("SECRET"), `${swap}: ${name} holds bytes read through the planted link`); + } + } + assert.ok(!answer.raw.includes("SECRET") && !answer.raw.includes("VICTIM")); + } finally { + await daemon.stop(); + } + } +}); + +test("the worker's environment is its named keys, and LD_LIBRARY_PATH reaches Blender only", async () => { + const daemon = await startDaemon({ extraEnv: { LD_LIBRARY_PATH: "/canary-lib" } }); + try { + const job = (await generate(daemon, exportRequest({ clipName: "wave" }))).body; + assert.equal(job.ok, true); + const named = ["HF_HOME", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "VILI_EMBEDDINGS_DIR", "VILI_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS", + "PYTHONPATH", "PYTHONDONTWRITEBYTECODE", "PYTHONUNBUFFERED", "VILI_KIMODO_COMMIT", "VILI_PYTHON_ENV"]; + const forwarded = ["PATH", "HOME", "LANG"].filter((name) => process.env[name] !== undefined || name === "LANG"); + assert.deepEqual(job.worker.envKeys, [...named, ...forwarded].sort(), "the worker gets exactly these keys, LD_LIBRARY_PATH not among them"); + const run = daemon.starts().find((start) => start.argv.includes("--")); + assert.equal(run.env.LD_LIBRARY_PATH, path.join(daemon.blenderDir, "lib"), "Blender gets the release's libraries, not the host's"); + } finally { + await daemon.stop(); + } +}); + +test("the lease goes between rest-fbx's directory and its copy: nothing is copied, nothing is exported", async () => { + const rest = restPython(); + const daemon = await idunnDaemon({ leased: true, loseWhenExists: "artifacts/*", python: rest.python }); + try { + await until(() => existsSync(path.join(daemon.stateRoot, "vili.service.cc")), "the service store", 10); + await until(async () => /^Blender /.test((await backendOf(daemon)).blender), "the Blender probe"); + const answer = await restRequest(daemon); + assert.equal(answer.status, 503); + assert.equal(answer.body.code, "write-lease-lost"); + for (const job of readdirSync(path.join(daemon.stateRoot, "artifacts"))) { + assert.deepEqual(readdirSync(path.join(daemon.stateRoot, "artifacts", job)), [], "the job's directory is empty"); + } + assert.equal(exportStarts(daemon).length, 0); + } finally { + await daemon.stop(); + rmSync(rest.root, { recursive: true, force: true }); + } +}); + +test("the job record lists regular files only: a planted symlink, to a file or to a directory, is not listed", async () => { + const daemon = await startDaemon(); + try { + const job = (await generate(daemon, exportRequest({ model: "plant-link", clipName: "plant" }))).body; + assert.equal(job.ok, true); + assert.equal(readdirSync(job.artifactDir).filter((name) => name.includes("link")).length, 2, "both symlinks are in the job directory"); + assert.deepEqual(job.files.map((file) => file.name), ["meta.json", "motion.bvh", "motion.fbx", "motion.npz", "request.json"]); + } finally { + await daemon.stop(); + } +}); + +// Soul s2: the job id's random suffix is what carries a burst. The clock alone repeats inside one millisecond, and +// the exclusive mkdir's few retries are spent there, so a constant suffix fails a burst and passes every small test. +test("four hundred simultaneous generates each get their own job id, directory and record", async () => { + const daemon = await startDaemon(); + try { + const answers = await Promise.all(Array.from({ length: 400 }, () => generate(daemon))); + assert.deepEqual([...new Set(answers.map((answer) => answer.status))], [200]); + const ids = answers.map((answer) => answer.body.jobId); + for (const id of ids) assert.match(id, /^vili-\d{13}-[0-9a-f]{8}$/); + assert.equal(new Set(ids).size, 400, "no two jobs share an id"); + assert.equal(readdirSync(path.join(daemon.stateRoot, "artifacts")).length, 400); + assert.equal(jobFiles(daemon).length, 400); + } finally { + await daemon.stop(); + } +}); + +// Soul s2: opening a path that blocks (a FIFO with no writer) must not wait. The open is non-blocking and the check +// is made on the open descriptor, so the listed node is refused at once and the filesystem threads stay free. +test("a listed BVH that is a FIFO is refused promptly and other file work proceeds", async () => { + const daemon = await startDaemon(); + const within = (promise, ms, what) => Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error(`${what} did not settle in ${ms} ms`)), ms)), + ]); + try { + // more than libuv's four filesystem threads: a blocking open per request would starve all of them + const answers = await within(Promise.all(Array.from({ length: 6 }, () => + generate(daemon, exportRequest({ model: "bvh-fifo", clipName: "canary-fifo-clip" })))), 8000, "the FIFO jobs"); + for (const answer of answers) { + assert.equal(answer.status, 502); + assert.equal(answer.body.ok, false); + const { request: recorded, ...outcome } = answer.body; // the record keeps the request; the refusal itself must not echo it + assert.ok(recorded && !JSON.stringify(outcome).includes("canary-fifo-clip"), "the refusal does not echo the clip name"); + } + assert.equal(exportStarts(daemon).length, 0, "no Blender for a node that is not a regular file"); + const other = await within(generate(daemon, exportRequest({ model: "stub-model", clipName: "after" })), 8000, "an ordinary job"); + assert.equal(other.status, 200); + assert.ok(!daemon.stdout().includes("canary-fifo-clip") && !daemon.stderr().includes("canary-fifo-clip")); + } finally { + await daemon.stop(); + } +}); + +// Soul s2: work/ is a declared state slot, and a daemon that holds the state root sweeps the export directories a +// killed daemon left, those and nothing else. +test("the recipe declares the work directory as a state slot", () => { + const recipe = readFileSync(path.join(here, "..", "deployment", "idunn", "recipe.toml"), "utf8"); + assert.match(recipe, /\[\[state\.slots\]\]\nid = "work"\nrelative_path = "work"\nkind = "external-directory"/); +}); + +test("a restarted daemon sweeps the export directories a killed one left, and touches nothing else", async () => { + const first = await startDaemon(); + let second = null; + try { + const pending = generate(first, exportRequest({ clipName: "slow" })).catch(() => null); + await until(() => exportStarts(first).length === 1, "the export to start"); + const work = path.join(first.stateRoot, "work"); + const [stale] = readdirSync(work); + assert.match(stale, /^export-[A-Za-z0-9]{6}$/); + first.proc.kill("SIGKILL"); + await until(() => first.proc.signalCode !== null, "the daemon to die"); + await pending; + assert.deepEqual(readdirSync(work), [stale], "the killed export left its directory"); + // things under the state root that are not stale export directories + const decoys = { + "work/keep": "K", + "work/export-short/in.bvh": "S", + "work/exports-keep/in.bvh": "E", + "jobs/decoy.json": "J", + "artifacts/decoy/motion.bvh": "A", + "logs/decoy.log": "L", + "decoy-root-file": "R", + }; + for (const [relative, text] of Object.entries(decoys)) { + mkdirSync(path.dirname(path.join(first.stateRoot, relative)), { recursive: true }); + writeFileSync(path.join(first.stateRoot, relative), text); + } + second = await startDaemon({ root: first.root }); + assert.deepEqual(readdirSync(work).sort(), ["export-short", "exports-keep", "keep"], "the stale export is gone, the rest stays"); + for (const [relative, text] of Object.entries(decoys)) assert.equal(readFileSync(path.join(second.stateRoot, relative), "utf8"), text, relative); + assert.equal((await generate(second, exportRequest({ clipName: "after" }))).status, 200); + } finally { + if (first.proc.signalCode === null) first.proc.kill("SIGKILL"); + await (second ?? first).stop(); + } +}); + +test("a daemon that does not hold the state root sweeps nothing", async () => { + const first = await startDaemon(); + let second = null; + try { + const pending = generate(first, exportRequest({ clipName: "slow" })); + await until(() => exportStarts(first).length === 1, "the export to start"); + const work = path.join(first.stateRoot, "work"); + const live = readdirSync(work); + assert.equal(live.length, 1); + second = await startDaemon({ root: first.root, hold: false }); + await until(() => second.proc.exitCode !== null, "the second daemon to refuse", 20); + assert.equal(second.proc.exitCode, 2); + assert.deepEqual(readdirSync(work), live, "the live export's directory is still there"); + assert.equal((await pending).status, 200, "the live export finished"); + } finally { + if (second && second.proc.exitCode === null && second.proc.signalCode === null) second.proc.kill("SIGKILL"); + await first.stop(); + } +}); diff --git a/scripts/vili-daemon.mjs b/scripts/vili-daemon.mjs index 77e4966..67fa254 100644 --- a/scripts/vili-daemon.mjs +++ b/scripts/vili-daemon.mjs @@ -1,1236 +1,1243 @@ -#!/usr/bin/env node -import http from "node:http"; -import os from "node:os"; -import { createHash, randomBytes, randomInt } from "node:crypto"; -import { closeSync, constants, lstatSync, openSync, readdirSync, readFileSync, realpathSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; -import { copyFile, mkdir, mkdtemp, open, readdir, readFile, rm, writeFile } from "node:fs/promises"; -import path from "node:path"; -import readline from "node:readline"; -import net from "node:net"; -import { fileURLToPath, pathToFileURL } from "node:url"; -import { createRequire } from "node:module"; -import { spawn } from "node:child_process"; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const require = createRequire(import.meta.url); -const { - CultCache, - SingleFileMessagePackBackingStore, - defineDocumentRegistry, - defineDocumentType, -} = require("@gamecult/cultcache-ts"); -const projectRoot = path.resolve(__dirname, ".."); -const providerId = "vili.animation"; -const viliStoreFileName = "vili.service.cc"; -const objectSchema = { parse: (value) => value }; -const operatorStateDefinition = defineDocumentType({ - type: "gamecult.vili.operator_state", - schemaName: "gamecult.vili.operator_state", - schemaId: "gamecult.vili.operator_state.v1", - schemaVersion: "gamecult.vili.operator_state.v1", - name: (value) => value?.service || "vili", - schema: objectSchema, - members: [ - { slot: 0, memberName: "service", typeName: "string", isName: true }, - { slot: 1, memberName: "providerId", typeName: "string" }, - { slot: 2, memberName: "updatedAt", typeName: "string" }, - { slot: 3, memberName: "backend", typeName: "object" }, - ], -}); -const commandBoundaryDefinition = defineDocumentType({ - type: "gamecult.vili.command_boundary", - schemaName: "gamecult.vili.command_boundary", - schemaId: "gamecult.vili.command_boundary.v1", - schemaVersion: "gamecult.vili.command_boundary.v1", - name: (value) => value?.boundaryId || "vili", - schema: objectSchema, - members: [ - { slot: 0, memberName: "boundaryId", typeName: "string", isName: true }, - { slot: 1, memberName: "owner", typeName: "string" }, - { slot: 2, memberName: "commandAuthority", typeName: "object" }, - { slot: 3, memberName: "forbiddenAuthority", typeName: "string" }, - { slot: 4, memberName: "updatedAt", typeName: "string" }, - ], -}); -const transportProfileDefinition = defineDocumentType({ - type: "gamecult.vili.transport_profile", - schemaName: "gamecult.vili.transport_profile", - schemaId: "gamecult.vili.transport_profile.v1", - schemaVersion: "gamecult.vili.transport_profile.v1", - name: (value) => value?.profileId || "vili", - schema: objectSchema, - members: [ - { slot: 0, memberName: "profileId", typeName: "string", isName: true }, - { slot: 1, memberName: "daemonId", typeName: "string" }, - { slot: 2, memberName: "targetTransport", typeName: "string" }, - { slot: 3, memberName: "state", typeName: "string" }, - { slot: 4, memberName: "updatedAt", typeName: "string" }, - ], -}); -const viliDocumentRegistry = defineDocumentRegistry( - operatorStateDefinition, - commandBoundaryDefinition, - transportProfileDefinition, -); - -const MAX_THREADS = 16; -const CLIP_NAME = /^[A-Za-z0-9_-]{1,64}$/; - -// The state root's one identity: symlinks resolved as far as the path exists. The hand-run lock's name, reclaim's -// marker and the worker's environment all derive from args.stateRoot, so two spellings of one root are one root. -function canonicalPath(target) { - try { - return realpathSync(target); - } catch { - const parent = path.dirname(target); - return parent === target ? target : path.join(canonicalPath(parent), path.basename(target)); - } -} - -function parseArgs(argv) { - const env = process.env; - const args = { - host: env.VILI_DAEMON_HOST || "127.0.0.1", - port: Number(env.VILI_DAEMON_PORT || 8824), - stateRoot: path.resolve(env.VILI_STATE_ROOT || path.join(projectRoot, ".vili")), - python: env.VILI_PYTHON || "python3", - workerEntry: path.resolve(env.VILI_WORKER_ENTRY || path.join(__dirname, "kimodo-resident-worker.py")), - modelsRoot: env.VILI_MODELS_ROOT || "", - threads: env.VILI_THREADS || "8", - kimodoCommit: env.VILI_KIMODO_COMMIT || "", - pythonEnv: env.VILI_PYTHON_ENV || "", - blender: env.VILI_BLENDER || "", - }; - - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--host") args.host = argv[++index]; - else if (arg === "--port") args.port = Number(argv[++index]); - else if (arg === "--state-root") args.stateRoot = path.resolve(argv[++index]); - else if (arg === "--python") args.python = argv[++index]; - else if (arg === "--worker-entry") args.workerEntry = path.resolve(argv[++index]); - else if (arg === "--models-root") args.modelsRoot = argv[++index]; - else if (arg === "--threads") args.threads = argv[++index]; - else if (arg === "--kimodo-commit") args.kimodoCommit = argv[++index]; - else if (arg === "--python-env") args.pythonEnv = argv[++index]; - else if (arg === "--blender") args.blender = argv[++index]; - } - if (!/^[0-9]{1,2}$/.test(String(args.threads)) || Number(args.threads) < 1 || Number(args.threads) > MAX_THREADS) { - console.error(`threads must be an integer from 1 to ${MAX_THREADS}`); - process.exit(2); - } - args.threads = Number(args.threads); - args.stateRoot = canonicalPath(args.stateRoot); - if (args.modelsRoot) args.modelsRoot = path.resolve(args.modelsRoot); - if (args.blender) args.blender = path.resolve(args.blender); - return args; -} - -const args = parseArgs(process.argv.slice(2)); -const startedAt = new Date().toISOString(); - -// The rest pose Kimodo ships, in the worker interpreter's own tree (/lib/python3.12/site-packages), once. -const restBvh = path.isAbsolute(args.python) - ? path.join(path.dirname(path.dirname(args.python)), "lib", "python3.12", "site-packages", "kimodo", "assets", "skeletons", "somaskel77", "somaskel77_standard_tpose.bvh") - : null; - -// The worker child. Display state only: nothing reads `worker` to decide anything but ensureWorker. -const worker = { state: "stopped", proc: null, pid: null, port: null, servedModels: [], lastError: null }; -let starting = null; - -// Blender, probed once after the lease. Display and admission state: it decides only whether an fbx request is -// refused early. `version` is the first line of `blender --version`, or null (unconfigured, or the probe failed). -const blender = { version: null }; -const blenderReady = () => blender.version !== null; - -function firstDefined(...values) { - return values.find((value) => value !== undefined && value !== null); -} - -function boolValue(value) { - return value === true || String(value).toLowerCase() === "true" || String(value) === "1"; -} - -function json(response, statusCode, body) { - const payload = JSON.stringify(body, null, 2); - response.writeHead(statusCode, { - "content-type": "application/json; charset=utf-8", - "cache-control": "no-store", - }); - response.end(payload); -} - -function text(response, statusCode, body) { - response.writeHead(statusCode, { - "content-type": "text/plain; charset=utf-8", - "cache-control": "no-store", - }); - response.end(body); -} - -// A browser page can POST to localhost; a browser cannot send application/json cross-site without a preflight, -// and no route answers a preflight. So a POST that carries an Origin, or is not JSON, never reaches the body. -function refuseCrossSite(request, response) { - const contentType = String(request.headers["content-type"] || "").split(";")[0].trim().toLowerCase(); - if (request.headers.origin !== undefined || contentType !== "application/json") { - json(response, 403, { ok: false, code: "cross-site" }); - return true; - } - return false; -} - -function backendStatus() { - return { - state: worker.state, - pid: worker.pid, - port: worker.port, - threads: args.threads, - pythonEnv: args.pythonEnv || null, - kimodoCommit: args.kimodoCommit || null, - modelsRoot: args.modelsRoot || null, - lastError: worker.lastError, - blender: blender.version ?? (args.blender ? "unavailable" : "unconfigured"), - }; -} - -function operatorState() { - return { - schema: "gamecult.vili.operator_state.v0", - service: "vili", - providerId, - host: os.hostname(), - startedAt, - updatedAt: new Date().toISOString(), - backend: backendStatus(), - authority: { - owns: [ - "animation job intake", - "worker process lifecycle", - "motion artifact references", - "clip export to FBX", - ], - doesNotOwn: [ - "Persona speech generation", - "Weksa interlingua lowering", - "retargeting and playback", - "Odin discovery truth", - ], - }, - routes: { - health: "/health", - operatorState: "/operator-state", - generate: "/motion/generate", - artifacts: "/motion/artifacts/{jobId}/{name}", - restFbx: "/motion/rest-fbx", - }, - }; -} - -function commandBoundary() { - return { - schema: "gamecult.vili.command_boundary.v1", - boundaryId: "vili", - daemonId: "vili", - owner: "Vili animation runtime", - updatedAt: new Date().toISOString(), - commandAuthority: { - intake: { - method: "POST", - path: "/motion/generate", - owns: [ - "animation job identity", - "request persistence", - "worker process lifecycle", - "timeout handling", - "motion artifact references", - "clip export", - ], - }, - restFbx: { - method: "POST", - path: "/motion/rest-fbx", - owns: ["the rest-pose FBX job"], - }, - artifacts: { - method: "GET", - path: "/motion/artifacts/{jobId}/{name}", - owns: ["read-only artifact bytes"], - doesNotOwn: ["any path outside the job's artifact directory"], - }, - liveness: { - method: "GET", - path: "/health", - owns: ["compatibility liveness witness"], - doesNotOwn: [], - }, - }, - forbiddenAuthority: - "HTTP health and the operator-state projection are compatibility lowerings; they do not own daemon health or provider truth when CultCache/CultNet records exist.", - }; -} - -function transportProfile() { - return { - schema: "gamecult.vili.transport_profile.v1", - profileId: "vili", - daemonId: "vili", - targetTransport: "cultnet.transport.rudp.v0", - currentTransport: "CultCache local state + compatibility HTTP lowerings", - state: "local-cultcache-only", - operatorStateSchema: "gamecult.vili.operator_state.v1", - commandBoundarySchema: "gamecult.vili.command_boundary.v1", - compatibilityMechanisms: ["/health", "/operator-state"], - cutLine: - "Vili owns typed operator, command, and transport state. HTTP projections are lowerings until Odin consumes Vili records over CultNet/RUDP.", - updatedAt: new Date().toISOString(), - }; -} - -async function persistSurfaces() { - guard(); - await mkdir(args.stateRoot, { recursive: true }); - const cache = CultCache.builder() - .withRegistry(viliDocumentRegistry) - .withGenericStore(new SingleFileMessagePackBackingStore(path.join(args.stateRoot, viliStoreFileName))) - .build(); - guard(); - await cache.put(operatorStateDefinition, "vili", operatorState()); - guard(); - await cache.put(commandBoundaryDefinition, "vili", commandBoundary()); - guard(); - await cache.put(transportProfileDefinition, "vili", transportProfile()); -} - -// The whole child environment, in one place. Nothing else from process.env reaches the worker. -function childEnv() { - const threads = String(args.threads); - const env = { - HF_HOME: args.modelsRoot, - HF_HUB_OFFLINE: "1", - TRANSFORMERS_OFFLINE: "1", - VILI_EMBEDDINGS_DIR: path.join(args.stateRoot, "embeddings"), - VILI_THREADS: threads, - OMP_NUM_THREADS: threads, - MKL_NUM_THREADS: threads, - PYTHONPATH: path.join(projectRoot, "vendor", "CultLib", "packages", "cultcache-py", "src"), - PYTHONDONTWRITEBYTECODE: "1", - PYTHONUNBUFFERED: "1", - VILI_KIMODO_COMMIT: args.kimodoCommit, - VILI_PYTHON_ENV: args.pythonEnv, - }; - for (const name of ["PATH", "HOME", "LANG"]) { - if (process.env[name] !== undefined) env[name] = process.env[name]; - } - return env; -} - -// The whole Blender environment, in one place: nothing else from process.env reaches it. The libraries the sealed -// release carries sit in blender/lib; PYTHONHASHSEED=0 is half of byte-reproducible export (scripts/bvh-to-fbx.py). -function exportEnv() { - const env = { - HOME: "/nonexistent", - PYTHONHASHSEED: "0", - PYTHONNOUSERSITE: "1", - LD_LIBRARY_PATH: path.join(path.dirname(args.blender), "lib"), - }; - for (const name of ["PATH", "LANG"]) { - if (process.env[name] !== undefined) env[name] = process.env[name]; - } - return env; -} - -function workerUnconfigured() { - return !args.modelsRoot || !args.kimodoCommit || !args.pythonEnv; -} - -function waitForExit(proc, timeoutMs) { - return new Promise((resolve) => { - if (proc.exitCode !== null || proc.signalCode !== null) return resolve(true); - const timer = setTimeout(() => resolve(false), timeoutMs); - proc.once("exit", () => { - clearTimeout(timer); - resolve(true); - }); - }); -} - -// The worker leads its own process group (its pid is the group id), so the encoder child it spawns is in the group -// too. The daemon owns the whole group: it is killed when the worker exits for any reason and on a requested stop. -// A crashed hand-run daemon's group is reclaimed by the next hand-run daemon (handRunRuntime). Under Idunn the -// unit's cgroup is the same ownership from outside the process. -function killGroup(pgid, signal) { - try { - process.kill(-pgid, signal); - } catch { - // no such group - } -} - -// The hand-run way of owning a state root (tests, development): a Unix socket lock, the worker's pid file and the -// /proc reclaim. Idunn's runtime (scripts/idunn-entry.mjs) replaces all three with its process write lease. -export function handRunRuntime(stateRoot) { - const pidFile = path.join(stateRoot, "worker.pid"); - let lock = null; - - // The lock is a listening socket in Linux's abstract namespace, named by the state root, which is canonical (its real path, see canonicalPath). A bind is - // atomic and exclusive and the kernel drops it with its owner, so there is no lock file, no stale lock to probe - // or take over, and no takeover to race: a bind either succeeds (this daemon owns the root) or it does not. - const listen = () => new Promise((resolve, reject) => { - const name = createHash("sha256").update(stateRoot).digest("hex"); - const server = net.createServer((socket) => socket.destroy()); - server.once("error", reject); - server.listen(`\0vili-state-root:${name}`, () => { - server.removeListener("error", reject); - lock = server; - resolve(); - }); - }); - - // A worker left by a crashed daemon: the pid file names its group, and the group is ours only if some member - // still carries this state root's embeddings directory in its environment (the pid may have been reused). - function reclaimStrayWorker() { - let pgid; - try { - pgid = Number(readFileSync(pidFile, "utf8").trim()); - } catch { - return; - } - const marker = `VILI_EMBEDDINGS_DIR=${path.join(stateRoot, "embeddings")}`; - let ours = false; - try { - for (const entry of readdirSync("/proc")) { - if (!/^\d+$/.test(entry)) continue; - try { - const stat = readFileSync(`/proc/${entry}/stat`, "utf8"); - const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); - if (Number(fields[2]) !== pgid) continue; - if (readFileSync(`/proc/${entry}/environ`, "utf8").split("\0").includes(marker)) ours = true; - } catch { - // the process exited while we looked - } - } - } catch { - // no /proc: nothing can be verified, so nothing is killed - } - if (ours) killGroup(pgid, "SIGKILL"); - try { - unlinkSync(pidFile); - } catch { - // already gone - } - } - - return { - async acquire() { - await mkdir(stateRoot, { recursive: true }); - try { - await listen(); - } catch (error) { - if (error?.code !== "EADDRINUSE") throw error; - console.error("the state root is held by a running daemon"); - process.exit(2); - } - reclaimStrayWorker(); // synchronous: no request is served before a predecessor's worker is gone - }, - held: () => lock !== null, - refusal: () => "write-lease-pending", - watch() {}, - recordWorker(pid) { - writeFileSync(pidFile, `${pid}\n`); - }, - forgetWorker(pid) { - try { - if (readFileSync(pidFile, "utf8").trim() === String(pid)) unlinkSync(pidFile); - } catch { - // no pid file, or another worker's - } - }, - release() { - if (lock) lock.close(); - lock = null; - }, - }; -} - -// A requested stop gives a process group its grace (SIGTERM, then SIGKILL); a lost writer gets SIGKILL at once. The -// worker and the exports are stopped by this one path. -async function stopGroups(procs, signals) { - for (const signal of signals) { - for (const proc of procs) if (proc.pid !== undefined) killGroup(proc.pid, signal); - const gone = await Promise.all(procs.map((proc) => waitForExit(proc, signal === "SIGTERM" ? 10000 : 5000))); - if (gone.every(Boolean)) break; - } -} - -const STOP_SIGNALS = ["SIGTERM", "SIGKILL"]; - -async function stopWorker(signals = STOP_SIGNALS) { - const proc = worker.proc; - if (!proc || proc.pid === undefined) return; - worker.state = "stopped"; - await stopGroups([proc], signals); -} - -// Blender's process groups in flight. A finished export removes itself; a model switch (ensureWorker) leaves them alone. -const exportGroups = new Set(); -const stopExports = (signals = STOP_SIGNALS) => stopGroups([...exportGroups], signals); - -function isRegularFile(file) { - try { - return lstatSync(file).isFile(); - } catch { - return false; - } -} - -// A regular file (no symlink, in the file or any directory above it) strictly inside `directory`, which is canonical. -function regularFileInside(file, directory) { - const relative = path.relative(directory, file); - if (!relative || path.isAbsolute(relative) || relative.split(path.sep)[0] === "..") return false; - try { - return lstatSync(file).isFile() && realpathSync(file) === file; - } catch { - return false; - } -} - -const exportScript = path.join(__dirname, "bvh-to-fbx.py"); -// The state slot "work" (deployment/idunn/recipe.toml): one private directory per export, removed by the export itself -// and, if the daemon died mid-export, swept by the next daemon that holds the state root (sweepWork). -const WORK_DIR = "work"; -const EXPORT_PREFIX = "export-"; -const EXPORT_NAME = new RegExp(`^${EXPORT_PREFIX}[A-Za-z0-9]{6}$`); -const EXPORT_SETTINGS = "standard-tpose soma77 metres hashseed0 time1970"; -const EXPORT_TIMEOUT_MS = 120000; -let exportScriptDigest = null; -const scriptDigest = () => (exportScriptDigest ??= `sha256:${createHash("sha256").update(readFileSync(exportScript)).digest("hex")}`); - -// Runs the sealed Blender in its own process group, niced, under the write gate. Both streams go to -// logs/export.log line by line through mayWrite(), and only there; the version probe is not logged, so it writes -// nothing to the state root. Resolves { code, signal, first, result }: the exit, the first stdout line and the last -// `VILI-FBX ` line's payload. It rejects only with LeaseLost. -async function runBlender(argv, timeoutMs, logged = true) { - guard(); - let sink = null; - if (logged) { - await mkdir(path.join(args.stateRoot, "logs"), { recursive: true }); - guard(); // synchronous with the log open and the spawn below: no await between them - sink = openSync(path.join(args.stateRoot, "logs", "export.log"), "a"); - } - const proc = spawn(args.blender, argv, { - cwd: __dirname, - env: exportEnv(), - stdio: ["ignore", "pipe", "pipe"], - shell: false, - detached: true, - }); - exportGroups.add(proc); - const seen = { first: null, result: null }; - let sinkOpen = sink !== null; - for (const [stream, isStdout] of [[proc.stdout, true], [proc.stderr, false]]) { - if (!stream) continue; - readline.createInterface({ input: stream }).on("line", (line) => { - if (isStdout) { - seen.first ??= line; - if (line.startsWith("VILI-FBX ")) seen.result = line.slice("VILI-FBX ".length); - } - if (sinkOpen && mayWrite()) writeSync(sink, `${line}\n`); - }); - } - const finished = new Promise((resolve) => { - proc.once("error", () => resolve({ code: null, signal: null })); - proc.once("close", (code, signal) => resolve({ code, signal })); - }); - proc.once("exit", () => { - exportGroups.delete(proc); - if (proc.pid !== undefined) killGroup(proc.pid, "SIGKILL"); // whatever Blender left running - }); - try { - os.setPriority(proc.pid, 10); - } catch { - if (proc.pid !== undefined) killGroup(proc.pid, "SIGKILL"); - } - const timer = setTimeout(() => { - if (proc.pid !== undefined) killGroup(proc.pid, "SIGKILL"); - }, timeoutMs); - const exit = await finished; - clearTimeout(timer); - exportGroups.delete(proc); - if (sink !== null) closeSync(sink); - sinkOpen = false; - return { ...exit, ...seen }; -} - -// Blender answers once at start: `blender --version`, 15 s. Ready only when it exits 0 and names itself. -async function probeBlender() { - if (!args.blender) return; - const run = await runBlender(["-b", "--factory-startup", "--version"], 15000, false); - guard(); - blender.version = run.code === 0 && run.first?.startsWith("Blender ") ? run.first : null; -} - -// The bytes of a regular file strictly inside `directory`, or null. The file is opened without following a symlink -// and without blocking (a FIFO or a device would otherwise hold the open, and a filesystem thread, indefinitely), and -// the open file is the one checked, so a path swapped after the check cannot redirect the read. -async function readInside(file, directory) { - let handle = null; - try { - handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); - const opened = await handle.stat(); - if (!opened.isFile() || !regularFileInside(file, directory)) return null; - const named = lstatSync(file); - if (named.dev !== opened.dev || named.ino !== opened.ino) return null; - return await handle.readFile(); - } catch { - return null; - } finally { - await handle?.close(); - } -} - -// One BVH to one FBX beside it. Blender never sees a path in the job's artifact directory: it reads a private copy of -// the BVH and writes a private FBX in a directory made for this export alone, and the FBX is then created beside the -// BVH exclusively, so nothing planted at either path is read, written through or replaced. The FBX counts only if -// Blender exits 0, prints its result line and its file exists. A failure names the exit or the signal and never -// Blender's output. A window {first, last} (BVH frame indices, last inclusive) is handed to Blender as the frame -// range to export, in Blender's frames, which count from 1; the daemon never chooses one. -async function exportClip(bvh, fbx, clipName, timeoutMs, artifactDir, window = null) { - const input = await readInside(bvh, artifactDir); - if (input === null) return { error: "fbx-export-refused" }; - guard(); - const work = await mkdtemp(path.join(args.stateRoot, WORK_DIR, EXPORT_PREFIX)); - try { - const privateBvh = path.join(work, "in.bvh"); - const privateFbx = path.join(work, "out.fbx"); - guard(); - await writeFile(privateBvh, input, { flag: "wx" }); - const run = await runBlender([ - "-b", "--factory-startup", "--python-use-system-env", "--python-exit-code", "1", - "--python", exportScript, "--", privateBvh, privateFbx, clipName, - ...(window ? ["--frames", String(window.first + 1), String(window.last + 1)] : []), - ], timeoutMs); - let info = null; - if (run.code === 0 && run.result !== null) { - try { - const parsed = JSON.parse(run.result); - if (Number.isInteger(parsed.bones) && Array.isArray(parsed.frames) && parsed.frames.length === 2 - && parsed.frames.every(Number.isInteger) && Number.isFinite(parsed.fps)) info = parsed; - } catch { - // not the result line - } - } - const made = info ? await readInside(privateFbx, work) : null; - if (made !== null) { - guard(); - try { - await writeFile(fbx, made, { flag: "wx" }); - return { clip: { file: fbx, clipName, bones: info.bones, frames: info.frames, fps: info.fps } }; - } catch (error) { - if (error instanceof LeaseLost) throw error; - return { error: "fbx-export-refused" }; - } - } - const how = run.code === 0 ? "no result" : run.code !== null ? `exit ${run.code}` : `signal ${run.signal ?? "none"}`; - return { error: `fbx-export-failed (${how})` }; - } finally { - if (mayWrite()) await rm(work, { recursive: true, force: true }); - } -} - -// The export directories a killed daemon left: only directories named as mkdtemp names them, only under work/. Called -// once, after the runtime holds the state root, so a live daemon's export is never swept. -async function sweepWork() { - const root = path.join(args.stateRoot, WORK_DIR); - for (const entry of await readdir(root, { withFileTypes: true })) { - if (!entry.isDirectory() || !EXPORT_NAME.test(entry.name)) continue; - guard(); - await rm(path.join(root, entry.name), { recursive: true, force: true }); - } -} - -// Every BVH the worker listed becomes an FBX, or the job fails. A listed BVH must be a regular file inside the job's -// artifact directory, and that is decided when its export begins (exportClip), never earlier. Sample N of several is -// _NN. With loops (the worker's answer, an array when the request asked for a loop, else undefined), sample k -// is exported over loops[k], and a job whose answer has no usable window for a sample fails. -async function exportListed(listed, artifactDir, clipName, timeoutMs, loops = undefined) { - const bvhs = []; - for (const file of Array.isArray(listed) ? listed : []) { - if (typeof file !== "string" || !file.endsWith(".bvh")) continue; - bvhs.push(path.resolve(artifactDir, file)); - } - if (bvhs.length === 0) return { clips: [], error: "fbx-export-failed (no bvh)" }; - bvhs.sort(); - const clips = []; - for (const [index, bvh] of bvhs.entries()) { - const name = bvhs.length === 1 ? clipName : `${clipName}_${String(index).padStart(2, "0")}`; - const window = loops === undefined ? null : loops[index]; - if (window !== null && !(Number.isInteger(window?.first) && Number.isInteger(window?.last) && window.first >= 0 && window.last > window.first)) { - return { clips, error: "fbx-export-failed (no loop window)" }; - } - const done = await exportClip(bvh, `${bvh.slice(0, -".bvh".length)}.fbx`, name, timeoutMs, artifactDir, window); - if (done.error) return { clips, error: done.error }; - clips.push({ ...done.clip, file: path.relative(artifactDir, done.clip.file).split(path.sep).join("/") }); - } - return { clips, error: null }; -} - -// Every regular file under the artifact directory, named relative to it, sorted. -async function artifactFiles(directory) { - const files = []; - const walk = async (relative) => { - for (const entry of readdirSync(path.join(directory, relative), { withFileTypes: true })) { - const name = path.join(relative, entry.name); - if (entry.isDirectory()) await walk(name); - else if (entry.isFile()) { - const bytes = await readFile(path.join(directory, name)); - files.push({ name: name.split(path.sep).join("/"), bytes: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }); - } - } - }; - await walk(""); - return files.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); -} - -// The one job record, for a generate job and a rest job alike: what the job was, which files it left and, for an -// export, the exact Blender and script that made them. -async function writeJobRecord(job, exported) { - guard(); - const record = { schema: "gamecult.vili.motion_job.v1", ...job, files: await artifactFiles(job.artifactDir) }; - if (exported) record.export = { blender: blender.version, script: scriptDigest(), settings: EXPORT_SETTINGS, clips: exported.clips }; - guard(); - await writeFile(path.join(args.stateRoot, "jobs", `${job.jobId}.json`), JSON.stringify(record, null, 2)); - return record; -} - -async function startWorker(model, timeoutMs) { - guard(); - await mkdir(path.join(args.stateRoot, "logs"), { recursive: true }); - guard(); - await mkdir(path.join(args.stateRoot, "embeddings"), { recursive: true }); - guard(); // synchronous with the log open and the spawn below: no await between them - const logFd = openSync(path.join(args.stateRoot, "logs", "worker.log"), "a"); - const argv = [args.workerEntry, "--host", "127.0.0.1", "--port", "0"]; - if (model) argv.push("--model", model); - const proc = spawn(args.python, argv, { - cwd: __dirname, - env: childEnv(), - stdio: ["ignore", "pipe", logFd], - shell: false, - detached: true, - }); - closeSync(logFd); - if (proc.pid !== undefined) { - runtime.recordWorker(proc.pid); - proc.once("exit", () => { - killGroup(proc.pid, "SIGKILL"); // whatever the worker left running, an encoder child included - runtime.forgetWorker(proc.pid); - }); - } - Object.assign(worker, { state: "starting", proc, pid: proc.pid ?? null, port: null, servedModels: [], lastError: null }); - const exited = new Promise((resolve) => { - proc.once("error", () => resolve("the worker could not be spawned")); - proc.once("exit", (code, signal) => resolve(`the worker exited (${signal ?? `code ${code}`})`)); - }); - exited.then((reason) => { - if (worker.proc !== proc) return; - Object.assign(worker, { state: "stopped", proc: null, pid: null, port: null, lastError: worker.state === "stopped" ? worker.lastError : reason }); - }); - - const fail = async (reason) => { - worker.lastError = reason; - await stopWorker(); - Object.assign(worker, { state: "failed", proc: null, pid: null, port: null, lastError: reason }); - throw new Error(reason); - }; - - if (proc.pid === undefined) await fail(await exited); - try { - os.setPriority(proc.pid, 10); - } catch { - await fail("the worker priority could not be lowered"); - } - - // Stdout lines are the worker's log, until the one that says it is ready. - const deadline = Date.now() + timeoutMs; - let readyTimer; - const ready = await Promise.race([ - new Promise((resolve) => { - const lines = readline.createInterface({ input: proc.stdout }); - const sink = openSync(path.join(args.stateRoot, "logs", "worker.log"), "a"); - lines.on("line", (line) => { - try { - const event = JSON.parse(line); - if (event?.event === "kimodo-worker-ready" && Number.isInteger(event.port)) return resolve(event); - } catch { - // not JSON: an ordinary log line - } - if (mayWrite()) writeSync(sink, `${line}\n`); - }); - lines.on("close", () => closeSync(sink)); - }), - exited.then((reason) => ({ failed: reason })), - new Promise((resolve) => { readyTimer = setTimeout(() => resolve({ failed: "the worker was not ready in time" }), timeoutMs); }), - ]); - clearTimeout(readyTimer); - if (ready.failed) await fail(ready.failed); - worker.port = ready.port; - - for (;;) { - try { - const health = await fetch(`http://127.0.0.1:${worker.port}/health`, { signal: AbortSignal.timeout(10000) }); - const body = await health.json(); - if (health.ok && body?.ok) { - worker.servedModels = [body.model, body.displayModel].filter(Boolean); - worker.state = "ready"; - return; - } - } catch { - // not answering yet - } - if (worker.proc !== proc) throw new Error(worker.lastError || "the worker exited"); - if (Date.now() > deadline) await fail("the worker did not become healthy in time"); - await new Promise((resolve) => setTimeout(resolve, 500)); - } -} - -async function ensureWorker({ model = null, timeoutMs = 1000 * 60 * 8 } = {}) { - if (starting) await starting; - if (worker.state === "ready" && model && !worker.servedModels.includes(model)) await stopWorker(); - if (worker.state === "ready") return; - if (!starting) starting = startWorker(model, timeoutMs).finally(() => { starting = null; }); - await starting; -} - -async function readRequestBody(request) { - const chunks = []; - for await (const chunk of request) chunks.push(chunk); - const raw = Buffer.concat(chunks).toString("utf8"); - return raw ? JSON.parse(raw) : {}; -} - -async function underLease(response, run) { - try { - await run(); - } catch (error) { - if (!(error instanceof LeaseLost)) throw error; - json(response, 503, { ok: false, code: stopping ? "stopping" : runtime.refusal() }); - } -} - -// A seed is an integer Kimodo can take: the worker seeds with int(seed), so the daemon records exactly what it sends. -const SEED_LIMIT = 2 ** 31; - -// The one source of job ids, for a generate job and a rest job alike: the clock for order, random bits for -// uniqueness, and an exclusive mkdir that decides, so two jobs never share a directory, a record or a file. -async function newJobDir(prefix) { - guard(); - await mkdir(path.join(args.stateRoot, "artifacts"), { recursive: true }); - for (let attempt = 0; attempt < 8; attempt += 1) { - const jobId = `${prefix}-${Date.now()}-${randomBytes(4).toString("hex")}`; - const artifactDir = path.join(args.stateRoot, "artifacts", jobId); - guard(); - try { - await mkdir(artifactDir); - return { jobId, artifactDir }; - } catch (error) { - if (error.code !== "EEXIST") throw error; - } - } - throw new Error("no free job id"); -} - -async function generate(request, response) { - guard(); - if (workerUnconfigured()) { - json(response, 503, { ok: false, code: "worker-unconfigured" }); - return; - } - let body; - try { - body = await readRequestBody(request); - } catch { - json(response, 400, { ok: false, error: "Invalid JSON." }); - return; - } - - const prompt = String(body.prompt || body.utterance || "").trim(); - const hasPromptInput = Boolean(prompt) - || Array.isArray(body.texts) - || typeof body.meta === "object" - || body.inputFolder - || body.input_folder; - if (!hasPromptInput) { - json(response, 400, { ok: false, error: "Missing prompt, utterance, texts, meta, or inputFolder." }); - return; - } - if (Array.isArray(body.texts) && !Array.isArray(body.durations)) { - json(response, 400, { ok: false, error: "Batch text requests require a durations array." }); - return; - } - - // One export plan: fbx implies the BVH and its standard T-pose, which the FBX is made from. - const fbx = boolValue(body.fbx); - const askedTpose = firstDefined(body.bvhStandardTpose, body.bvh_standard_tpose); - const loop = body.loop === true; - if ((body.loop !== undefined && typeof body.loop !== "boolean") || (loop && !fbx) - || (fbx && ((askedTpose !== undefined && !boolValue(askedTpose)) - || (body.clipName !== undefined && !(typeof body.clipName === "string" && CLIP_NAME.test(body.clipName)))))) { - json(response, 400, { ok: false, error: "Invalid fbx request." }); - return; - } - if (fbx && !blenderReady()) { - json(response, 503, { ok: false, code: "blender-unavailable" }); - return; - } - const plan = { - bvh: fbx || boolValue(body.bvh), - bvhStandardTpose: fbx || boolValue(askedTpose), - }; - - const bodyMeta = typeof body.meta === "object" && body.meta ? body.meta : {}; - const askedSeed = firstDefined(body.seed, bodyMeta.seed); - if (askedSeed !== undefined && !(Number.isInteger(askedSeed) && askedSeed >= 0 && askedSeed < SEED_LIMIT)) { - json(response, 400, { ok: false, error: "Invalid seed." }); - return; - } - - const { jobId, artifactDir } = await newJobDir("vili"); - const clipName = fbx ? (body.clipName ?? jobId) : undefined; - const duration = firstDefined(body.durationSeconds, body.duration, 4); - const diffusionSteps = Math.max(1, Number(firstDefined( - body.diffusionSteps, - body.diffusion_steps, - bodyMeta.diffusion_steps, - 20, - ))); - const seed = askedSeed ?? randomInt(0, SEED_LIMIT); - const numSamples = Math.max(1, Number(firstDefined( - body.numSamples, - body.num_samples, - bodyMeta.num_samples, - 1, - ))); - const meta = Object.keys(bodyMeta).length > 0 - ? { ...bodyMeta } - : Array.isArray(body.texts) - ? { texts: body.texts, durations: body.durations } - : { - text: prompt, - duration, - }; - meta.num_samples = Math.max(1, Number(firstDefined(body.numSamples, body.num_samples, meta.num_samples, 1))); - meta.diffusion_steps = diffusionSteps; - meta.seed = seed; - if (body.cfg) meta.cfg = body.cfg; - const constraintsPath = (() => { - if (typeof body.constraints === "string") return body.constraints; - if (body.constraints_path) return body.constraints_path; - if (body.constraintsPath) return body.constraintsPath; - if (body.constraints !== undefined) return path.join(artifactDir, "constraints.json"); - return undefined; - })(); - if (body.constraints !== undefined && typeof body.constraints !== "string") { - guard(); - await writeFile(path.join(artifactDir, "constraints.json"), JSON.stringify(body.constraints, null, 2)); - } - const requestRecord = { - schema: "gamecult.vili.motion_request.v0", - prompt: prompt || undefined, - texts: Array.isArray(body.texts) ? body.texts : undefined, - meta, - model: body.model, - duration, - numSamples, - diffusion_steps: diffusionSteps, - numTransitionFrames: firstDefined(body.numTransitionFrames, body.num_transition_frames), - constraints: body.constraints !== undefined && typeof body.constraints !== "string" ? body.constraints : constraintsPath, - seed, - fbx: fbx || undefined, - loop: loop || undefined, - clipName, - cfg: body.cfg, - cfgType: firstDefined(body.cfgType, body.cfg_type), - cfgWeight: firstDefined(body.cfgWeight, body.cfg_weight), - ...plan, - noPostprocess: boolValue(firstDefined(body.noPostprocess, body.no_postprocess)), - saveExampleDir: boolValue(firstDefined(body.saveExampleDir, body.save_example_dir)), - inputFolder: firstDefined(body.inputFolder, body.input_folder), - }; - guard(); - await writeFile(path.join(artifactDir, "meta.json"), JSON.stringify(meta, null, 2)); - - let result; - try { - await ensureWorker({ - timeoutMs: Number(firstDefined(body.workerTimeoutMs, body.worker_timeout_ms, 1000 * 60 * 8)), - model: body.model || null, - }); - const workerRequest = { - model: body.model, - prompt: prompt || undefined, - texts: Array.isArray(body.texts) ? body.texts : undefined, - meta, - inputFolder: firstDefined(body.inputFolder, body.input_folder), - duration, - diffusionSteps, - numSamples, - numTransitionFrames: firstDefined(body.numTransitionFrames, body.num_transition_frames), - constraints: constraintsPath, - seed, - loop: loop || undefined, - cfg: body.cfg, - cfgType: firstDefined(body.cfgType, body.cfg_type), - cfgWeight: firstDefined(body.cfgWeight, body.cfg_weight), - ...plan, - noPostprocess: boolValue(firstDefined(body.noPostprocess, body.no_postprocess)), - saveExampleDir: boolValue(firstDefined(body.saveExampleDir, body.save_example_dir)), - output: path.join(artifactDir, "motion"), - }; - guard(); - await writeFile(path.join(artifactDir, "request.json"), JSON.stringify(workerRequest, null, 2)); - const answer = await fetch(`http://127.0.0.1:${worker.port}/generate`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(workerRequest), - signal: AbortSignal.timeout(Number(body.timeoutMs || 1000 * 60 * 30)), - }); - const workerBody = await answer.json(); - result = { ok: answer.ok && workerBody?.ok !== false, error: answer.ok ? null : `the worker answered ${answer.status}`, body: workerBody }; - } catch (error) { - if (error instanceof LeaseLost) throw error; - result = { ok: false, error: `the worker request failed (${error?.name || "Error"})`, body: null }; - } - - let exported = null; - if (fbx && result.ok) { - const timeoutMs = Math.min(EXPORT_TIMEOUT_MS, Math.max(1, Number(firstDefined(body.exportTimeoutMs, EXPORT_TIMEOUT_MS)) || EXPORT_TIMEOUT_MS)); - exported = await exportListed(result.body?.files, artifactDir, clipName, timeoutMs, loop ? (Array.isArray(result.body?.loops) ? result.body.loops : []) : undefined); - if (exported.error) result = { ok: false, error: exported.error, body: result.body }; - } - - guard(); // the writer may have been lost while the job ran - const job = await writeJobRecord({ - jobId, - createdAt: new Date().toISOString(), - prompt: prompt || null, - model: body.model || result.body?.model || null, - request: requestRecord, - durationSeconds: duration, - diffusionSteps, - seed, - artifactDir, - ok: result.ok, - error: result.error, - worker: result.body, - }, exported); - json(response, result.ok ? 200 : 502, job); -} - -// The release's own rest pose, exported as the avatar cut's source: a job like any other, made from Kimodo's -// standard-T-pose BVH that the worker interpreter's site-packages carries. -async function restFbx(response) { - guard(); - if (!blenderReady()) { - json(response, 503, { ok: false, code: "blender-unavailable" }); - return; - } - if (!restBvh || !isRegularFile(restBvh)) { - json(response, 503, { ok: false, code: "rest-skeleton-missing" }); - return; - } - const { jobId, artifactDir } = await newJobDir("rest"); - guard(); - await copyFile(restBvh, path.join(artifactDir, "SOMA77_Rest.bvh")); - const exported = await exportListed([path.join(artifactDir, "SOMA77_Rest.bvh")], artifactDir, "SOMA77_Rest", EXPORT_TIMEOUT_MS); - guard(); - const job = await writeJobRecord({ - jobId, - createdAt: new Date().toISOString(), - prompt: null, - model: null, - request: null, - durationSeconds: null, - diffusionSteps: null, - seed: null, - artifactDir, - ok: !exported.error, - error: exported.error, - worker: null, - }, exported); - json(response, job.ok ? 200 : 502, job); -} - -// Reads only: a job's own FBX and BVH files, by exact name, regular files inside the job's artifact directory. -const ARTIFACT_ROUTE = /^\/motion\/artifacts\/((?:vili|rest)-[0-9]{13}-[0-9a-f]{8})\/(motion\.fbx|motion\.bvh|SOMA77_Rest\.fbx|motion\/motion_[0-9]{2}\.(?:fbx|bvh))$/; - -async function serveArtifact(response, jobId, name) { - const directory = path.join(args.stateRoot, "artifacts", jobId); - const file = path.join(directory, name); - let bytes = null; - if (regularFileInside(file, directory)) { - try { - bytes = await readFile(file); - } catch { - // gone between the check and the read - } - } - if (bytes === null) { - text(response, 404, "not found\n"); - return; - } - response.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "no-store" }); - response.end(bytes); -} - -const SNAPSHOT_BODY_MAX = 64 * 1024; - -// Idunn's route challenge: a msgpack POST from the host, answered from local authority. A browser page can send -// it too, so an Origin is refused exactly as on every other POST. Only the Idunn runtime answers; the hand-run -// daemon has no such route. -async function answerRouteChallenge(request, response) { - if (!runtime.answerSnapshot) { - text(response, 404, "not found\n"); - return; - } - if (request.headers.origin !== undefined) { - json(response, 403, { ok: false, code: "cross-site" }); - return; - } - const chunks = []; - let size = 0; - for await (const chunk of request) { - size += chunk.length; - if (size > SNAPSHOT_BODY_MAX) { - json(response, 400, { ok: false, code: "invalid-snapshot-request" }); - return; - } - chunks.push(chunk); - } - let answer; - try { - answer = runtime.answerSnapshot(Buffer.concat(chunks)); - } catch (error) { - if (error?.code === "invalid-snapshot-request") json(response, 400, { ok: false, code: error.code }); - else json(response, 503, { ok: false, code: "runtime-presence-unavailable" }); - return; - } - response.writeHead(200, { "content-type": "application/msgpack", "cache-control": "no-store" }); - response.end(answer); -} - -async function handleRequest(request, response) { - const url = new URL(request.url, `http://${request.headers.host || `${args.host}:${args.port}`}`); - if (url.pathname === "/cultnet/snapshot" && request.method === "POST") { - await answerRouteChallenge(request, response); - return; - } - const artifact = request.method === "GET" ? ARTIFACT_ROUTE.exec(url.pathname) : null; - if (artifact) { - await serveArtifact(response, artifact[1], artifact[2]); - return; - } - if (request.method === "POST" && refuseCrossSite(request, response)) return; - - if (url.pathname === "/" || url.pathname === "/health") { - json(response, 200, { ok: true, service: "vili", providerId, state: operatorState() }); - } else if (url.pathname === "/operator-state") { - json(response, 200, operatorState()); - } else if (url.pathname === "/motion/generate" && request.method === "POST") { - await underLease(response, () => generate(request, response)); - } else if (url.pathname === "/motion/rest-fbx" && request.method === "POST") { - await underLease(response, () => restFbx(response)); - } else { - text(response, 404, "not found\n"); - } -} - -let runtime; -let stopping = false; - -class LeaseLost extends Error {} - -// The one write authority: the runtime holds the state root (the lease, or the lock) and the daemon is not stopping. -// held() is read at the moment of every ask, so a loss is seen by the first gate to ask. -function mayWrite() { - return !stopping && runtime.held(); -} - -// The one write gate. Every write to the state root and every spawn calls it, immediately before it happens (not once -// per request), so a request admitted before the lease went, or before SIGTERM, still stops at its next write. -function guard() { - if (!mayWrite()) throw new LeaseLost("the writer was lost"); -} - -// How often the lease is asked while nothing else asks, and so how long a worker can outlive a loss (plus the -// SIGKILL): a generate can await its worker for 30 minutes. The service documents are rewritten every PERSIST_EVERY asks. -const WATCH_MS = 250; -const PERSIST_EVERY = 20; - -// The one daemon. `rt` owns the state root: the hand-run runtime (socket lock) or Idunn's (process write lease). -// The server listens first, so Idunn's challenge can be answered while the lease is still pending; nothing is -// written to the state root until the runtime holds it. -export async function serve(rt) { - runtime = rt; - const server = http.createServer((request, response) => { - handleRequest(request, response).catch((error) => { - // The error's message can carry the request; name only its class. - console.error(`request failed: ${error?.name || "Error"}`); - if (!response.headersSent) json(response, 500, { ok: false, code: "internal" }); - else response.end(); - }); - }); - await new Promise((resolve) => server.listen(args.port, args.host, resolve)); - console.log(`Vili listening on http://${args.host}:${server.address().port}`); - - let tick = null; - // From the signal on nothing is accepted, written or spawned (guard() refuses what is already in flight); then the - // worker tree is stopped, and only then does the process end. - const shutdown = async () => { - if (stopping) return; - stopping = true; - server.close(); - server.closeIdleConnections(); - clearInterval(tick); - await Promise.all([stopWorker(), stopExports()]); - runtime.release(); - process.exit(0); - }; - process.on("SIGTERM", shutdown); - process.on("SIGINT", shutdown); - - await runtime.acquire(); - let lost = false; - const onLost = () => { - if (lost) return; - lost = true; - clearInterval(tick); - Promise.all([stopWorker(["SIGKILL"]), stopExports(["SIGKILL"])]).finally(() => runtime.reportHealth?.("degraded")); - }; - runtime.watch(onLost); // held() reports a loss through this the moment any gate sees it - try { - guard(); - await mkdir(path.join(args.stateRoot, "jobs"), { recursive: true }); - await mkdir(path.join(args.stateRoot, WORK_DIR), { recursive: true }); - await sweepWork(); - await persistSurfaces(); - await probeBlender(); - } catch (error) { - if (error instanceof LeaseLost) return; // lost while opening the state root: onLost has run, and nothing is active - throw error; - } - let asks = 0; - tick = setInterval(() => { - runtime.watch(onLost); - if (lost || ++asks % PERSIST_EVERY !== 0) return; - persistSurfaces().catch((error) => { - if (!(error instanceof LeaseLost)) console.error(`persist failed: ${error?.name || "Error"}`); - }); - }, WATCH_MS); - if (lost) return; - runtime.reportHealth?.("active"); - console.log("Vili holds the state root"); -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - await serve(handRunRuntime(args.stateRoot)); -} +#!/usr/bin/env node +import http from "node:http"; +import os from "node:os"; +import { createHash, randomBytes, randomInt } from "node:crypto"; +import { closeSync, constants, lstatSync, openSync, readdirSync, readFileSync, realpathSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; +import { copyFile, mkdir, mkdtemp, open, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import readline from "node:readline"; +import net from "node:net"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { createRequire } from "node:module"; +import { spawn } from "node:child_process"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); +const { + CultCache, + SingleFileMessagePackBackingStore, + defineDocumentRegistry, + defineDocumentType, +} = require("@gamecult/cultcache-ts"); +const projectRoot = path.resolve(__dirname, ".."); +const providerId = "vili.animation"; +const viliStoreFileName = "vili.service.cc"; +const objectSchema = { parse: (value) => value }; +const operatorStateDefinition = defineDocumentType({ + type: "gamecult.vili.operator_state", + schemaName: "gamecult.vili.operator_state", + schemaId: "gamecult.vili.operator_state.v1", + schemaVersion: "gamecult.vili.operator_state.v1", + name: (value) => value?.service || "vili", + schema: objectSchema, + members: [ + { slot: 0, memberName: "service", typeName: "string", isName: true }, + { slot: 1, memberName: "providerId", typeName: "string" }, + { slot: 2, memberName: "updatedAt", typeName: "string" }, + { slot: 3, memberName: "backend", typeName: "object" }, + ], +}); +const commandBoundaryDefinition = defineDocumentType({ + type: "gamecult.vili.command_boundary", + schemaName: "gamecult.vili.command_boundary", + schemaId: "gamecult.vili.command_boundary.v1", + schemaVersion: "gamecult.vili.command_boundary.v1", + name: (value) => value?.boundaryId || "vili", + schema: objectSchema, + members: [ + { slot: 0, memberName: "boundaryId", typeName: "string", isName: true }, + { slot: 1, memberName: "owner", typeName: "string" }, + { slot: 2, memberName: "commandAuthority", typeName: "object" }, + { slot: 3, memberName: "forbiddenAuthority", typeName: "string" }, + { slot: 4, memberName: "updatedAt", typeName: "string" }, + ], +}); +const transportProfileDefinition = defineDocumentType({ + type: "gamecult.vili.transport_profile", + schemaName: "gamecult.vili.transport_profile", + schemaId: "gamecult.vili.transport_profile.v1", + schemaVersion: "gamecult.vili.transport_profile.v1", + name: (value) => value?.profileId || "vili", + schema: objectSchema, + members: [ + { slot: 0, memberName: "profileId", typeName: "string", isName: true }, + { slot: 1, memberName: "daemonId", typeName: "string" }, + { slot: 2, memberName: "targetTransport", typeName: "string" }, + { slot: 3, memberName: "state", typeName: "string" }, + { slot: 4, memberName: "updatedAt", typeName: "string" }, + ], +}); +const viliDocumentRegistry = defineDocumentRegistry( + operatorStateDefinition, + commandBoundaryDefinition, + transportProfileDefinition, +); + +const MAX_THREADS = 16; +const CLIP_NAME = /^[A-Za-z0-9_-]{1,64}$/; + +// The state root's one identity: symlinks resolved as far as the path exists. The hand-run lock's name, reclaim's +// marker and the worker's environment all derive from args.stateRoot, so two spellings of one root are one root. +function canonicalPath(target) { + try { + return realpathSync(target); + } catch { + const parent = path.dirname(target); + return parent === target ? target : path.join(canonicalPath(parent), path.basename(target)); + } +} + +function parseArgs(argv) { + const env = process.env; + const args = { + host: env.VILI_DAEMON_HOST || "127.0.0.1", + port: Number(env.VILI_DAEMON_PORT || 8824), + stateRoot: path.resolve(env.VILI_STATE_ROOT || path.join(projectRoot, ".vili")), + python: env.VILI_PYTHON || "python3", + workerEntry: path.resolve(env.VILI_WORKER_ENTRY || path.join(__dirname, "kimodo-resident-worker.py")), + modelsRoot: env.VILI_MODELS_ROOT || "", + threads: env.VILI_THREADS || "8", + kimodoCommit: env.VILI_KIMODO_COMMIT || "", + pythonEnv: env.VILI_PYTHON_ENV || "", + blender: env.VILI_BLENDER || "", + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--host") args.host = argv[++index]; + else if (arg === "--port") args.port = Number(argv[++index]); + else if (arg === "--state-root") args.stateRoot = path.resolve(argv[++index]); + else if (arg === "--python") args.python = argv[++index]; + else if (arg === "--worker-entry") args.workerEntry = path.resolve(argv[++index]); + else if (arg === "--models-root") args.modelsRoot = argv[++index]; + else if (arg === "--threads") args.threads = argv[++index]; + else if (arg === "--kimodo-commit") args.kimodoCommit = argv[++index]; + else if (arg === "--python-env") args.pythonEnv = argv[++index]; + else if (arg === "--blender") args.blender = argv[++index]; + } + if (!/^[0-9]{1,2}$/.test(String(args.threads)) || Number(args.threads) < 1 || Number(args.threads) > MAX_THREADS) { + console.error(`threads must be an integer from 1 to ${MAX_THREADS}`); + process.exit(2); + } + args.threads = Number(args.threads); + args.stateRoot = canonicalPath(args.stateRoot); + if (args.modelsRoot) args.modelsRoot = path.resolve(args.modelsRoot); + if (args.blender) args.blender = path.resolve(args.blender); + return args; +} + +const args = parseArgs(process.argv.slice(2)); +const startedAt = new Date().toISOString(); + +// The rest pose Kimodo ships, in the worker interpreter's own tree (/lib/python3.12/site-packages), once. +const restBvh = path.isAbsolute(args.python) + ? path.join(path.dirname(path.dirname(args.python)), "lib", "python3.12", "site-packages", "kimodo", "assets", "skeletons", "somaskel77", "somaskel77_standard_tpose.bvh") + : null; + +// The worker child. Display state only: nothing reads `worker` to decide anything but ensureWorker. +const worker = { state: "stopped", proc: null, pid: null, port: null, servedModels: [], lastError: null }; +let starting = null; + +// Blender, probed once after the lease. Display and admission state: it decides only whether an fbx request is +// refused early. `version` is the first line of `blender --version`, or null (unconfigured, or the probe failed). +const blender = { version: null }; +const blenderReady = () => blender.version !== null; + +function firstDefined(...values) { + return values.find((value) => value !== undefined && value !== null); +} + +function boolValue(value) { + return value === true || String(value).toLowerCase() === "true" || String(value) === "1"; +} + +function json(response, statusCode, body) { + const payload = JSON.stringify(body, null, 2); + response.writeHead(statusCode, { + "content-type": "application/json; charset=utf-8", + "cache-control": "no-store", + }); + response.end(payload); +} + +function text(response, statusCode, body) { + response.writeHead(statusCode, { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + }); + response.end(body); +} + +// A browser page can POST to localhost; a browser cannot send application/json cross-site without a preflight, +// and no route answers a preflight. So a POST that carries an Origin, or is not JSON, never reaches the body. +function refuseCrossSite(request, response) { + const contentType = String(request.headers["content-type"] || "").split(";")[0].trim().toLowerCase(); + if (request.headers.origin !== undefined || contentType !== "application/json") { + json(response, 403, { ok: false, code: "cross-site" }); + return true; + } + return false; +} + +function backendStatus() { + return { + state: worker.state, + pid: worker.pid, + port: worker.port, + threads: args.threads, + pythonEnv: args.pythonEnv || null, + kimodoCommit: args.kimodoCommit || null, + modelsRoot: args.modelsRoot || null, + lastError: worker.lastError, + blender: blender.version ?? (args.blender ? "unavailable" : "unconfigured"), + }; +} + +function operatorState() { + return { + schema: "gamecult.vili.operator_state.v0", + service: "vili", + providerId, + host: os.hostname(), + startedAt, + updatedAt: new Date().toISOString(), + backend: backendStatus(), + authority: { + owns: [ + "animation job intake", + "worker process lifecycle", + "motion artifact references", + "clip export to FBX", + ], + doesNotOwn: [ + "Persona speech generation", + "Weksa interlingua lowering", + "retargeting and playback", + "Odin discovery truth", + ], + }, + routes: { + health: "/health", + operatorState: "/operator-state", + generate: "/motion/generate", + artifacts: "/motion/artifacts/{jobId}/{name}", + restFbx: "/motion/rest-fbx", + }, + }; +} + +function commandBoundary() { + return { + schema: "gamecult.vili.command_boundary.v1", + boundaryId: "vili", + daemonId: "vili", + owner: "Vili animation runtime", + updatedAt: new Date().toISOString(), + commandAuthority: { + intake: { + method: "POST", + path: "/motion/generate", + owns: [ + "animation job identity", + "request persistence", + "worker process lifecycle", + "timeout handling", + "motion artifact references", + "clip export", + ], + }, + restFbx: { + method: "POST", + path: "/motion/rest-fbx", + owns: ["the rest-pose FBX job"], + }, + artifacts: { + method: "GET", + path: "/motion/artifacts/{jobId}/{name}", + owns: ["read-only artifact bytes"], + doesNotOwn: ["any path outside the job's artifact directory"], + }, + liveness: { + method: "GET", + path: "/health", + owns: ["compatibility liveness witness"], + doesNotOwn: [], + }, + }, + forbiddenAuthority: + "HTTP health and the operator-state projection are compatibility lowerings; they do not own daemon health or provider truth when CultCache/CultNet records exist.", + }; +} + +function transportProfile() { + return { + schema: "gamecult.vili.transport_profile.v1", + profileId: "vili", + daemonId: "vili", + targetTransport: "cultnet.transport.rudp.v0", + currentTransport: "CultCache local state + compatibility HTTP lowerings", + state: "local-cultcache-only", + operatorStateSchema: "gamecult.vili.operator_state.v1", + commandBoundarySchema: "gamecult.vili.command_boundary.v1", + compatibilityMechanisms: ["/health", "/operator-state"], + cutLine: + "Vili owns typed operator, command, and transport state. HTTP projections are lowerings until Odin consumes Vili records over CultNet/RUDP.", + updatedAt: new Date().toISOString(), + }; +} + +async function persistSurfaces() { + guard(); + await mkdir(args.stateRoot, { recursive: true }); + const cache = CultCache.builder() + .withRegistry(viliDocumentRegistry) + .withGenericStore(new SingleFileMessagePackBackingStore(path.join(args.stateRoot, viliStoreFileName))) + .build(); + guard(); + await cache.put(operatorStateDefinition, "vili", operatorState()); + guard(); + await cache.put(commandBoundaryDefinition, "vili", commandBoundary()); + guard(); + await cache.put(transportProfileDefinition, "vili", transportProfile()); +} + +// The whole child environment, in one place. Nothing else from process.env reaches the worker. +function childEnv() { + const threads = String(args.threads); + const env = { + HF_HOME: args.modelsRoot, + HF_HUB_OFFLINE: "1", + TRANSFORMERS_OFFLINE: "1", + VILI_EMBEDDINGS_DIR: path.join(args.stateRoot, "embeddings"), + VILI_THREADS: threads, + OMP_NUM_THREADS: threads, + MKL_NUM_THREADS: threads, + PYTHONPATH: path.join(projectRoot, "vendor", "CultLib", "packages", "cultcache-py", "src"), + PYTHONDONTWRITEBYTECODE: "1", + PYTHONUNBUFFERED: "1", + VILI_KIMODO_COMMIT: args.kimodoCommit, + VILI_PYTHON_ENV: args.pythonEnv, + }; + for (const name of ["PATH", "HOME", "LANG"]) { + if (process.env[name] !== undefined) env[name] = process.env[name]; + } + return env; +} + +// The whole Blender environment, in one place: nothing else from process.env reaches it. The libraries the sealed +// release carries sit in blender/lib; PYTHONHASHSEED=0 is half of byte-reproducible export (scripts/bvh-to-fbx.py). +function exportEnv() { + const env = { + HOME: "/nonexistent", + PYTHONHASHSEED: "0", + PYTHONNOUSERSITE: "1", + LD_LIBRARY_PATH: path.join(path.dirname(args.blender), "lib"), + }; + for (const name of ["PATH", "LANG"]) { + if (process.env[name] !== undefined) env[name] = process.env[name]; + } + return env; +} + +function workerUnconfigured() { + return !args.modelsRoot || !args.kimodoCommit || !args.pythonEnv; +} + +function waitForExit(proc, timeoutMs) { + return new Promise((resolve) => { + if (proc.exitCode !== null || proc.signalCode !== null) return resolve(true); + const timer = setTimeout(() => resolve(false), timeoutMs); + proc.once("exit", () => { + clearTimeout(timer); + resolve(true); + }); + }); +} + +// The worker leads its own process group (its pid is the group id), so the encoder child it spawns is in the group +// too. The daemon owns the whole group: it is killed when the worker exits for any reason and on a requested stop. +// A crashed hand-run daemon's group is reclaimed by the next hand-run daemon (handRunRuntime). Under Idunn the +// unit's cgroup is the same ownership from outside the process. +function killGroup(pgid, signal) { + try { + process.kill(-pgid, signal); + } catch { + // no such group + } +} + +// The hand-run way of owning a state root (tests, development): a Unix socket lock, the worker's pid file and the +// /proc reclaim. Idunn's runtime (scripts/idunn-entry.mjs) replaces all three with its process write lease. +export function handRunRuntime(stateRoot) { + const pidFile = path.join(stateRoot, "worker.pid"); + let lock = null; + + // The lock is a listening socket in Linux's abstract namespace, named by the state root, which is canonical (its real path, see canonicalPath). A bind is + // atomic and exclusive and the kernel drops it with its owner, so there is no lock file, no stale lock to probe + // or take over, and no takeover to race: a bind either succeeds (this daemon owns the root) or it does not. + const listen = () => new Promise((resolve, reject) => { + const name = createHash("sha256").update(stateRoot).digest("hex"); + const server = net.createServer((socket) => socket.destroy()); + server.once("error", reject); + server.listen(`\0vili-state-root:${name}`, () => { + server.removeListener("error", reject); + lock = server; + resolve(); + }); + }); + + // A worker left by a crashed daemon: the pid file names its group, and the group is ours only if some member + // still carries this state root's embeddings directory in its environment (the pid may have been reused). + function reclaimStrayWorker() { + let pgid; + try { + pgid = Number(readFileSync(pidFile, "utf8").trim()); + } catch { + return; + } + const marker = `VILI_EMBEDDINGS_DIR=${path.join(stateRoot, "embeddings")}`; + let ours = false; + try { + for (const entry of readdirSync("/proc")) { + if (!/^\d+$/.test(entry)) continue; + try { + const stat = readFileSync(`/proc/${entry}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + if (Number(fields[2]) !== pgid) continue; + if (readFileSync(`/proc/${entry}/environ`, "utf8").split("\0").includes(marker)) ours = true; + } catch { + // the process exited while we looked + } + } + } catch { + // no /proc: nothing can be verified, so nothing is killed + } + if (ours) killGroup(pgid, "SIGKILL"); + try { + unlinkSync(pidFile); + } catch { + // already gone + } + } + + return { + async acquire() { + await mkdir(stateRoot, { recursive: true }); + try { + await listen(); + } catch (error) { + if (error?.code !== "EADDRINUSE") throw error; + console.error("the state root is held by a running daemon"); + process.exit(2); + } + reclaimStrayWorker(); // synchronous: no request is served before a predecessor's worker is gone + }, + held: () => lock !== null, + refusal: () => "write-lease-pending", + watch() {}, + recordWorker(pid) { + writeFileSync(pidFile, `${pid}\n`); + }, + forgetWorker(pid) { + try { + if (readFileSync(pidFile, "utf8").trim() === String(pid)) unlinkSync(pidFile); + } catch { + // no pid file, or another worker's + } + }, + release() { + if (lock) lock.close(); + lock = null; + }, + }; +} + +// A requested stop gives a process group its grace (SIGTERM, then SIGKILL); a lost writer gets SIGKILL at once. The +// worker and the exports are stopped by this one path. +async function stopGroups(procs, signals) { + for (const signal of signals) { + for (const proc of procs) if (proc.pid !== undefined) killGroup(proc.pid, signal); + const gone = await Promise.all(procs.map((proc) => waitForExit(proc, signal === "SIGTERM" ? 10000 : 5000))); + if (gone.every(Boolean)) break; + } +} + +const STOP_SIGNALS = ["SIGTERM", "SIGKILL"]; + +async function stopWorker(signals = STOP_SIGNALS) { + const proc = worker.proc; + if (!proc || proc.pid === undefined) return; + worker.state = "stopped"; + await stopGroups([proc], signals); +} + +// Blender's process groups in flight. A finished export removes itself; a model switch (ensureWorker) leaves them alone. +const exportGroups = new Set(); +const stopExports = (signals = STOP_SIGNALS) => stopGroups([...exportGroups], signals); + +function isRegularFile(file) { + try { + return lstatSync(file).isFile(); + } catch { + return false; + } +} + +// A regular file (no symlink, in the file or any directory above it) strictly inside `directory`, which is canonical. +function regularFileInside(file, directory) { + const relative = path.relative(directory, file); + if (!relative || path.isAbsolute(relative) || relative.split(path.sep)[0] === "..") return false; + try { + return lstatSync(file).isFile() && realpathSync(file) === file; + } catch { + return false; + } +} + +const exportScript = path.join(__dirname, "bvh-to-fbx.py"); +// The state slot "work" (deployment/idunn/recipe.toml): one private directory per export, removed by the export itself +// and, if the daemon died mid-export, swept by the next daemon that holds the state root (sweepWork). +const WORK_DIR = "work"; +const EXPORT_PREFIX = "export-"; +const EXPORT_NAME = new RegExp(`^${EXPORT_PREFIX}[A-Za-z0-9]{6}$`); +const EXPORT_SETTINGS = "standard-tpose soma77 metres hashseed0 time1970"; +const EXPORT_TIMEOUT_MS = 120000; +let exportScriptDigest = null; +const scriptDigest = () => (exportScriptDigest ??= `sha256:${createHash("sha256").update(readFileSync(exportScript)).digest("hex")}`); + +// Runs the sealed Blender in its own process group, niced, under the write gate. Both streams go to +// logs/export.log line by line through mayWrite(), and only there; the version probe is not logged, so it writes +// nothing to the state root. Resolves { code, signal, first, result }: the exit, the first stdout line and the last +// `VILI-FBX ` line's payload. It rejects only with LeaseLost. +async function runBlender(argv, timeoutMs, logged = true) { + guard(); + let sink = null; + if (logged) { + await mkdir(path.join(args.stateRoot, "logs"), { recursive: true }); + guard(); // synchronous with the log open and the spawn below: no await between them + sink = openSync(path.join(args.stateRoot, "logs", "export.log"), "a"); + } + const proc = spawn(args.blender, argv, { + cwd: __dirname, + env: exportEnv(), + stdio: ["ignore", "pipe", "pipe"], + shell: false, + detached: true, + }); + exportGroups.add(proc); + const seen = { first: null, result: null }; + let sinkOpen = sink !== null; + for (const [stream, isStdout] of [[proc.stdout, true], [proc.stderr, false]]) { + if (!stream) continue; + readline.createInterface({ input: stream }).on("line", (line) => { + if (isStdout) { + seen.first ??= line; + if (line.startsWith("VILI-FBX ")) seen.result = line.slice("VILI-FBX ".length); + } + if (sinkOpen && mayWrite()) writeSync(sink, `${line}\n`); + }); + } + const finished = new Promise((resolve) => { + proc.once("error", () => resolve({ code: null, signal: null })); + proc.once("close", (code, signal) => resolve({ code, signal })); + }); + proc.once("exit", () => { + exportGroups.delete(proc); + if (proc.pid !== undefined) killGroup(proc.pid, "SIGKILL"); // whatever Blender left running + }); + try { + os.setPriority(proc.pid, 10); + } catch { + if (proc.pid !== undefined) killGroup(proc.pid, "SIGKILL"); + } + const timer = setTimeout(() => { + if (proc.pid !== undefined) killGroup(proc.pid, "SIGKILL"); + }, timeoutMs); + const exit = await finished; + clearTimeout(timer); + exportGroups.delete(proc); + if (sink !== null) closeSync(sink); + sinkOpen = false; + return { ...exit, ...seen }; +} + +// Blender answers once at start: `blender --version`, 15 s. Ready only when it exits 0 and names itself. +async function probeBlender() { + if (!args.blender) return; + const run = await runBlender(["-b", "--factory-startup", "--version"], 15000, false); + guard(); + blender.version = run.code === 0 && run.first?.startsWith("Blender ") ? run.first : null; +} + +// The bytes of a regular file strictly inside `directory`, or null. The file is opened without following a symlink +// and without blocking (a FIFO or a device would otherwise hold the open, and a filesystem thread, indefinitely), and +// the open file is the one checked, so a path swapped after the check cannot redirect the read. +async function readInside(file, directory) { + let handle = null; + try { + handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const opened = await handle.stat(); + if (!opened.isFile() || !regularFileInside(file, directory)) return null; + const named = lstatSync(file); + if (named.dev !== opened.dev || named.ino !== opened.ino) return null; + return await handle.readFile(); + } catch { + return null; + } finally { + await handle?.close(); + } +} + +// One BVH to one FBX beside it. Blender never sees a path in the job's artifact directory: it reads a private copy of +// the BVH and writes a private FBX in a directory made for this export alone, and the FBX is then created beside the +// BVH exclusively, so nothing planted at either path is read, written through or replaced. The FBX counts only if +// Blender exits 0, prints its result line and its file exists. A failure names the exit or the signal and never +// Blender's output. A window {first, last} (BVH frame indices, last inclusive) is handed to Blender as the frame +// range to export, in Blender's frames, which count from 1; the daemon never chooses one. +async function exportClip(bvh, fbx, clipName, timeoutMs, artifactDir, window = null) { + const input = await readInside(bvh, artifactDir); + if (input === null) return { error: "fbx-export-refused" }; + guard(); + const work = await mkdtemp(path.join(args.stateRoot, WORK_DIR, EXPORT_PREFIX)); + try { + const privateBvh = path.join(work, "in.bvh"); + const privateFbx = path.join(work, "out.fbx"); + guard(); + await writeFile(privateBvh, input, { flag: "wx" }); + const run = await runBlender([ + "-b", "--factory-startup", "--python-use-system-env", "--python-exit-code", "1", + "--python", exportScript, "--", privateBvh, privateFbx, clipName, + ...(window ? ["--frames", String(window.first + 1), String(window.last + 1)] : []), + ], timeoutMs); + let info = null; + if (run.code === 0 && run.result !== null) { + try { + const parsed = JSON.parse(run.result); + if (Number.isInteger(parsed.bones) && Array.isArray(parsed.frames) && parsed.frames.length === 2 + && parsed.frames.every(Number.isInteger) && Number.isFinite(parsed.fps)) info = parsed; + } catch { + // not the result line + } + } + const made = info ? await readInside(privateFbx, work) : null; + if (made !== null) { + guard(); + try { + await writeFile(fbx, made, { flag: "wx" }); + return { clip: { file: fbx, clipName, bones: info.bones, frames: info.frames, fps: info.fps } }; + } catch (error) { + if (error instanceof LeaseLost) throw error; + return { error: "fbx-export-refused" }; + } + } + const how = run.code === 0 ? "no result" : run.code !== null ? `exit ${run.code}` : `signal ${run.signal ?? "none"}`; + return { error: `fbx-export-failed (${how})` }; + } finally { + if (mayWrite()) await rm(work, { recursive: true, force: true }); + } +} + +// The export directories a killed daemon left: only directories named as mkdtemp names them, only under work/. Called +// once, after the runtime holds the state root, so a live daemon's export is never swept. +async function sweepWork() { + const root = path.join(args.stateRoot, WORK_DIR); + for (const entry of await readdir(root, { withFileTypes: true })) { + if (!entry.isDirectory() || !EXPORT_NAME.test(entry.name)) continue; + guard(); + await rm(path.join(root, entry.name), { recursive: true, force: true }); + } +} + +// Every BVH the worker listed becomes an FBX, or the job fails. A listed BVH must be a regular file inside the job's +// artifact directory, and that is decided when its export begins (exportClip), never earlier. Sample N of several is +// _NN. With loops (the worker's answer, an array when the request asked for a loop, else undefined), sample k +// is exported over loops[k], and a job whose answer has no usable window for a sample fails. +async function exportListed(listed, artifactDir, clipName, timeoutMs, loops = undefined) { + const bvhs = []; + for (const file of Array.isArray(listed) ? listed : []) { + if (typeof file !== "string" || !file.endsWith(".bvh")) continue; + bvhs.push(path.resolve(artifactDir, file)); + } + if (bvhs.length === 0) return { clips: [], error: "fbx-export-failed (no bvh)" }; + bvhs.sort(); + const clips = []; + for (const [index, bvh] of bvhs.entries()) { + const name = bvhs.length === 1 ? clipName : `${clipName}_${String(index).padStart(2, "0")}`; + const window = loops === undefined ? null : loops[index]; + if (window !== null && !(Number.isInteger(window?.first) && Number.isInteger(window?.last) && window.first >= 0 && window.last > window.first)) { + return { clips, error: "fbx-export-failed (no loop window)" }; + } + const done = await exportClip(bvh, `${bvh.slice(0, -".bvh".length)}.fbx`, name, timeoutMs, artifactDir, window); + if (done.error) return { clips, error: done.error }; + clips.push({ ...done.clip, file: path.relative(artifactDir, done.clip.file).split(path.sep).join("/") }); + } + return { clips, error: null }; +} + +// Every regular file under the artifact directory, named relative to it, sorted. +async function artifactFiles(directory) { + const files = []; + const walk = async (relative) => { + for (const entry of readdirSync(path.join(directory, relative), { withFileTypes: true })) { + const name = path.join(relative, entry.name); + if (entry.isDirectory()) await walk(name); + else if (entry.isFile()) { + const bytes = await readFile(path.join(directory, name)); + files.push({ name: name.split(path.sep).join("/"), bytes: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }); + } + } + }; + await walk(""); + return files.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); +} + +// The one job record, for a generate job and a rest job alike: what the job was, which files it left and, for an +// export, the exact Blender and script that made them. +async function writeJobRecord(job, exported) { + guard(); + const record = { schema: "gamecult.vili.motion_job.v1", ...job, files: await artifactFiles(job.artifactDir) }; + if (exported) record.export = { blender: blender.version, script: scriptDigest(), settings: EXPORT_SETTINGS, clips: exported.clips }; + guard(); + await writeFile(path.join(args.stateRoot, "jobs", `${job.jobId}.json`), JSON.stringify(record, null, 2)); + return record; +} + +async function startWorker(model, timeoutMs) { + guard(); + await mkdir(path.join(args.stateRoot, "logs"), { recursive: true }); + guard(); + await mkdir(path.join(args.stateRoot, "embeddings"), { recursive: true }); + guard(); // synchronous with the log open and the spawn below: no await between them + const logFd = openSync(path.join(args.stateRoot, "logs", "worker.log"), "a"); + const argv = [args.workerEntry, "--host", "127.0.0.1", "--port", "0"]; + if (model) argv.push("--model", model); + const proc = spawn(args.python, argv, { + cwd: __dirname, + env: childEnv(), + stdio: ["ignore", "pipe", logFd], + shell: false, + detached: true, + }); + closeSync(logFd); + if (proc.pid !== undefined) { + runtime.recordWorker(proc.pid); + proc.once("exit", () => { + killGroup(proc.pid, "SIGKILL"); // whatever the worker left running, an encoder child included + runtime.forgetWorker(proc.pid); + }); + } + Object.assign(worker, { state: "starting", proc, pid: proc.pid ?? null, port: null, servedModels: [], lastError: null }); + const exited = new Promise((resolve) => { + proc.once("error", () => resolve("the worker could not be spawned")); + proc.once("exit", (code, signal) => resolve(`the worker exited (${signal ?? `code ${code}`})`)); + }); + exited.then((reason) => { + if (worker.proc !== proc) return; + Object.assign(worker, { state: "stopped", proc: null, pid: null, port: null, lastError: worker.state === "stopped" ? worker.lastError : reason }); + }); + + const fail = async (reason) => { + worker.lastError = reason; + await stopWorker(); + Object.assign(worker, { state: "failed", proc: null, pid: null, port: null, lastError: reason }); + throw new Error(reason); + }; + + if (proc.pid === undefined) await fail(await exited); + try { + os.setPriority(proc.pid, 10); + } catch { + await fail("the worker priority could not be lowered"); + } + + // Stdout lines are the worker's log, until the one that says it is ready. + const deadline = Date.now() + timeoutMs; + let readyTimer; + const ready = await Promise.race([ + new Promise((resolve) => { + const lines = readline.createInterface({ input: proc.stdout }); + const sink = openSync(path.join(args.stateRoot, "logs", "worker.log"), "a"); + lines.on("line", (line) => { + try { + const event = JSON.parse(line); + if (event?.event === "kimodo-worker-ready" && Number.isInteger(event.port)) return resolve(event); + } catch { + // not JSON: an ordinary log line + } + if (mayWrite()) writeSync(sink, `${line}\n`); + }); + lines.on("close", () => closeSync(sink)); + }), + exited.then((reason) => ({ failed: reason })), + new Promise((resolve) => { readyTimer = setTimeout(() => resolve({ failed: "the worker was not ready in time" }), timeoutMs); }), + ]); + clearTimeout(readyTimer); + if (ready.failed) await fail(ready.failed); + worker.port = ready.port; + + for (;;) { + try { + const health = await fetch(`http://127.0.0.1:${worker.port}/health`, { signal: AbortSignal.timeout(10000) }); + const body = await health.json(); + if (health.ok && body?.ok) { + worker.servedModels = [body.model, body.displayModel].filter(Boolean); + worker.state = "ready"; + return; + } + } catch { + // not answering yet + } + if (worker.proc !== proc) throw new Error(worker.lastError || "the worker exited"); + if (Date.now() > deadline) await fail("the worker did not become healthy in time"); + await new Promise((resolve) => setTimeout(resolve, 500)); + } +} + +async function ensureWorker({ model = null, timeoutMs = 1000 * 60 * 8 } = {}) { + if (starting) await starting; + if (worker.state === "ready" && model && !worker.servedModels.includes(model)) await stopWorker(); + if (worker.state === "ready") return; + if (!starting) starting = startWorker(model, timeoutMs).finally(() => { starting = null; }); + await starting; +} + +async function readRequestBody(request) { + const chunks = []; + for await (const chunk of request) chunks.push(chunk); + const raw = Buffer.concat(chunks).toString("utf8"); + return raw ? JSON.parse(raw) : {}; +} + +async function underLease(response, run) { + try { + await run(); + } catch (error) { + if (!(error instanceof LeaseLost)) throw error; + json(response, 503, { ok: false, code: stopping ? "stopping" : runtime.refusal() }); + } +} + +// A seed is an integer Kimodo can take: the worker seeds with int(seed), so the daemon records exactly what it sends. +const SEED_LIMIT = 2 ** 31; + +// The one source of job ids, for a generate job and a rest job alike: the clock for order, random bits for +// uniqueness, and an exclusive mkdir that decides, so two jobs never share a directory, a record or a file. +async function newJobDir(prefix) { + guard(); + await mkdir(path.join(args.stateRoot, "artifacts"), { recursive: true }); + for (let attempt = 0; attempt < 8; attempt += 1) { + const jobId = `${prefix}-${Date.now()}-${randomBytes(4).toString("hex")}`; + const artifactDir = path.join(args.stateRoot, "artifacts", jobId); + guard(); + try { + await mkdir(artifactDir); + return { jobId, artifactDir }; + } catch (error) { + if (error.code !== "EEXIST") throw error; + } + } + throw new Error("no free job id"); +} + +async function generate(request, response) { + guard(); + if (workerUnconfigured()) { + json(response, 503, { ok: false, code: "worker-unconfigured" }); + return; + } + let body; + try { + body = await readRequestBody(request); + } catch { + json(response, 400, { ok: false, error: "Invalid JSON." }); + return; + } + + const prompt = String(body.prompt || body.utterance || "").trim(); + const hasPromptInput = Boolean(prompt) + || Array.isArray(body.texts) + || typeof body.meta === "object" + || body.inputFolder + || body.input_folder; + if (!hasPromptInput) { + json(response, 400, { ok: false, error: "Missing prompt, utterance, texts, meta, or inputFolder." }); + return; + } + if (Array.isArray(body.texts) && !Array.isArray(body.durations)) { + json(response, 400, { ok: false, error: "Batch text requests require a durations array." }); + return; + } + + // One export plan: fbx implies the BVH and its standard T-pose, which the FBX is made from. + const fbx = boolValue(body.fbx); + const askedTpose = firstDefined(body.bvhStandardTpose, body.bvh_standard_tpose); + const loop = body.loop === true; + if ((body.loop !== undefined && typeof body.loop !== "boolean") || (loop && !fbx) + || (fbx && ((askedTpose !== undefined && !boolValue(askedTpose)) + || (body.clipName !== undefined && !(typeof body.clipName === "string" && CLIP_NAME.test(body.clipName)))))) { + json(response, 400, { ok: false, error: "Invalid fbx request." }); + return; + } + if (fbx && !blenderReady()) { + json(response, 503, { ok: false, code: "blender-unavailable" }); + return; + } + const plan = { + bvh: fbx || boolValue(body.bvh), + bvhStandardTpose: fbx || boolValue(askedTpose), + }; + + // One way to give constraints: the worker reads exactly one path, so two inputs would leave the record naming + // constraints that did not make the clip. + if ([body.constraints, body.constraints_path, body.constraintsPath].filter((given) => given !== undefined).length > 1) { + json(response, 400, { ok: false, error: "Invalid constraints request." }); + return; + } + + const bodyMeta = typeof body.meta === "object" && body.meta ? body.meta : {}; + const askedSeed = firstDefined(body.seed, bodyMeta.seed); + if (askedSeed !== undefined && !(Number.isInteger(askedSeed) && askedSeed >= 0 && askedSeed < SEED_LIMIT)) { + json(response, 400, { ok: false, error: "Invalid seed." }); + return; + } + + const { jobId, artifactDir } = await newJobDir("vili"); + const clipName = fbx ? (body.clipName ?? jobId) : undefined; + const duration = firstDefined(body.durationSeconds, body.duration, 4); + const diffusionSteps = Math.max(1, Number(firstDefined( + body.diffusionSteps, + body.diffusion_steps, + bodyMeta.diffusion_steps, + 20, + ))); + const seed = askedSeed ?? randomInt(0, SEED_LIMIT); + const numSamples = Math.max(1, Number(firstDefined( + body.numSamples, + body.num_samples, + bodyMeta.num_samples, + 1, + ))); + const meta = Object.keys(bodyMeta).length > 0 + ? { ...bodyMeta } + : Array.isArray(body.texts) + ? { texts: body.texts, durations: body.durations } + : { + text: prompt, + duration, + }; + meta.num_samples = Math.max(1, Number(firstDefined(body.numSamples, body.num_samples, meta.num_samples, 1))); + meta.diffusion_steps = diffusionSteps; + meta.seed = seed; + if (body.cfg) meta.cfg = body.cfg; + const constraintsPath = (() => { + if (typeof body.constraints === "string") return body.constraints; + if (body.constraints_path) return body.constraints_path; + if (body.constraintsPath) return body.constraintsPath; + if (body.constraints !== undefined) return path.join(artifactDir, "constraints.json"); + return undefined; + })(); + if (body.constraints !== undefined && typeof body.constraints !== "string") { + guard(); + await writeFile(path.join(artifactDir, "constraints.json"), JSON.stringify(body.constraints, null, 2)); + } + const requestRecord = { + schema: "gamecult.vili.motion_request.v0", + prompt: prompt || undefined, + texts: Array.isArray(body.texts) ? body.texts : undefined, + meta, + model: body.model, + duration, + numSamples, + diffusion_steps: diffusionSteps, + numTransitionFrames: firstDefined(body.numTransitionFrames, body.num_transition_frames), + constraints: body.constraints !== undefined && typeof body.constraints !== "string" ? body.constraints : constraintsPath, + seed, + fbx: fbx || undefined, + loop: loop || undefined, + clipName, + cfg: body.cfg, + cfgType: firstDefined(body.cfgType, body.cfg_type), + cfgWeight: firstDefined(body.cfgWeight, body.cfg_weight), + ...plan, + noPostprocess: boolValue(firstDefined(body.noPostprocess, body.no_postprocess)), + saveExampleDir: boolValue(firstDefined(body.saveExampleDir, body.save_example_dir)), + inputFolder: firstDefined(body.inputFolder, body.input_folder), + }; + guard(); + await writeFile(path.join(artifactDir, "meta.json"), JSON.stringify(meta, null, 2)); + + let result; + try { + await ensureWorker({ + timeoutMs: Number(firstDefined(body.workerTimeoutMs, body.worker_timeout_ms, 1000 * 60 * 8)), + model: body.model || null, + }); + const workerRequest = { + model: body.model, + prompt: prompt || undefined, + texts: Array.isArray(body.texts) ? body.texts : undefined, + meta, + inputFolder: firstDefined(body.inputFolder, body.input_folder), + duration, + diffusionSteps, + numSamples, + numTransitionFrames: firstDefined(body.numTransitionFrames, body.num_transition_frames), + constraints: constraintsPath, + seed, + loop: loop || undefined, + cfg: body.cfg, + cfgType: firstDefined(body.cfgType, body.cfg_type), + cfgWeight: firstDefined(body.cfgWeight, body.cfg_weight), + ...plan, + noPostprocess: boolValue(firstDefined(body.noPostprocess, body.no_postprocess)), + saveExampleDir: boolValue(firstDefined(body.saveExampleDir, body.save_example_dir)), + output: path.join(artifactDir, "motion"), + }; + guard(); + await writeFile(path.join(artifactDir, "request.json"), JSON.stringify(workerRequest, null, 2)); + const answer = await fetch(`http://127.0.0.1:${worker.port}/generate`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(workerRequest), + signal: AbortSignal.timeout(Number(body.timeoutMs || 1000 * 60 * 30)), + }); + const workerBody = await answer.json(); + result = { ok: answer.ok && workerBody?.ok !== false, error: answer.ok ? null : `the worker answered ${answer.status}`, body: workerBody }; + } catch (error) { + if (error instanceof LeaseLost) throw error; + result = { ok: false, error: `the worker request failed (${error?.name || "Error"})`, body: null }; + } + + let exported = null; + if (fbx && result.ok) { + const timeoutMs = Math.min(EXPORT_TIMEOUT_MS, Math.max(1, Number(firstDefined(body.exportTimeoutMs, EXPORT_TIMEOUT_MS)) || EXPORT_TIMEOUT_MS)); + exported = await exportListed(result.body?.files, artifactDir, clipName, timeoutMs, loop ? (Array.isArray(result.body?.loops) ? result.body.loops : []) : undefined); + if (exported.error) result = { ok: false, error: exported.error, body: result.body }; + } + + guard(); // the writer may have been lost while the job ran + const job = await writeJobRecord({ + jobId, + createdAt: new Date().toISOString(), + prompt: prompt || null, + model: body.model || result.body?.model || null, + request: requestRecord, + durationSeconds: duration, + diffusionSteps, + seed, + artifactDir, + ok: result.ok, + error: result.error, + worker: result.body, + }, exported); + json(response, result.ok ? 200 : 502, job); +} + +// The release's own rest pose, exported as the avatar cut's source: a job like any other, made from Kimodo's +// standard-T-pose BVH that the worker interpreter's site-packages carries. +async function restFbx(response) { + guard(); + if (!blenderReady()) { + json(response, 503, { ok: false, code: "blender-unavailable" }); + return; + } + if (!restBvh || !isRegularFile(restBvh)) { + json(response, 503, { ok: false, code: "rest-skeleton-missing" }); + return; + } + const { jobId, artifactDir } = await newJobDir("rest"); + guard(); + await copyFile(restBvh, path.join(artifactDir, "SOMA77_Rest.bvh")); + const exported = await exportListed([path.join(artifactDir, "SOMA77_Rest.bvh")], artifactDir, "SOMA77_Rest", EXPORT_TIMEOUT_MS); + guard(); + const job = await writeJobRecord({ + jobId, + createdAt: new Date().toISOString(), + prompt: null, + model: null, + request: null, + durationSeconds: null, + diffusionSteps: null, + seed: null, + artifactDir, + ok: !exported.error, + error: exported.error, + worker: null, + }, exported); + json(response, job.ok ? 200 : 502, job); +} + +// Reads only: a job's own FBX and BVH files, by exact name, regular files inside the job's artifact directory. +const ARTIFACT_ROUTE = /^\/motion\/artifacts\/((?:vili|rest)-[0-9]{13}-[0-9a-f]{8})\/(motion\.fbx|motion\.bvh|SOMA77_Rest\.fbx|motion\/motion_[0-9]{2}\.(?:fbx|bvh))$/; + +async function serveArtifact(response, jobId, name) { + const directory = path.join(args.stateRoot, "artifacts", jobId); + const file = path.join(directory, name); + let bytes = null; + if (regularFileInside(file, directory)) { + try { + bytes = await readFile(file); + } catch { + // gone between the check and the read + } + } + if (bytes === null) { + text(response, 404, "not found\n"); + return; + } + response.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "no-store" }); + response.end(bytes); +} + +const SNAPSHOT_BODY_MAX = 64 * 1024; + +// Idunn's route challenge: a msgpack POST from the host, answered from local authority. A browser page can send +// it too, so an Origin is refused exactly as on every other POST. Only the Idunn runtime answers; the hand-run +// daemon has no such route. +async function answerRouteChallenge(request, response) { + if (!runtime.answerSnapshot) { + text(response, 404, "not found\n"); + return; + } + if (request.headers.origin !== undefined) { + json(response, 403, { ok: false, code: "cross-site" }); + return; + } + const chunks = []; + let size = 0; + for await (const chunk of request) { + size += chunk.length; + if (size > SNAPSHOT_BODY_MAX) { + json(response, 400, { ok: false, code: "invalid-snapshot-request" }); + return; + } + chunks.push(chunk); + } + let answer; + try { + answer = runtime.answerSnapshot(Buffer.concat(chunks)); + } catch (error) { + if (error?.code === "invalid-snapshot-request") json(response, 400, { ok: false, code: error.code }); + else json(response, 503, { ok: false, code: "runtime-presence-unavailable" }); + return; + } + response.writeHead(200, { "content-type": "application/msgpack", "cache-control": "no-store" }); + response.end(answer); +} + +async function handleRequest(request, response) { + const url = new URL(request.url, `http://${request.headers.host || `${args.host}:${args.port}`}`); + if (url.pathname === "/cultnet/snapshot" && request.method === "POST") { + await answerRouteChallenge(request, response); + return; + } + const artifact = request.method === "GET" ? ARTIFACT_ROUTE.exec(url.pathname) : null; + if (artifact) { + await serveArtifact(response, artifact[1], artifact[2]); + return; + } + if (request.method === "POST" && refuseCrossSite(request, response)) return; + + if (url.pathname === "/" || url.pathname === "/health") { + json(response, 200, { ok: true, service: "vili", providerId, state: operatorState() }); + } else if (url.pathname === "/operator-state") { + json(response, 200, operatorState()); + } else if (url.pathname === "/motion/generate" && request.method === "POST") { + await underLease(response, () => generate(request, response)); + } else if (url.pathname === "/motion/rest-fbx" && request.method === "POST") { + await underLease(response, () => restFbx(response)); + } else { + text(response, 404, "not found\n"); + } +} + +let runtime; +let stopping = false; + +class LeaseLost extends Error {} + +// The one write authority: the runtime holds the state root (the lease, or the lock) and the daemon is not stopping. +// held() is read at the moment of every ask, so a loss is seen by the first gate to ask. +function mayWrite() { + return !stopping && runtime.held(); +} + +// The one write gate. Every write to the state root and every spawn calls it, immediately before it happens (not once +// per request), so a request admitted before the lease went, or before SIGTERM, still stops at its next write. +function guard() { + if (!mayWrite()) throw new LeaseLost("the writer was lost"); +} + +// How often the lease is asked while nothing else asks, and so how long a worker can outlive a loss (plus the +// SIGKILL): a generate can await its worker for 30 minutes. The service documents are rewritten every PERSIST_EVERY asks. +const WATCH_MS = 250; +const PERSIST_EVERY = 20; + +// The one daemon. `rt` owns the state root: the hand-run runtime (socket lock) or Idunn's (process write lease). +// The server listens first, so Idunn's challenge can be answered while the lease is still pending; nothing is +// written to the state root until the runtime holds it. +export async function serve(rt) { + runtime = rt; + const server = http.createServer((request, response) => { + handleRequest(request, response).catch((error) => { + // The error's message can carry the request; name only its class. + console.error(`request failed: ${error?.name || "Error"}`); + if (!response.headersSent) json(response, 500, { ok: false, code: "internal" }); + else response.end(); + }); + }); + await new Promise((resolve) => server.listen(args.port, args.host, resolve)); + console.log(`Vili listening on http://${args.host}:${server.address().port}`); + + let tick = null; + // From the signal on nothing is accepted, written or spawned (guard() refuses what is already in flight); then the + // worker tree is stopped, and only then does the process end. + const shutdown = async () => { + if (stopping) return; + stopping = true; + server.close(); + server.closeIdleConnections(); + clearInterval(tick); + await Promise.all([stopWorker(), stopExports()]); + runtime.release(); + process.exit(0); + }; + process.on("SIGTERM", shutdown); + process.on("SIGINT", shutdown); + + await runtime.acquire(); + let lost = false; + const onLost = () => { + if (lost) return; + lost = true; + clearInterval(tick); + Promise.all([stopWorker(["SIGKILL"]), stopExports(["SIGKILL"])]).finally(() => runtime.reportHealth?.("degraded")); + }; + runtime.watch(onLost); // held() reports a loss through this the moment any gate sees it + try { + guard(); + await mkdir(path.join(args.stateRoot, "jobs"), { recursive: true }); + await mkdir(path.join(args.stateRoot, WORK_DIR), { recursive: true }); + await sweepWork(); + await persistSurfaces(); + await probeBlender(); + } catch (error) { + if (error instanceof LeaseLost) return; // lost while opening the state root: onLost has run, and nothing is active + throw error; + } + let asks = 0; + tick = setInterval(() => { + runtime.watch(onLost); + if (lost || ++asks % PERSIST_EVERY !== 0) return; + persistSurfaces().catch((error) => { + if (!(error instanceof LeaseLost)) console.error(`persist failed: ${error?.name || "Error"}`); + }); + }, WATCH_MS); + if (lost) return; + runtime.reportHealth?.("active"); + console.log("Vili holds the state root"); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + await serve(handRunRuntime(args.stateRoot)); +}