From 3843a772f8bba32986bf6051bc7a5dfdf2a5b38e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 13:16:50 +0200 Subject: [PATCH] Tests: worker failure paths, env twins, group kill, job record, store refresh Co-Authored-By: Claude Opus 5.5 --- scripts/fixtures/stub-worker.mjs | 129 ++++++---- scripts/test-vili-daemon.mjs | 399 +++++++++++++++++++++++++++---- 2 files changed, 439 insertions(+), 89 deletions(-) diff --git a/scripts/fixtures/stub-worker.mjs b/scripts/fixtures/stub-worker.mjs index 24d2c9a..9725218 100644 --- a/scripts/fixtures/stub-worker.mjs +++ b/scripts/fixtures/stub-worker.mjs @@ -1,57 +1,96 @@ // Stands in for kimodo-resident-worker.py in the daemon's tests. It binds port 0, prints the ready line the -// daemon waits for, and answers /health and /generate with what the daemon gave it. +// daemon waits for, and answers /health and /generate with what the daemon gave it. The --model value picks a +// misbehaviour: crash-before-ready, hang-before-ready, sick, slow-health, stubborn, fail-generate, +// refuse-generate, drop-generate, slow-generate. Any other value is just the model the worker serves. import http from "node:http"; import os from "node:os"; import path from "node:path"; +import { spawn } from "node:child_process"; import { appendFileSync } from "node:fs"; const argv = process.argv.slice(2); -const flag = (name) => argv[argv.indexOf(name) + 1]; -const host = flag("--host"); +const flag = (name) => { + const index = argv.indexOf(name); + return index < 0 ? undefined : argv[index + 1]; +}; +const model = flag("--model"); +const spawnLog = path.join(process.env.VILI_EMBEDDINGS_DIR, "stub-spawns.log"); let requests = 0; +let healthCalls = 0; -appendFileSync(path.join(process.env.VILI_EMBEDDINGS_DIR, "stub-spawns.log"), `${process.pid}\n`); +appendFileSync(spawnLog, `${process.pid}\n`); +console.log("stub-stdout-line"); +console.error("stub-stderr-line"); +// decoys: neither is the ready line +console.log(JSON.stringify({ event: "progress", port: 1 })); +console.log(JSON.stringify({ event: "kimodo-worker-ready", port: "not-a-port" })); -const server = http.createServer((request, response) => { - const chunks = []; - request.on("data", (chunk) => chunks.push(chunk)); - request.on("end", () => { - const send = (body) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(JSON.stringify(body)); - }; - if (request.method === "GET" && request.url === "/health") { - send({ ok: true, model: "stub-model", displayModel: "Stub Model", pid: process.pid, requests }); - } else if (request.method === "POST" && request.url === "/generate") { - requests += 1; - const payload = JSON.parse(Buffer.concat(chunks).toString("utf8")); - send({ - ok: true, - pid: process.pid, - requests, - output: payload.output, - priority: os.getPriority(), - env: { - OMP_NUM_THREADS: process.env.OMP_NUM_THREADS, - MKL_NUM_THREADS: process.env.MKL_NUM_THREADS, - VILI_THREADS: process.env.VILI_THREADS, - HF_HUB_OFFLINE: process.env.HF_HUB_OFFLINE, - TRANSFORMERS_OFFLINE: process.env.TRANSFORMERS_OFFLINE, - HF_HOME: process.env.HF_HOME, - PYTHONPATH: process.env.PYTHONPATH, - VILI_KIMODO_COMMIT: process.env.VILI_KIMODO_COMMIT, - VILI_PYTHON_ENV: process.env.VILI_PYTHON_ENV, - LEAK_CHECK: process.env.LEAK_CHECK, - }, - argv, - }); - } else { - response.writeHead(404); - response.end(); - } +if (model === "crash-before-ready") process.exit(3); + +if (model === "stubborn") { + process.on("SIGTERM", () => {}); + const grandchild = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" }); + appendFileSync(spawnLog, `grandchild ${grandchild.pid}\n`); +} + +if (model === "hang-before-ready") { + setInterval(() => {}, 1000); +} else { + const server = http.createServer((request, response) => { + const chunks = []; + request.on("data", (chunk) => chunks.push(chunk)); + request.on("end", () => { + const send = (status, body) => { + response.writeHead(status, { "content-type": "application/json" }); + response.end(JSON.stringify(body)); + }; + if (request.method === "GET" && request.url === "/health") { + healthCalls += 1; + if (model === "sick" || (model === "slow-health" && healthCalls <= 2)) return send(200, { ok: false }); + send(200, { ok: true, model: model || "stub-model", displayModel: model || "Stub Model", pid: process.pid, requests }); + } else if (request.method === "POST" && request.url === "/generate") { + requests += 1; + const payload = JSON.parse(Buffer.concat(chunks).toString("utf8")); + if (model === "fail-generate") return send(500, { ok: false }); + if (model === "refuse-generate") return send(200, { ok: false }); + if (model === "drop-generate") return request.socket.destroy(); + if (model === "slow-generate") return; + send(200, { + ok: true, + model: model || "stub-model", + pid: process.pid, + requests, + output: payload.output, + prompt: payload.prompt, + constraints: payload.constraints, + priority: os.getPriority(), + env: { + OMP_NUM_THREADS: process.env.OMP_NUM_THREADS, + MKL_NUM_THREADS: process.env.MKL_NUM_THREADS, + VILI_THREADS: process.env.VILI_THREADS, + HF_HUB_OFFLINE: process.env.HF_HUB_OFFLINE, + TRANSFORMERS_OFFLINE: process.env.TRANSFORMERS_OFFLINE, + HF_HOME: process.env.HF_HOME, + PYTHONPATH: process.env.PYTHONPATH, + PYTHONDONTWRITEBYTECODE: process.env.PYTHONDONTWRITEBYTECODE, + PYTHONUNBUFFERED: process.env.PYTHONUNBUFFERED, + VILI_KIMODO_COMMIT: process.env.VILI_KIMODO_COMMIT, + VILI_PYTHON_ENV: process.env.VILI_PYTHON_ENV, + PATH: process.env.PATH, + HOME: process.env.HOME, + LANG: process.env.LANG, + LEAK_CHECK: process.env.LEAK_CHECK, + }, + argv, + }); + } else { + response.writeHead(404); + response.end(); + } + }); }); -}); -server.listen(Number(flag("--port")), host, () => { - console.log(JSON.stringify({ event: "kimodo-worker-ready", port: server.address().port })); -}); + server.listen(Number(flag("--port")), flag("--host"), () => { + console.log(JSON.stringify({ event: "kimodo-worker-ready", port: server.address().port })); + }); +} diff --git a/scripts/test-vili-daemon.mjs b/scripts/test-vili-daemon.mjs index 0d2e5e5..58fb55c 100644 --- a/scripts/test-vili-daemon.mjs +++ b/scripts/test-vili-daemon.mjs @@ -14,8 +14,8 @@ const daemonPath = path.join(here, "vili-daemon.mjs"); const stubPath = path.join(here, "fixtures", "stub-worker.mjs"); const THREADS = 5; -async function until(condition, what) { - const deadline = Date.now() + 15000; +async function until(condition, what, seconds = 15) { + const deadline = Date.now() + seconds * 1000; for (;;) { const value = await condition(); if (value) return value; @@ -24,33 +24,67 @@ async function until(condition, what) { } } -async function startDaemon({ configured = true } = {}) { - const stateRoot = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); - const modelsRoot = path.join(stateRoot, "models"); - const argv = [ - daemonPath, "--port", "0", "--state-root", stateRoot, "--python", process.execPath, "--worker-entry", stubPath, - "--threads", String(THREADS), "--python-env", "test-env", - ]; - if (configured) argv.push("--models-root", modelsRoot, "--kimodo-commit", "abc1234"); - const proc = spawn(process.execPath, argv, { env: { ...process.env, LEAK_CHECK: "leaked" }, stdio: ["ignore", "pipe", "pipe"] }); +// A zombie is dead for our purposes: its parent (or init) has simply not reaped it yet. +function isAlive(pid) { + try { + process.kill(pid, 0); + } catch { + return false; + } + try { + return !/^\d+ \(.*\) Z/.test(readFileSync(`/proc/${pid}/stat`, "utf8")); + } catch { + return false; + } +} + +async function startDaemon({ omit = [], viaEnv = false, threads = String(THREADS), python = process.execPath, nested = false } = {}) { + const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-")); + const stateRoot = nested ? path.join(root, "a", "b", "state") : root; + const modelsRoot = path.join(root, "models"); + const settings = { + host: "127.0.0.1", + port: "0", + "state-root": stateRoot, + python, + "worker-entry": stubPath, + "models-root": modelsRoot, + "kimodo-commit": "abc1234", + "python-env": "test-env", + }; + if (threads !== null) settings.threads = threads; + for (const name of omit) delete settings[name]; + const env = { ...process.env, LEAK_CHECK: "leaked", LANG: "xx_TEST.UTF-8" }; + for (const name of Object.keys(env)) if (name.startsWith("VILI_")) delete env[name]; + const argv = [daemonPath]; + for (const [name, value] of Object.entries(settings)) { + if (viaEnv) env[`VILI_${{ host: "DAEMON_HOST", port: "DAEMON_PORT", "state-root": "STATE_ROOT", "worker-entry": "WORKER_ENTRY" }[name] ?? name.toUpperCase().replace("-", "_")}`] = value; + else argv.push(`--${name}`, value); + } + const proc = spawn(process.execPath, argv, { env, stdio: ["ignore", "pipe", "pipe"] }); let output = ""; + let errors = ""; proc.stdout.on("data", (chunk) => { output += chunk; }); + proc.stderr.on("data", (chunk) => { errors += chunk; }); const port = await until(() => /listening on http:\/\/127\.0\.0\.1:(\d+)/.exec(output)?.[1], "the daemon to listen"); + const spawnLog = () => { + const log = path.join(stateRoot, "embeddings", "stub-spawns.log"); + return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean) : []; + }; return { stateRoot, modelsRoot, proc, port: Number(port), - spawns: () => { - const log = path.join(stateRoot, "embeddings", "stub-spawns.log"); - return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean).map(Number) : []; - }, + stderr: () => errors, + spawns: () => spawnLog().filter((line) => /^\d+$/.test(line)).map(Number), + grandchildren: () => spawnLog().filter((line) => line.startsWith("grandchild ")).map((line) => Number(line.split(" ")[1])), async stop() { if (proc.exitCode === null && proc.signalCode === null) { proc.kill("SIGTERM"); - await until(() => proc.exitCode !== null || proc.signalCode !== null, "the daemon to exit"); + await until(() => proc.exitCode !== null || proc.signalCode !== null, "the daemon to exit", 30); } - rmSync(stateRoot, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true }); }, }; } @@ -82,18 +116,20 @@ const generate = (daemon, payload = { prompt: "a person waves", duration: 2 }, h headers: { "content-type": "application/json", ...headers }, }); const operatorState = async (daemon) => (await request(daemon.port, "GET", "/operator-state")).body; +const backendOf = async (daemon) => (await operatorState(daemon)).backend; test("the worker's port comes from its ready line", async () => { const daemon = await startDaemon(); try { const answer = await generate(daemon); assert.equal(answer.status, 200); - const backend = (await operatorState(daemon)).backend; + const backend = await backendOf(daemon); assert.equal(backend.state, "ready"); - assert.ok(Number.isInteger(backend.port) && backend.port > 0 && backend.port !== 8825); + assert.notEqual(daemon.port, 8824, "--port 0 binds a free port"); + assert.ok(Number.isInteger(backend.port) && backend.port > 1 && backend.port !== 8825); assert.equal(answer.body.worker.pid, backend.pid); assert.deepEqual(answer.body.worker.argv.slice(-4), ["--host", "127.0.0.1", "--port", "0"]); - // the request reached the stub on the port the stub printed + // the request reached the stub on the port the stub printed, not on a decoy line's port assert.equal((await request(backend.port, "GET", "/health")).body.requests, 1); } finally { await daemon.stop(); @@ -109,13 +145,15 @@ test("one child, respawned after exit", async () => { assert.equal(daemon.spawns().length, 1); const [pid] = daemon.spawns(); process.kill(pid, "SIGKILL"); - await until(async () => (await operatorState(daemon)).backend.state === "stopped", "the daemon to see the exit"); + await until(async () => (await backendOf(daemon)).state === "stopped", "the daemon to see the exit"); + assert.match((await backendOf(daemon)).lastError, /^the worker exited \(SIGKILL\)$/); const third = await generate(daemon); assert.equal(third.status, 200); const spawns = daemon.spawns(); assert.equal(spawns.length, 2); assert.notEqual(spawns[1], pid); assert.equal(third.body.worker.pid, spawns[1]); + assert.equal((await backendOf(daemon)).lastError, null); } finally { await daemon.stop(); } @@ -133,6 +171,11 @@ test("environment and nice", async () => { assert.equal(env.HF_HOME, daemon.modelsRoot); assert.equal(env.VILI_KIMODO_COMMIT, "abc1234"); assert.equal(env.VILI_PYTHON_ENV, "test-env"); + assert.equal(env.PYTHONDONTWRITEBYTECODE, "1"); + assert.equal(env.PYTHONUNBUFFERED, "1"); + assert.equal(env.PATH, process.env.PATH); + assert.equal(env.HOME, process.env.HOME); + assert.equal(env.LANG, "xx_TEST.UTF-8"); assert.ok(env.PYTHONPATH.endsWith(path.join("vendor", "CultLib", "packages", "cultcache-py", "src"))); assert.equal(env.LEAK_CHECK, undefined, "the daemon's own environment does not reach the worker"); assert.equal(priority, 10); @@ -141,15 +184,125 @@ test("environment and nice", async () => { } }); -test("SIGTERM leaves no child", async () => { +test("every flag has an environment twin, and the thread cap defaults to 8", async () => { + const viaEnv = await startDaemon({ viaEnv: true, threads: "16" }); + try { + const answer = await generate(viaEnv); + assert.equal(answer.status, 200); + assert.equal(answer.body.worker.env.OMP_NUM_THREADS, "16"); + assert.deepEqual(await backendOf(viaEnv), { + state: "ready", pid: answer.body.worker.pid, port: (await backendOf(viaEnv)).port, threads: 16, pythonEnv: "test-env", + kimodoCommit: "abc1234", modelsRoot: viaEnv.modelsRoot, lastError: null, + }); + } finally { + await viaEnv.stop(); + } + const byDefault = await startDaemon({ threads: null }); + try { + const answer = await generate(byDefault); + assert.equal(answer.body.worker.env.OMP_NUM_THREADS, "8"); + assert.equal((await backendOf(byDefault)).threads, 8); + } finally { + await byDefault.stop(); + } + const one = await startDaemon({ threads: "1", nested: true }); + try { + assert.equal((await generate(one)).body.worker.env.OMP_NUM_THREADS, "1"); + } finally { + await one.stop(); + } +}); + +test("the worker's output lands in its log, and the ready line does not", async () => { const daemon = await startDaemon(); try { assert.equal((await generate(daemon)).status, 200); + const log = path.join(daemon.stateRoot, "logs", "worker.log"); + const text = await until(() => { + const content = existsSync(log) ? readFileSync(log, "utf8") : ""; + return content.includes("stub-stdout-line") && content.includes("stub-stderr-line") ? content : null; + }, "the worker log"); + assert.ok(!text.includes(`"port":${(await backendOf(daemon)).port}`), "the real ready line is not logged"); + } finally { + await daemon.stop(); + } +}); + +test("a job is a record on disk", async () => { + const daemon = await startDaemon(); + try { + const answer = await generate(daemon, { prompt: "a person waves", duration: 2, constraints: [{ type: "root2d" }] }); + assert.equal(answer.status, 200); + const job = answer.body; + assert.equal(job.schema, "gamecult.vili.motion_job.v0"); + assert.equal(job.request.schema, "gamecult.vili.motion_request.v0"); + assert.equal(job.prompt, "a person waves"); + assert.equal(job.model, "stub-model"); + assert.equal(job.ok, true); + assert.equal(job.error, null); + assert.equal(job.artifactDir, path.join(daemon.stateRoot, "artifacts", job.jobId)); + assert.equal(job.worker.output, path.join(job.artifactDir, "motion")); + assert.equal(job.worker.prompt, "a person waves"); + assert.equal(job.worker.constraints, path.join(job.artifactDir, "constraints.json")); + assert.deepEqual(JSON.parse(readFileSync(job.worker.constraints, "utf8")), [{ type: "root2d" }]); + assert.deepEqual(JSON.parse(readFileSync(path.join(daemon.stateRoot, "jobs", `${job.jobId}.json`), "utf8")), job); + const batch = await generate(daemon, { texts: ["a", "b"], durations: [1, 1] }); + assert.equal(batch.status, 200); + assert.equal(batch.body.prompt, null); + assert.equal(batch.body.worker.prompt, undefined); + } finally { + await daemon.stop(); + } +}); + +test("a requested model restarts the worker only when it is not the one serving", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon)).status, 200); + assert.equal((await generate(daemon, { prompt: "x", model: "stub-model" })).status, 200); + assert.equal((await generate(daemon, { prompt: "x", model: "Stub Model" })).status, 200); + assert.equal(daemon.spawns().length, 1); + const [first] = daemon.spawns(); + const other = await generate(daemon, { prompt: "x", model: "other-model" }); + assert.equal(other.status, 200); + assert.equal(other.body.model, "other-model"); + assert.deepEqual(other.body.worker.argv.slice(-2), ["--model", "other-model"]); + assert.equal(daemon.spawns().length, 2); + await until(() => !isAlive(first), "the first worker to be gone"); + assert.equal((await generate(daemon)).status, 200); + assert.equal(daemon.spawns().length, 2, "a request that names no model keeps the one serving"); + } finally { + await daemon.stop(); + } +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + test(`${signal} leaves no child`, async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon)).status, 200); + const [pid] = daemon.spawns(); + assert.ok(isAlive(pid)); + daemon.proc.kill(signal); + await until(() => daemon.proc.exitCode !== null, "the daemon to exit"); + assert.equal(isAlive(pid), false); + } finally { + await daemon.stop(); + } + }); +} + +test("a worker that ignores SIGTERM is killed with everything it started", async () => { + const daemon = await startDaemon(); + try { + assert.equal((await generate(daemon, { prompt: "x", model: "stubborn" })).status, 200); const [pid] = daemon.spawns(); - process.kill(pid, 0); + const [grandchild] = daemon.grandchildren(); + assert.ok(isAlive(pid) && isAlive(grandchild)); daemon.proc.kill("SIGTERM"); - await until(() => daemon.proc.exitCode !== null, "the daemon to exit"); - assert.throws(() => process.kill(pid, 0), { code: "ESRCH" }); + await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 30); + assert.equal(isAlive(pid), false); + assert.equal(isAlive(grandchild), false); } finally { await daemon.stop(); } @@ -161,58 +314,216 @@ test("cross-site is refused before the worker", async () => { assert.equal((await generate(daemon)).status, 200); const withOrigin = await generate(daemon, undefined, { origin: "https://example.test" }); assert.equal(withOrigin.status, 403); - assert.equal(withOrigin.body.code, "cross-site"); + assert.deepEqual(withOrigin.body, { ok: false, code: "cross-site" }); const plain = await generate(daemon, undefined, { "content-type": "text/plain" }); assert.equal(plain.status, 403); - assert.equal(plain.body.code, "cross-site"); + assert.deepEqual(plain.body, { ok: false, code: "cross-site" }); + const untyped = await request(daemon.port, "POST", "/motion/generate", { body: JSON.stringify({ prompt: "x" }) }); + assert.equal(untyped.status, 403); const options = await request(daemon.port, "OPTIONS", "/motion/generate", { headers: { origin: "https://example.test" } }); assert.equal(options.status, 404); const health = await request(daemon.port, "GET", "/health"); - for (const reply of [withOrigin, plain, options, health]) { + for (const reply of [withOrigin, plain, untyped, options, health]) { assert.equal(Object.keys(reply.headers).filter((name) => name.startsWith("access-control-")).length, 0); } - const backend = (await operatorState(daemon)).backend; + const backend = await backendOf(daemon); assert.equal((await request(backend.port, "GET", "/health")).body.requests, 1, "only the same-site request reached the worker"); + // the media type is matched as a type, not as a string + assert.equal((await generate(daemon, undefined, { "content-type": "Application/JSON; charset=utf-8" })).status, 200); } finally { await daemon.stop(); } }); -test("unconfigured refuses before spawning", async () => { - const daemon = await startDaemon({ configured: false }); - try { - const answer = await generate(daemon); - assert.equal(answer.status, 503); - assert.equal(answer.body.code, "worker-unconfigured"); - assert.deepEqual(daemon.spawns(), []); - assert.equal((await operatorState(daemon)).backend.state, "stopped"); - } finally { - await daemon.stop(); - } -}); +for (const omitted of ["models-root", "kimodo-commit", "python-env"]) { + test(`unconfigured (no ${omitted}) refuses before spawning`, async () => { + const daemon = await startDaemon({ omit: [omitted] }); + try { + const answer = await generate(daemon); + assert.equal(answer.status, 503); + assert.deepEqual(answer.body, { ok: false, code: "worker-unconfigured" }); + assert.deepEqual(daemon.spawns(), []); + assert.equal((await backendOf(daemon)).state, "stopped"); + } finally { + await daemon.stop(); + } + }); +} test("an error never echoes the request", async () => { const daemon = await startDaemon(); try { const malformed = await generate(daemon, "{\"prompt\": \"the launch code is swordfish\", "); assert.equal(malformed.status, 400); - assert.ok(!malformed.raw.includes("swordfish")); + assert.deepEqual(malformed.body, { ok: false, error: "Invalid JSON." }); const missing = await generate(daemon, { texts: ["the launch code is swordfish"] }); assert.equal(missing.status, 400); assert.ok(!missing.raw.includes("swordfish")); + // a body the handler cannot use at all: the answer names nothing, and neither does the log + const unusable = await generate(daemon, "null"); + assert.equal(unusable.status, 500); + assert.deepEqual(unusable.body, { ok: false, code: "internal" }); + await until(() => daemon.stderr().includes("request failed: TypeError"), "the daemon to log the class"); + assert.ok(!daemon.stderr().includes("swordfish")); + } finally { + await daemon.stop(); + } +}); + +test("a worker that dies before it is ready fails the job, and the next job tries again", async () => { + const daemon = await startDaemon(); + try { + const failed = await generate(daemon, { prompt: "x", model: "crash-before-ready" }); + assert.equal(failed.status, 502); + assert.equal(failed.body.ok, false); + assert.equal(failed.body.error, "the worker request failed (Error)"); + const backend = await backendOf(daemon); + assert.equal(backend.state, "failed"); + assert.equal(backend.lastError, "the worker exited (code 3)"); + assert.equal(backend.pid, null); + assert.equal(backend.port, null); + assert.equal((await generate(daemon)).status, 200); + assert.equal((await backendOf(daemon)).state, "ready"); + assert.equal(daemon.spawns().length, 2); + } finally { + await daemon.stop(); + } +}); + +test("a worker that never says it is ready is killed at the deadline", async () => { + const daemon = await startDaemon(); + try { + const pending = generate(daemon, { prompt: "x", model: "hang-before-ready", workerTimeoutMs: 1500 }); + const starting = await until(async () => { + const backend = await backendOf(daemon); + return backend.state === "starting" ? backend : null; + }, "the starting state"); + assert.ok(Number.isInteger(starting.pid) && starting.port === null); + const failed = await pending; + assert.equal(failed.status, 502); + const backend = await backendOf(daemon); + assert.equal(backend.state, "failed"); + assert.equal(backend.lastError, "the worker was not ready in time"); + assert.equal(isAlive(daemon.spawns()[0]), false); + } finally { + await daemon.stop(); + } +}); + +test("a worker that is ready but not healthy fails at the deadline; one that gets healthy in time serves", async () => { + const daemon = await startDaemon(); + try { + const failed = await generate(daemon, { prompt: "x", model: "sick", workerTimeoutMs: 1500 }); + assert.equal(failed.status, 502); + const backend = await backendOf(daemon); + assert.equal(backend.state, "failed"); + assert.equal(backend.lastError, "the worker did not become healthy in time"); + assert.equal(isAlive(daemon.spawns()[0]), false); + const slow = await generate(daemon, { prompt: "x", model: "slow-health" }); + assert.equal(slow.status, 200); + assert.equal((await backendOf(daemon)).state, "ready"); + } finally { + await daemon.stop(); + } +}); + +test("a missing interpreter fails the job at once", async () => { + const daemon = await startDaemon({ python: "/no/such/python" }); + try { + const began = Date.now(); + const failed = await generate(daemon); + assert.equal(failed.status, 502); + assert.ok(Date.now() - began < 5000, "nothing waits on a process that never existed"); + const backend = await backendOf(daemon); + assert.equal(backend.state, "failed"); + assert.equal(backend.lastError, "the worker could not be spawned"); + } finally { + await daemon.stop(); + } +}); + +test("a worker that refuses, errors or drops the request fails the job without echoing it", async () => { + const daemon = await startDaemon(); + try { + const refused = await generate(daemon, { prompt: "swordfish", model: "refuse-generate" }); + assert.equal(refused.status, 502); + assert.equal(refused.body.ok, false); + assert.equal(refused.body.error, null); + const errored = await generate(daemon, { prompt: "swordfish", model: "fail-generate" }); + assert.equal(errored.status, 502); + assert.equal(errored.body.error, "the worker answered 500"); + const dropped = await generate(daemon, { prompt: "swordfish", model: "drop-generate" }); + assert.equal(dropped.status, 502); + assert.equal(dropped.body.error, "the worker request failed (TypeError)"); + const stalled = await generate(daemon, { prompt: "swordfish", model: "slow-generate", timeoutMs: 700 }); + assert.equal(stalled.status, 502); + assert.equal(stalled.body.error, "the worker request failed (TimeoutError)"); + for (const answer of [refused, errored, dropped, stalled]) assert.ok(!answer.body.error?.includes("swordfish")); + } finally { + await daemon.stop(); + } +}); + +test("health, operator state and the unknown", async () => { + const daemon = await startDaemon(); + try { + for (const route of ["/", "/health"]) { + const answer = await request(daemon.port, "GET", route); + assert.equal(answer.status, 200); + assert.equal(answer.body.ok, true); + assert.equal(answer.body.service, "vili"); + assert.equal(answer.body.state.backend.threads, THREADS); + } + assert.equal((await request(daemon.port, "GET", "/motion/generate")).status, 404); + const unknown = await request(daemon.port, "GET", "/eve/operator"); + assert.equal(unknown.status, 404); + assert.equal(unknown.raw, "not found\n"); + const state = await operatorState(daemon); + assert.equal(state.host, os.hostname()); + assert.deepEqual(Object.keys(state.routes), ["health", "operatorState", "generate"]); + } finally { + await daemon.stop(); + } +}); + +test("the typed service state is refreshed while the daemon runs", async () => { + const daemon = await startDaemon({ nested: true }); + try { + const store = path.join(daemon.stateRoot, "vili.service.cc"); + assert.ok(existsSync(store)); + assert.ok(!readFileSync(store).includes("ready")); + assert.equal((await generate(daemon)).status, 200); + await until(() => readFileSync(store).includes("ready"), "the store to show the worker ready", 20); + } finally { + await daemon.stop(); + } +}); + +test("--health asks a running daemon, and fails when there is none", async () => { + const daemon = await startDaemon(); + const check = (port) => new Promise((resolve) => { + const proc = spawn(process.execPath, [daemonPath, "--health", "--host", "127.0.0.1", "--port", String(port)], { stdio: "ignore" }); + proc.on("exit", resolve); + }); + try { + assert.equal(await check(daemon.port), 0); + const gone = daemon.port; + await daemon.stop(); + assert.equal(await check(gone), 1); } finally { await daemon.stop(); } }); test("a thread count outside 1 to 16 stops the daemon without echoing it", async () => { - for (const threads of ["0", "17", "99", "8x"]) { + for (const threads of ["0", "17", "99", "100", "8x", "x8", "-1", "1.5"]) { const proc = spawn(process.execPath, [daemonPath, "--threads", threads], { stdio: ["ignore", "pipe", "pipe"] }); let output = ""; proc.stdout.on("data", (chunk) => { output += chunk; }); proc.stderr.on("data", (chunk) => { output += chunk; }); const code = await new Promise((resolve) => proc.on("exit", resolve)); assert.equal(code, 2); - assert.ok(!output.includes(threads), `the refusal repeated ${threads}`); + assert.match(output, /threads must be an integer from 1 to 16/); + assert.ok(!output.replace("from 1 to 16", "").includes(threads), `the refusal repeated ${threads}`); } });