Daemon: drop the unread worker.model; tests for grace, host default, relative models root, early death
Mutation triage (StrykerJS, 81 survivors on the first full run) showed worker.model was written and never read. The new tests pin the survivors that were behaviour: the loopback default, --host, stray flags, the models-root resolution, the SIGTERM grace, a worker that dies while the daemon waits for its health, the POST route, and lastError after a requested stop. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
3843a772f8
commit
c3b048a2ab
3 changed files with 90 additions and 11 deletions
|
|
@ -1,7 +1,7 @@
|
|||
// Stands in for kimodo-resident-worker.py in the daemon's tests. It binds port 0, prints the ready line the
|
||||
// daemon waits for, and answers /health and /generate with what the daemon gave it. The --model value picks a
|
||||
// misbehaviour: crash-before-ready, hang-before-ready, sick, slow-health, stubborn, fail-generate,
|
||||
// refuse-generate, drop-generate, slow-generate. Any other value is just the model the worker serves.
|
||||
// refuse-generate, drop-generate, slow-generate, graceful (exits 0.5 s after SIGTERM), die-after-ready. Any other value is just the model the worker serves.
|
||||
import http from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
|
@ -33,6 +33,15 @@ if (model === "stubborn") {
|
|||
appendFileSync(spawnLog, `grandchild ${grandchild.pid}\n`);
|
||||
}
|
||||
|
||||
if (model === "graceful") {
|
||||
process.on("SIGTERM", () => {
|
||||
setTimeout(() => {
|
||||
appendFileSync(spawnLog, "graceful-exit\n");
|
||||
process.exit(0);
|
||||
}, 500);
|
||||
});
|
||||
}
|
||||
|
||||
if (model === "hang-before-ready") {
|
||||
setInterval(() => {}, 1000);
|
||||
} else {
|
||||
|
|
@ -46,7 +55,7 @@ if (model === "hang-before-ready") {
|
|||
};
|
||||
if (request.method === "GET" && request.url === "/health") {
|
||||
healthCalls += 1;
|
||||
if (model === "sick" || (model === "slow-health" && healthCalls <= 2)) return send(200, { ok: false });
|
||||
if (model === "sick" || model === "die-after-ready" || (model === "slow-health" && healthCalls <= 2)) return send(200, { ok: false });
|
||||
send(200, { ok: true, model: model || "stub-model", displayModel: model || "Stub Model", pid: process.pid, requests });
|
||||
} else if (request.method === "POST" && request.url === "/generate") {
|
||||
requests += 1;
|
||||
|
|
@ -60,6 +69,7 @@ if (model === "hang-before-ready") {
|
|||
model: model || "stub-model",
|
||||
pid: process.pid,
|
||||
requests,
|
||||
contentType: request.headers["content-type"],
|
||||
output: payload.output,
|
||||
prompt: payload.prompt,
|
||||
constraints: payload.constraints,
|
||||
|
|
@ -92,5 +102,6 @@ if (model === "hang-before-ready") {
|
|||
|
||||
server.listen(Number(flag("--port")), flag("--host"), () => {
|
||||
console.log(JSON.stringify({ event: "kimodo-worker-ready", port: server.address().port }));
|
||||
if (model === "die-after-ready") setTimeout(() => process.exit(4), 300);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,17 +38,20 @@ function isAlive(pid) {
|
|||
}
|
||||
}
|
||||
|
||||
async function startDaemon({ omit = [], viaEnv = false, threads = String(THREADS), python = process.execPath, nested = false } = {}) {
|
||||
async function startDaemon({
|
||||
omit = [], viaEnv = false, threads = String(THREADS), python = process.execPath, nested = false, host = "127.0.0.1",
|
||||
relativeModels = false, extraArgs = [],
|
||||
} = {}) {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "vili-test-"));
|
||||
const stateRoot = nested ? path.join(root, "a", "b", "state") : root;
|
||||
const modelsRoot = path.join(root, "models");
|
||||
const settings = {
|
||||
host: "127.0.0.1",
|
||||
host,
|
||||
port: "0",
|
||||
"state-root": stateRoot,
|
||||
python,
|
||||
"worker-entry": stubPath,
|
||||
"models-root": modelsRoot,
|
||||
"models-root": relativeModels ? "models-rel" : modelsRoot,
|
||||
"kimodo-commit": "abc1234",
|
||||
"python-env": "test-env",
|
||||
};
|
||||
|
|
@ -56,17 +59,18 @@ async function startDaemon({ omit = [], viaEnv = false, threads = String(THREADS
|
|||
for (const name of omit) delete settings[name];
|
||||
const env = { ...process.env, LEAK_CHECK: "leaked", LANG: "xx_TEST.UTF-8" };
|
||||
for (const name of Object.keys(env)) if (name.startsWith("VILI_")) delete env[name];
|
||||
const argv = [daemonPath];
|
||||
const argv = [daemonPath, ...extraArgs];
|
||||
for (const [name, value] of Object.entries(settings)) {
|
||||
if (viaEnv) env[`VILI_${{ host: "DAEMON_HOST", port: "DAEMON_PORT", "state-root": "STATE_ROOT", "worker-entry": "WORKER_ENTRY" }[name] ?? name.toUpperCase().replace("-", "_")}`] = value;
|
||||
else argv.push(`--${name}`, value);
|
||||
}
|
||||
const proc = spawn(process.execPath, argv, { env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
const proc = spawn(process.execPath, argv, { env, cwd: root, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let output = "";
|
||||
let errors = "";
|
||||
proc.stdout.on("data", (chunk) => { output += chunk; });
|
||||
proc.stderr.on("data", (chunk) => { errors += chunk; });
|
||||
const port = await until(() => /listening on http:\/\/127\.0\.0\.1:(\d+)/.exec(output)?.[1], "the daemon to listen");
|
||||
const listening = await until(() => /listening on http:\/\/([^:]+):(\d+)/.exec(output), "the daemon to listen");
|
||||
const port = listening[2];
|
||||
const spawnLog = () => {
|
||||
const log = path.join(stateRoot, "embeddings", "stub-spawns.log");
|
||||
return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean) : [];
|
||||
|
|
@ -75,6 +79,7 @@ async function startDaemon({ omit = [], viaEnv = false, threads = String(THREADS
|
|||
stateRoot,
|
||||
modelsRoot,
|
||||
proc,
|
||||
boundHost: listening[1],
|
||||
port: Number(port),
|
||||
stderr: () => errors,
|
||||
spawns: () => spawnLog().filter((line) => /^\d+$/.test(line)).map(Number),
|
||||
|
|
@ -242,6 +247,7 @@ test("a job is a record on disk", async () => {
|
|||
assert.equal(job.error, null);
|
||||
assert.equal(job.artifactDir, path.join(daemon.stateRoot, "artifacts", job.jobId));
|
||||
assert.equal(job.worker.output, path.join(job.artifactDir, "motion"));
|
||||
assert.equal(job.worker.contentType, "application/json");
|
||||
assert.equal(job.worker.prompt, "a person waves");
|
||||
assert.equal(job.worker.constraints, path.join(job.artifactDir, "constraints.json"));
|
||||
assert.deepEqual(JSON.parse(readFileSync(job.worker.constraints, "utf8")), [{ type: "root2d" }]);
|
||||
|
|
@ -269,6 +275,7 @@ test("a requested model restarts the worker only when it is not the one serving"
|
|||
assert.deepEqual(other.body.worker.argv.slice(-2), ["--model", "other-model"]);
|
||||
assert.equal(daemon.spawns().length, 2);
|
||||
await until(() => !isAlive(first), "the first worker to be gone");
|
||||
assert.equal((await backendOf(daemon)).lastError, null, "a stop the daemon asked for is not an error");
|
||||
assert.equal((await generate(daemon)).status, 200);
|
||||
assert.equal(daemon.spawns().length, 2, "a request that names no model keeps the one serving");
|
||||
} finally {
|
||||
|
|
@ -284,7 +291,7 @@ for (const signal of ["SIGTERM", "SIGINT"]) {
|
|||
const [pid] = daemon.spawns();
|
||||
assert.ok(isAlive(pid));
|
||||
daemon.proc.kill(signal);
|
||||
await until(() => daemon.proc.exitCode !== null, "the daemon to exit");
|
||||
await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 8);
|
||||
assert.equal(isAlive(pid), false);
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
|
|
@ -292,6 +299,60 @@ for (const signal of ["SIGTERM", "SIGINT"]) {
|
|||
});
|
||||
}
|
||||
|
||||
test("a worker that is stopping gets its grace, not a SIGKILL", async () => {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
assert.equal((await generate(daemon, { prompt: "x", model: "graceful" })).status, 200);
|
||||
daemon.proc.kill("SIGTERM");
|
||||
await until(() => daemon.proc.exitCode !== null, "the daemon to exit", 8);
|
||||
assert.ok(readFileSync(path.join(daemon.stateRoot, "embeddings", "stub-spawns.log"), "utf8").includes("graceful-exit"));
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("a worker that dies while the daemon waits for its health fails the job at once", async () => {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
const began = Date.now();
|
||||
const failed = await generate(daemon, { prompt: "x", model: "die-after-ready" });
|
||||
assert.equal(failed.status, 502);
|
||||
assert.ok(Date.now() - began < 20000, "the default deadline is minutes; a dead worker does not wait for it");
|
||||
const backend = await backendOf(daemon);
|
||||
assert.equal(backend.state, "stopped");
|
||||
assert.equal(backend.lastError, "the worker exited (code 4)");
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("the daemon binds loopback unless told otherwise, and ignores flags it does not know", async () => {
|
||||
const byDefault = await startDaemon({ omit: ["host"], extraArgs: ["--bogus"] });
|
||||
try {
|
||||
assert.equal(byDefault.boundHost, "127.0.0.1");
|
||||
assert.equal((await request(byDefault.port, "GET", "/health")).status, 200);
|
||||
} finally {
|
||||
await byDefault.stop();
|
||||
}
|
||||
const named = await startDaemon({ host: "127.0.0.2" });
|
||||
try {
|
||||
assert.equal(named.boundHost, "127.0.0.2");
|
||||
assert.equal((await fetch(`http://127.0.0.2:${named.port}/health`)).status, 200);
|
||||
} finally {
|
||||
await named.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("a relative models root reaches the worker as an absolute path", async () => {
|
||||
const daemon = await startDaemon({ relativeModels: true });
|
||||
try {
|
||||
const { env } = (await generate(daemon)).body.worker;
|
||||
assert.equal(env.HF_HOME, path.join(path.dirname(daemon.modelsRoot), "models-rel"));
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("a worker that ignores SIGTERM is killed with everything it started", async () => {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
|
|
@ -330,6 +391,13 @@ test("cross-site is refused before the worker", async () => {
|
|||
assert.equal((await request(backend.port, "GET", "/health")).body.requests, 1, "only the same-site request reached the worker");
|
||||
// the media type is matched as a type, not as a string
|
||||
assert.equal((await generate(daemon, undefined, { "content-type": "Application/JSON; charset=utf-8" })).status, 200);
|
||||
assert.equal((await generate(daemon, undefined, { "content-type": "application/json ; charset=utf-8" })).status, 200);
|
||||
// and only the generate route answers a POST
|
||||
const stray = await request(daemon.port, "POST", "/motion/other", {
|
||||
body: JSON.stringify({ prompt: "x" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
assert.equal(stray.status, 404);
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ const args = parseArgs(process.argv.slice(2));
|
|||
const startedAt = new Date().toISOString();
|
||||
|
||||
// The worker child. Display state only: nothing reads `worker` to decide anything but ensureWorker.
|
||||
const worker = { state: "stopped", proc: null, pid: null, port: null, model: null, servedModels: [], lastError: null };
|
||||
const worker = { state: "stopped", proc: null, pid: null, port: null, servedModels: [], lastError: null };
|
||||
let starting = null;
|
||||
|
||||
function firstDefined(...values) {
|
||||
|
|
@ -322,7 +322,7 @@ async function startWorker(model, timeoutMs) {
|
|||
detached: true,
|
||||
});
|
||||
closeSync(logFd);
|
||||
Object.assign(worker, { state: "starting", proc, pid: proc.pid ?? null, port: null, model: model || null, servedModels: [], lastError: null });
|
||||
Object.assign(worker, { state: "starting", proc, pid: proc.pid ?? null, port: null, servedModels: [], lastError: null });
|
||||
const exited = new Promise((resolve) => {
|
||||
proc.once("error", () => resolve("the worker could not be spawned"));
|
||||
proc.once("exit", (code, signal) => resolve(`the worker exited (${signal ?? `code ${code}`})`));
|
||||
|
|
|
|||
Loading…
Reference in a new issue