Vili/scripts/test-gpu-tenancy.mjs

import assert from "node:assert/strict";
import test from "node:test";
import { admitThen, createGpuTenancy, gpuTenancyOptionsFromEnv, placementRefusal } from "./gpu-tenancy.mjs";

const SEC = 1000;
const CLEAR = { quns: 1, freeMiB: 14000, util3d: 1, engines: 8 };

function rig(options = {}) {
  let t = 1_000_000;
  const gate = createGpuTenancy({ now: () => t, ...options });
  return {
    gate,
    t0: t,
    at: () => t,
    advance: (ms) => { t += ms; },
    feed(sample, afterMs = 15 * SEC) {
      t += afterMs;
      gate.observe(sample);
    },
  };
}

function freeRig(options) {
  const r = rig(options);
  for (let i = 0; i < 13; i += 1) r.feed(CLEAR); // the 13th sample is 180 s after the first
  assert.equal(r.gate.status().inUse, false, "precondition: gate is free");
  return r;
}

test("hysteresis: busy holds for the clear window", () => {
  const r = freeRig();
  r.feed({ ...CLEAR, quns: 3 });
  const busyAt = r.at();
  assert.deepEqual(r.gate.status(), { inUse: true, since: busyAt, checkedAt: busyAt, probeOk: true });
  for (let elapsed = 15; elapsed < 180; elapsed += 15) {
    r.feed(CLEAR);
    assert.equal(r.gate.status().inUse, true, `still held ${elapsed} s into the hold`);
    assert.equal(r.gate.status().since, busyAt);
  }
  // a busy sample late in the hold restarts it
  r.feed({ ...CLEAR, quns: 2 });
  const restartAt = r.at();
  r.feed(CLEAR); // the hold counts from the first clear sample
  const holdStart = r.at();
  for (let elapsed = 15; elapsed <= 165; elapsed += 15) {
    r.feed(CLEAR);
    assert.equal(r.gate.status().inUse, true, `restarted hold, ${elapsed} s clear`);
  }
  r.feed(CLEAR);
  assert.equal(r.at() - holdStart, 180 * SEC);
  assert.ok(r.at() > restartAt);
  const freedAt = r.at();
  assert.deepEqual(r.gate.status(), { inUse: false, since: freedAt, checkedAt: freedAt, probeOk: true });
  r.feed(CLEAR);
  assert.equal(r.gate.status().since, freedAt, "since moves only when the state turns");
  r.feed({ ...CLEAR, freeMiB: 100 });
  assert.deepEqual(r.gate.status(), { inUse: true, since: r.at(), checkedAt: r.at(), probeOk: true });
});

test("hysteresis: a busy sample just before the window ends restarts it", () => {
  const r = rig();
  r.feed(CLEAR);
  for (let i = 0; i < 11; i += 1) r.feed(CLEAR);
  assert.equal(r.gate.status().inUse, true, "165 s clear is not enough");
  r.feed({ ...CLEAR, quns: 3 });
  r.feed(CLEAR);
  r.feed(CLEAR);
  assert.equal(r.gate.status().inUse, true);
});

test("sustained 3D, immediate full-screen and VRAM", () => {
  const spike = freeRig();
  spike.feed({ ...CLEAR, util3d: 80 });
  spike.feed(CLEAR);
  assert.equal(spike.gate.status().inUse, false, "one 3D spike between clear samples");

  const sustained = freeRig();
  sustained.feed({ ...CLEAR, util3d: 80 });
  assert.equal(sustained.gate.status().inUse, false, "first high sample alone");
  sustained.feed({ ...CLEAR, util3d: 80 });
  assert.equal(sustained.gate.status().inUse, true, "two consecutive high samples");

  const edge = freeRig();
  edge.feed({ ...CLEAR, util3d: 20 });
  edge.feed({ ...CLEAR, util3d: 20 });
  assert.equal(edge.gate.status().inUse, false, "exactly the threshold is not above it");
  edge.feed({ ...CLEAR, util3d: 80 });
  edge.feed({ ...CLEAR, util3d: 20 });
  assert.equal(edge.gate.status().inUse, false, "high then exactly the threshold");
  const edgeUp = freeRig();
  edgeUp.feed({ ...CLEAR, util3d: 20 });
  edgeUp.feed({ ...CLEAR, util3d: 80 });
  assert.equal(edgeUp.gate.status().inUse, false, "exactly the threshold then high");

  for (const [name, sample] of [
    ["QUNS_BUSY", { ...CLEAR, quns: 2 }],
    ["QUNS_RUNNING_D3D_FULL_SCREEN", { ...CLEAR, quns: 3 }],
    ["low free VRAM", { ...CLEAR, freeMiB: 10239 }],
  ]) {
    const r = freeRig();
    r.feed(sample);
    assert.equal(r.gate.status().inUse, true, name);
  }

  const enough = freeRig();
  enough.feed({ ...CLEAR, freeMiB: 10240 });
  assert.equal(enough.gate.status().inUse, false, "free VRAM equal to the minimum is enough");

  const otherStates = freeRig();
  for (const quns of [1, 4, 5, 6, 7]) otherStates.feed({ ...CLEAR, quns });
  assert.equal(otherStates.gate.status().inUse, false, "other notification states are not household use");
});

test("thresholds from the options are honored", () => {
  const r = rig({ minFreeMiB: 500, util3dBusyPct: 90, clearHoldMs: 30 * SEC });
  r.feed({ ...CLEAR, freeMiB: 600, util3d: 85 });
  r.feed({ ...CLEAR, freeMiB: 600, util3d: 85 });
  r.feed({ ...CLEAR, freeMiB: 600, util3d: 85 });
  assert.equal(r.gate.status().inUse, false);
  r.feed({ ...CLEAR, freeMiB: 400 });
  assert.equal(r.gate.status().inUse, true);
});

test("start and stale fail closed", () => {
  const r = rig();
  assert.deepEqual(r.gate.status(), { inUse: true, since: r.t0, checkedAt: null, probeOk: true });
  for (let i = 0; i < 13; i += 1) r.feed(CLEAR);
  assert.equal(r.gate.status().inUse, false);
  const lastSample = r.at();

  r.advance(45 * SEC);
  assert.equal(r.gate.status().probeOk, true, "exactly 3 intervals is not stale yet");
  r.advance(1);
  const stale = r.gate.status();
  assert.equal(stale.inUse, true);
  assert.equal(stale.probeOk, false);
  assert.equal(stale.since, lastSample + 45 * SEC);
  assert.equal(stale.checkedAt, lastSample);

  r.feed(CLEAR, 0);
  assert.equal(r.gate.status().inUse, true, "samples resumed but the hold has not passed");
  assert.equal(r.gate.status().probeOk, true);
  for (let i = 0; i < 11; i += 1) r.feed(CLEAR);
  assert.equal(r.gate.status().inUse, true);
  r.feed(CLEAR);
  assert.equal(r.gate.status().inUse, false);
});

test("a stale start keeps since at the start when samples resume busy", () => {
  const r = rig();
  r.advance(60 * SEC);
  r.feed({ ...CLEAR, quns: 3 }, 0);
  assert.deepEqual(r.gate.status(), { inUse: true, since: r.t0, checkedAt: r.at(), probeOk: true });
});

test("a gate that never hears from its probe stays in use and not ok", () => {
  const r = rig();
  r.advance(46 * SEC);
  assert.deepEqual(r.gate.status(), { inUse: true, since: r.t0, checkedAt: null, probeOk: false });
});

test("defer, not kill", async () => {
  const r = freeRig();
  let release;
  let started = 0;
  const running = admitThen(r.gate, () => {
    started += 1;
    return new Promise((resolve) => { release = () => resolve({ ok: true, value: "finished" }); });
  });
  assert.equal(started, 1);
  r.feed({ ...CLEAR, quns: 3 });
  assert.equal(r.gate.status().inUse, true, "the gate went busy while the job runs");

  let secondCalls = 0;
  const second = await admitThen(r.gate, async () => { secondCalls += 1; });
  assert.equal(secondCalls, 0, "a second job is refused without being called");
  assert.equal(second.code, "gpu-busy");

  release();
  assert.deepEqual(await running, { ok: true, value: "finished" });
});

test("a job that fails is not swallowed by the gate", async () => {
  const r = freeRig();
  await assert.rejects(admitThen(r.gate, async () => { throw new Error("job failed"); }), /job failed/);
});

test("typed busy result", async () => {
  const r = rig();
  r.feed({ ...CLEAR, quns: 3 });
  const since = r.gate.status().since;
  r.feed({ ...CLEAR, quns: 3 });
  r.feed({ ...CLEAR, quns: 3 });
  let calls = 0;
  const refusal = await admitThen(r.gate, async () => { calls += 1; });
  assert.deepEqual(refusal, {
    ok: false,
    code: "gpu-busy",
    source: "admission",
    gpu: { inUse: true, since, checkedAt: r.at(), probeOk: true },
  });
  assert.equal(calls, 0);
});

test("placement refusal is keyed on the typed code, never on text, and carries no gate status", () => {
  assert.deepEqual(placementRefusal({ ok: false, code: "gpu-busy", error: "x" }, "vili-1"), {
    ok: false, code: "gpu-busy", source: "placement", jobId: "vili-1",
  });
  assert.equal(placementRefusal({ ok: false, error: "gpu-busy: 0.50 GiB of VRAM free" }, "vili-1"), null);
  assert.equal(placementRefusal({ ok: false, code: null }, "vili-1"), null);
  assert.equal(placementRefusal(null, "vili-1"), null);
});


// ---- the slot: one job at a time, judged where the GPU is taken ----

const OWN = { ...CLEAR, freeMiB: 1800 }; // the encoder child holds the VRAM

test("slot: a second request while a job runs is refused as a slot refusal and never waits", async () => {
  const r = freeRig();
  let release;
  const running = admitThen(r.gate, () => new Promise((resolve) => { release = () => resolve("done"); }));
  let calls = 0;
  const refused = await admitThen(r.gate, async () => { calls += 1; });
  assert.deepEqual(refused, { ok: false, code: "gpu-busy", source: "slot" });
  assert.equal(calls, 0);
  const third = await admitThen(r.gate, async () => { calls += 1; });
  assert.equal(third.source, "slot");
  release();
  assert.equal(await running, "done");
  assert.equal(await admitThen(r.gate, async () => "next"), "next", "the slot frees when the job ends");
});

test("slot: a failing job still frees the slot", async () => {
  const r = freeRig();
  await assert.rejects(admitThen(r.gate, async () => { throw new Error("boom"); }), /boom/);
  assert.equal(await admitThen(r.gate, async () => "after"), "after");
});

test("slot: the household's answer comes before the slot's", async () => {
  const r = freeRig();
  const running = admitThen(r.gate, () => new Promise(() => {}));
  r.feed({ ...OWN, quns: 3 });
  const refused = await admitThen(r.gate, async () => {});
  assert.equal(refused.source, "admission");
  assert.equal(refused.gpu.inUse, true);
  void running;
});

test("own VRAM: a sequential batch of misses is never refused for Vili's own load", async () => {
  const r = freeRig();
  for (let miss = 1; miss <= 8; miss += 1) {
    const result = await admitThen(r.gate, async () => {
      for (let i = 0; i < 9; i += 1) r.feed(OWN); // about 2 minutes of child VRAM
      return `miss ${miss}`;
    });
    assert.equal(result, `miss ${miss}`, `miss ${miss} was admitted`);
    r.feed(CLEAR); // the child's VRAM has drained
    assert.equal(r.gate.status().inUse, false, `gate free after miss ${miss}`);
  }
});

test("own VRAM: the next request right after a job is admitted while the child's VRAM drains", async () => {
  const r = freeRig();
  await admitThen(r.gate, async () => { r.feed(OWN); r.feed(OWN); });
  r.feed(OWN); // still draining after the slot freed
  assert.equal(r.gate.status().inUse, false);
  const second = await admitThen(r.gate, async () => {
    r.feed(OWN);
    return "second";
  });
  assert.equal(second, "second");
  r.feed(CLEAR);
  assert.equal(r.gate.status().inUse, false, "the baseline survives back-to-back jobs, so recovery is judged against the first");
});

test("own VRAM: low VRAM that outlives the drain is the household's, and refuses", async () => {
  const r = freeRig({ sampleIntervalMs: 15 * SEC });
  await admitThen(r.gate, async () => { r.feed(OWN); });
  for (let i = 0; i < 3; i += 1) {
    r.feed(OWN);
    assert.equal(r.gate.status().inUse, false, `draining ${i}`);
  }
  r.feed(OWN); // 60 s after the release: the drain cap
  assert.equal(r.gate.status().inUse, true);
  assert.equal((await admitThen(r.gate, async () => {})).source, "admission");
});

test("own VRAM: partial recovery within the tolerance ends the drain, below it does not", async () => {
  const within = freeRig();
  await admitThen(within.gate, async () => { within.feed(OWN); });
  within.feed({ ...CLEAR, freeMiB: 14000 - 1024 });
  within.feed({ ...CLEAR, freeMiB: 5000 });
  assert.equal(within.gate.status().inUse, true, "recovered, then VRAM below the minimum counts again");

  const below = freeRig();
  await admitThen(below.gate, async () => { below.feed(OWN); });
  below.feed({ ...CLEAR, freeMiB: 14000 - 1025 });
  below.feed({ ...CLEAR, freeMiB: 5000 });
  assert.equal(below.gate.status().inUse, false, "still draining: not recovered to within the tolerance");
});

test("household signals mid-job count, and the next job is refused", async () => {
  for (const [name, sample] of [
    ["full-screen", { ...OWN, quns: 2 }],
    ["exclusive full-screen", { ...OWN, quns: 3 }],
  ]) {
    const r = freeRig();
    await admitThen(r.gate, async () => {
      r.feed(OWN);
      r.feed(sample);
    });
    const refused = await admitThen(r.gate, async () => {});
    assert.equal(refused.code, "gpu-busy", name);
    assert.equal(refused.source, "admission", name);
  }
  const sustained = freeRig();
  await admitThen(sustained.gate, async () => {
    sustained.feed({ ...OWN, util3d: 90 });
    sustained.feed({ ...OWN, util3d: 90 });
  });
  assert.equal((await admitThen(sustained.gate, async () => {})).source, "admission", "sustained 3D mid-job");
});

test("household VRAM that arrives mid-job is still there when the drain ends", async () => {
  const r = freeRig();
  await admitThen(r.gate, async () => { r.feed(OWN); });
  for (let i = 0; i < 4; i += 1) r.feed({ ...OWN, freeMiB: 600 });
  assert.equal(r.gate.status().inUse, true);
  assert.equal((await admitThen(r.gate, async () => {})).source, "admission");
});

// ---- the sample that cannot see the 3D engine ----

test("a sample with no resolved 3D counters is a probe fault, not a clear sample", () => {
  const r = rig();
  for (let i = 0; i < 20; i += 1) r.feed({ ...CLEAR, engines: 0 });
  assert.deepEqual(r.gate.status(), { inUse: true, since: r.t0, checkedAt: null, probeOk: false });
  r.feed({ ...CLEAR, engines: 1 });
  assert.equal(r.gate.status().checkedAt, r.at(), "one resolved counter is enough");
});

// ---- operator overrides ----

test("env overrides: valid values are read, absent ones are left to the defaults", () => {
  assert.deepEqual(gpuTenancyOptionsFromEnv({}), {});
  assert.deepEqual(gpuTenancyOptionsFromEnv({
    VILI_GPU_SAMPLE_MS: "5000",
    VILI_GPU_CLEAR_HOLD_MS: "0",
    VILI_GPU_MIN_FREE_MIB: "8192.5",
    VILI_GPU_UTIL3D_BUSY_PCT: "35",
  }), { sampleIntervalMs: 5000, clearHoldMs: 0, minFreeMiB: 8192.5, util3dBusyPct: 35 });
});

test("env overrides: a malformed value refuses to start and names the variable, not the value", () => {
  for (const name of ["VILI_GPU_SAMPLE_MS", "VILI_GPU_CLEAR_HOLD_MS", "VILI_GPU_MIN_FREE_MIB", "VILI_GPU_UTIL3D_BUSY_PCT"]) {
    for (const bad of ["10240MiB", "", "  ", "NaN", "Infinity", "-5"]) {
      assert.throws(
        () => gpuTenancyOptionsFromEnv({ [name]: bad }),
        (error) => error.message.includes(name) && (bad.trim() === "" || !error.message.includes(bad)),
        `${name}=${JSON.stringify(bad)}`,
      );
    }
  }
  assert.throws(() => gpuTenancyOptionsFromEnv({ VILI_GPU_SAMPLE_MS: "0" }), /VILI_GPU_SAMPLE_MS/, "a zero interval would spin the probe");
  assert.doesNotThrow(() => gpuTenancyOptionsFromEnv({ VILI_GPU_CLEAR_HOLD_MS: "0" }));
});