Probe: own module, 3D counter path as a tested constant, engines count, hung-child watchdog

The probe's sample plumbing leaves the gate module (the gate never kills; the probe may kill
its own hung child). The counter path is one exported constant passed to the script as
-Util3dPath, the script reports how many 3D engine counters resolved, and a child that is
alive but silent for 3 intervals is killed and respawned.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/gpu-probe.mjs

@@ -0,0 +1,84 @@
+// The sample source for the GPU tenancy gate: one long-lived hidden PowerShell child in the daemon's
+// own session. It decides nothing; it only turns the child's stdout into samples and keeps the child
+// alive.
+
+// The 3D-engine counter the probe reads. A wrong path reads as zero without an error, so the path is
+// one constant, pinned by a test, and the probe reports how many engine counters it resolved.
+export const UTIL3D_COUNTER_PATH = String.raw`\GPU Engine(*engtype_3D)\Utilization Percentage`;
+
+// A probe line is JSON; only the four finite numbers survive, so nothing else can reach the gate.
+export function parseSample(line) {
+  let value;
+  try {
+    value = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  if (value === null || typeof value !== "object") return null;
+  const { quns, freeMiB, util3d, engines } = value;
+  for (const field of [quns, freeMiB, util3d, engines]) {
+    if (typeof field !== "number" || !Number.isFinite(field)) return null;
+  }
+  return { quns, freeMiB, util3d, engines };
+}
+
+// The child exits with the daemon (the script watches the parent pid). If it exits on its own it is
+// respawned at most once per interval. A child that stays alive but sends no sample for staleIntervals
+// intervals is hung: it is killed and respawned the same way.
+export function startProbe({ spawn, scriptPath, intervalMs, onSample, parentPid = process.pid, staleIntervals = 3 }) {
+  let stopped = false;
+  let lastSpawnAt = 0;
+  let lastSampleAt = 0;
+  let current = null;
+
+  function scheduleRespawn(entry) {
+    if (entry.respawned) return;
+    entry.respawned = true;
+    if (current === entry) current = null;
+    setTimeout(launch, Math.max(0, lastSpawnAt + intervalMs - Date.now())).unref();
+  }
+
+  function launch() {
+    if (stopped) return;
+    lastSpawnAt = Date.now();
+    lastSampleAt = lastSpawnAt;
+    const child = spawn("powershell.exe", [
+      "-NoProfile",
+      "-NonInteractive",
+      "-ExecutionPolicy", "Bypass",
+      "-File", scriptPath,
+      "-ParentPid", String(parentPid),
+      "-IntervalMs", String(intervalMs),
+      "-Util3dPath", UTIL3D_COUNTER_PATH,
+    ], { windowsHide: true, stdio: ["ignore", "pipe", "ignore"] });
+    const entry = { child, respawned: false };
+    current = entry;
+    let pending = "";
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      pending += chunk;
+      const lines = pending.split("\n");
+      pending = lines.pop();
+      for (const line of lines) {
+        const sample = parseSample(line.trim());
+        if (sample) {
+          lastSampleAt = Date.now();
+          onSample(sample);
+        }
+      }
+    });
+    child.on("exit", () => scheduleRespawn(entry));
+    child.on("error", () => scheduleRespawn(entry));
+  }
+
+  const watchdog = setInterval(() => {
+    if (stopped || current === null || Date.now() - lastSampleAt <= staleIntervals * intervalMs) return;
+    const hung = current;
+    scheduleRespawn(hung);
+    hung.child.kill();
+  }, intervalMs);
+  watchdog.unref();
+
+  launch();
+  return { stop: () => { stopped = true; clearInterval(watchdog); } };
+}

scripts/gpu-tenancy-probe.ps1

@@ -1,5 +1,7 @@
-# One sample line per interval: {"quns":n,"freeMiB":n,"util3d":n}. Numbers only; no process identity is read.
-param([int]$ParentPid, [int]$IntervalMs = 15000)
+# One sample line per interval: {"quns":n,"freeMiB":n,"util3d":n,"engines":n}. Numbers only; no process
+# identity is read. engines is how many 3D-engine counters the path resolved: zero means the path is
+# wrong, and the gate treats that as a probe fault.
+param([int]$ParentPid, [int]$IntervalMs = 15000, [string]$Util3dPath)
 $ErrorActionPreference = 'Stop'
 Add-Type -Namespace Vili -Name Shell -MemberDefinition '[DllImport("shell32.dll")] public static extern int SHQueryUserNotificationState(out int state);'
 while ($true) {
@@ -11,8 +13,9 @@ while ($true) {
     $smi = @(nvidia-smi.exe --query-gpu=memory.free --format=csv,noheader,nounits)
     if ($LASTEXITCODE -ne 0) { throw 'smi' }
     $free = $smi[0]
-    $sum = (Get-Counter '\GPU Engine(*engtype_3D)\Utilization Percentage' -SampleInterval 1 -MaxSamples 1).CounterSamples | Measure-Object -Property CookedValue -Sum
-    $line = [ordered]@{ quns = [int]$quns; freeMiB = [int]$free; util3d = [math]::Min(100, [math]::Round([double]$sum.Sum, 1)) } | ConvertTo-Json -Compress
+    $counters = @((Get-Counter $Util3dPath -SampleInterval 1 -MaxSamples 1).CounterSamples)
+    $sum = $counters | Measure-Object -Property CookedValue -Sum
+    $line = [ordered]@{ quns = [int]$quns; freeMiB = [int]$free; util3d = [math]::Min(100, [math]::Round([double]$sum.Sum, 1)); engines = $counters.Count } | ConvertTo-Json -Compress
     [Console]::Out.WriteLine($line)
     [Console]::Out.Flush()
   } catch { }

scripts/test-gpu-probe.mjs

@@ -0,0 +1,167 @@
+import assert from "node:assert/strict";
+import { EventEmitter } from "node:events";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import { UTIL3D_COUNTER_PATH, parseSample, startProbe } from "./gpu-probe.mjs";
+import { createGpuTenancy } from "./gpu-tenancy.mjs";
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+test("no program name canary", () => {
+  const canary = "canary-vili-7f3a";
+  const line = JSON.stringify({ quns: 3, freeMiB: 2048, util3d: 55.5, engines: 8, name: canary, process: canary, instance: canary });
+  const sample = parseSample(line);
+  assert.deepEqual(sample, { quns: 3, freeMiB: 2048, util3d: 55.5, engines: 8 });
+  assert.deepEqual(Object.keys(sample).sort(), ["engines", "freeMiB", "quns", "util3d"]);
+  const gate = createGpuTenancy({ now: () => 1_000_000 });
+  gate.observe(sample);
+  assert.equal(JSON.stringify(gate.status()).includes(canary), false);
+});
+
+test("parseSample drops anything that is not four finite numbers", () => {
+  const ok = { quns: 1, freeMiB: 1, util3d: 1, engines: 1 };
+  assert.deepEqual(parseSample(JSON.stringify(ok)), ok);
+  assert.equal(parseSample("not json"), null);
+  assert.equal(parseSample("null"), null);
+  assert.equal(parseSample("[1,2,3]"), null);
+  assert.equal(parseSample(""), null);
+  for (const field of Object.keys(ok)) {
+    const missing = { ...ok };
+    delete missing[field];
+    assert.equal(parseSample(JSON.stringify(missing)), null, `${field} missing`);
+    assert.equal(parseSample(JSON.stringify({ ...ok, [field]: "1" })), null, `${field} as a string`);
+  }
+  assert.equal(parseSample('{"quns":1,"freeMiB":1,"util3d":1e999,"engines":1}'), null);
+});
+
+test("the 3D counter path is the one constant, and the script has no copy of it", async () => {
+  assert.equal(UTIL3D_COUNTER_PATH, "\\GPU Engine(*engtype_3D)\\Utilization Percentage");
+  const script = await readFile(new URL("./gpu-tenancy-probe.ps1", import.meta.url), "utf8");
+  assert.equal(/GPU Engine|engtype/.test(script.replace(/^#.*$/gm, "")), false, "the path reaches the script as -Util3dPath only");
+  assert.match(script, /Get-Counter \$Util3dPath/);
+  assert.match(script, /engines = \$counters\.Count/);
+});
+
+test("a probe that resolved no 3D counters keeps the gate failing closed", () => {
+  let t = 1_000_000;
+  const gate = createGpuTenancy({ now: () => t });
+  const line = (engines) => JSON.stringify({ quns: 1, freeMiB: 14000, util3d: 0, engines });
+  for (let i = 0; i < 20; i += 1) {
+    t += 15_000;
+    gate.observe(parseSample(line(0)));
+  }
+  assert.deepEqual(gate.status(), { inUse: true, since: 1_000_000, checkedAt: null, probeOk: false });
+  for (let i = 0; i < 13; i += 1) {
+    t += 15_000;
+    gate.observe(parseSample(line(8)));
+  }
+  assert.equal(gate.status().inUse, false, "the same clear samples with a resolved counter do clear the gate");
+  assert.equal(gate.status().probeOk, true);
+});
+
+function fakeSpawn() {
+  const spawned = [];
+  const spawn = (command, args, options) => {
+    const child = new EventEmitter();
+    child.stdout = new EventEmitter();
+    child.stdout.setEncoding = (encoding) => { child.encoding = encoding; };
+    child.kills = 0;
+    child.kill = () => { child.kills += 1; };
+    spawned.push({ command, args, options, child });
+    return child;
+  };
+  return { spawn, spawned };
+}
+
+test("probe: one hidden child, lines become samples, junk is dropped", () => {
+  const { spawn, spawned } = fakeSpawn();
+  const samples = [];
+  const probe = startProbe({ spawn, scriptPath: "gpu-tenancy-probe.ps1", intervalMs: 15000, onSample: (s) => samples.push(s), parentPid: 4242 });
+  assert.equal(spawned.length, 1);
+  const { command, args, options, child } = spawned[0];
+  assert.equal(command, "powershell.exe");
+  assert.equal(options.windowsHide, true);
+  assert.deepEqual(options.stdio, ["ignore", "pipe", "ignore"]);
+  assert.deepEqual(args, [
+    "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
+    "-File", "gpu-tenancy-probe.ps1", "-ParentPid", "4242", "-IntervalMs", "15000",
+    "-Util3dPath", UTIL3D_COUNTER_PATH,
+  ]);
+  assert.equal(child.encoding, "utf8");
+
+  child.stdout.emit("data", '{"quns":1,"freeMiB":14000,"util3d":2,"engines":4}\n{"quns":2,"free');
+  child.stdout.emit("data", 'MiB":9000,"util3d":3,"engines":4}\r\ngarbage\n');
+  assert.deepEqual(samples, [
+    { quns: 1, freeMiB: 14000, util3d: 2, engines: 4 },
+    { quns: 2, freeMiB: 9000, util3d: 3, engines: 4 },
+  ]);
+  probe.stop();
+});
+
+test("probe: respawns at most once per interval after the child exits", async () => {
+  const { spawn, spawned } = fakeSpawn();
+  const probe = startProbe({ spawn, scriptPath: "p.ps1", intervalMs: 80, onSample: () => {} });
+  spawned[0].child.emit("exit", 1);
+  spawned[0].child.emit("error", new Error("again"));
+  spawned[0].child.emit("exit", 1);
+  await sleep(30);
+  assert.equal(spawned.length, 1, "no respawn inside the interval");
+  await sleep(170);
+  assert.equal(spawned.length, 2, "one respawn for one death, however many events");
+  probe.stop();
+  spawned[1].child.emit("exit", 1);
+  await sleep(200);
+  assert.equal(spawned.length, 2, "a stopped probe does not respawn");
+});
+
+test("probe: a plain exit and a spawn error each respawn", async () => {
+  const exited = fakeSpawn();
+  const a = startProbe({ spawn: exited.spawn, scriptPath: "p.ps1", intervalMs: 20, onSample: () => {} });
+  exited.spawned[0].child.emit("exit", 0);
+  await sleep(120);
+  assert.equal(exited.spawned.length, 2);
+  a.stop();
+  const failed = fakeSpawn();
+  const b = startProbe({ spawn: failed.spawn, scriptPath: "p.ps1", intervalMs: 20, onSample: () => {} });
+  failed.spawned[0].child.emit("error", new Error("ENOENT"));
+  await sleep(120);
+  assert.equal(failed.spawned.length, 2);
+  b.stop();
+});
+
+test("probe watchdog: a child that stays alive but sends nothing is killed and respawned once", async () => {
+  const { spawn, spawned } = fakeSpawn();
+  const probe = startProbe({ spawn, scriptPath: "p.ps1", intervalMs: 40, onSample: () => {} });
+  await sleep(90);
+  assert.equal(spawned[0].child.kills, 0, "under 3 intervals of silence the child is left alone");
+  assert.equal(spawned.length, 1);
+  await sleep(200);
+  assert.equal(spawned[0].child.kills, 1, "the hung child is killed exactly once");
+  assert.equal(spawned.length, 2, "and replaced");
+  spawned[0].child.emit("exit", 1);
+  const line = '{"quns":1,"freeMiB":14000,"util3d":1,"engines":4}
+';
+  for (let i = 0; i < 4; i += 1) {
+    await sleep(40);
+    spawned[1].child.stdout.emit("data", line);
+  }
+  assert.equal(spawned.length, 2, "the killed child's exit adds no second respawn");
+  assert.equal(spawned[1].child.kills, 0);
+  probe.stop();
+});
+
+test("probe watchdog: samples keep a child alive, and a dead child is not killed again", async () => {
+  const { spawn, spawned } = fakeSpawn();
+  const probe = startProbe({ spawn, scriptPath: "p.ps1", intervalMs: 40, onSample: () => {} });
+  const line = '{"quns":1,"freeMiB":14000,"util3d":1,"engines":4}\n';
+  for (let i = 0; i < 10; i += 1) {
+    await sleep(40);
+    spawned[0].child.stdout.emit("data", line);
+  }
+  assert.equal(spawned[0].child.kills, 0, "a child that samples is never killed");
+  assert.equal(spawned.length, 1);
+  spawned[0].child.emit("exit", 1);
+  await sleep(400);
+  assert.equal(spawned[0].child.kills, 0, "a child that already exited is not killed");
+  probe.stop();
+});