VoidBot/scripts/inspect-model-output-logs.mjs

#!/usr/bin/env node
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";

const repoRoot = resolve(import.meta.dirname, "..");

async function main() {
  const options = parseArgs(process.argv.slice(2));
  const hours = Number.parseFloat(options["since-hours"] ?? "1");
  const sinceMs = Date.now() - Math.max(0.01, hours) * 60 * 60 * 1000;
  const logPath = resolve(repoRoot, options.log ?? ".voidbot/logs/model-outputs.jsonl");
  const markdownOut = resolve(repoRoot, options.out ?? ".voidbot/status/model-output-inspection.md");
  const jsonOut = resolve(repoRoot, options["json-out"] ?? ".voidbot/status/model-output-inspection.json");
  const records = (await readJsonl(logPath))
    .filter((record) => Date.parse(String(record.loggedAt ?? record.finishedAt ?? "")) >= sinceMs);
  const findings = inspect(records);
  const report = {
    generatedAt: new Date().toISOString(),
    logPath,
    sinceHours: hours,
    recordCount: records.length,
    ...findings,
  };

  await mkdir(dirname(markdownOut), { recursive: true });
  await writeFile(jsonOut, `${JSON.stringify(report, null, 2)}\n`, "utf8");
  await writeFile(markdownOut, renderMarkdown(report), "utf8");
  process.stdout.write(`${JSON.stringify({ ok: true, markdownOut, jsonOut, recordCount: records.length, warningCount: findings.warnings.length }, null, 2)}\n`);
}

async function readJsonl(path) {
  try {
    const text = await readFile(path, "utf8");
    return text
      .split(/\r?\n/)
      .filter(Boolean)
      .map((line) => {
        try {
          return JSON.parse(line);
        } catch {
          return { malformed: true, raw: line };
        }
      });
  } catch (error) {
    if (error && typeof error === "object" && error.code === "ENOENT") {
      return [];
    }
    throw error;
  }
}

function inspect(records) {
  const warnings = [];
  const byModel = countBy(records, (record) => record.model ?? "(unknown)");
  const byCommand = countBy(records, (record) => record.command ?? "(unknown)");
  const byPrompt = countBy(records, (record) => record.promptMarker ?? "(none)");
  const failures = records.filter((record) => record.exitCode !== 0 || record.timedOut || record.handoffReason);
  const noFinalMessage = records.filter((record) => !String(record.finalMessage ?? "").trim());
  const forbiddenToolRecords = records.filter((record) =>
    (record.toolCalls ?? []).some((call) =>
      ["read_repo_face_state", "list_mcp_resources", "read_mcp_resource", "post_repo_identity_message", "apply_repo_face_state_operation", "notify_owner"].includes(call.tool),
    ),
  );
  const speechSmells = records.filter((record) =>
    /repo-face heartbeat|heartbeat from|bright bridge note|tiny fish sorting note|librarian note|maintenance pass/i.test(String(record.finalMessage ?? "")),
  );
  const quotaSignals = records.filter((record) =>
    /quota|rate limit|rate-limit|usage limit|capacity|too many requests|(?:http|status|code|error)\s*429|429\s*(?:too many requests|rate)|insufficient_quota|model.*unavailable|model.*access|limit exceeded/i.test(
      `${record.stderrTail ?? ""}\n${record.stdoutTail ?? ""}\n${record.handoffReason ?? ""}`,
    ),
  );

  pushWarning(warnings, failures.length, "model run failures/timeouts/handoffs", failures);
  pushWarning(warnings, noFinalMessage.length, "model runs with no final message", noFinalMessage);
  pushWarning(warnings, forbiddenToolRecords.length, "Face/model runs using forbidden substrate tools", forbiddenToolRecords);
  pushWarning(warnings, speechSmells.length, "robotic/provenance speech smells", speechSmells);
  pushWarning(warnings, quotaSignals.length, "quota/rate/capacity fallback signals", quotaSignals);

  return {
    byModel,
    byCommand,
    byPrompt,
    failures: summarizeRecords(failures),
    noFinalMessage: summarizeRecords(noFinalMessage),
    forbiddenToolRecords: summarizeRecords(forbiddenToolRecords),
    speechSmells: summarizeRecords(speechSmells),
    quotaSignals: summarizeRecords(quotaSignals),
    warnings,
    recent: summarizeRecords(records.slice(-12)),
  };
}

function pushWarning(warnings, count, label, records) {
  if (count <= 0) {
    return;
  }
  warnings.push({
    label,
    count,
    examples: summarizeRecords(records.slice(0, 5)),
  });
}

function summarizeRecords(records) {
  return records.map((record) => ({
    loggedAt: record.loggedAt ?? null,
    jobId: record.jobId ?? null,
    command: record.command ?? null,
    turn: record.turn ?? null,
    model: record.model ?? null,
    promptMarker: record.promptMarker ?? null,
    exitCode: record.exitCode ?? null,
    timedOut: Boolean(record.timedOut),
    handoffReason: record.handoffReason ?? null,
    tools: (record.toolCalls ?? []).map((call) => call.tool).filter(Boolean),
    finalPreview: preview(record.finalMessage ?? record.stdoutTail ?? ""),
  }));
}

function countBy(records, keyFn) {
  const counts = {};
  for (const record of records) {
    const key = String(keyFn(record));
    counts[key] = (counts[key] ?? 0) + 1;
  }
  return counts;
}

function preview(value) {
  const collapsed = String(value).replace(/\s+/g, " ").trim();
  return collapsed.length > 240 ? `${collapsed.slice(0, 237)}...` : collapsed;
}

function renderMarkdown(report) {
  const lines = [
    "# Model Output Inspection",
    "",
    `Generated: ${report.generatedAt}`,
    `Window: last ${report.sinceHours} hour(s)`,
    `Log: ${report.logPath}`,
    `Records: ${report.recordCount}`,
    "",
    "## Counts",
    "",
    `- By model: ${formatCounts(report.byModel)}`,
    `- By command: ${formatCounts(report.byCommand)}`,
    `- By prompt: ${formatCounts(report.byPrompt)}`,
    "",
    "## Warnings",
    "",
  ];

  if (report.warnings.length === 0) {
    lines.push("- No inspection warnings in this window.");
  } else {
    for (const warning of report.warnings) {
      lines.push(`- ${warning.label}: ${warning.count}`);
      for (const example of warning.examples) {
        lines.push(`  - ${example.loggedAt} ${example.model} ${example.command} ${example.promptMarker}: ${example.finalPreview || example.handoffReason || "(no preview)"}`);
      }
    }
  }

  lines.push("", "## Recent Records", "");
  for (const record of report.recent) {
    lines.push(`- ${record.loggedAt} ${record.model} ${record.command} ${record.promptMarker} tools=[${record.tools.join(", ")}]`);
    if (record.finalPreview) {
      lines.push(`  ${record.finalPreview}`);
    }
  }

  return `${lines.join("\n")}\n`;
}

function formatCounts(counts) {
  const entries = Object.entries(counts);
  return entries.length > 0
    ? entries.map(([key, value]) => `${key}=${value}`).join(", ")
    : "(none)";
}

function parseArgs(args) {
  const options = {};
  for (let index = 0; index < args.length; index += 1) {
    const token = args[index];
    if (!token.startsWith("--")) {
      continue;
    }
    const key = token.slice(2);
    const next = args[index + 1];
    if (!next || next.startsWith("--")) {
      options[key] = true;
      continue;
    }
    options[key] = next;
    index += 1;
  }
  return options;
}

await main();