Add model output log inspection

This commit is contained in:
Meta Vens 2026-05-21 19:12:30 +01:00
commit cf804334d1
4 changed files with 278 additions and 0 deletions

View file

@ -29,6 +29,7 @@
"state:install-mood-task": "powershell -ExecutionPolicy Bypass -File scripts/install-void-mood-drift-task.ps1",
"state:install-repo-face-heartbeat-task": "powershell -ExecutionPolicy Bypass -File scripts/install-repo-face-heartbeat-task.ps1",
"agents:inspect-requests": "tsx scripts/inspect-agent-state-requests.ts",
"agents:inspect-model-outputs": "node scripts/inspect-model-output-logs.mjs",
"agents:assemble-character-prompt": "tsx scripts/run-repo-face-heartbeats.ts --assemble-prompt",
"agents:heartbeat-model-smoke": "node scripts/smoke-repo-face-heartbeat-model-scenarios.mjs",
"agents:interpreter-loop-smoke": "node scripts/smoke-repo-face-interpreter-loop.mjs",

View file

@ -1,3 +1,6 @@
import { appendFile, mkdir } from "node:fs/promises";
import { join } from "node:path";
import type {
Actor,
ContextBundle,
@ -11,6 +14,7 @@ import type {
import {
buildDefaultHandoffNotice,
buildRequestPayload,
type CodexRunResult,
type HistoryLookupTool,
MAX_HISTORY_TOOL_CALLS,
type OwnerCodexProviderOptions,
@ -188,6 +192,14 @@ export class OwnerCodexProvider implements ProviderAdapter {
prompt: codexPrompt,
});
turnResults.push(result);
await appendModelOutputLog({
workingDirectory: this.options.workingDirectory,
request,
command,
turn: turn + 1,
prompt: codexPrompt,
result,
});
artifacts.push(
{
@ -451,6 +463,69 @@ function isRetryableCodexModelCapacityFailure(result: { exitCode: number | null;
return /quota|rate limit|rate-limit|usage limit|capacity|too many requests|429|insufficient_quota|model.*unavailable|model.*access|limit exceeded/.test(text);
}
async function appendModelOutputLog(input: {
workingDirectory: string;
request: ProviderRequest;
command: string;
turn: number;
prompt: string;
result: CodexRunResult;
}): Promise<void> {
const jobId = getStringOption(input.request.options?.jobId) ?? "unknown-job";
const logPath = join(input.workingDirectory, ".voidbot", "logs", "model-outputs.jsonl");
const finalMessage = [...input.result.traceEvents]
.reverse()
.find((event) => event.kind === "agent_message" && event.message?.trim())?.message?.trim();
const toolCalls = input.result.traceEvents
.filter((event) => event.kind === "mcp_tool_started")
.map((event) => ({
server: event.server,
tool: event.tool,
arguments: event.arguments,
}));
const commandExecutions = input.result.traceEvents
.filter((event) => event.kind === "command_started" || event.kind === "command_completed")
.map((event) => ({
kind: event.kind,
command: event.command,
status: event.status,
exitCode: event.exitCode,
}));
const promptMarker = input.prompt.match(/<!--\s*prompt:([^>\s]+)\s*-->/)?.[1];
const record = {
schemaVersion: 1,
loggedAt: new Date().toISOString(),
jobId,
command: input.command,
turn: input.turn,
model: input.result.model,
promptMarker: promptMarker ?? null,
promptLength: input.prompt.length,
startedAt: input.result.startedAt,
finishedAt: input.result.finishedAt,
durationMs: input.result.durationMs,
exitCode: input.result.exitCode,
timedOut: input.result.timedOut,
handoffReason: input.result.handoffReason ?? null,
usage: input.result.usage ?? null,
finalMessage: finalMessage ?? null,
stdoutTail: input.result.stdout.slice(-4000),
stderrTail: input.result.stderr.slice(-4000),
toolCalls,
commandExecutions,
artifactRefs: {
prompt: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-prompt.md`,
stdout: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-stdout.txt`,
stderr: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-stderr.txt`,
trace: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-trace.json`,
debug: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-debug.md`,
},
};
await mkdir(join(input.workingDirectory, ".voidbot", "logs"), { recursive: true });
await appendFile(logPath, `${JSON.stringify(record)}\n`, "utf8");
}
function getReasoningEffortOption(value: unknown): "low" | "medium" | "high" | "xhigh" | undefined {
if (value === "low" || value === "medium" || value === "high" || value === "xhigh") {
return value;

View file

@ -0,0 +1,201 @@
#!/usr/bin/env node
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
const repoRoot = resolve(import.meta.dirname, "..");
async function main() {
const options = parseArgs(process.argv.slice(2));
const hours = Number.parseFloat(options["since-hours"] ?? "1");
const sinceMs = Date.now() - Math.max(0.01, hours) * 60 * 60 * 1000;
const logPath = resolve(repoRoot, options.log ?? ".voidbot/logs/model-outputs.jsonl");
const markdownOut = resolve(repoRoot, options.out ?? ".voidbot/status/model-output-inspection.md");
const jsonOut = resolve(repoRoot, options["json-out"] ?? ".voidbot/status/model-output-inspection.json");
const records = (await readJsonl(logPath))
.filter((record) => Date.parse(String(record.loggedAt ?? record.finishedAt ?? "")) >= sinceMs);
const findings = inspect(records);
const report = {
generatedAt: new Date().toISOString(),
logPath,
sinceHours: hours,
recordCount: records.length,
...findings,
};
await mkdir(dirname(markdownOut), { recursive: true });
await writeFile(jsonOut, `${JSON.stringify(report, null, 2)}\n`, "utf8");
await writeFile(markdownOut, renderMarkdown(report), "utf8");
process.stdout.write(`${JSON.stringify({ ok: true, markdownOut, jsonOut, recordCount: records.length, warningCount: findings.warnings.length }, null, 2)}\n`);
}
async function readJsonl(path) {
try {
const text = await readFile(path, "utf8");
return text
.split(/\r?\n/)
.filter(Boolean)
.map((line) => {
try {
return JSON.parse(line);
} catch {
return { malformed: true, raw: line };
}
});
} catch (error) {
if (error && typeof error === "object" && error.code === "ENOENT") {
return [];
}
throw error;
}
}
function inspect(records) {
const warnings = [];
const byModel = countBy(records, (record) => record.model ?? "(unknown)");
const byCommand = countBy(records, (record) => record.command ?? "(unknown)");
const byPrompt = countBy(records, (record) => record.promptMarker ?? "(none)");
const failures = records.filter((record) => record.exitCode !== 0 || record.timedOut || record.handoffReason);
const noFinalMessage = records.filter((record) => !String(record.finalMessage ?? "").trim());
const forbiddenToolRecords = records.filter((record) =>
(record.toolCalls ?? []).some((call) =>
["read_repo_face_state", "list_mcp_resources", "read_mcp_resource", "post_repo_identity_message", "apply_repo_face_state_operation", "notify_owner"].includes(call.tool),
),
);
const speechSmells = records.filter((record) =>
/repo-face heartbeat|heartbeat from|bright bridge note|tiny fish sorting note|librarian note|maintenance pass/i.test(String(record.finalMessage ?? "")),
);
const quotaSignals = records.filter((record) =>
/quota|rate limit|rate-limit|usage limit|capacity|too many requests|429|insufficient_quota|model.*unavailable|model.*access|limit exceeded/i.test(
`${record.stderrTail ?? ""}\n${record.stdoutTail ?? ""}\n${record.handoffReason ?? ""}`,
),
);
pushWarning(warnings, failures.length, "model run failures/timeouts/handoffs", failures);
pushWarning(warnings, noFinalMessage.length, "model runs with no final message", noFinalMessage);
pushWarning(warnings, forbiddenToolRecords.length, "Face/model runs using forbidden substrate tools", forbiddenToolRecords);
pushWarning(warnings, speechSmells.length, "robotic/provenance speech smells", speechSmells);
pushWarning(warnings, quotaSignals.length, "quota/rate/capacity fallback signals", quotaSignals);
return {
byModel,
byCommand,
byPrompt,
failures: summarizeRecords(failures),
noFinalMessage: summarizeRecords(noFinalMessage),
forbiddenToolRecords: summarizeRecords(forbiddenToolRecords),
speechSmells: summarizeRecords(speechSmells),
quotaSignals: summarizeRecords(quotaSignals),
warnings,
recent: summarizeRecords(records.slice(-12)),
};
}
function pushWarning(warnings, count, label, records) {
if (count <= 0) {
return;
}
warnings.push({
label,
count,
examples: summarizeRecords(records.slice(0, 5)),
});
}
function summarizeRecords(records) {
return records.map((record) => ({
loggedAt: record.loggedAt ?? null,
jobId: record.jobId ?? null,
command: record.command ?? null,
turn: record.turn ?? null,
model: record.model ?? null,
promptMarker: record.promptMarker ?? null,
exitCode: record.exitCode ?? null,
timedOut: Boolean(record.timedOut),
handoffReason: record.handoffReason ?? null,
tools: (record.toolCalls ?? []).map((call) => call.tool).filter(Boolean),
finalPreview: preview(record.finalMessage ?? record.stdoutTail ?? ""),
}));
}
function countBy(records, keyFn) {
const counts = {};
for (const record of records) {
const key = String(keyFn(record));
counts[key] = (counts[key] ?? 0) + 1;
}
return counts;
}
function preview(value) {
const collapsed = String(value).replace(/\s+/g, " ").trim();
return collapsed.length > 240 ? `${collapsed.slice(0, 237)}...` : collapsed;
}
function renderMarkdown(report) {
const lines = [
"# Model Output Inspection",
"",
`Generated: ${report.generatedAt}`,
`Window: last ${report.sinceHours} hour(s)`,
`Log: ${report.logPath}`,
`Records: ${report.recordCount}`,
"",
"## Counts",
"",
`- By model: ${formatCounts(report.byModel)}`,
`- By command: ${formatCounts(report.byCommand)}`,
`- By prompt: ${formatCounts(report.byPrompt)}`,
"",
"## Warnings",
"",
];
if (report.warnings.length === 0) {
lines.push("- No inspection warnings in this window.");
} else {
for (const warning of report.warnings) {
lines.push(`- ${warning.label}: ${warning.count}`);
for (const example of warning.examples) {
lines.push(` - ${example.loggedAt} ${example.model} ${example.command} ${example.promptMarker}: ${example.finalPreview || example.handoffReason || "(no preview)"}`);
}
}
}
lines.push("", "## Recent Records", "");
for (const record of report.recent) {
lines.push(`- ${record.loggedAt} ${record.model} ${record.command} ${record.promptMarker} tools=[${record.tools.join(", ")}]`);
if (record.finalPreview) {
lines.push(` ${record.finalPreview}`);
}
}
return `${lines.join("\n")}\n`;
}
function formatCounts(counts) {
const entries = Object.entries(counts);
return entries.length > 0
? entries.map(([key, value]) => `${key}=${value}`).join(", ")
: "(none)";
}
function parseArgs(args) {
const options = {};
for (let index = 0; index < args.length; index += 1) {
const token = args[index];
if (!token.startsWith("--")) {
continue;
}
const key = token.slice(2);
const next = args[index + 1];
if (!next || next.startsWith("--")) {
options[key] = true;
continue;
}
options[key] = next;
index += 1;
}
return options;
}
await main();

View file

@ -242,3 +242,4 @@
{"ts":"2026-05-21T16:28:10Z","type":"implementation","status":"ok","note":"Added universal repo Face alienation/social-placement affect need. Newborn and active Faces now carry a social_contact need that should resolve through bonds, rivalries, status reads, and trusted roles; social embodiment prompt now explicitly tells Faces to form real relationships when projected alienation is live."}
{"ts":"2026-05-21T17:44:48Z","type":"implementation","status":"ok","note":"Switched repo Face heartbeat/dry-run defaults from gpt-5.3-codex-spark to gpt-5.4-mini after pricing check; dry-run harness now defaults to real read-only retrieval with forbidden child-tool detection, and worker retries/drops Face turns that touch private state or MCP inventory tools."}
{"ts":"2026-05-21T17:53:10Z","type":"implementation","status":"ok","note":"Added repo Face Codex model preference chain. Heartbeat jobs now pass REPO_FACE_HEARTBEAT_CODEX_MODELS through owner Codex, which tries Spark first and automatically falls back to gpt-5.4-mini on quota/rate/capacity/model-access failures while recording the model used in trace artifacts."}
{"ts":"2026-05-21T18:12:10Z","type":"implementation","status":"ok","note":"Added central model-output JSONL logging for owner-Codex model attempts and an hourly Codex automation that runs scripts/inspect-model-output-logs.mjs over the last hour for observability-only warnings."}