Add model output log inspection
This commit is contained in:
parent
8de9c3c3e0
commit
cf804334d1
4 changed files with 278 additions and 0 deletions
|
|
@ -29,6 +29,7 @@
|
|||
"state:install-mood-task": "powershell -ExecutionPolicy Bypass -File scripts/install-void-mood-drift-task.ps1",
|
||||
"state:install-repo-face-heartbeat-task": "powershell -ExecutionPolicy Bypass -File scripts/install-repo-face-heartbeat-task.ps1",
|
||||
"agents:inspect-requests": "tsx scripts/inspect-agent-state-requests.ts",
|
||||
"agents:inspect-model-outputs": "node scripts/inspect-model-output-logs.mjs",
|
||||
"agents:assemble-character-prompt": "tsx scripts/run-repo-face-heartbeats.ts --assemble-prompt",
|
||||
"agents:heartbeat-model-smoke": "node scripts/smoke-repo-face-heartbeat-model-scenarios.mjs",
|
||||
"agents:interpreter-loop-smoke": "node scripts/smoke-repo-face-interpreter-loop.mjs",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
import { appendFile, mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import type {
|
||||
Actor,
|
||||
ContextBundle,
|
||||
|
|
@ -11,6 +14,7 @@ import type {
|
|||
import {
|
||||
buildDefaultHandoffNotice,
|
||||
buildRequestPayload,
|
||||
type CodexRunResult,
|
||||
type HistoryLookupTool,
|
||||
MAX_HISTORY_TOOL_CALLS,
|
||||
type OwnerCodexProviderOptions,
|
||||
|
|
@ -188,6 +192,14 @@ export class OwnerCodexProvider implements ProviderAdapter {
|
|||
prompt: codexPrompt,
|
||||
});
|
||||
turnResults.push(result);
|
||||
await appendModelOutputLog({
|
||||
workingDirectory: this.options.workingDirectory,
|
||||
request,
|
||||
command,
|
||||
turn: turn + 1,
|
||||
prompt: codexPrompt,
|
||||
result,
|
||||
});
|
||||
|
||||
artifacts.push(
|
||||
{
|
||||
|
|
@ -451,6 +463,69 @@ function isRetryableCodexModelCapacityFailure(result: { exitCode: number | null;
|
|||
return /quota|rate limit|rate-limit|usage limit|capacity|too many requests|429|insufficient_quota|model.*unavailable|model.*access|limit exceeded/.test(text);
|
||||
}
|
||||
|
||||
async function appendModelOutputLog(input: {
|
||||
workingDirectory: string;
|
||||
request: ProviderRequest;
|
||||
command: string;
|
||||
turn: number;
|
||||
prompt: string;
|
||||
result: CodexRunResult;
|
||||
}): Promise<void> {
|
||||
const jobId = getStringOption(input.request.options?.jobId) ?? "unknown-job";
|
||||
const logPath = join(input.workingDirectory, ".voidbot", "logs", "model-outputs.jsonl");
|
||||
const finalMessage = [...input.result.traceEvents]
|
||||
.reverse()
|
||||
.find((event) => event.kind === "agent_message" && event.message?.trim())?.message?.trim();
|
||||
const toolCalls = input.result.traceEvents
|
||||
.filter((event) => event.kind === "mcp_tool_started")
|
||||
.map((event) => ({
|
||||
server: event.server,
|
||||
tool: event.tool,
|
||||
arguments: event.arguments,
|
||||
}));
|
||||
const commandExecutions = input.result.traceEvents
|
||||
.filter((event) => event.kind === "command_started" || event.kind === "command_completed")
|
||||
.map((event) => ({
|
||||
kind: event.kind,
|
||||
command: event.command,
|
||||
status: event.status,
|
||||
exitCode: event.exitCode,
|
||||
}));
|
||||
const promptMarker = input.prompt.match(/<!--\s*prompt:([^>\s]+)\s*-->/)?.[1];
|
||||
const record = {
|
||||
schemaVersion: 1,
|
||||
loggedAt: new Date().toISOString(),
|
||||
jobId,
|
||||
command: input.command,
|
||||
turn: input.turn,
|
||||
model: input.result.model,
|
||||
promptMarker: promptMarker ?? null,
|
||||
promptLength: input.prompt.length,
|
||||
startedAt: input.result.startedAt,
|
||||
finishedAt: input.result.finishedAt,
|
||||
durationMs: input.result.durationMs,
|
||||
exitCode: input.result.exitCode,
|
||||
timedOut: input.result.timedOut,
|
||||
handoffReason: input.result.handoffReason ?? null,
|
||||
usage: input.result.usage ?? null,
|
||||
finalMessage: finalMessage ?? null,
|
||||
stdoutTail: input.result.stdout.slice(-4000),
|
||||
stderrTail: input.result.stderr.slice(-4000),
|
||||
toolCalls,
|
||||
commandExecutions,
|
||||
artifactRefs: {
|
||||
prompt: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-prompt.md`,
|
||||
stdout: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-stdout.txt`,
|
||||
stderr: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-stderr.txt`,
|
||||
trace: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-trace.json`,
|
||||
debug: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-debug.md`,
|
||||
},
|
||||
};
|
||||
|
||||
await mkdir(join(input.workingDirectory, ".voidbot", "logs"), { recursive: true });
|
||||
await appendFile(logPath, `${JSON.stringify(record)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function getReasoningEffortOption(value: unknown): "low" | "medium" | "high" | "xhigh" | undefined {
|
||||
if (value === "low" || value === "medium" || value === "high" || value === "xhigh") {
|
||||
return value;
|
||||
|
|
|
|||
201
scripts/inspect-model-output-logs.mjs
Normal file
201
scripts/inspect-model-output-logs.mjs
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
#!/usr/bin/env node
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
const repoRoot = resolve(import.meta.dirname, "..");
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const hours = Number.parseFloat(options["since-hours"] ?? "1");
|
||||
const sinceMs = Date.now() - Math.max(0.01, hours) * 60 * 60 * 1000;
|
||||
const logPath = resolve(repoRoot, options.log ?? ".voidbot/logs/model-outputs.jsonl");
|
||||
const markdownOut = resolve(repoRoot, options.out ?? ".voidbot/status/model-output-inspection.md");
|
||||
const jsonOut = resolve(repoRoot, options["json-out"] ?? ".voidbot/status/model-output-inspection.json");
|
||||
const records = (await readJsonl(logPath))
|
||||
.filter((record) => Date.parse(String(record.loggedAt ?? record.finishedAt ?? "")) >= sinceMs);
|
||||
const findings = inspect(records);
|
||||
const report = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
logPath,
|
||||
sinceHours: hours,
|
||||
recordCount: records.length,
|
||||
...findings,
|
||||
};
|
||||
|
||||
await mkdir(dirname(markdownOut), { recursive: true });
|
||||
await writeFile(jsonOut, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
await writeFile(markdownOut, renderMarkdown(report), "utf8");
|
||||
process.stdout.write(`${JSON.stringify({ ok: true, markdownOut, jsonOut, recordCount: records.length, warningCount: findings.warnings.length }, null, 2)}\n`);
|
||||
}
|
||||
|
||||
async function readJsonl(path) {
|
||||
try {
|
||||
const text = await readFile(path, "utf8");
|
||||
return text
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return { malformed: true, raw: line };
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
if (error && typeof error === "object" && error.code === "ENOENT") {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function inspect(records) {
|
||||
const warnings = [];
|
||||
const byModel = countBy(records, (record) => record.model ?? "(unknown)");
|
||||
const byCommand = countBy(records, (record) => record.command ?? "(unknown)");
|
||||
const byPrompt = countBy(records, (record) => record.promptMarker ?? "(none)");
|
||||
const failures = records.filter((record) => record.exitCode !== 0 || record.timedOut || record.handoffReason);
|
||||
const noFinalMessage = records.filter((record) => !String(record.finalMessage ?? "").trim());
|
||||
const forbiddenToolRecords = records.filter((record) =>
|
||||
(record.toolCalls ?? []).some((call) =>
|
||||
["read_repo_face_state", "list_mcp_resources", "read_mcp_resource", "post_repo_identity_message", "apply_repo_face_state_operation", "notify_owner"].includes(call.tool),
|
||||
),
|
||||
);
|
||||
const speechSmells = records.filter((record) =>
|
||||
/repo-face heartbeat|heartbeat from|bright bridge note|tiny fish sorting note|librarian note|maintenance pass/i.test(String(record.finalMessage ?? "")),
|
||||
);
|
||||
const quotaSignals = records.filter((record) =>
|
||||
/quota|rate limit|rate-limit|usage limit|capacity|too many requests|429|insufficient_quota|model.*unavailable|model.*access|limit exceeded/i.test(
|
||||
`${record.stderrTail ?? ""}\n${record.stdoutTail ?? ""}\n${record.handoffReason ?? ""}`,
|
||||
),
|
||||
);
|
||||
|
||||
pushWarning(warnings, failures.length, "model run failures/timeouts/handoffs", failures);
|
||||
pushWarning(warnings, noFinalMessage.length, "model runs with no final message", noFinalMessage);
|
||||
pushWarning(warnings, forbiddenToolRecords.length, "Face/model runs using forbidden substrate tools", forbiddenToolRecords);
|
||||
pushWarning(warnings, speechSmells.length, "robotic/provenance speech smells", speechSmells);
|
||||
pushWarning(warnings, quotaSignals.length, "quota/rate/capacity fallback signals", quotaSignals);
|
||||
|
||||
return {
|
||||
byModel,
|
||||
byCommand,
|
||||
byPrompt,
|
||||
failures: summarizeRecords(failures),
|
||||
noFinalMessage: summarizeRecords(noFinalMessage),
|
||||
forbiddenToolRecords: summarizeRecords(forbiddenToolRecords),
|
||||
speechSmells: summarizeRecords(speechSmells),
|
||||
quotaSignals: summarizeRecords(quotaSignals),
|
||||
warnings,
|
||||
recent: summarizeRecords(records.slice(-12)),
|
||||
};
|
||||
}
|
||||
|
||||
function pushWarning(warnings, count, label, records) {
|
||||
if (count <= 0) {
|
||||
return;
|
||||
}
|
||||
warnings.push({
|
||||
label,
|
||||
count,
|
||||
examples: summarizeRecords(records.slice(0, 5)),
|
||||
});
|
||||
}
|
||||
|
||||
function summarizeRecords(records) {
|
||||
return records.map((record) => ({
|
||||
loggedAt: record.loggedAt ?? null,
|
||||
jobId: record.jobId ?? null,
|
||||
command: record.command ?? null,
|
||||
turn: record.turn ?? null,
|
||||
model: record.model ?? null,
|
||||
promptMarker: record.promptMarker ?? null,
|
||||
exitCode: record.exitCode ?? null,
|
||||
timedOut: Boolean(record.timedOut),
|
||||
handoffReason: record.handoffReason ?? null,
|
||||
tools: (record.toolCalls ?? []).map((call) => call.tool).filter(Boolean),
|
||||
finalPreview: preview(record.finalMessage ?? record.stdoutTail ?? ""),
|
||||
}));
|
||||
}
|
||||
|
||||
function countBy(records, keyFn) {
|
||||
const counts = {};
|
||||
for (const record of records) {
|
||||
const key = String(keyFn(record));
|
||||
counts[key] = (counts[key] ?? 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
function preview(value) {
|
||||
const collapsed = String(value).replace(/\s+/g, " ").trim();
|
||||
return collapsed.length > 240 ? `${collapsed.slice(0, 237)}...` : collapsed;
|
||||
}
|
||||
|
||||
function renderMarkdown(report) {
|
||||
const lines = [
|
||||
"# Model Output Inspection",
|
||||
"",
|
||||
`Generated: ${report.generatedAt}`,
|
||||
`Window: last ${report.sinceHours} hour(s)`,
|
||||
`Log: ${report.logPath}`,
|
||||
`Records: ${report.recordCount}`,
|
||||
"",
|
||||
"## Counts",
|
||||
"",
|
||||
`- By model: ${formatCounts(report.byModel)}`,
|
||||
`- By command: ${formatCounts(report.byCommand)}`,
|
||||
`- By prompt: ${formatCounts(report.byPrompt)}`,
|
||||
"",
|
||||
"## Warnings",
|
||||
"",
|
||||
];
|
||||
|
||||
if (report.warnings.length === 0) {
|
||||
lines.push("- No inspection warnings in this window.");
|
||||
} else {
|
||||
for (const warning of report.warnings) {
|
||||
lines.push(`- ${warning.label}: ${warning.count}`);
|
||||
for (const example of warning.examples) {
|
||||
lines.push(` - ${example.loggedAt} ${example.model} ${example.command} ${example.promptMarker}: ${example.finalPreview || example.handoffReason || "(no preview)"}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lines.push("", "## Recent Records", "");
|
||||
for (const record of report.recent) {
|
||||
lines.push(`- ${record.loggedAt} ${record.model} ${record.command} ${record.promptMarker} tools=[${record.tools.join(", ")}]`);
|
||||
if (record.finalPreview) {
|
||||
lines.push(` ${record.finalPreview}`);
|
||||
}
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function formatCounts(counts) {
|
||||
const entries = Object.entries(counts);
|
||||
return entries.length > 0
|
||||
? entries.map(([key, value]) => `${key}=${value}`).join(", ")
|
||||
: "(none)";
|
||||
}
|
||||
|
||||
function parseArgs(args) {
|
||||
const options = {};
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const token = args[index];
|
||||
if (!token.startsWith("--")) {
|
||||
continue;
|
||||
}
|
||||
const key = token.slice(2);
|
||||
const next = args[index + 1];
|
||||
if (!next || next.startsWith("--")) {
|
||||
options[key] = true;
|
||||
continue;
|
||||
}
|
||||
options[key] = next;
|
||||
index += 1;
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
await main();
|
||||
|
|
@ -242,3 +242,4 @@
|
|||
{"ts":"2026-05-21T16:28:10Z","type":"implementation","status":"ok","note":"Added universal repo Face alienation/social-placement affect need. Newborn and active Faces now carry a social_contact need that should resolve through bonds, rivalries, status reads, and trusted roles; social embodiment prompt now explicitly tells Faces to form real relationships when projected alienation is live."}
|
||||
{"ts":"2026-05-21T17:44:48Z","type":"implementation","status":"ok","note":"Switched repo Face heartbeat/dry-run defaults from gpt-5.3-codex-spark to gpt-5.4-mini after pricing check; dry-run harness now defaults to real read-only retrieval with forbidden child-tool detection, and worker retries/drops Face turns that touch private state or MCP inventory tools."}
|
||||
{"ts":"2026-05-21T17:53:10Z","type":"implementation","status":"ok","note":"Added repo Face Codex model preference chain. Heartbeat jobs now pass REPO_FACE_HEARTBEAT_CODEX_MODELS through owner Codex, which tries Spark first and automatically falls back to gpt-5.4-mini on quota/rate/capacity/model-access failures while recording the model used in trace artifacts."}
|
||||
{"ts":"2026-05-21T18:12:10Z","type":"implementation","status":"ok","note":"Added central model-output JSONL logging for owner-Codex model attempts and an hourly Codex automation that runs scripts/inspect-model-output-logs.mjs over the last hour for observability-only warnings."}
|
||||
|
|
|
|||
Loading…
Reference in a new issue