From cf804334d16f56621074990b6c8b48676f7d4914 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 21 May 2026 19:12:30 +0100 Subject: [PATCH] Add model output log inspection --- package.json | 1 + .../providers/src/owner-codex-provider.ts | 75 +++++++ scripts/inspect-model-output-logs.mjs | 201 ++++++++++++++++++ state/evidence.jsonl | 1 + 4 files changed, 278 insertions(+) create mode 100644 scripts/inspect-model-output-logs.mjs diff --git a/package.json b/package.json index 8c8087d..8e51099 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "state:install-mood-task": "powershell -ExecutionPolicy Bypass -File scripts/install-void-mood-drift-task.ps1", "state:install-repo-face-heartbeat-task": "powershell -ExecutionPolicy Bypass -File scripts/install-repo-face-heartbeat-task.ps1", "agents:inspect-requests": "tsx scripts/inspect-agent-state-requests.ts", + "agents:inspect-model-outputs": "node scripts/inspect-model-output-logs.mjs", "agents:assemble-character-prompt": "tsx scripts/run-repo-face-heartbeats.ts --assemble-prompt", "agents:heartbeat-model-smoke": "node scripts/smoke-repo-face-heartbeat-model-scenarios.mjs", "agents:interpreter-loop-smoke": "node scripts/smoke-repo-face-interpreter-loop.mjs", diff --git a/packages/providers/src/owner-codex-provider.ts b/packages/providers/src/owner-codex-provider.ts index a9fc1bd..6b420f1 100644 --- a/packages/providers/src/owner-codex-provider.ts +++ b/packages/providers/src/owner-codex-provider.ts @@ -1,3 +1,6 @@ +import { appendFile, mkdir } from "node:fs/promises"; +import { join } from "node:path"; + import type { Actor, ContextBundle, @@ -11,6 +14,7 @@ import type { import { buildDefaultHandoffNotice, buildRequestPayload, + type CodexRunResult, type HistoryLookupTool, MAX_HISTORY_TOOL_CALLS, type OwnerCodexProviderOptions, @@ -188,6 +192,14 @@ export class OwnerCodexProvider implements ProviderAdapter { prompt: codexPrompt, }); turnResults.push(result); + await appendModelOutputLog({ + workingDirectory: this.options.workingDirectory, + request, + command, + turn: turn + 1, + prompt: codexPrompt, + result, + }); artifacts.push( { @@ -451,6 +463,69 @@ function isRetryableCodexModelCapacityFailure(result: { exitCode: number | null; return /quota|rate limit|rate-limit|usage limit|capacity|too many requests|429|insufficient_quota|model.*unavailable|model.*access|limit exceeded/.test(text); } +async function appendModelOutputLog(input: { + workingDirectory: string; + request: ProviderRequest; + command: string; + turn: number; + prompt: string; + result: CodexRunResult; +}): Promise { + const jobId = getStringOption(input.request.options?.jobId) ?? "unknown-job"; + const logPath = join(input.workingDirectory, ".voidbot", "logs", "model-outputs.jsonl"); + const finalMessage = [...input.result.traceEvents] + .reverse() + .find((event) => event.kind === "agent_message" && event.message?.trim())?.message?.trim(); + const toolCalls = input.result.traceEvents + .filter((event) => event.kind === "mcp_tool_started") + .map((event) => ({ + server: event.server, + tool: event.tool, + arguments: event.arguments, + })); + const commandExecutions = input.result.traceEvents + .filter((event) => event.kind === "command_started" || event.kind === "command_completed") + .map((event) => ({ + kind: event.kind, + command: event.command, + status: event.status, + exitCode: event.exitCode, + })); + const promptMarker = input.prompt.match(//)?.[1]; + const record = { + schemaVersion: 1, + loggedAt: new Date().toISOString(), + jobId, + command: input.command, + turn: input.turn, + model: input.result.model, + promptMarker: promptMarker ?? null, + promptLength: input.prompt.length, + startedAt: input.result.startedAt, + finishedAt: input.result.finishedAt, + durationMs: input.result.durationMs, + exitCode: input.result.exitCode, + timedOut: input.result.timedOut, + handoffReason: input.result.handoffReason ?? null, + usage: input.result.usage ?? null, + finalMessage: finalMessage ?? null, + stdoutTail: input.result.stdout.slice(-4000), + stderrTail: input.result.stderr.slice(-4000), + toolCalls, + commandExecutions, + artifactRefs: { + prompt: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-prompt.md`, + stdout: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-stdout.txt`, + stderr: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-stderr.txt`, + trace: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-trace.json`, + debug: `.voidbot/artifacts/${jobId}/codex-turn-${input.turn}-debug.md`, + }, + }; + + await mkdir(join(input.workingDirectory, ".voidbot", "logs"), { recursive: true }); + await appendFile(logPath, `${JSON.stringify(record)}\n`, "utf8"); +} + function getReasoningEffortOption(value: unknown): "low" | "medium" | "high" | "xhigh" | undefined { if (value === "low" || value === "medium" || value === "high" || value === "xhigh") { return value; diff --git a/scripts/inspect-model-output-logs.mjs b/scripts/inspect-model-output-logs.mjs new file mode 100644 index 0000000..3729c54 --- /dev/null +++ b/scripts/inspect-model-output-logs.mjs @@ -0,0 +1,201 @@ +#!/usr/bin/env node +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +const repoRoot = resolve(import.meta.dirname, ".."); + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const hours = Number.parseFloat(options["since-hours"] ?? "1"); + const sinceMs = Date.now() - Math.max(0.01, hours) * 60 * 60 * 1000; + const logPath = resolve(repoRoot, options.log ?? ".voidbot/logs/model-outputs.jsonl"); + const markdownOut = resolve(repoRoot, options.out ?? ".voidbot/status/model-output-inspection.md"); + const jsonOut = resolve(repoRoot, options["json-out"] ?? ".voidbot/status/model-output-inspection.json"); + const records = (await readJsonl(logPath)) + .filter((record) => Date.parse(String(record.loggedAt ?? record.finishedAt ?? "")) >= sinceMs); + const findings = inspect(records); + const report = { + generatedAt: new Date().toISOString(), + logPath, + sinceHours: hours, + recordCount: records.length, + ...findings, + }; + + await mkdir(dirname(markdownOut), { recursive: true }); + await writeFile(jsonOut, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await writeFile(markdownOut, renderMarkdown(report), "utf8"); + process.stdout.write(`${JSON.stringify({ ok: true, markdownOut, jsonOut, recordCount: records.length, warningCount: findings.warnings.length }, null, 2)}\n`); +} + +async function readJsonl(path) { + try { + const text = await readFile(path, "utf8"); + return text + .split(/\r?\n/) + .filter(Boolean) + .map((line) => { + try { + return JSON.parse(line); + } catch { + return { malformed: true, raw: line }; + } + }); + } catch (error) { + if (error && typeof error === "object" && error.code === "ENOENT") { + return []; + } + throw error; + } +} + +function inspect(records) { + const warnings = []; + const byModel = countBy(records, (record) => record.model ?? "(unknown)"); + const byCommand = countBy(records, (record) => record.command ?? "(unknown)"); + const byPrompt = countBy(records, (record) => record.promptMarker ?? "(none)"); + const failures = records.filter((record) => record.exitCode !== 0 || record.timedOut || record.handoffReason); + const noFinalMessage = records.filter((record) => !String(record.finalMessage ?? "").trim()); + const forbiddenToolRecords = records.filter((record) => + (record.toolCalls ?? []).some((call) => + ["read_repo_face_state", "list_mcp_resources", "read_mcp_resource", "post_repo_identity_message", "apply_repo_face_state_operation", "notify_owner"].includes(call.tool), + ), + ); + const speechSmells = records.filter((record) => + /repo-face heartbeat|heartbeat from|bright bridge note|tiny fish sorting note|librarian note|maintenance pass/i.test(String(record.finalMessage ?? "")), + ); + const quotaSignals = records.filter((record) => + /quota|rate limit|rate-limit|usage limit|capacity|too many requests|429|insufficient_quota|model.*unavailable|model.*access|limit exceeded/i.test( + `${record.stderrTail ?? ""}\n${record.stdoutTail ?? ""}\n${record.handoffReason ?? ""}`, + ), + ); + + pushWarning(warnings, failures.length, "model run failures/timeouts/handoffs", failures); + pushWarning(warnings, noFinalMessage.length, "model runs with no final message", noFinalMessage); + pushWarning(warnings, forbiddenToolRecords.length, "Face/model runs using forbidden substrate tools", forbiddenToolRecords); + pushWarning(warnings, speechSmells.length, "robotic/provenance speech smells", speechSmells); + pushWarning(warnings, quotaSignals.length, "quota/rate/capacity fallback signals", quotaSignals); + + return { + byModel, + byCommand, + byPrompt, + failures: summarizeRecords(failures), + noFinalMessage: summarizeRecords(noFinalMessage), + forbiddenToolRecords: summarizeRecords(forbiddenToolRecords), + speechSmells: summarizeRecords(speechSmells), + quotaSignals: summarizeRecords(quotaSignals), + warnings, + recent: summarizeRecords(records.slice(-12)), + }; +} + +function pushWarning(warnings, count, label, records) { + if (count <= 0) { + return; + } + warnings.push({ + label, + count, + examples: summarizeRecords(records.slice(0, 5)), + }); +} + +function summarizeRecords(records) { + return records.map((record) => ({ + loggedAt: record.loggedAt ?? null, + jobId: record.jobId ?? null, + command: record.command ?? null, + turn: record.turn ?? null, + model: record.model ?? null, + promptMarker: record.promptMarker ?? null, + exitCode: record.exitCode ?? null, + timedOut: Boolean(record.timedOut), + handoffReason: record.handoffReason ?? null, + tools: (record.toolCalls ?? []).map((call) => call.tool).filter(Boolean), + finalPreview: preview(record.finalMessage ?? record.stdoutTail ?? ""), + })); +} + +function countBy(records, keyFn) { + const counts = {}; + for (const record of records) { + const key = String(keyFn(record)); + counts[key] = (counts[key] ?? 0) + 1; + } + return counts; +} + +function preview(value) { + const collapsed = String(value).replace(/\s+/g, " ").trim(); + return collapsed.length > 240 ? `${collapsed.slice(0, 237)}...` : collapsed; +} + +function renderMarkdown(report) { + const lines = [ + "# Model Output Inspection", + "", + `Generated: ${report.generatedAt}`, + `Window: last ${report.sinceHours} hour(s)`, + `Log: ${report.logPath}`, + `Records: ${report.recordCount}`, + "", + "## Counts", + "", + `- By model: ${formatCounts(report.byModel)}`, + `- By command: ${formatCounts(report.byCommand)}`, + `- By prompt: ${formatCounts(report.byPrompt)}`, + "", + "## Warnings", + "", + ]; + + if (report.warnings.length === 0) { + lines.push("- No inspection warnings in this window."); + } else { + for (const warning of report.warnings) { + lines.push(`- ${warning.label}: ${warning.count}`); + for (const example of warning.examples) { + lines.push(` - ${example.loggedAt} ${example.model} ${example.command} ${example.promptMarker}: ${example.finalPreview || example.handoffReason || "(no preview)"}`); + } + } + } + + lines.push("", "## Recent Records", ""); + for (const record of report.recent) { + lines.push(`- ${record.loggedAt} ${record.model} ${record.command} ${record.promptMarker} tools=[${record.tools.join(", ")}]`); + if (record.finalPreview) { + lines.push(` ${record.finalPreview}`); + } + } + + return `${lines.join("\n")}\n`; +} + +function formatCounts(counts) { + const entries = Object.entries(counts); + return entries.length > 0 + ? entries.map(([key, value]) => `${key}=${value}`).join(", ") + : "(none)"; +} + +function parseArgs(args) { + const options = {}; + for (let index = 0; index < args.length; index += 1) { + const token = args[index]; + if (!token.startsWith("--")) { + continue; + } + const key = token.slice(2); + const next = args[index + 1]; + if (!next || next.startsWith("--")) { + options[key] = true; + continue; + } + options[key] = next; + index += 1; + } + return options; +} + +await main(); diff --git a/state/evidence.jsonl b/state/evidence.jsonl index bdc5016..d3b2aab 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -242,3 +242,4 @@ {"ts":"2026-05-21T16:28:10Z","type":"implementation","status":"ok","note":"Added universal repo Face alienation/social-placement affect need. Newborn and active Faces now carry a social_contact need that should resolve through bonds, rivalries, status reads, and trusted roles; social embodiment prompt now explicitly tells Faces to form real relationships when projected alienation is live."} {"ts":"2026-05-21T17:44:48Z","type":"implementation","status":"ok","note":"Switched repo Face heartbeat/dry-run defaults from gpt-5.3-codex-spark to gpt-5.4-mini after pricing check; dry-run harness now defaults to real read-only retrieval with forbidden child-tool detection, and worker retries/drops Face turns that touch private state or MCP inventory tools."} {"ts":"2026-05-21T17:53:10Z","type":"implementation","status":"ok","note":"Added repo Face Codex model preference chain. Heartbeat jobs now pass REPO_FACE_HEARTBEAT_CODEX_MODELS through owner Codex, which tries Spark first and automatically falls back to gpt-5.4-mini on quota/rate/capacity/model-access failures while recording the model used in trace artifacts."} +{"ts":"2026-05-21T18:12:10Z","type":"implementation","status":"ok","note":"Added central model-output JSONL logging for owner-Codex model attempts and an hourly Codex automation that runs scripts/inspect-model-output-logs.mjs over the last hour for observability-only warnings."}