diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 770f582..38d6ef3 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -745,6 +745,9 @@ function repoFaceHeartbeatCodexOptions(job: JobRecord): Record { return { ...(config.repoFaceHeartbeats.codexModel ? { model: config.repoFaceHeartbeats.codexModel } : {}), + ...(config.repoFaceHeartbeats.codexModels.length > 0 + ? { models: config.repoFaceHeartbeats.codexModels.join(",") } + : {}), ...(config.repoFaceHeartbeats.codexModelReasoningEffort ? { reasoningEffort: config.repoFaceHeartbeats.codexModelReasoningEffort } : {}), diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index de10598..3a191d9 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -116,7 +116,8 @@ const envSchema = z.object({ REPO_FACE_HEARTBEAT_GLOBAL_HEAT: z.coerce.number().positive().default(1), REPO_FACE_HEARTBEAT_SPEED_OVERRIDES: z.string().default(""), REPO_FACE_HEARTBEAT_HEAT_OVERRIDES: z.string().default(""), - REPO_FACE_HEARTBEAT_CODEX_MODEL: z.string().min(1).default("gpt-5.4-mini"), + REPO_FACE_HEARTBEAT_CODEX_MODEL: z.string().min(1).default("gpt-5.3-codex-spark"), + REPO_FACE_HEARTBEAT_CODEX_MODELS: z.string().default("gpt-5.3-codex-spark,gpt-5.4-mini"), REPO_FACE_HEARTBEAT_CODEX_REASONING_EFFORT: z .enum(["low", "medium", "high", "xhigh"]) .optional(), @@ -172,6 +173,7 @@ export interface AppConfig { speedOverrides: Record; heatOverrides: Record; codexModel?: string; + codexModels: string[]; codexModelReasoningEffort?: "low" | "medium" | "high" | "xhigh"; }; epiphanyAgentRoot: string; @@ -394,6 +396,9 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig { speedOverrides: parseNumericMap(parsed.REPO_FACE_HEARTBEAT_SPEED_OVERRIDES), heatOverrides: parseNumericMap(parsed.REPO_FACE_HEARTBEAT_HEAT_OVERRIDES), codexModel: parsed.REPO_FACE_HEARTBEAT_CODEX_MODEL, + codexModels: parseList(parsed.REPO_FACE_HEARTBEAT_CODEX_MODELS).length > 0 + ? parseList(parsed.REPO_FACE_HEARTBEAT_CODEX_MODELS) + : [parsed.REPO_FACE_HEARTBEAT_CODEX_MODEL], codexModelReasoningEffort: parsed.REPO_FACE_HEARTBEAT_CODEX_REASONING_EFFORT, }, epiphanyAgentRoot: resolve(parsed.EPIPHANY_AGENT_ROOT), diff --git a/packages/providers/src/owner-codex-provider.ts b/packages/providers/src/owner-codex-provider.ts index a2d5e86..a9fc1bd 100644 --- a/packages/providers/src/owner-codex-provider.ts +++ b/packages/providers/src/owner-codex-provider.ts @@ -182,17 +182,10 @@ export class OwnerCodexProvider implements ProviderAdapter { false, ); - result = await runCodexExec({ - executable: this.options.executable, - executableArgs: this.options.executableArgs, - model: getStringOption(request.options?.model) ?? this.options.model, - reasoningEffort: getReasoningEffortOption(request.options?.reasoningEffort) ?? this.options.reasoningEffort, - timeoutMs: this.options.timeoutMs, - workingDirectory: this.options.workingDirectory, + result = await this.runCodexExecWithModelFallback({ + request, + command, prompt: codexPrompt, - mcpServers: command === "repo-face-rumination" - ? restrictMcpServersToRepoFaceExploration(this.options.mcpServers ?? []) - : this.options.mcpServers ?? [], }); turnResults.push(result); @@ -220,6 +213,7 @@ export class OwnerCodexProvider implements ProviderAdapter { startedAt: result.startedAt, finishedAt: result.finishedAt, durationMs: result.durationMs, + model: result.model, usage: result.usage ?? null, events: result.traceEvents, }, @@ -388,12 +382,75 @@ export class OwnerCodexProvider implements ProviderAdapter { notifications: normalizedReply.notifications, }; } + + private async runCodexExecWithModelFallback(input: { + request: ProviderRequest; + command: string; + prompt: string; + }) { + const models = getStringListOption(input.request.options?.models); + const candidates = models.length > 0 + ? models + : [getStringOption(input.request.options?.model) ?? this.options.model]; + let lastResult; + + for (const model of candidates) { + const result = await runCodexExec({ + executable: this.options.executable, + executableArgs: this.options.executableArgs, + model, + reasoningEffort: getReasoningEffortOption(input.request.options?.reasoningEffort) ?? this.options.reasoningEffort, + timeoutMs: this.options.timeoutMs, + workingDirectory: this.options.workingDirectory, + prompt: input.prompt, + mcpServers: input.command === "repo-face-rumination" + ? restrictMcpServersToRepoFaceExploration(this.options.mcpServers ?? []) + : this.options.mcpServers ?? [], + }); + lastResult = result; + if (!isRetryableCodexModelCapacityFailure(result) || model === candidates.at(-1)) { + return result; + } + console.warn( + `Codex model ${model} failed with retryable quota/capacity signal; falling back to ${candidates[candidates.indexOf(model) + 1]}.`, + ); + } + + return lastResult ?? runCodexExec({ + executable: this.options.executable, + executableArgs: this.options.executableArgs, + model: this.options.model, + reasoningEffort: this.options.reasoningEffort, + timeoutMs: this.options.timeoutMs, + workingDirectory: this.options.workingDirectory, + prompt: input.prompt, + mcpServers: this.options.mcpServers ?? [], + }); + } } function getStringOption(value: unknown): string | undefined { return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; } +function getStringListOption(value: unknown): string[] { + if (typeof value !== "string") { + return []; + } + return value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function isRetryableCodexModelCapacityFailure(result: { exitCode: number | null; stdout: string; stderr: string; handoffReason?: string }): boolean { + if (result.exitCode === 0 || result.handoffReason || result.stdout.includes("item.completed")) { + return false; + } + const text = `${result.stderr}\n${result.stdout}`.toLowerCase(); + return /quota|rate limit|rate-limit|usage limit|capacity|too many requests|429|insufficient_quota|model.*unavailable|model.*access|limit exceeded/.test(text); +} + function getReasoningEffortOption(value: unknown): "low" | "medium" | "high" | "xhigh" | undefined { if (value === "low" || value === "medium" || value === "high" || value === "xhigh") { return value; diff --git a/packages/providers/src/owner-codex-runtime.ts b/packages/providers/src/owner-codex-runtime.ts index 76c2648..04fd7eb 100644 --- a/packages/providers/src/owner-codex-runtime.ts +++ b/packages/providers/src/owner-codex-runtime.ts @@ -86,6 +86,7 @@ export async function runCodexExec(input: { result: Omit< CodexRunResult, "startedAt" | "finishedAt" | "durationMs" | "traceEvents" | "usage" + | "model" >, ) => { if (resolved) { @@ -98,6 +99,7 @@ export async function runCodexExec(input: { resolve({ ...result, + model: input.model, startedAt, finishedAt: new Date(finishedAtMs).toISOString(), durationMs: finishedAtMs - startedAtMs, diff --git a/packages/providers/src/owner-codex-shared.ts b/packages/providers/src/owner-codex-shared.ts index 755653f..6bca8cc 100644 --- a/packages/providers/src/owner-codex-shared.ts +++ b/packages/providers/src/owner-codex-shared.ts @@ -47,6 +47,7 @@ export interface CodexRunResult { stdout: string; stderr: string; exitCode: number | null; + model: string; handoffReason?: string; toolRequest?: HistoryToolRequest; timedOut: boolean; diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 9d045c2..bdc5016 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -241,3 +241,4 @@ {"ts":"2026-05-21T16:24:00Z","type":"implementation","status":"ok","note":"Audited active Face projection surfaces for social drama. Wired repo Face typed state into live heartbeat prompt assembly as character-facing memory prose, seeded cross-Face bonds/status reads for Nibu, Aqua, Mimir, Epiphany, Libby, Bifrost, and Heimdall, and tightened the interpreter smoke social-pressure check so lore/social-status prose does not falsely require a relationship STATE NOTE."} {"ts":"2026-05-21T16:28:10Z","type":"implementation","status":"ok","note":"Added universal repo Face alienation/social-placement affect need. Newborn and active Faces now carry a social_contact need that should resolve through bonds, rivalries, status reads, and trusted roles; social embodiment prompt now explicitly tells Faces to form real relationships when projected alienation is live."} {"ts":"2026-05-21T17:44:48Z","type":"implementation","status":"ok","note":"Switched repo Face heartbeat/dry-run defaults from gpt-5.3-codex-spark to gpt-5.4-mini after pricing check; dry-run harness now defaults to real read-only retrieval with forbidden child-tool detection, and worker retries/drops Face turns that touch private state or MCP inventory tools."} +{"ts":"2026-05-21T17:53:10Z","type":"implementation","status":"ok","note":"Added repo Face Codex model preference chain. Heartbeat jobs now pass REPO_FACE_HEARTBEAT_CODEX_MODELS through owner Codex, which tries Spark first and automatically falls back to gpt-5.4-mini on quota/rate/capacity/model-access failures while recording the model used in trace artifacts."}