Add Face model fallback chain

apps/worker/src/index.ts

@@ -745,6 +745,9 @@ function repoFaceHeartbeatCodexOptions(job: JobRecord): Record<string, string> {
 
   return {
     ...(config.repoFaceHeartbeats.codexModel ? { model: config.repoFaceHeartbeats.codexModel } : {}),
+    ...(config.repoFaceHeartbeats.codexModels.length > 0
+      ? { models: config.repoFaceHeartbeats.codexModels.join(",") }
+      : {}),
     ...(config.repoFaceHeartbeats.codexModelReasoningEffort
       ? { reasoningEffort: config.repoFaceHeartbeats.codexModelReasoningEffort }
       : {}),

packages/config/src/index.ts

@@ -116,7 +116,8 @@ const envSchema = z.object({
   REPO_FACE_HEARTBEAT_GLOBAL_HEAT: z.coerce.number().positive().default(1),
   REPO_FACE_HEARTBEAT_SPEED_OVERRIDES: z.string().default(""),
   REPO_FACE_HEARTBEAT_HEAT_OVERRIDES: z.string().default(""),
-  REPO_FACE_HEARTBEAT_CODEX_MODEL: z.string().min(1).default("gpt-5.4-mini"),
+  REPO_FACE_HEARTBEAT_CODEX_MODEL: z.string().min(1).default("gpt-5.3-codex-spark"),
+  REPO_FACE_HEARTBEAT_CODEX_MODELS: z.string().default("gpt-5.3-codex-spark,gpt-5.4-mini"),
   REPO_FACE_HEARTBEAT_CODEX_REASONING_EFFORT: z
     .enum(["low", "medium", "high", "xhigh"])
     .optional(),
@@ -172,6 +173,7 @@ export interface AppConfig {
     speedOverrides: Record<string, number>;
     heatOverrides: Record<string, number>;
     codexModel?: string;
+    codexModels: string[];
     codexModelReasoningEffort?: "low" | "medium" | "high" | "xhigh";
   };
   epiphanyAgentRoot: string;
@@ -394,6 +396,9 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig {
       speedOverrides: parseNumericMap(parsed.REPO_FACE_HEARTBEAT_SPEED_OVERRIDES),
       heatOverrides: parseNumericMap(parsed.REPO_FACE_HEARTBEAT_HEAT_OVERRIDES),
       codexModel: parsed.REPO_FACE_HEARTBEAT_CODEX_MODEL,
+      codexModels: parseList(parsed.REPO_FACE_HEARTBEAT_CODEX_MODELS).length > 0
+        ? parseList(parsed.REPO_FACE_HEARTBEAT_CODEX_MODELS)
+        : [parsed.REPO_FACE_HEARTBEAT_CODEX_MODEL],
       codexModelReasoningEffort: parsed.REPO_FACE_HEARTBEAT_CODEX_REASONING_EFFORT,
     },
     epiphanyAgentRoot: resolve(parsed.EPIPHANY_AGENT_ROOT),

packages/providers/src/owner-codex-provider.ts

@@ -182,17 +182,10 @@ export class OwnerCodexProvider implements ProviderAdapter {
         false,
       );
 
-      result = await runCodexExec({
-        executable: this.options.executable,
-        executableArgs: this.options.executableArgs,
-        model: getStringOption(request.options?.model) ?? this.options.model,
-        reasoningEffort: getReasoningEffortOption(request.options?.reasoningEffort) ?? this.options.reasoningEffort,
-        timeoutMs: this.options.timeoutMs,
-        workingDirectory: this.options.workingDirectory,
+      result = await this.runCodexExecWithModelFallback({
+        request,
+        command,
         prompt: codexPrompt,
-        mcpServers: command === "repo-face-rumination"
-          ? restrictMcpServersToRepoFaceExploration(this.options.mcpServers ?? [])
-          : this.options.mcpServers ?? [],
       });
       turnResults.push(result);
 
@@ -220,6 +213,7 @@ export class OwnerCodexProvider implements ProviderAdapter {
               startedAt: result.startedAt,
               finishedAt: result.finishedAt,
               durationMs: result.durationMs,
+              model: result.model,
               usage: result.usage ?? null,
               events: result.traceEvents,
             },
@@ -388,12 +382,75 @@ export class OwnerCodexProvider implements ProviderAdapter {
       notifications: normalizedReply.notifications,
     };
   }
+
+  private async runCodexExecWithModelFallback(input: {
+    request: ProviderRequest;
+    command: string;
+    prompt: string;
+  }) {
+    const models = getStringListOption(input.request.options?.models);
+    const candidates = models.length > 0
+      ? models
+      : [getStringOption(input.request.options?.model) ?? this.options.model];
+    let lastResult;
+
+    for (const model of candidates) {
+      const result = await runCodexExec({
+        executable: this.options.executable,
+        executableArgs: this.options.executableArgs,
+        model,
+        reasoningEffort: getReasoningEffortOption(input.request.options?.reasoningEffort) ?? this.options.reasoningEffort,
+        timeoutMs: this.options.timeoutMs,
+        workingDirectory: this.options.workingDirectory,
+        prompt: input.prompt,
+        mcpServers: input.command === "repo-face-rumination"
+          ? restrictMcpServersToRepoFaceExploration(this.options.mcpServers ?? [])
+          : this.options.mcpServers ?? [],
+      });
+      lastResult = result;
+      if (!isRetryableCodexModelCapacityFailure(result) || model === candidates.at(-1)) {
+        return result;
+      }
+      console.warn(
+        `Codex model ${model} failed with retryable quota/capacity signal; falling back to ${candidates[candidates.indexOf(model) + 1]}.`,
+      );
+    }
+
+    return lastResult ?? runCodexExec({
+      executable: this.options.executable,
+      executableArgs: this.options.executableArgs,
+      model: this.options.model,
+      reasoningEffort: this.options.reasoningEffort,
+      timeoutMs: this.options.timeoutMs,
+      workingDirectory: this.options.workingDirectory,
+      prompt: input.prompt,
+      mcpServers: this.options.mcpServers ?? [],
+    });
+  }
 }
 
 function getStringOption(value: unknown): string | undefined {
   return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
 }
 
+function getStringListOption(value: unknown): string[] {
+  if (typeof value !== "string") {
+    return [];
+  }
+  return value
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter(Boolean);
+}
+
+function isRetryableCodexModelCapacityFailure(result: { exitCode: number | null; stdout: string; stderr: string; handoffReason?: string }): boolean {
+  if (result.exitCode === 0 || result.handoffReason || result.stdout.includes("item.completed")) {
+    return false;
+  }
+  const text = `${result.stderr}\n${result.stdout}`.toLowerCase();
+  return /quota|rate limit|rate-limit|usage limit|capacity|too many requests|429|insufficient_quota|model.*unavailable|model.*access|limit exceeded/.test(text);
+}
+
 function getReasoningEffortOption(value: unknown): "low" | "medium" | "high" | "xhigh" | undefined {
   if (value === "low" || value === "medium" || value === "high" || value === "xhigh") {
     return value;

packages/providers/src/owner-codex-runtime.ts

@@ -86,6 +86,7 @@ export async function runCodexExec(input: {
       result: Omit<
         CodexRunResult,
         "startedAt" | "finishedAt" | "durationMs" | "traceEvents" | "usage"
+        | "model"
       >,
     ) => {
       if (resolved) {
@@ -98,6 +99,7 @@ export async function runCodexExec(input: {
 
       resolve({
         ...result,
+        model: input.model,
         startedAt,
         finishedAt: new Date(finishedAtMs).toISOString(),
         durationMs: finishedAtMs - startedAtMs,

packages/providers/src/owner-codex-shared.ts

@@ -47,6 +47,7 @@ export interface CodexRunResult {
   stdout: string;
   stderr: string;
   exitCode: number | null;
+  model: string;
   handoffReason?: string;
   toolRequest?: HistoryToolRequest;
   timedOut: boolean;

state/evidence.jsonl

@@ -241,3 +241,4 @@
 {"ts":"2026-05-21T16:24:00Z","type":"implementation","status":"ok","note":"Audited active Face projection surfaces for social drama. Wired repo Face typed state into live heartbeat prompt assembly as character-facing memory prose, seeded cross-Face bonds/status reads for Nibu, Aqua, Mimir, Epiphany, Libby, Bifrost, and Heimdall, and tightened the interpreter smoke social-pressure check so lore/social-status prose does not falsely require a relationship STATE NOTE."}
 {"ts":"2026-05-21T16:28:10Z","type":"implementation","status":"ok","note":"Added universal repo Face alienation/social-placement affect need. Newborn and active Faces now carry a social_contact need that should resolve through bonds, rivalries, status reads, and trusted roles; social embodiment prompt now explicitly tells Faces to form real relationships when projected alienation is live."}
 {"ts":"2026-05-21T17:44:48Z","type":"implementation","status":"ok","note":"Switched repo Face heartbeat/dry-run defaults from gpt-5.3-codex-spark to gpt-5.4-mini after pricing check; dry-run harness now defaults to real read-only retrieval with forbidden child-tool detection, and worker retries/drops Face turns that touch private state or MCP inventory tools."}
+{"ts":"2026-05-21T17:53:10Z","type":"implementation","status":"ok","note":"Added repo Face Codex model preference chain. Heartbeat jobs now pass REPO_FACE_HEARTBEAT_CODEX_MODELS through owner Codex, which tries Spark first and automatically falls back to gpt-5.4-mini on quota/rate/capacity/model-access failures while recording the model used in trace artifacts."}