Isolate Persona active-turn liveness
This commit is contained in:
parent
84ff635c7a
commit
343a4ad90c
6 changed files with 119 additions and 89 deletions
|
|
@ -63,16 +63,11 @@ import {
|
|||
writePersonaSchedulerState as writeHeartbeatState,
|
||||
type PersonaSchedulerState as FaceHeartbeatState,
|
||||
} from "../apps/persona-scheduler/dist/state-store.js";
|
||||
import { scanActivePersonaTurns } from "../apps/persona-scheduler/dist/active-turn-source.js";
|
||||
|
||||
const HEARTBEAT_COMMAND = "repo-face-rumination";
|
||||
const MIN_STALE_ACTIVE_JOB_MS = 45 * 60_000;
|
||||
const MAX_FACE_IMAGE_ATTACHMENT_BYTES = 12 * 1024 * 1024;
|
||||
|
||||
interface ActiveTurnScan {
|
||||
active: Map<string, string>;
|
||||
staleRecovered: StaleActiveTurn[];
|
||||
}
|
||||
|
||||
interface IdleCoolingSnapshot {
|
||||
enabled: boolean;
|
||||
active: boolean;
|
||||
|
|
@ -87,15 +82,6 @@ interface IdleCoolingSnapshot {
|
|||
observedHumanMessages: Array<SourceMessage & { channelId: string }>;
|
||||
}
|
||||
|
||||
interface StaleActiveTurn {
|
||||
identityId: string;
|
||||
jobId: string;
|
||||
requestMessageId?: string;
|
||||
state: string;
|
||||
updatedAt?: string;
|
||||
ageMinutes: number;
|
||||
}
|
||||
|
||||
interface RepoFaceChannelPlan {
|
||||
primaryChannelId?: string;
|
||||
snapshotChannelIds: string[];
|
||||
|
|
@ -213,7 +199,7 @@ async function main(): Promise<void> {
|
|||
advanceInitiativeClockFromWallClock(state, now);
|
||||
const activeTurnScan = dryRun
|
||||
? { active: new Map<string, string>(), staleRecovered: [] }
|
||||
: await listExistingActiveTurns(config.databaseDsn, config.stateStorageBackend, config);
|
||||
: await scanActivePersonaTurns(config);
|
||||
for (const stale of activeTurnScan.staleRecovered) {
|
||||
state.history.push({
|
||||
type: "stale_active_turn_recovered",
|
||||
|
|
@ -1162,76 +1148,6 @@ function sleep(ms: number): Promise<void> {
|
|||
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
||||
}
|
||||
|
||||
async function listExistingActiveTurns(
|
||||
databaseDsn: string,
|
||||
stateStorageBackend: "file" | "postgres",
|
||||
config: ReturnType<typeof loadConfig>,
|
||||
): Promise<ActiveTurnScan> {
|
||||
const storage = await createStateStorage({
|
||||
backend: stateStorageBackend,
|
||||
databaseDsn,
|
||||
jobsFile: config.jobsFile,
|
||||
auditLogFile: config.auditLogFile,
|
||||
interactionMemoryFile: config.interactionMemoryFile,
|
||||
rateLimitStateFile: config.rateLimitStateFile,
|
||||
});
|
||||
try {
|
||||
const jobs = await storage.jobQueue.listByStates(["approved", "running"]);
|
||||
const active = new Map<string, string>();
|
||||
const staleRecovered: StaleActiveTurn[] = [];
|
||||
const staleAfterMs = Math.max(MIN_STALE_ACTIVE_JOB_MS, config.codexExecTimeoutMs * 3);
|
||||
const nowMs = Date.now();
|
||||
for (const job of jobs) {
|
||||
if (job.command !== HEARTBEAT_COMMAND) {
|
||||
continue;
|
||||
}
|
||||
const match =
|
||||
job.requestMessageId?.match(/^agent-turn:([^:]+):/) ??
|
||||
job.requestMessageId?.match(/^agent-heartbeat:([^:]+):/) ??
|
||||
job.requestMessageId?.match(/^repo-face-heartbeat:([^:]+):/) ??
|
||||
job.requestMessageId?.match(/:repo-face:([^:]+):\d+$/);
|
||||
if (match) {
|
||||
const updatedMs = Date.parse(job.updatedAt);
|
||||
const ageMs = Number.isFinite(updatedMs) ? nowMs - updatedMs : Number.POSITIVE_INFINITY;
|
||||
if (ageMs > staleAfterMs) {
|
||||
const ageMinutes = Number.isFinite(ageMs) ? Math.round((ageMs / 60_000) * 10) / 10 : -1;
|
||||
await storage.jobQueue.markFailed(
|
||||
job.id,
|
||||
`Repo Face CTB recovered stale active turn job after ${ageMinutes} minutes without progress.`,
|
||||
);
|
||||
staleRecovered.push({
|
||||
identityId: match[1],
|
||||
jobId: job.id,
|
||||
requestMessageId: job.requestMessageId,
|
||||
state: job.state,
|
||||
updatedAt: job.updatedAt,
|
||||
ageMinutes,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
active.set(match[1], job.id);
|
||||
}
|
||||
}
|
||||
const voidLock = await readRecentLock(resolve(config.storageRoot, "status", "moderation-rumination.lock"), 20);
|
||||
if (voidLock) {
|
||||
active.set("void", "lock:moderation-rumination");
|
||||
}
|
||||
return { active, staleRecovered };
|
||||
} finally {
|
||||
await storage.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function readRecentLock(path: string, maxAgeMinutes: number): Promise<boolean> {
|
||||
try {
|
||||
const info = await stat(path);
|
||||
const ageMs = Date.now() - info.mtimeMs;
|
||||
return ageMs >= 0 && ageMs < maxAgeMinutes * 60_000;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function applySemanticResponsePressure(input: {
|
||||
state: FaceHeartbeatState;
|
||||
specs: ParticipantSpec[];
|
||||
|
|
|
|||
|
|
@ -24,6 +24,10 @@ import {
|
|||
readPersonaSchedulerState,
|
||||
writePersonaSchedulerState,
|
||||
} from "../apps/persona-scheduler/dist/state-store.js";
|
||||
import {
|
||||
parsePersonaTurnIdentity,
|
||||
staleActiveTurnThresholdMs,
|
||||
} from "../apps/persona-scheduler/dist/active-turn-source.js";
|
||||
|
||||
function participant(identityId: string, nextTurnAt: number): InitiativeParticipant {
|
||||
return {
|
||||
|
|
@ -161,6 +165,10 @@ const inspected = participant("inspected", 69);
|
|||
recordDryRunSelection(inspected, queueState, "2026-07-15T21:02:00.000Z", 0);
|
||||
assert.equal(inspected.nextTurnAt, 100, "dry-run selection uses the same recovery ownership without claiming active load");
|
||||
assert.equal(inspected.currentLoad, 0, "inspection cannot impersonate a live job");
|
||||
assert.equal(parsePersonaTurnIdentity("agent-turn:nibu:2026-07-15"), "nibu", "the queue adapter recognizes current Persona turn witnesses");
|
||||
assert.equal(parsePersonaTurnIdentity("unrelated:job"), undefined, "unrelated jobs cannot freeze Persona initiative");
|
||||
assert.equal(staleActiveTurnThresholdMs(20 * 60_000), 60 * 60_000, "runtime timeout expands the stale-claim boundary");
|
||||
assert.equal(staleActiveTurnThresholdMs(5 * 60_000), 45 * 60_000, "stale recovery never becomes an impatient repair loop");
|
||||
|
||||
const stateDirectory = await mkdtemp(join(tmpdir(), "voidbot-persona-scheduler-"));
|
||||
try {
|
||||
|
|
|
|||
Loading…
Reference in a new issue