import "dotenv/config"; import { createHash } from "node:crypto"; import { resolve } from "node:path"; import { loadVoidSelfStateTypedDocuments, writeVoidSelfStateTypedDocuments, type VoidSelfStateTypedProjection, } from "@voidbot/core"; import { loadConfig } from "@voidbot/config"; import { createTextEmbedder, createVectorStores } from "@voidbot/rag"; import type { EmbeddingChunk } from "@voidbot/shared"; const CHUNK_TARGET_CHARS = 2600; const CHUNK_OVERLAP_CHARS = 240; interface CliOptions { canonicalPath: string; personaId: string; publicName: string; query?: string; limit: number; } interface PersonaMemoryChunkPlan { memoryId: string; contentHash: string; chunks: EmbeddingChunk[]; } async function main(): Promise { if (!process.env.DISCORD_OWNER_ID) { process.env.DISCORD_OWNER_ID = "__persona_memory_index__"; } const options = parseArgs(process.argv.slice(2)); const config = loadConfig(); const embedder = createTextEmbedder({ backend: config.ragEmbeddingBackend, hashDimensions: config.ragEmbeddingDimensions, ollamaBaseUrl: config.ragOllamaBaseUrl, ollamaModel: config.ragOllamaModel, ollamaTimeoutMs: config.ragOllamaTimeoutMs, queryInstruction: "Given a Persona memory question, retrieve the relevant typed Persona memories.", }); const vectorStores = createVectorStores({ kind: config.vectorStore.kind, historyPath: config.vectorStore.path, sourceRoot: config.sourceVectorStoreRoot, qdrant: config.qdrant, historyEmbedder: embedder, sourceEmbedder: embedder, personaEmbedder: embedder, }); const state = await loadVoidSelfStateTypedDocuments({ canonicalPath: options.canonicalPath, identity: { agentId: options.personaId, publicName: options.publicName, }, }); const plans = buildPersonaMemoryChunkPlans(state, options.personaId, options.publicName); const chunks = plans.flatMap((plan) => plan.chunks); await vectorStores.persona.upsert(chunks); updateSemanticIndexMetadata(state, plans, { vectorStore: config.vectorStore.kind, collectionName: config.vectorStore.kind === "qdrant" ? config.qdrant.personaCollection : config.vectorStore.path.replace(/\.json$/i, "-persona-memory.json"), embedderId: embedder.id, }); await writeVoidSelfStateTypedDocuments({ canonicalPath: options.canonicalPath, identity: { agentId: options.personaId, publicName: options.publicName, }, }, state); console.log(`Persona memory indexed: ${plans.length} memories, ${chunks.length} chunks`); console.log(`Vector store: ${config.vectorStore.kind}`); console.log(`Collection/path: ${config.vectorStore.kind === "qdrant" ? config.qdrant.personaCollection : config.vectorStore.path.replace(/\.json$/i, "-persona-memory.json")}`); console.log(`Embedder: ${embedder.id}`); if (options.query) { const results = await vectorStores.persona.query(options.query, options.limit, { corpusKind: "persona_memory", repoName: options.personaId, }); console.log(JSON.stringify({ query: options.query, results: results.map((result) => ({ score: result.score, memoryId: result.metadata.memoryId, chunkId: result.chunkId, target: result.metadata.targetLabel, text: result.text.slice(0, 500), })), }, null, 2)); } } function parseArgs(args: string[]): CliOptions { const options: CliOptions = { canonicalPath: "state/personas/metacrat.cc", personaId: "metacrat", publicName: "Metacrat", limit: 5, }; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; const next = args[index + 1]; if (arg === "--state" && next) { options.canonicalPath = next; index += 1; } else if (arg === "--persona-id" && next) { options.personaId = next; index += 1; } else if (arg === "--public-name" && next) { options.publicName = next; index += 1; } else if (arg === "--query" && next) { options.query = next; index += 1; } else if (arg === "--limit" && next) { options.limit = Number.parseInt(next, 10); index += 1; } else if (arg === "--help") { printHelpAndExit(); } else { throw new Error(`Unknown or incomplete argument: ${arg}`); } } options.canonicalPath = resolve(options.canonicalPath); if (!Number.isFinite(options.limit) || options.limit < 1) { throw new Error("--limit must be a positive integer."); } return options; } function printHelpAndExit(): never { console.log([ "Usage: npm run persona:memory-index -- [options]", "", "Options:", " --state Persona .cc path. Defaults to state/personas/metacrat.cc", " --persona-id Persona id stored in vector metadata. Defaults to metacrat", " --public-name Persona public name. Defaults to Metacrat", " --query Optional smoke query after indexing.", " --limit Query result limit. Defaults to 5", ].join("\n")); process.exit(0); } function buildPersonaMemoryChunkPlans( state: VoidSelfStateTypedProjection, personaId: string, publicName: string, ): PersonaMemoryChunkPlan[] { return state.thoughtMemory.memories .filter((memory) => !memory.retiredAt) .map((memory) => { const text = renderMemoryForEmbedding(memory); const contentHash = hashText(text); const parts = splitText(text); const chunks = parts.map((part, index): EmbeddingChunk => ({ id: `${personaId}:${memory.memoryId}:chunk-${index}`, sourceId: `${personaId}:${memory.memoryId}`, sourceKind: "persona_memory", text: part, normalizedText: normalizeWhitespace(part), metadata: { corpusKind: "persona_memory", sourceId: `${personaId}:${memory.memoryId}`, personaId, publicName, memoryId: memory.memoryId, memoryKind: memory.kind, targetKind: memory.target.kind, targetId: memory.target.id, targetLabel: memory.target.label ?? memory.target.id, repoName: personaId, contentHash, chunkIndex: String(index), chunkCount: String(parts.length), tags: memory.tags.join(","), }, })); return { memoryId: memory.memoryId, contentHash, chunks, }; }); } function updateSemanticIndexMetadata( state: VoidSelfStateTypedProjection, plans: PersonaMemoryChunkPlan[], index: { vectorStore: "qdrant" | "local_json"; collectionName: string; embedderId: string; }, ): void { const indexedAt = new Date().toISOString(); const planByMemoryId = new Map(plans.map((plan) => [plan.memoryId, plan])); for (const memory of state.thoughtMemory.memories) { const plan = planByMemoryId.get(memory.memoryId); if (!plan) { if (memory.semanticIndex) { memory.semanticIndex = { ...memory.semanticIndex, stale: true, }; } continue; } memory.semanticIndex = { corpusKind: "persona_memory", vectorStore: index.vectorStore, collectionName: index.collectionName, embedderId: index.embedderId, contentHash: plan.contentHash, chunkIds: plan.chunks.map((chunk) => chunk.id), indexedAt, stale: false, }; memory.updatedAt = indexedAt; } state.thoughtMemory.updatedAt = indexedAt; } function renderMemoryForEmbedding( memory: VoidSelfStateTypedProjection["thoughtMemory"]["memories"][number], ): string { return [ `Memory: ${memory.memoryId}`, `Kind: ${memory.kind}`, `Target: ${memory.target.label ?? memory.target.id} (${memory.target.kind}:${memory.target.id})`, `Summary: ${memory.summary}`, memory.claim ? `Claim: ${memory.claim}` : undefined, memory.question ? `Question: ${memory.question}` : undefined, memory.tension ? `Tension: ${memory.tension}` : undefined, memory.actionImplication ? `Action: ${memory.actionImplication}` : undefined, memory.tags.length > 0 ? `Tags: ${memory.tags.join(", ")}` : undefined, memory.anchorRefs.length > 0 ? `Anchors: ${memory.anchorRefs.map((anchor) => anchor.ref).join(", ")}` : undefined, ].filter(Boolean).join("\n"); } function splitText(text: string): string[] { if (text.length <= CHUNK_TARGET_CHARS) { return [text]; } const chunks: string[] = []; let start = 0; while (start < text.length) { let end = Math.min(text.length, start + CHUNK_TARGET_CHARS); if (end < text.length) { const paragraphBreak = text.lastIndexOf("\n", end); if (paragraphBreak > start + CHUNK_TARGET_CHARS * 0.5) { end = paragraphBreak; } } chunks.push(text.slice(start, end).trim()); start = Math.max(end - CHUNK_OVERLAP_CHARS, end); } return chunks.filter((chunk) => chunk.length > 0); } function hashText(text: string): string { return `sha256:${createHash("sha256").update(normalizeWhitespace(text)).digest("hex")}`; } function normalizeWhitespace(text: string): string { return text.replace(/\s+/g, " ").trim(); } void main().catch((error) => { console.error(error); process.exitCode = 1; });