diff --git a/scripts/index-source-repos.ts b/apps/worker/src/source-index-main.ts similarity index 86% rename from scripts/index-source-repos.ts rename to apps/worker/src/source-index-main.ts index 613f502..f319a15 100644 --- a/scripts/index-source-repos.ts +++ b/apps/worker/src/source-index-main.ts @@ -1,9 +1,11 @@ import "dotenv/config"; import { loadConfig } from "@voidbot/config"; - -import { discoverSourceRepos, selectSourceRepos } from "./source-repo-discovery"; -import { indexSourceRepos } from "./source-repo-indexing"; +import { + discoverSourceRepos, + indexSourceRepos, + selectSourceRepos, +} from "@voidbot/rag"; interface ScriptOptions { repos?: string[]; @@ -22,8 +24,7 @@ async function main(): Promise { throw new Error("SOURCE_REPO_ROOT is not configured."); } - const repoNames = - await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns); + const repoNames = await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns); const repos = selectSourceRepos(repoNames, options.repos); if (repos.length === 0) { diff --git a/package.json b/package.json index 802a1b6..1c27f5c 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "moderation:affect-state:fixture": "powershell -ExecutionPolicy Bypass -File scripts/smoke-void-affect-state-fixture.ps1", "agents:repo-face-affect-mcp-fixture": "node scripts/smoke-repo-face-affect-mcp-fixture.mjs", "rag:scrub-bot-prompts": "tsx scripts/scrub-bot-prompts-from-history.ts", - "rag:index-sources": "tsx scripts/index-source-repos.ts", + "rag:index-sources": "node apps/worker/dist/source-index-main.js", "rag:install-push-hooks": "tsx scripts/install-source-index-post-push-hooks.ts", "rag:reconcile-sources": "tsx scripts/reconcile-source-repos.ts", "rag:migrate-qdrant": "tsx scripts/migrate-local-vectors-to-qdrant.ts", diff --git a/packages/rag/src/index.ts b/packages/rag/src/index.ts index 5bc6267..f29d0a3 100644 --- a/packages/rag/src/index.ts +++ b/packages/rag/src/index.ts @@ -17,5 +17,7 @@ export * from "./source-document-archive"; export * from "./source-document-ingester"; export * from "./source-rag-pipeline"; export * from "./source-repo-crawler"; +export * from "./source-repo-discovery"; +export * from "./source-repo-indexing"; export * from "./source-vector-store"; export * from "./vector-store-factory"; diff --git a/scripts/source-repo-discovery.ts b/packages/rag/src/source-repo-discovery.ts similarity index 99% rename from scripts/source-repo-discovery.ts rename to packages/rag/src/source-repo-discovery.ts index 8d3d56d..22e2e7f 100644 --- a/scripts/source-repo-discovery.ts +++ b/packages/rag/src/source-repo-discovery.ts @@ -189,3 +189,4 @@ function matchesPattern(value: string, pattern: string): boolean { const regex = new RegExp(`^${escaped}$`, "i"); return regex.test(value); } + diff --git a/scripts/source-repo-indexing.ts b/packages/rag/src/source-repo-indexing.ts similarity index 74% rename from scripts/source-repo-indexing.ts rename to packages/rag/src/source-repo-indexing.ts index 71dbce0..530ea22 100644 --- a/scripts/source-repo-indexing.ts +++ b/packages/rag/src/source-repo-indexing.ts @@ -1,14 +1,35 @@ -import { type AppConfig } from "@voidbot/config"; -import { - crawlRepositoryDocuments, - createTextEmbedder, - createVectorStores, - FileSourceDocumentArchiveRepository, - SourceDocumentIngester, - SourceRagPipeline, -} from "@voidbot/rag"; - +import { createTextEmbedder } from "./embedder-factory"; +import { FileSourceDocumentArchiveRepository } from "./source-document-archive"; +import { SourceDocumentIngester } from "./source-document-ingester"; +import { SourceRagPipeline } from "./source-rag-pipeline"; +import { crawlRepositoryDocuments } from "./source-repo-crawler"; import { type SourceRepoMatch } from "./source-repo-discovery"; +import { createVectorStores } from "./vector-store-factory"; + +export interface SourceRepoIndexConfig { + ragSourceArchivePath: string; + ragEmbeddingBackend: "hash" | "ollama"; + ragEmbeddingDimensions: number; + ragOllamaBaseUrl: string; + ragOllamaModel: string; + ragOllamaTimeoutMs: number; + ragQueryInstruction: string; + sourceRepoIncludePrefixes: Record; + vectorStore: { + kind: "local_json" | "qdrant"; + path: string; + personaMemoryPath: string; + }; + sourceVectorStoreRoot: string; + qdrant: { + url: string; + apiKey?: string; + timeoutMs: number; + historyCollection: string; + sourceCollection: string; + personaMemoryCollection: string; + }; +} export interface IndexedRepoResult { repoName: string; @@ -28,7 +49,7 @@ export interface SourceIndexRunSummary { results: IndexedRepoResult[]; } -export function createSourceVectorStore(config: AppConfig) { +export function createSourceVectorStore(config: SourceRepoIndexConfig) { const embedder = createTextEmbedder({ backend: config.ragEmbeddingBackend, hashDimensions: config.ragEmbeddingDimensions, @@ -51,7 +72,7 @@ export function createSourceVectorStore(config: AppConfig) { } export async function indexSourceRepos( - config: AppConfig, + config: SourceRepoIndexConfig, repos: SourceRepoMatch[], options: { forceReindex?: boolean; diff --git a/scripts/git-post-push-index.mjs b/scripts/git-post-push-index.mjs index 2d331a5..f1942e5 100644 --- a/scripts/git-post-push-index.mjs +++ b/scripts/git-post-push-index.mjs @@ -126,7 +126,7 @@ async function runWorker() { const child = spawn( process.execPath, - [tsxCliPath, join(voidbotRoot, "scripts", "index-source-repos.ts"), "--repo", args.repoName], + [tsxCliPath, join(voidbotRoot, "apps", "worker", "src", "source-index-main.ts"), "--repo", args.repoName], { cwd: voidbotRoot, windowsHide: true, diff --git a/scripts/install-source-index-post-push-hooks.ts b/scripts/install-source-index-post-push-hooks.ts index 359078e..ec7d7a2 100644 --- a/scripts/install-source-index-post-push-hooks.ts +++ b/scripts/install-source-index-post-push-hooks.ts @@ -1,7 +1,7 @@ import "dotenv/config"; import { loadConfig } from "@voidbot/config"; -import { discoverSourceRepos, selectSourceRepos } from "./source-repo-discovery"; +import { discoverSourceRepos, selectSourceRepos } from "@voidbot/rag"; import { installSourceIndexPushHooks } from "./source-repo-hooks"; interface ScriptOptions { diff --git a/scripts/reconcile-source-repos.ts b/scripts/reconcile-source-repos.ts index a23569c..7238e40 100644 --- a/scripts/reconcile-source-repos.ts +++ b/scripts/reconcile-source-repos.ts @@ -5,11 +5,16 @@ import { fileURLToPath } from "node:url"; import { config as loadDotEnv } from "dotenv"; import { loadConfig } from "@voidbot/config"; -import { FileSourceDocumentArchiveRepository } from "@voidbot/rag"; +import { + createSourceVectorStore, + discoverSourceRepos, + FileSourceDocumentArchiveRepository, + indexSourceRepos, + selectSourceRepos, + type SourceRepoMatch, +} from "@voidbot/rag"; -import { discoverSourceRepos, selectSourceRepos, type SourceRepoMatch } from "./source-repo-discovery"; import { installSourceIndexPushHooks } from "./source-repo-hooks"; -import { createSourceVectorStore, indexSourceRepos } from "./source-repo-indexing"; interface ScriptOptions { repos?: string[]; diff --git a/scripts/source-repo-hooks.ts b/scripts/source-repo-hooks.ts index 42a54ef..dd6aea2 100644 --- a/scripts/source-repo-hooks.ts +++ b/scripts/source-repo-hooks.ts @@ -1,7 +1,7 @@ import { chmod, mkdir, readFile, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { type SourceRepoMatch } from "./source-repo-discovery"; +import { type SourceRepoMatch } from "@voidbot/rag"; export const HOOK_MARKER = "# voidbot-source-index-push-hook";