Compile the source indexing command into the runtime

apps/worker/src/source-index-main.ts

@@ -1,9 +1,11 @@
 import "dotenv/config";
 
 import { loadConfig } from "@voidbot/config";
-
-import { discoverSourceRepos, selectSourceRepos } from "./source-repo-discovery";
-import { indexSourceRepos } from "./source-repo-indexing";
+import {
+  discoverSourceRepos,
+  indexSourceRepos,
+  selectSourceRepos,
+} from "@voidbot/rag";
 
 interface ScriptOptions {
   repos?: string[];
@@ -22,8 +24,7 @@ async function main(): Promise<void> {
     throw new Error("SOURCE_REPO_ROOT is not configured.");
   }
 
-  const repoNames =
-    await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns);
+  const repoNames = await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns);
   const repos = selectSourceRepos(repoNames, options.repos);
 
   if (repos.length === 0) {

package.json

@@ -66,7 +66,7 @@
     "moderation:affect-state:fixture": "powershell -ExecutionPolicy Bypass -File scripts/smoke-void-affect-state-fixture.ps1",
     "agents:repo-face-affect-mcp-fixture": "node scripts/smoke-repo-face-affect-mcp-fixture.mjs",
     "rag:scrub-bot-prompts": "tsx scripts/scrub-bot-prompts-from-history.ts",
-    "rag:index-sources": "tsx scripts/index-source-repos.ts",
+    "rag:index-sources": "node apps/worker/dist/source-index-main.js",
     "rag:install-push-hooks": "tsx scripts/install-source-index-post-push-hooks.ts",
     "rag:reconcile-sources": "tsx scripts/reconcile-source-repos.ts",
     "rag:migrate-qdrant": "tsx scripts/migrate-local-vectors-to-qdrant.ts",

packages/rag/src/index.ts

@@ -17,5 +17,7 @@ export * from "./source-document-archive";
 export * from "./source-document-ingester";
 export * from "./source-rag-pipeline";
 export * from "./source-repo-crawler";
+export * from "./source-repo-discovery";
+export * from "./source-repo-indexing";
 export * from "./source-vector-store";
 export * from "./vector-store-factory";

packages/rag/src/source-repo-discovery.ts

@@ -189,3 +189,4 @@ function matchesPattern(value: string, pattern: string): boolean {
   const regex = new RegExp(`^${escaped}$`, "i");
   return regex.test(value);
 }
+

packages/rag/src/source-repo-indexing.ts

@@ -1,14 +1,35 @@
-import { type AppConfig } from "@voidbot/config";
-import {
-  crawlRepositoryDocuments,
-  createTextEmbedder,
-  createVectorStores,
-  FileSourceDocumentArchiveRepository,
-  SourceDocumentIngester,
-  SourceRagPipeline,
-} from "@voidbot/rag";
-
+import { createTextEmbedder } from "./embedder-factory";
+import { FileSourceDocumentArchiveRepository } from "./source-document-archive";
+import { SourceDocumentIngester } from "./source-document-ingester";
+import { SourceRagPipeline } from "./source-rag-pipeline";
+import { crawlRepositoryDocuments } from "./source-repo-crawler";
 import { type SourceRepoMatch } from "./source-repo-discovery";
+import { createVectorStores } from "./vector-store-factory";
+
+export interface SourceRepoIndexConfig {
+  ragSourceArchivePath: string;
+  ragEmbeddingBackend: "hash" | "ollama";
+  ragEmbeddingDimensions: number;
+  ragOllamaBaseUrl: string;
+  ragOllamaModel: string;
+  ragOllamaTimeoutMs: number;
+  ragQueryInstruction: string;
+  sourceRepoIncludePrefixes: Record<string, string[]>;
+  vectorStore: {
+    kind: "local_json" | "qdrant";
+    path: string;
+    personaMemoryPath: string;
+  };
+  sourceVectorStoreRoot: string;
+  qdrant: {
+    url: string;
+    apiKey?: string;
+    timeoutMs: number;
+    historyCollection: string;
+    sourceCollection: string;
+    personaMemoryCollection: string;
+  };
+}
 
 export interface IndexedRepoResult {
   repoName: string;
@@ -28,7 +49,7 @@ export interface SourceIndexRunSummary {
   results: IndexedRepoResult[];
 }
 
-export function createSourceVectorStore(config: AppConfig) {
+export function createSourceVectorStore(config: SourceRepoIndexConfig) {
   const embedder = createTextEmbedder({
     backend: config.ragEmbeddingBackend,
     hashDimensions: config.ragEmbeddingDimensions,
@@ -51,7 +72,7 @@ export function createSourceVectorStore(config: AppConfig) {
 }
 
 export async function indexSourceRepos(
-  config: AppConfig,
+  config: SourceRepoIndexConfig,
   repos: SourceRepoMatch[],
   options: {
     forceReindex?: boolean;

scripts/git-post-push-index.mjs

@@ -126,7 +126,7 @@ async function runWorker() {
 
     const child = spawn(
       process.execPath,
-      [tsxCliPath, join(voidbotRoot, "scripts", "index-source-repos.ts"), "--repo", args.repoName],
+      [tsxCliPath, join(voidbotRoot, "apps", "worker", "src", "source-index-main.ts"), "--repo", args.repoName],
       {
         cwd: voidbotRoot,
         windowsHide: true,

scripts/install-source-index-post-push-hooks.ts

@@ -1,7 +1,7 @@
 import "dotenv/config";
 
 import { loadConfig } from "@voidbot/config";
-import { discoverSourceRepos, selectSourceRepos } from "./source-repo-discovery";
+import { discoverSourceRepos, selectSourceRepos } from "@voidbot/rag";
 import { installSourceIndexPushHooks } from "./source-repo-hooks";
 
 interface ScriptOptions {

scripts/reconcile-source-repos.ts

@@ -5,11 +5,16 @@ import { fileURLToPath } from "node:url";
 import { config as loadDotEnv } from "dotenv";
 
 import { loadConfig } from "@voidbot/config";
-import { FileSourceDocumentArchiveRepository } from "@voidbot/rag";
+import {
+  createSourceVectorStore,
+  discoverSourceRepos,
+  FileSourceDocumentArchiveRepository,
+  indexSourceRepos,
+  selectSourceRepos,
+  type SourceRepoMatch,
+} from "@voidbot/rag";
 
-import { discoverSourceRepos, selectSourceRepos, type SourceRepoMatch } from "./source-repo-discovery";
 import { installSourceIndexPushHooks } from "./source-repo-hooks";
-import { createSourceVectorStore, indexSourceRepos } from "./source-repo-indexing";
 
 interface ScriptOptions {
   repos?: string[];

scripts/source-repo-hooks.ts

@@ -1,7 +1,7 @@
 import { chmod, mkdir, readFile, stat, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 
-import { type SourceRepoMatch } from "./source-repo-discovery";
+import { type SourceRepoMatch } from "@voidbot/rag";
 
 export const HOOK_MARKER = "# voidbot-source-index-push-hook";