Index from admitted source catalog

apps/worker/src/source-index-main.ts

@@ -3,6 +3,7 @@ import "dotenv/config";
 import { loadConfig } from "@voidbot/config";
 import {
   discoverSourceRepos,
+  discoverSourceReposFromCatalog,
   indexSourceRepos,
   selectSourceRepos,
 } from "@voidbot/rag";
@@ -10,6 +11,7 @@ import {
 interface ScriptOptions {
   repos?: string[];
   forceReindex?: boolean;
+  catalogPath?: string;
 }
 
 async function main(): Promise<void> {
@@ -24,7 +26,9 @@ async function main(): Promise<void> {
     throw new Error("SOURCE_REPO_ROOT is not configured.");
   }
 
-  const repoNames = await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns);
+  const repoNames = options.catalogPath
+    ? await discoverSourceReposFromCatalog(config.sourceRepoRoot, options.catalogPath)
+    : await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns);
   const repos = selectSourceRepos(repoNames, options.repos);
 
   if (repos.length === 0) {
@@ -58,6 +62,12 @@ function parseArgs(argv: string[]): ScriptOptions {
 
     if (argument === "--force") {
       options.forceReindex = true;
+      continue;
+    }
+
+    if (argument === "--catalog" && argv[index + 1]) {
+      options.catalogPath = argv[index + 1];
+      index += 1;
     }
   }
 

packages/rag/src/source-repo-discovery.ts

@@ -1,7 +1,7 @@
 import { execFile as execFileCallback } from "node:child_process";
 import { promisify } from "node:util";
 import { readdir, readFile, stat } from "node:fs/promises";
-import { join, resolve } from "node:path";
+import { isAbsolute, join, relative, resolve } from "node:path";
 
 const execFile = promisify(execFileCallback);
 const GAMECULT_ORIGIN_PATTERN = /github\.com[:/]GameCult\//i;
@@ -56,6 +56,54 @@ export async function discoverSourceRepos(
   return selectCanonicalGameCultRepos(approved);
 }
 
+export async function discoverSourceReposFromCatalog(
+  root: string,
+  catalogPath: string,
+): Promise<SourceRepoMatch[]> {
+  const resolvedRoot = resolve(root);
+  const catalog = await readFile(catalogPath, "utf8");
+  const candidates: Array<{ repo: SourceRepoMatch; originUrl: string }> = [];
+
+  for (const [index, rawLine] of catalog.split(/\r?\n/).entries()) {
+    const line = rawLine.trim();
+
+    if (line.length === 0) {
+      continue;
+    }
+
+    const [localRepoName, originUrl] = line.split("\t");
+
+    if (!localRepoName || !originUrl || !GAMECULT_ORIGIN_PATTERN.test(originUrl)) {
+      throw new Error(`Invalid source catalog entry at line ${index + 1}.`);
+    }
+
+    const repoPath = resolve(resolvedRoot, localRepoName);
+    const relativePath = relative(resolvedRoot, repoPath);
+
+    if (relativePath.startsWith("..") || isAbsolute(relativePath)) {
+      throw new Error(`Source catalog entry escapes SOURCE_REPO_ROOT: ${localRepoName}`);
+    }
+
+    const gitDir = await resolveGitDir(repoPath);
+
+    if (!gitDir) {
+      throw new Error(`Source catalog repository is not materialized: ${localRepoName}`);
+    }
+
+    candidates.push({
+      repo: {
+        repoName: localRepoName,
+        localRepoName,
+        repoPath,
+        gitDir,
+      },
+      originUrl,
+    });
+  }
+
+  return selectCanonicalGameCultRepos(candidates);
+}
+
 async function readOriginUrl(repoPath: string): Promise<string | undefined> {
   try {
     const { stdout } = await execFile("git", ["-C", repoPath, "remote", "get-url", "origin"], {
@@ -189,4 +237,3 @@ function matchesPattern(value: string, pattern: string): boolean {
   const regex = new RegExp(`^${escaped}$`, "i");
   return regex.test(value);
 }
-

scripts/smoke-source-rag-commit-boundary.ts

@@ -1,4 +1,4 @@
-import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
@@ -7,6 +7,7 @@ import {
   InMemoryVectorStore,
   SourceDocumentIngester,
   SourceRagPipeline,
+  discoverSourceReposFromCatalog,
   type ArchivedSourceDocument,
 } from "@voidbot/rag";
 
@@ -51,6 +52,13 @@ async function main(): Promise<void> {
 
     const manifest = JSON.parse(await readFile(archivePath, "utf8")) as { repos?: unknown[] };
     assertEqual(manifest.repos?.length, 1, "manifest summary was not committed");
+
+    const repoRoot = join(root, "repos");
+    await mkdir(join(repoRoot, "Fixture", ".git"), { recursive: true });
+    const catalogPath = join(root, "catalog.tsv");
+    await writeFile(catalogPath, "Fixture\thttps://github.com/GameCult/Fixture.git\tmain\n", "utf8");
+    const catalogRepos = await discoverSourceReposFromCatalog(repoRoot, catalogPath);
+    assertEqual(catalogRepos[0]?.repoName, "Fixture", "catalog discovery lost canonical repo identity");
     console.log("Source RAG commit-boundary smoke passed.");
   } finally {
     await rm(root, { recursive: true, force: true });