Compile the source indexing command into the runtime
This commit is contained in:
Meta Vens
parent
1103255277
commit
11568671a0
9 changed files with 54 additions and 24 deletions
|
|
@ -126,7 +126,7 @@ async function runWorker() {
|
|||
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[tsxCliPath, join(voidbotRoot, "scripts", "index-source-repos.ts"), "--repo", args.repoName],
|
||||
[tsxCliPath, join(voidbotRoot, "apps", "worker", "src", "source-index-main.ts"), "--repo", args.repoName],
|
||||
{
|
||||
cwd: voidbotRoot,
|
||||
windowsHide: true,
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import { loadConfig } from "@voidbot/config";
|
||||
|
||||
import { discoverSourceRepos, selectSourceRepos } from "./source-repo-discovery";
|
||||
import { indexSourceRepos } from "./source-repo-indexing";
|
||||
|
||||
interface ScriptOptions {
|
||||
repos?: string[];
|
||||
forceReindex?: boolean;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (!process.env.DISCORD_OWNER_ID) {
|
||||
process.env.DISCORD_OWNER_ID = "__source_reindex__";
|
||||
}
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const config = loadConfig();
|
||||
|
||||
if (!config.sourceRepoRoot) {
|
||||
throw new Error("SOURCE_REPO_ROOT is not configured.");
|
||||
}
|
||||
|
||||
const repoNames =
|
||||
await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns);
|
||||
const repos = selectSourceRepos(repoNames, options.repos);
|
||||
|
||||
if (repos.length === 0) {
|
||||
throw new Error("No source repositories matched SOURCE_REPO_PATTERNS.");
|
||||
}
|
||||
|
||||
const summary = await indexSourceRepos(config, repos, {
|
||||
forceReindex: options.forceReindex,
|
||||
logger: (line) => console.log(line),
|
||||
});
|
||||
|
||||
console.log(`Indexed repositories: ${summary.indexedRepositories}`);
|
||||
console.log(`Indexed documents: ${summary.totalDocuments}`);
|
||||
console.log(`Indexed chunks: ${summary.totalChunks}`);
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ScriptOptions {
|
||||
const options: ScriptOptions = {};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const argument = argv[index];
|
||||
|
||||
if ((argument === "--repo" || argument === "--repos") && argv[index + 1]) {
|
||||
options.repos = argv[index + 1]
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry.length > 0);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (argument === "--force") {
|
||||
options.forceReindex = true;
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
void main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import { loadConfig } from "@voidbot/config";
|
||||
import { discoverSourceRepos, selectSourceRepos } from "./source-repo-discovery";
|
||||
import { discoverSourceRepos, selectSourceRepos } from "@voidbot/rag";
|
||||
import { installSourceIndexPushHooks } from "./source-repo-hooks";
|
||||
|
||||
interface ScriptOptions {
|
||||
|
|
|
|||
|
|
@ -5,11 +5,16 @@ import { fileURLToPath } from "node:url";
|
|||
import { config as loadDotEnv } from "dotenv";
|
||||
|
||||
import { loadConfig } from "@voidbot/config";
|
||||
import { FileSourceDocumentArchiveRepository } from "@voidbot/rag";
|
||||
import {
|
||||
createSourceVectorStore,
|
||||
discoverSourceRepos,
|
||||
FileSourceDocumentArchiveRepository,
|
||||
indexSourceRepos,
|
||||
selectSourceRepos,
|
||||
type SourceRepoMatch,
|
||||
} from "@voidbot/rag";
|
||||
|
||||
import { discoverSourceRepos, selectSourceRepos, type SourceRepoMatch } from "./source-repo-discovery";
|
||||
import { installSourceIndexPushHooks } from "./source-repo-hooks";
|
||||
import { createSourceVectorStore, indexSourceRepos } from "./source-repo-indexing";
|
||||
|
||||
interface ScriptOptions {
|
||||
repos?: string[];
|
||||
|
|
|
|||
|
|
@ -1,191 +0,0 @@
|
|||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { readdir, readFile, stat } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
const GAMECULT_ORIGIN_PATTERN = /github\.com[:/]GameCult\//i;
|
||||
|
||||
export interface SourceRepoMatch {
|
||||
/** Canonical GameCult upstream slug: the source archive identity. */
|
||||
repoName: string;
|
||||
/** Local checkout label; diagnostic only, never an archive key. */
|
||||
localRepoName: string;
|
||||
repoPath: string;
|
||||
gitDir: string;
|
||||
}
|
||||
|
||||
export async function discoverSourceRepos(
|
||||
root: string,
|
||||
patterns: string[],
|
||||
): Promise<SourceRepoMatch[]> {
|
||||
const entries = await readdir(root, { withFileTypes: true });
|
||||
const repoMatches: SourceRepoMatch[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (patterns.length > 0 && !patterns.some((pattern) => matchesPattern(entry.name, pattern))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const repoPath = join(root, entry.name);
|
||||
const gitDir = await resolveGitDir(repoPath);
|
||||
|
||||
if (!gitDir) {
|
||||
continue;
|
||||
}
|
||||
|
||||
repoMatches.push({
|
||||
repoName: entry.name,
|
||||
localRepoName: entry.name,
|
||||
repoPath,
|
||||
gitDir,
|
||||
});
|
||||
}
|
||||
|
||||
const approved = await Promise.all(
|
||||
repoMatches.map(async (repo) => ({
|
||||
repo,
|
||||
originUrl: await readOriginUrl(repo.repoPath),
|
||||
})),
|
||||
);
|
||||
|
||||
return selectCanonicalGameCultRepos(approved);
|
||||
}
|
||||
|
||||
async function readOriginUrl(repoPath: string): Promise<string | undefined> {
|
||||
try {
|
||||
const { stdout } = await execFile("git", ["-C", repoPath, "remote", "get-url", "origin"], {
|
||||
windowsHide: true,
|
||||
});
|
||||
const originUrl = stdout.trim();
|
||||
return originUrl.length > 0 ? originUrl : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function selectCanonicalGameCultRepos(
|
||||
candidates: Array<{ repo: SourceRepoMatch; originUrl?: string }>,
|
||||
): SourceRepoMatch[] {
|
||||
const byOrigin = new Map<string, Array<{ repo: SourceRepoMatch; originUrl: string }>>();
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (!candidate.originUrl || !GAMECULT_ORIGIN_PATTERN.test(candidate.originUrl)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const originKey = normalizeOrigin(candidate.originUrl);
|
||||
const entries = byOrigin.get(originKey) ?? [];
|
||||
entries.push({ repo: candidate.repo, originUrl: candidate.originUrl });
|
||||
byOrigin.set(originKey, entries);
|
||||
}
|
||||
|
||||
return [...byOrigin.values()]
|
||||
.map((entries) => {
|
||||
const canonical = chooseCanonicalRepo(entries);
|
||||
return {
|
||||
...canonical,
|
||||
repoName: repoNameFromOrigin(entries[0].originUrl),
|
||||
};
|
||||
})
|
||||
.sort((left, right) => left.repoName.localeCompare(right.repoName));
|
||||
}
|
||||
|
||||
function chooseCanonicalRepo(entries: Array<{ repo: SourceRepoMatch; originUrl: string }>): SourceRepoMatch {
|
||||
const remoteName = repoNameFromOrigin(entries[0].originUrl);
|
||||
return entries
|
||||
.slice()
|
||||
.sort((left, right) => canonicalRank(left.repo, remoteName) - canonicalRank(right.repo, remoteName)
|
||||
|| left.repo.repoName.localeCompare(right.repo.repoName))[0].repo;
|
||||
}
|
||||
|
||||
function canonicalRank(repo: SourceRepoMatch, remoteName: string): number {
|
||||
if (repo.localRepoName.localeCompare(remoteName, undefined, { sensitivity: "accent" }) === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return repo.gitDir.replace(/\\/g, "/").toLowerCase().endsWith("/.git") ? 1 : 2;
|
||||
}
|
||||
|
||||
function normalizeOrigin(originUrl: string): string {
|
||||
return originUrl.trim().replace(/\\/g, "/").replace(/\.git$/i, "").toLowerCase();
|
||||
}
|
||||
|
||||
function repoNameFromOrigin(originUrl: string): string {
|
||||
const normalized = originUrl.trim().replace(/\\/g, "/").replace(/\.git$/i, "");
|
||||
return normalized.slice(normalized.lastIndexOf("/") + 1);
|
||||
}
|
||||
|
||||
export function selectSourceRepos(
|
||||
availableRepos: SourceRepoMatch[],
|
||||
requestedRepoNames?: string[],
|
||||
): SourceRepoMatch[] {
|
||||
if (!requestedRepoNames || requestedRepoNames.length === 0) {
|
||||
return availableRepos;
|
||||
}
|
||||
|
||||
const lookup = new Map(
|
||||
availableRepos.flatMap((repo) => [
|
||||
[repo.repoName.toLowerCase(), repo] as const,
|
||||
[repo.localRepoName.toLowerCase(), repo] as const,
|
||||
]),
|
||||
);
|
||||
const selectedRepos: SourceRepoMatch[] = [];
|
||||
const missingRepoNames: string[] = [];
|
||||
|
||||
for (const requestedRepoName of requestedRepoNames) {
|
||||
const repo = lookup.get(requestedRepoName.toLowerCase());
|
||||
|
||||
if (!repo) {
|
||||
missingRepoNames.push(requestedRepoName);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!selectedRepos.some((entry) => entry.repoName === repo.repoName)) {
|
||||
selectedRepos.push(repo);
|
||||
}
|
||||
}
|
||||
|
||||
if (missingRepoNames.length > 0) {
|
||||
throw new Error(
|
||||
`Requested source repos were not found under SOURCE_REPO_ROOT: ${missingRepoNames.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
return selectedRepos;
|
||||
}
|
||||
|
||||
export async function resolveGitDir(repoPath: string): Promise<string | undefined> {
|
||||
const dotGitPath = join(repoPath, ".git");
|
||||
|
||||
try {
|
||||
const dotGitStats = await stat(dotGitPath);
|
||||
|
||||
if (dotGitStats.isDirectory()) {
|
||||
return dotGitPath;
|
||||
}
|
||||
|
||||
if (dotGitStats.isFile()) {
|
||||
const pointer = await readFile(dotGitPath, "utf8");
|
||||
const match = pointer.match(/^gitdir:\s*(.+)$/im);
|
||||
|
||||
if (match?.[1]) {
|
||||
return resolve(repoPath, match[1].trim());
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function matchesPattern(value: string, pattern: string): boolean {
|
||||
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
||||
const regex = new RegExp(`^${escaped}$`, "i");
|
||||
return regex.test(value);
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import { chmod, mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { type SourceRepoMatch } from "./source-repo-discovery";
|
||||
import { type SourceRepoMatch } from "@voidbot/rag";
|
||||
|
||||
export const HOOK_MARKER = "# voidbot-source-index-push-hook";
|
||||
|
||||
|
|
|
|||
|
|
@ -1,114 +0,0 @@
|
|||
import { type AppConfig } from "@voidbot/config";
|
||||
import {
|
||||
crawlRepositoryDocuments,
|
||||
createTextEmbedder,
|
||||
createVectorStores,
|
||||
FileSourceDocumentArchiveRepository,
|
||||
SourceDocumentIngester,
|
||||
SourceRagPipeline,
|
||||
} from "@voidbot/rag";
|
||||
|
||||
import { type SourceRepoMatch } from "./source-repo-discovery";
|
||||
|
||||
export interface IndexedRepoResult {
|
||||
repoName: string;
|
||||
documents: number;
|
||||
createdDocuments: number;
|
||||
updatedDocuments: number;
|
||||
unchangedDocuments: number;
|
||||
deletedDocuments: number;
|
||||
indexedChunks: number;
|
||||
skippedFiles: number;
|
||||
}
|
||||
|
||||
export interface SourceIndexRunSummary {
|
||||
indexedRepositories: number;
|
||||
totalDocuments: number;
|
||||
totalChunks: number;
|
||||
results: IndexedRepoResult[];
|
||||
}
|
||||
|
||||
export function createSourceVectorStore(config: AppConfig) {
|
||||
const embedder = createTextEmbedder({
|
||||
backend: config.ragEmbeddingBackend,
|
||||
hashDimensions: config.ragEmbeddingDimensions,
|
||||
ollamaBaseUrl: config.ragOllamaBaseUrl,
|
||||
ollamaModel: config.ragOllamaModel,
|
||||
ollamaTimeoutMs: config.ragOllamaTimeoutMs,
|
||||
queryInstruction: config.ragQueryInstruction,
|
||||
});
|
||||
|
||||
return createVectorStores({
|
||||
kind: config.vectorStore.kind,
|
||||
historyPath: config.vectorStore.path,
|
||||
personaMemoryPath: config.vectorStore.personaMemoryPath,
|
||||
sourceRoot: config.sourceVectorStoreRoot,
|
||||
qdrant: config.qdrant,
|
||||
historyEmbedder: embedder,
|
||||
sourceEmbedder: embedder,
|
||||
personaMemoryEmbedder: embedder,
|
||||
}).source;
|
||||
}
|
||||
|
||||
export async function indexSourceRepos(
|
||||
config: AppConfig,
|
||||
repos: SourceRepoMatch[],
|
||||
options: {
|
||||
forceReindex?: boolean;
|
||||
logger?: (line: string) => void;
|
||||
} = {},
|
||||
): Promise<SourceIndexRunSummary> {
|
||||
const vectorStore = createSourceVectorStore(config);
|
||||
const archiveRepository = new FileSourceDocumentArchiveRepository(config.ragSourceArchivePath);
|
||||
const sourceIngester = new SourceDocumentIngester();
|
||||
const pipeline = new SourceRagPipeline(archiveRepository, sourceIngester, vectorStore);
|
||||
const results: IndexedRepoResult[] = [];
|
||||
let indexedRepositories = 0;
|
||||
let totalDocuments = 0;
|
||||
let totalChunks = 0;
|
||||
|
||||
for (const repo of repos) {
|
||||
const scan = await crawlRepositoryDocuments(repo.repoPath, repo.repoName, {
|
||||
includePathPrefixes: config.sourceRepoIncludePrefixes[repo.repoName]
|
||||
?? config.sourceRepoIncludePrefixes[repo.localRepoName],
|
||||
});
|
||||
const result = await pipeline.syncRepoDocuments(repo.repoName, scan.documents, {
|
||||
forceReindex: options.forceReindex,
|
||||
});
|
||||
const repoResult: IndexedRepoResult = {
|
||||
repoName: repo.repoName,
|
||||
documents: scan.documents.length,
|
||||
createdDocuments: result.createdDocuments,
|
||||
updatedDocuments: result.updatedDocuments,
|
||||
unchangedDocuments: result.unchangedDocuments,
|
||||
deletedDocuments: result.deletedDocuments,
|
||||
indexedChunks: result.indexedChunks,
|
||||
skippedFiles: scan.skippedFiles.length,
|
||||
};
|
||||
|
||||
results.push(repoResult);
|
||||
indexedRepositories += 1;
|
||||
totalDocuments += scan.documents.length;
|
||||
totalChunks += result.indexedChunks;
|
||||
|
||||
options.logger?.(
|
||||
[
|
||||
`Repo ${repo.repoName}:`,
|
||||
`documents=${repoResult.documents}`,
|
||||
`created=${repoResult.createdDocuments}`,
|
||||
`updated=${repoResult.updatedDocuments}`,
|
||||
`unchanged=${repoResult.unchangedDocuments}`,
|
||||
`deleted=${repoResult.deletedDocuments}`,
|
||||
`indexedChunks=${repoResult.indexedChunks}`,
|
||||
`skippedFiles=${repoResult.skippedFiles}`,
|
||||
].join(" "),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
indexedRepositories,
|
||||
totalDocuments,
|
||||
totalChunks,
|
||||
results,
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue