Auto-reconcile source repo indexing
This commit is contained in:
Meta Vens
parent
32e4a1f184
commit
de223f0fff
16 changed files with 1125 additions and 286 deletions
|
|
@ -235,6 +235,46 @@ function Invoke-CommandWithTimeout {
|
|||
}
|
||||
}
|
||||
|
||||
function Invoke-SourceRepoReconcile {
|
||||
param(
|
||||
[switch] $CheckOnly,
|
||||
[switch] $Detached
|
||||
)
|
||||
|
||||
$tsxCommand = Join-Path $repoRoot "node_modules\.bin\tsx.cmd"
|
||||
|
||||
if (-not (Test-Path -LiteralPath $tsxCommand)) {
|
||||
throw "Local tsx launcher is missing at $tsxCommand."
|
||||
}
|
||||
|
||||
$arguments = @("scripts/reconcile-source-repos.ts", "--json")
|
||||
|
||||
if ($CheckOnly) {
|
||||
$arguments += "--check"
|
||||
}
|
||||
|
||||
if ($Detached) {
|
||||
$arguments += "--detached"
|
||||
}
|
||||
|
||||
$stdout = (& $tsxCommand @arguments 2>&1 | Out-String).Trim()
|
||||
$exitCode = $LASTEXITCODE
|
||||
|
||||
if ($exitCode -ne 0) {
|
||||
if ([string]::IsNullOrWhiteSpace($stdout)) {
|
||||
$stdout = "Source repo reconcile exited with code $exitCode."
|
||||
}
|
||||
|
||||
throw $stdout
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($stdout)) {
|
||||
throw "Source repo reconcile returned no JSON output."
|
||||
}
|
||||
|
||||
return $stdout | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function Ensure-DockerCliOnPath {
|
||||
$dockerCliDir = "C:\Program Files\Docker\Docker\resources\bin"
|
||||
|
||||
|
|
@ -652,6 +692,7 @@ try {
|
|||
$vectorStoreKind = if ($config.ContainsKey("VECTOR_STORE_KIND")) { $config["VECTOR_STORE_KIND"] } else { "local_json" }
|
||||
$historyCollection = if ($config.ContainsKey("QDRANT_HISTORY_COLLECTION")) { $config["QDRANT_HISTORY_COLLECTION"] } else { "voidbot_discord_history_chunks" }
|
||||
$sourceCollection = if ($config.ContainsKey("QDRANT_SOURCE_COLLECTION")) { $config["QDRANT_SOURCE_COLLECTION"] } else { "voidbot_repository_source_chunks" }
|
||||
$qdrantReady = $vectorStoreKind -ne "qdrant"
|
||||
|
||||
Set-WatchdogStep -CurrentStep "qdrant_checks" -Detail "Verifying Qdrant reachability and required collections."
|
||||
if ($vectorStoreKind -eq "qdrant") {
|
||||
|
|
@ -659,6 +700,7 @@ try {
|
|||
try {
|
||||
$collectionsResponse = Get-Json -Url "$($qdrantUrl.TrimEnd('/'))/collections"
|
||||
$collectionNames = @($collectionsResponse.result.collections | ForEach-Object { [string]$_.name })
|
||||
$qdrantReady = $true
|
||||
Add-Check -Name "qdrant.reachable" -Status "passed" -Detail "Qdrant is reachable at $qdrantUrl." -Data @{
|
||||
url = $qdrantUrl
|
||||
collections = $collectionNames
|
||||
|
|
@ -742,6 +784,7 @@ try {
|
|||
|
||||
$ragOllamaUrl = if ($config.ContainsKey("RAG_OLLAMA_BASE_URL")) { $config["RAG_OLLAMA_BASE_URL"] } else { "http://127.0.0.1:11434" }
|
||||
$ragOllamaModel = if ($config.ContainsKey("RAG_OLLAMA_MODEL")) { $config["RAG_OLLAMA_MODEL"] } else { "qwen3-embedding:0.6b" }
|
||||
$ragOllamaReady = $false
|
||||
|
||||
Set-WatchdogStep -CurrentStep "ollama_checks" -Detail "Checking embedding and local LLM Ollama endpoints."
|
||||
try {
|
||||
|
|
@ -749,6 +792,7 @@ try {
|
|||
$ragModels = @($ragTags.models | ForEach-Object { [string]$_.name })
|
||||
|
||||
if ($ragModels -contains $ragOllamaModel) {
|
||||
$ragOllamaReady = $true
|
||||
Add-Check -Name "rag_ollama.model" -Status "passed" -Detail "Embedding Ollama is reachable and has $ragOllamaModel." -Data @{
|
||||
url = $ragOllamaUrl
|
||||
model = $ragOllamaModel
|
||||
|
|
@ -805,6 +849,54 @@ try {
|
|||
}
|
||||
}
|
||||
|
||||
$sourceRepoRoot = if ($config.ContainsKey("SOURCE_REPO_ROOT")) { $config["SOURCE_REPO_ROOT"] } else { "" }
|
||||
if ([string]::IsNullOrWhiteSpace($sourceRepoRoot)) {
|
||||
Add-Check -Name "source.repos" -Status "passed" -Detail "Source repo reconciliation check skipped because SOURCE_REPO_ROOT is blank."
|
||||
} else {
|
||||
Set-WatchdogStep -CurrentStep "source_repo_checks" -Detail "Reconciling discovered source repos against the indexed source archive."
|
||||
|
||||
try {
|
||||
$sourceRepoPreview = Invoke-SourceRepoReconcile -CheckOnly
|
||||
$missingRepos = @($sourceRepoPreview.missingRepos | ForEach-Object { [string]$_ })
|
||||
$staleIndexedRepos = @($sourceRepoPreview.staleIndexedRepos | ForEach-Object { [string]$_ })
|
||||
|
||||
if ($missingRepos.Count -eq 0) {
|
||||
Add-Check -Name "source.repos" -Status "passed" -Detail "All discovered source repos are represented in the indexed source archive." -Data @{
|
||||
discoveredRepoCount = [int]$sourceRepoPreview.discoveredRepoCount
|
||||
indexedRepoCount = [int]$sourceRepoPreview.indexedRepoCount
|
||||
selectedRepoCount = [int]$sourceRepoPreview.selectedRepoCount
|
||||
}
|
||||
} elseif ($qdrantReady -and $ragOllamaReady) {
|
||||
$sourceRepoRepair = Invoke-SourceRepoReconcile -Detached
|
||||
$launchedRepos = @($sourceRepoRepair.launchedRepos | ForEach-Object { [string]$_ })
|
||||
Add-Check -Name "source.repos" -Status "warning" -Detail "Detected $($missingRepos.Count) unindexed source repos and launched detached reconciliation." -Data @{
|
||||
missingRepos = $missingRepos
|
||||
launchedRepos = $launchedRepos
|
||||
discoveredRepoCount = [int]$sourceRepoPreview.discoveredRepoCount
|
||||
indexedRepoCount = [int]$sourceRepoPreview.indexedRepoCount
|
||||
}
|
||||
} else {
|
||||
Add-Check -Name "source.repos" -Status "failed" -Detail "Detected $($missingRepos.Count) unindexed source repos, but source indexing dependencies are unhealthy." -Data @{
|
||||
missingRepos = $missingRepos
|
||||
qdrantReady = $qdrantReady
|
||||
ragOllamaReady = $ragOllamaReady
|
||||
}
|
||||
}
|
||||
|
||||
if ($staleIndexedRepos.Count -eq 0) {
|
||||
Add-Check -Name "source.stale_index" -Status "passed" -Detail "Indexed source archive has no repo summaries for repos that are no longer present."
|
||||
} else {
|
||||
Add-Check -Name "source.stale_index" -Status "warning" -Detail "Indexed source archive still carries repo summaries for repos that are no longer present." -Data @{
|
||||
staleIndexedRepos = $staleIndexedRepos
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
Add-Check -Name "source.repos" -Status "failed" -Detail "Source repo reconciliation check failed: $($_.Exception.Message)" -Data @{
|
||||
sourceRepoRoot = $sourceRepoRoot
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($config.ContainsKey("DISCORD_BOT_TOKEN") -and -not [string]::IsNullOrWhiteSpace($config["DISCORD_BOT_TOKEN"])) {
|
||||
Set-WatchdogStep -CurrentStep "discord_checks" -Detail "Validating Discord bot credentials."
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,17 +1,9 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { loadConfig } from "@voidbot/config";
|
||||
import {
|
||||
crawlRepositoryDocuments,
|
||||
createTextEmbedder,
|
||||
createVectorStores,
|
||||
FileSourceDocumentArchiveRepository,
|
||||
SourceDocumentIngester,
|
||||
SourceRagPipeline,
|
||||
} from "@voidbot/rag";
|
||||
|
||||
import { discoverSourceRepos, selectSourceRepos } from "./source-repo-discovery";
|
||||
import { indexSourceRepos } from "./source-repo-indexing";
|
||||
|
||||
interface ScriptOptions {
|
||||
repos?: string[];
|
||||
|
|
@ -31,67 +23,21 @@ async function main(): Promise<void> {
|
|||
}
|
||||
|
||||
const repoNames =
|
||||
options.repos && options.repos.length > 0
|
||||
? options.repos
|
||||
: await resolveRepoNames(config.sourceRepoRoot, config.sourceRepoPatterns);
|
||||
await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns);
|
||||
const repos = selectSourceRepos(repoNames, options.repos);
|
||||
|
||||
if (repoNames.length === 0) {
|
||||
if (repos.length === 0) {
|
||||
throw new Error("No source repositories matched SOURCE_REPO_PATTERNS.");
|
||||
}
|
||||
|
||||
const embedder = createTextEmbedder({
|
||||
backend: config.ragEmbeddingBackend,
|
||||
hashDimensions: config.ragEmbeddingDimensions,
|
||||
ollamaBaseUrl: config.ragOllamaBaseUrl,
|
||||
ollamaModel: config.ragOllamaModel,
|
||||
ollamaTimeoutMs: config.ragOllamaTimeoutMs,
|
||||
queryInstruction: config.ragQueryInstruction,
|
||||
const summary = await indexSourceRepos(config, repos, {
|
||||
forceReindex: options.forceReindex,
|
||||
logger: (line) => console.log(line),
|
||||
});
|
||||
const vectorStore = createVectorStores({
|
||||
kind: config.vectorStore.kind,
|
||||
historyPath: config.vectorStore.path,
|
||||
sourceRoot: config.sourceVectorStoreRoot,
|
||||
qdrant: config.qdrant,
|
||||
historyEmbedder: embedder,
|
||||
sourceEmbedder: embedder,
|
||||
}).source;
|
||||
const archiveRepository = new FileSourceDocumentArchiveRepository(config.ragSourceArchivePath);
|
||||
const sourceIngester = new SourceDocumentIngester();
|
||||
const pipeline = new SourceRagPipeline(archiveRepository, sourceIngester, vectorStore);
|
||||
|
||||
let indexedRepositories = 0;
|
||||
let totalDocuments = 0;
|
||||
let totalChunks = 0;
|
||||
|
||||
for (const repoName of repoNames) {
|
||||
const repoPath = join(config.sourceRepoRoot, repoName);
|
||||
const scan = await crawlRepositoryDocuments(repoPath, repoName, {
|
||||
includePathPrefixes: config.sourceRepoIncludePrefixes[repoName],
|
||||
});
|
||||
const result = await pipeline.syncRepoDocuments(repoName, scan.documents, {
|
||||
forceReindex: options.forceReindex,
|
||||
});
|
||||
indexedRepositories += 1;
|
||||
totalDocuments += scan.documents.length;
|
||||
totalChunks += result.indexedChunks;
|
||||
|
||||
console.log(
|
||||
[
|
||||
`Repo ${repoName}:`,
|
||||
`documents=${scan.documents.length}`,
|
||||
`created=${result.createdDocuments}`,
|
||||
`updated=${result.updatedDocuments}`,
|
||||
`unchanged=${result.unchangedDocuments}`,
|
||||
`deleted=${result.deletedDocuments}`,
|
||||
`indexedChunks=${result.indexedChunks}`,
|
||||
`skippedFiles=${scan.skippedFiles.length}`,
|
||||
].join(" "),
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`Indexed repositories: ${indexedRepositories}`);
|
||||
console.log(`Indexed documents: ${totalDocuments}`);
|
||||
console.log(`Indexed chunks: ${totalChunks}`);
|
||||
console.log(`Indexed repositories: ${summary.indexedRepositories}`);
|
||||
console.log(`Indexed documents: ${summary.totalDocuments}`);
|
||||
console.log(`Indexed chunks: ${summary.totalChunks}`);
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ScriptOptions {
|
||||
|
|
@ -117,23 +63,6 @@ function parseArgs(argv: string[]): ScriptOptions {
|
|||
return options;
|
||||
}
|
||||
|
||||
async function resolveRepoNames(root: string, patterns: string[]): Promise<string[]> {
|
||||
const entries = await readdir(root, { withFileTypes: true });
|
||||
const repoNames = entries
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.filter((name) => patterns.length === 0 || patterns.some((pattern) => matchesPattern(name, pattern)))
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
|
||||
return repoNames;
|
||||
}
|
||||
|
||||
function matchesPattern(value: string, pattern: string): boolean {
|
||||
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
||||
const regex = new RegExp(`^${escaped}$`, "i");
|
||||
return regex.test(value);
|
||||
}
|
||||
|
||||
void main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import { readdir, chmod, mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import { loadConfig } from "@voidbot/config";
|
||||
|
||||
const HOOK_MARKER = "# voidbot-source-index-push-hook";
|
||||
import { discoverSourceRepos, selectSourceRepos } from "./source-repo-discovery";
|
||||
import { installSourceIndexPushHooks } from "./source-repo-hooks";
|
||||
|
||||
interface ScriptOptions {
|
||||
repos?: string[];
|
||||
|
|
@ -23,77 +20,20 @@ async function main(): Promise<void> {
|
|||
}
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const repoNames =
|
||||
options.repos && options.repos.length > 0
|
||||
? options.repos
|
||||
: await resolveRepoNames(config.sourceRepoRoot, config.sourceRepoPatterns);
|
||||
const discoveredRepos = await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns);
|
||||
const selectedRepos = selectSourceRepos(discoveredRepos, options.repos);
|
||||
const summary = await installSourceIndexPushHooks(process.cwd(), selectedRepos);
|
||||
|
||||
let installed = 0;
|
||||
let updated = 0;
|
||||
let backedUp = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const repoName of repoNames) {
|
||||
const repoPath = join(config.sourceRepoRoot, repoName);
|
||||
const gitDir = await resolveGitDir(repoPath);
|
||||
|
||||
if (!gitDir) {
|
||||
console.log(`Skipping ${repoName}: no .git directory found.`);
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const hooksDir = join(gitDir, "hooks");
|
||||
const hookPath = join(hooksDir, "pre-push");
|
||||
const backupHookPath = join(hooksDir, "pre-push.voidbot.prev");
|
||||
|
||||
await mkdir(hooksDir, { recursive: true });
|
||||
|
||||
let existingContent: string | undefined;
|
||||
|
||||
try {
|
||||
existingContent = await readFile(hookPath, "utf8");
|
||||
} catch {
|
||||
existingContent = undefined;
|
||||
}
|
||||
|
||||
if (
|
||||
existingContent &&
|
||||
!existingContent.includes(HOOK_MARKER) &&
|
||||
!(await pathExists(backupHookPath))
|
||||
) {
|
||||
await writeFile(backupHookPath, existingContent, "utf8");
|
||||
await chmod(backupHookPath, 0o755);
|
||||
backedUp += 1;
|
||||
} else if (
|
||||
existingContent &&
|
||||
!existingContent.includes(HOOK_MARKER) &&
|
||||
(await pathExists(backupHookPath))
|
||||
) {
|
||||
console.log(`Skipping ${repoName}: existing pre-push hook already has a Void backup; not clobbering it.`);
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const hookScript = renderHookScript(process.cwd(), repoName);
|
||||
await writeFile(hookPath, hookScript, "utf8");
|
||||
await chmod(hookPath, 0o755);
|
||||
|
||||
if (existingContent?.includes(HOOK_MARKER)) {
|
||||
updated += 1;
|
||||
console.log(`Updated pre-push hook for ${repoName}.`);
|
||||
} else {
|
||||
installed += 1;
|
||||
console.log(`Installed pre-push hook for ${repoName}.`);
|
||||
}
|
||||
for (const message of summary.messages) {
|
||||
console.log(message);
|
||||
}
|
||||
|
||||
console.log(
|
||||
[
|
||||
`installed=${installed}`,
|
||||
`updated=${updated}`,
|
||||
`backedUp=${backedUp}`,
|
||||
`skipped=${skipped}`,
|
||||
`installed=${summary.installed}`,
|
||||
`updated=${summary.updated}`,
|
||||
`backedUp=${summary.backedUp}`,
|
||||
`skipped=${summary.skipped}`,
|
||||
].join(" "),
|
||||
);
|
||||
}
|
||||
|
|
@ -116,89 +56,6 @@ function parseArgs(argv: string[]): ScriptOptions {
|
|||
return options;
|
||||
}
|
||||
|
||||
async function resolveRepoNames(root: string, patterns: string[]): Promise<string[]> {
|
||||
const entries = await readdir(root, { withFileTypes: true });
|
||||
return entries
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.filter((name) => patterns.length === 0 || patterns.some((pattern) => matchesPattern(name, pattern)))
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
async function resolveGitDir(repoPath: string): Promise<string | undefined> {
|
||||
const dotGitPath = join(repoPath, ".git");
|
||||
|
||||
try {
|
||||
const dotGitStats = await stat(dotGitPath);
|
||||
|
||||
if (dotGitStats.isDirectory()) {
|
||||
return dotGitPath;
|
||||
}
|
||||
|
||||
if (dotGitStats.isFile()) {
|
||||
const pointer = await readFile(dotGitPath, "utf8");
|
||||
const match = pointer.match(/^gitdir:\s*(.+)$/im);
|
||||
|
||||
if (match?.[1]) {
|
||||
return resolve(repoPath, match[1].trim());
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function renderHookScript(voidbotRoot: string, repoName: string): string {
|
||||
const normalizedRoot = voidbotRoot.replace(/\\/g, "/");
|
||||
const launcherPath = `${normalizedRoot}/scripts/git-post-push-index.mjs`;
|
||||
const nodePath = process.execPath.replace(/\\/g, "/");
|
||||
const quotedNodePath = escapeForDoubleQuotedShell(nodePath);
|
||||
const quotedLauncherPath = escapeForDoubleQuotedShell(launcherPath);
|
||||
const quotedRoot = escapeForDoubleQuotedShell(normalizedRoot);
|
||||
const quotedRepoName = escapeForDoubleQuotedShell(repoName);
|
||||
|
||||
return [
|
||||
"#!/bin/sh",
|
||||
HOOK_MARKER,
|
||||
"HOOK_DIR=\"$(CDPATH= cd -- \"$(dirname -- \"$0\")\" && pwd)\"",
|
||||
"PREV_HOOK=\"$HOOK_DIR/pre-push.voidbot.prev\"",
|
||||
"",
|
||||
"if [ -x \"$PREV_HOOK\" ]; then",
|
||||
" \"$PREV_HOOK\" \"$@\"",
|
||||
" PREV_STATUS=$?",
|
||||
" if [ \"$PREV_STATUS\" -ne 0 ]; then",
|
||||
" exit \"$PREV_STATUS\"",
|
||||
" fi",
|
||||
"fi",
|
||||
"",
|
||||
`\"${quotedNodePath}\" \"${quotedLauncherPath}\" --voidbot-root \"${quotedRoot}\" --repo-name \"${quotedRepoName}\" >/dev/null 2>&1 || true`,
|
||||
"",
|
||||
"exit 0",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function escapeForDoubleQuotedShell(value: string): string {
|
||||
return value.replace(/(["`$\\])/g, "\\$1");
|
||||
}
|
||||
|
||||
function matchesPattern(value: string, pattern: string): boolean {
|
||||
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
||||
const regex = new RegExp(`^${escaped}$`, "i");
|
||||
return regex.test(value);
|
||||
}
|
||||
|
||||
async function pathExists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await stat(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
|
|
|
|||
267
scripts/reconcile-source-repos.ts
Normal file
267
scripts/reconcile-source-repos.ts
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { loadConfig } from "@voidbot/config";
|
||||
import { FileSourceDocumentArchiveRepository } from "@voidbot/rag";
|
||||
|
||||
import { discoverSourceRepos, selectSourceRepos, type SourceRepoMatch } from "./source-repo-discovery";
|
||||
import { installSourceIndexPushHooks } from "./source-repo-hooks";
|
||||
import { createSourceVectorStore, indexSourceRepos } from "./source-repo-indexing";
|
||||
|
||||
interface ScriptOptions {
|
||||
repos?: string[];
|
||||
forceReindex?: boolean;
|
||||
detached?: boolean;
|
||||
json?: boolean;
|
||||
checkOnly?: boolean;
|
||||
}
|
||||
|
||||
interface ReconcileSummary {
|
||||
discoveredRepoCount: number;
|
||||
indexedRepoCount: number;
|
||||
selectedRepoCount: number;
|
||||
missingRepos: string[];
|
||||
staleIndexedRepos: string[];
|
||||
hookInstall: {
|
||||
installed: number;
|
||||
updated: number;
|
||||
backedUp: number;
|
||||
skipped: number;
|
||||
};
|
||||
prunedRepos: string[];
|
||||
normalizedArchive: boolean;
|
||||
detached: boolean;
|
||||
forceReindex: boolean;
|
||||
checkOnly: boolean;
|
||||
indexedRepos: string[];
|
||||
launchedRepos: string[];
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (!process.env.DISCORD_OWNER_ID) {
|
||||
process.env.DISCORD_OWNER_ID = "__source_reconcile__";
|
||||
}
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const config = loadConfig();
|
||||
|
||||
if (!config.sourceRepoRoot) {
|
||||
throw new Error("SOURCE_REPO_ROOT is not configured.");
|
||||
}
|
||||
|
||||
const discoveredRepos = await discoverSourceRepos(config.sourceRepoRoot, config.sourceRepoPatterns);
|
||||
const selectedRepos = selectSourceRepos(discoveredRepos, options.repos);
|
||||
|
||||
if (selectedRepos.length === 0) {
|
||||
throw new Error("No source repositories matched SOURCE_REPO_PATTERNS.");
|
||||
}
|
||||
|
||||
const archiveRepository = new FileSourceDocumentArchiveRepository(config.ragSourceArchivePath);
|
||||
|
||||
if (!options.checkOnly) {
|
||||
await archiveRepository.normalizeStore();
|
||||
}
|
||||
|
||||
const indexedRepoSummaries = await archiveRepository.listRepoSummaries();
|
||||
const indexedRepoNames = new Set(indexedRepoSummaries.map((summary) => summary.repoName));
|
||||
const missingRepos = selectedRepos
|
||||
.filter((repo) => options.forceReindex || !indexedRepoNames.has(repo.repoName))
|
||||
.map((repo) => repo.repoName);
|
||||
const staleIndexedRepos = indexedRepoSummaries
|
||||
.map((summary) => summary.repoName)
|
||||
.filter((repoName) => !discoveredRepos.some((repo) => repo.repoName === repoName))
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
|
||||
let hookInstall = {
|
||||
installed: 0,
|
||||
updated: 0,
|
||||
backedUp: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
|
||||
if (!options.checkOnly) {
|
||||
const hookSummary = await installSourceIndexPushHooks(process.cwd(), selectedRepos);
|
||||
hookInstall = {
|
||||
installed: hookSummary.installed,
|
||||
updated: hookSummary.updated,
|
||||
backedUp: hookSummary.backedUp,
|
||||
skipped: hookSummary.skipped,
|
||||
};
|
||||
|
||||
if (!options.json) {
|
||||
for (const message of hookSummary.messages) {
|
||||
console.log(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const reposToIndex = selectedRepos.filter((repo) => missingRepos.includes(repo.repoName));
|
||||
const indexedRepos: string[] = [];
|
||||
const launchedRepos: string[] = [];
|
||||
const prunedRepos: string[] = [];
|
||||
|
||||
if (!options.checkOnly && reposToIndex.length > 0) {
|
||||
if (options.detached) {
|
||||
for (const repo of reposToIndex) {
|
||||
await launchDetachedRepoIndex(process.cwd(), repo.repoName);
|
||||
launchedRepos.push(repo.repoName);
|
||||
}
|
||||
} else {
|
||||
const runSummary = await indexSourceRepos(config, reposToIndex, {
|
||||
forceReindex: options.forceReindex,
|
||||
logger: options.json ? undefined : (line) => console.log(line),
|
||||
});
|
||||
indexedRepos.push(...runSummary.results.map((result) => result.repoName));
|
||||
|
||||
if (!options.json) {
|
||||
console.log(`Indexed repositories: ${runSummary.indexedRepositories}`);
|
||||
console.log(`Indexed documents: ${runSummary.totalDocuments}`);
|
||||
console.log(`Indexed chunks: ${runSummary.totalChunks}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!options.checkOnly && staleIndexedRepos.length > 0) {
|
||||
const sourceVectorStore = createSourceVectorStore(config);
|
||||
|
||||
for (const repoName of staleIndexedRepos) {
|
||||
await sourceVectorStore.deleteByFilters({
|
||||
corpusKind: "repository_source",
|
||||
repoName,
|
||||
});
|
||||
await archiveRepository.removeRepo(repoName);
|
||||
prunedRepos.push(repoName);
|
||||
}
|
||||
}
|
||||
|
||||
const summary: ReconcileSummary = {
|
||||
discoveredRepoCount: discoveredRepos.length,
|
||||
indexedRepoCount: indexedRepoSummaries.length,
|
||||
selectedRepoCount: selectedRepos.length,
|
||||
missingRepos,
|
||||
staleIndexedRepos,
|
||||
hookInstall,
|
||||
prunedRepos,
|
||||
normalizedArchive: !options.checkOnly,
|
||||
detached: options.detached,
|
||||
forceReindex: options.forceReindex,
|
||||
checkOnly: options.checkOnly,
|
||||
indexedRepos,
|
||||
launchedRepos,
|
||||
};
|
||||
|
||||
if (options.json) {
|
||||
process.stdout.write(`${JSON.stringify(summary)}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(
|
||||
[
|
||||
`discoveredRepos=${summary.discoveredRepoCount}`,
|
||||
`indexedRepos=${summary.indexedRepoCount}`,
|
||||
`selectedRepos=${summary.selectedRepoCount}`,
|
||||
`missingRepos=${summary.missingRepos.length}`,
|
||||
`staleIndexedRepos=${summary.staleIndexedRepos.length}`,
|
||||
`installedHooks=${summary.hookInstall.installed}`,
|
||||
`updatedHooks=${summary.hookInstall.updated}`,
|
||||
`backedUpHooks=${summary.hookInstall.backedUp}`,
|
||||
`skippedHooks=${summary.hookInstall.skipped}`,
|
||||
`prunedStaleRepos=${summary.prunedRepos.length}`,
|
||||
`indexedNow=${summary.indexedRepos.length}`,
|
||||
`launchedDetached=${summary.launchedRepos.length}`,
|
||||
].join(" "),
|
||||
);
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ScriptOptions {
|
||||
const options: ScriptOptions = {
|
||||
forceReindex: false,
|
||||
detached: false,
|
||||
json: false,
|
||||
checkOnly: false,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const argument = argv[index];
|
||||
|
||||
if ((argument === "--repo" || argument === "--repos") && argv[index + 1]) {
|
||||
options.repos = argv[index + 1]
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry.length > 0);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (argument === "--force") {
|
||||
options.forceReindex = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (argument === "--detached") {
|
||||
options.detached = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (argument === "--json") {
|
||||
options.json = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (argument === "--check") {
|
||||
options.checkOnly = true;
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
async function launchDetachedRepoIndex(voidbotRoot: string, repoName: string): Promise<void> {
|
||||
await new Promise<void>((resolvePromise, rejectPromise) => {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
join(voidbotRoot, "scripts", "git-post-push-index.mjs"),
|
||||
"--voidbot-root",
|
||||
voidbotRoot,
|
||||
"--repo-name",
|
||||
repoName,
|
||||
],
|
||||
{
|
||||
cwd: voidbotRoot,
|
||||
env: process.env,
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
windowsHide: true,
|
||||
},
|
||||
);
|
||||
|
||||
let stderr = "";
|
||||
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
|
||||
child.on("error", rejectPromise);
|
||||
child.on("close", (code) => {
|
||||
if ((code ?? 1) !== 0) {
|
||||
rejectPromise(
|
||||
new Error(
|
||||
stderr.trim().length > 0
|
||||
? stderr.trim()
|
||||
: `Detached source-index launcher failed for ${repoName}.`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
resolvePromise();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
109
scripts/source-repo-discovery.ts
Normal file
109
scripts/source-repo-discovery.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { readdir, readFile, stat } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
export interface SourceRepoMatch {
|
||||
repoName: string;
|
||||
repoPath: string;
|
||||
gitDir: string;
|
||||
}
|
||||
|
||||
export async function discoverSourceRepos(
|
||||
root: string,
|
||||
patterns: string[],
|
||||
): Promise<SourceRepoMatch[]> {
|
||||
const entries = await readdir(root, { withFileTypes: true });
|
||||
const repoMatches: SourceRepoMatch[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (patterns.length > 0 && !patterns.some((pattern) => matchesPattern(entry.name, pattern))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const repoPath = join(root, entry.name);
|
||||
const gitDir = await resolveGitDir(repoPath);
|
||||
|
||||
if (!gitDir) {
|
||||
continue;
|
||||
}
|
||||
|
||||
repoMatches.push({
|
||||
repoName: entry.name,
|
||||
repoPath,
|
||||
gitDir,
|
||||
});
|
||||
}
|
||||
|
||||
repoMatches.sort((left, right) => left.repoName.localeCompare(right.repoName));
|
||||
return repoMatches;
|
||||
}
|
||||
|
||||
export function selectSourceRepos(
|
||||
availableRepos: SourceRepoMatch[],
|
||||
requestedRepoNames?: string[],
|
||||
): SourceRepoMatch[] {
|
||||
if (!requestedRepoNames || requestedRepoNames.length === 0) {
|
||||
return availableRepos;
|
||||
}
|
||||
|
||||
const lookup = new Map(
|
||||
availableRepos.map((repo) => [repo.repoName.toLowerCase(), repo] as const),
|
||||
);
|
||||
const selectedRepos: SourceRepoMatch[] = [];
|
||||
const missingRepoNames: string[] = [];
|
||||
|
||||
for (const requestedRepoName of requestedRepoNames) {
|
||||
const repo = lookup.get(requestedRepoName.toLowerCase());
|
||||
|
||||
if (!repo) {
|
||||
missingRepoNames.push(requestedRepoName);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!selectedRepos.some((entry) => entry.repoName === repo.repoName)) {
|
||||
selectedRepos.push(repo);
|
||||
}
|
||||
}
|
||||
|
||||
if (missingRepoNames.length > 0) {
|
||||
throw new Error(
|
||||
`Requested source repos were not found under SOURCE_REPO_ROOT: ${missingRepoNames.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
return selectedRepos;
|
||||
}
|
||||
|
||||
export async function resolveGitDir(repoPath: string): Promise<string | undefined> {
|
||||
const dotGitPath = join(repoPath, ".git");
|
||||
|
||||
try {
|
||||
const dotGitStats = await stat(dotGitPath);
|
||||
|
||||
if (dotGitStats.isDirectory()) {
|
||||
return dotGitPath;
|
||||
}
|
||||
|
||||
if (dotGitStats.isFile()) {
|
||||
const pointer = await readFile(dotGitPath, "utf8");
|
||||
const match = pointer.match(/^gitdir:\s*(.+)$/im);
|
||||
|
||||
if (match?.[1]) {
|
||||
return resolve(repoPath, match[1].trim());
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function matchesPattern(value: string, pattern: string): boolean {
|
||||
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
||||
const regex = new RegExp(`^${escaped}$`, "i");
|
||||
return regex.test(value);
|
||||
}
|
||||
124
scripts/source-repo-hooks.ts
Normal file
124
scripts/source-repo-hooks.ts
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import { chmod, mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { type SourceRepoMatch } from "./source-repo-discovery";
|
||||
|
||||
export const HOOK_MARKER = "# voidbot-source-index-push-hook";
|
||||
|
||||
export interface SourceHookInstallSummary {
|
||||
installed: number;
|
||||
updated: number;
|
||||
backedUp: number;
|
||||
skipped: number;
|
||||
messages: string[];
|
||||
}
|
||||
|
||||
export async function installSourceIndexPushHooks(
|
||||
voidbotRoot: string,
|
||||
repos: SourceRepoMatch[],
|
||||
): Promise<SourceHookInstallSummary> {
|
||||
let installed = 0;
|
||||
let updated = 0;
|
||||
let backedUp = 0;
|
||||
let skipped = 0;
|
||||
const messages: string[] = [];
|
||||
|
||||
for (const repo of repos) {
|
||||
const hooksDir = join(repo.gitDir, "hooks");
|
||||
const hookPath = join(hooksDir, "pre-push");
|
||||
const backupHookPath = join(hooksDir, "pre-push.voidbot.prev");
|
||||
|
||||
await mkdir(hooksDir, { recursive: true });
|
||||
|
||||
let existingContent: string | undefined;
|
||||
|
||||
try {
|
||||
existingContent = await readFile(hookPath, "utf8");
|
||||
} catch {
|
||||
existingContent = undefined;
|
||||
}
|
||||
|
||||
if (
|
||||
existingContent &&
|
||||
!existingContent.includes(HOOK_MARKER) &&
|
||||
!(await pathExists(backupHookPath))
|
||||
) {
|
||||
await writeFile(backupHookPath, existingContent, "utf8");
|
||||
await chmod(backupHookPath, 0o755);
|
||||
backedUp += 1;
|
||||
} else if (
|
||||
existingContent &&
|
||||
!existingContent.includes(HOOK_MARKER) &&
|
||||
(await pathExists(backupHookPath))
|
||||
) {
|
||||
skipped += 1;
|
||||
messages.push(
|
||||
`Skipping ${repo.repoName}: existing pre-push hook already has a Void backup; not clobbering it.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const hookScript = renderHookScript(voidbotRoot, repo.repoName);
|
||||
await writeFile(hookPath, hookScript, "utf8");
|
||||
await chmod(hookPath, 0o755);
|
||||
|
||||
if (existingContent?.includes(HOOK_MARKER)) {
|
||||
updated += 1;
|
||||
messages.push(`Updated pre-push hook for ${repo.repoName}.`);
|
||||
} else {
|
||||
installed += 1;
|
||||
messages.push(`Installed pre-push hook for ${repo.repoName}.`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
installed,
|
||||
updated,
|
||||
backedUp,
|
||||
skipped,
|
||||
messages,
|
||||
};
|
||||
}
|
||||
|
||||
function renderHookScript(voidbotRoot: string, repoName: string): string {
|
||||
const normalizedRoot = voidbotRoot.replace(/\\/g, "/");
|
||||
const launcherPath = `${normalizedRoot}/scripts/git-post-push-index.mjs`;
|
||||
const nodePath = process.execPath.replace(/\\/g, "/");
|
||||
const quotedNodePath = escapeForDoubleQuotedShell(nodePath);
|
||||
const quotedLauncherPath = escapeForDoubleQuotedShell(launcherPath);
|
||||
const quotedRoot = escapeForDoubleQuotedShell(normalizedRoot);
|
||||
const quotedRepoName = escapeForDoubleQuotedShell(repoName);
|
||||
|
||||
return [
|
||||
"#!/bin/sh",
|
||||
HOOK_MARKER,
|
||||
"HOOK_DIR=\"$(CDPATH= cd -- \"$(dirname -- \"$0\")\" && pwd)\"",
|
||||
"PREV_HOOK=\"$HOOK_DIR/pre-push.voidbot.prev\"",
|
||||
"",
|
||||
"if [ -x \"$PREV_HOOK\" ]; then",
|
||||
" \"$PREV_HOOK\" \"$@\"",
|
||||
" PREV_STATUS=$?",
|
||||
" if [ \"$PREV_STATUS\" -ne 0 ]; then",
|
||||
" exit \"$PREV_STATUS\"",
|
||||
" fi",
|
||||
"fi",
|
||||
"",
|
||||
`\"${quotedNodePath}\" \"${quotedLauncherPath}\" --voidbot-root \"${quotedRoot}\" --repo-name \"${quotedRepoName}\" >/dev/null 2>&1 || true`,
|
||||
"",
|
||||
"exit 0",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function escapeForDoubleQuotedShell(value: string): string {
|
||||
return value.replace(/(["`$\\])/g, "\\$1");
|
||||
}
|
||||
|
||||
async function pathExists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await stat(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
111
scripts/source-repo-indexing.ts
Normal file
111
scripts/source-repo-indexing.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
import { type AppConfig } from "@voidbot/config";
|
||||
import {
|
||||
crawlRepositoryDocuments,
|
||||
createTextEmbedder,
|
||||
createVectorStores,
|
||||
FileSourceDocumentArchiveRepository,
|
||||
SourceDocumentIngester,
|
||||
SourceRagPipeline,
|
||||
} from "@voidbot/rag";
|
||||
|
||||
import { type SourceRepoMatch } from "./source-repo-discovery";
|
||||
|
||||
export interface IndexedRepoResult {
|
||||
repoName: string;
|
||||
documents: number;
|
||||
createdDocuments: number;
|
||||
updatedDocuments: number;
|
||||
unchangedDocuments: number;
|
||||
deletedDocuments: number;
|
||||
indexedChunks: number;
|
||||
skippedFiles: number;
|
||||
}
|
||||
|
||||
export interface SourceIndexRunSummary {
|
||||
indexedRepositories: number;
|
||||
totalDocuments: number;
|
||||
totalChunks: number;
|
||||
results: IndexedRepoResult[];
|
||||
}
|
||||
|
||||
export function createSourceVectorStore(config: AppConfig) {
|
||||
const embedder = createTextEmbedder({
|
||||
backend: config.ragEmbeddingBackend,
|
||||
hashDimensions: config.ragEmbeddingDimensions,
|
||||
ollamaBaseUrl: config.ragOllamaBaseUrl,
|
||||
ollamaModel: config.ragOllamaModel,
|
||||
ollamaTimeoutMs: config.ragOllamaTimeoutMs,
|
||||
queryInstruction: config.ragQueryInstruction,
|
||||
});
|
||||
|
||||
return createVectorStores({
|
||||
kind: config.vectorStore.kind,
|
||||
historyPath: config.vectorStore.path,
|
||||
sourceRoot: config.sourceVectorStoreRoot,
|
||||
qdrant: config.qdrant,
|
||||
historyEmbedder: embedder,
|
||||
sourceEmbedder: embedder,
|
||||
}).source;
|
||||
}
|
||||
|
||||
export async function indexSourceRepos(
|
||||
config: AppConfig,
|
||||
repos: SourceRepoMatch[],
|
||||
options: {
|
||||
forceReindex?: boolean;
|
||||
logger?: (line: string) => void;
|
||||
} = {},
|
||||
): Promise<SourceIndexRunSummary> {
|
||||
const vectorStore = createSourceVectorStore(config);
|
||||
const archiveRepository = new FileSourceDocumentArchiveRepository(config.ragSourceArchivePath);
|
||||
const sourceIngester = new SourceDocumentIngester();
|
||||
const pipeline = new SourceRagPipeline(archiveRepository, sourceIngester, vectorStore);
|
||||
const results: IndexedRepoResult[] = [];
|
||||
let indexedRepositories = 0;
|
||||
let totalDocuments = 0;
|
||||
let totalChunks = 0;
|
||||
|
||||
for (const repo of repos) {
|
||||
const scan = await crawlRepositoryDocuments(repo.repoPath, repo.repoName, {
|
||||
includePathPrefixes: config.sourceRepoIncludePrefixes[repo.repoName],
|
||||
});
|
||||
const result = await pipeline.syncRepoDocuments(repo.repoName, scan.documents, {
|
||||
forceReindex: options.forceReindex,
|
||||
});
|
||||
const repoResult: IndexedRepoResult = {
|
||||
repoName: repo.repoName,
|
||||
documents: scan.documents.length,
|
||||
createdDocuments: result.createdDocuments,
|
||||
updatedDocuments: result.updatedDocuments,
|
||||
unchangedDocuments: result.unchangedDocuments,
|
||||
deletedDocuments: result.deletedDocuments,
|
||||
indexedChunks: result.indexedChunks,
|
||||
skippedFiles: scan.skippedFiles.length,
|
||||
};
|
||||
|
||||
results.push(repoResult);
|
||||
indexedRepositories += 1;
|
||||
totalDocuments += scan.documents.length;
|
||||
totalChunks += result.indexedChunks;
|
||||
|
||||
options.logger?.(
|
||||
[
|
||||
`Repo ${repo.repoName}:`,
|
||||
`documents=${repoResult.documents}`,
|
||||
`created=${repoResult.createdDocuments}`,
|
||||
`updated=${repoResult.updatedDocuments}`,
|
||||
`unchanged=${repoResult.unchangedDocuments}`,
|
||||
`deleted=${repoResult.deletedDocuments}`,
|
||||
`indexedChunks=${repoResult.indexedChunks}`,
|
||||
`skippedFiles=${repoResult.skippedFiles}`,
|
||||
].join(" "),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
indexedRepositories,
|
||||
totalDocuments,
|
||||
totalChunks,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
|
@ -532,6 +532,16 @@ if ($LASTEXITCODE -ne 0) {
|
|||
|
||||
Stop-StaleRuntimeProcesses -RepoRoot $repoRoot
|
||||
|
||||
$status.stage = "source_reconcile"
|
||||
Write-StatusFile -Path $statusPath -Status $status
|
||||
|
||||
Write-Host "Reconciling source repos..."
|
||||
& $npmCommand.Source run rag:reconcile-sources
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "npm run rag:reconcile-sources failed."
|
||||
}
|
||||
|
||||
$botOut = Join-Path $logDir "bot.log"
|
||||
$botErr = Join-Path $logDir "bot.err.log"
|
||||
$workerOut = Join-Path $logDir "worker.log"
|
||||
|
|
|
|||
Loading…
Reference in a new issue