From 37765ce2463b2dfdad64a1ab74679250558f54d1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 3 May 2026 22:24:08 +0100 Subject: [PATCH] Harden local history fallback for repeated prompts --- apps/bot/src/discord-bot-handlers.ts | 26 +++-- apps/bot/src/discord-bot-support.ts | 137 ++++++++++++++++++++++ apps/bot/src/discord-bot.ts | 12 +- packages/rag/src/message-archive.ts | 162 ++++++++++++++++++++++++++- state/evidence.jsonl | 1 + 5 files changed, 328 insertions(+), 10 deletions(-) diff --git a/apps/bot/src/discord-bot-handlers.ts b/apps/bot/src/discord-bot-handlers.ts index 4ecaf7e..5d5401a 100644 --- a/apps/bot/src/discord-bot-handlers.ts +++ b/apps/bot/src/discord-bot-handlers.ts @@ -21,6 +21,7 @@ import { } from "@voidbot/core"; import type { ProviderRegistry } from "@voidbot/providers"; import { + FileMessageArchiveRepository, FileSourceDocumentArchiveRepository, RagPipeline, RetrievalService, @@ -41,11 +42,13 @@ import { buildChannelIndexingTarget, canSendMessages, convertDiscordMessageToArchive, + filterPromptEchoHistoryResults, getRecentMessages, inferSourceGroundingHint, renderRateLimitMessage, renderSystemMessage, renderInteractionProfileDisclosure, + searchHistoryWithArchiveFallback, sendDirectMessage, sendChunkedChannelMessage, splitDiscordContent, @@ -134,6 +137,7 @@ export interface PromptHandlerOptions { permissionEngine: PermissionEngine; contextBuilder: ContextBuilder; retrievalService: RetrievalService; + archiveRepository: FileMessageArchiveRepository; sourceArchiveRepository: FileSourceDocumentArchiveRepository; jobQueue: JobQueue; auditLog: AuditLog; @@ -262,14 +266,22 @@ export async function handlePrompt(options: PromptHandlerOptions): Promise await options.sourceArchiveRepository.listRepoSummaries(), ); const shouldAttachInitialRetrieval = - !provider.getCapabilities().includes("tool_driven_retrieval") && - (shouldIndexChannel(options.config.channelIndexing, buildChannelIndexingTarget(options.channel)) || - providerName !== "owner_codex"); + providerName === "local_llm" || + (!provider.getCapabilities().includes("tool_driven_retrieval") && + (shouldIndexChannel(options.config.channelIndexing, buildChannelIndexingTarget(options.channel)) || + providerName !== "owner_codex")); + const initialRetrievalLimit = providerName === "local_llm" ? 3 : 5; const retrieval = shouldAttachInitialRetrieval - ? await options.retrievalService.search(options.prompt, 5, { - guildId: options.guildContext.guildId, - corpusKind: "discord_history", - }) + ? filterPromptEchoHistoryResults( + await searchHistoryWithArchiveFallback({ + retrievalService: options.retrievalService, + archiveRepository: options.archiveRepository, + query: options.prompt, + limit: initialRetrievalLimit, + guildId: options.guildContext.guildId, + }), + options.prompt, + ).slice(0, initialRetrievalLimit) : []; const contextBundle = options.contextBuilder.build({ prompt: options.prompt, diff --git a/apps/bot/src/discord-bot-support.ts b/apps/bot/src/discord-bot-support.ts index f0ce888..8707ee9 100644 --- a/apps/bot/src/discord-bot-support.ts +++ b/apps/bot/src/discord-bot-support.ts @@ -17,6 +17,7 @@ import { import type { ProviderRegistry } from "@voidbot/providers"; import { RagPipeline, + FileMessageArchiveRepository, RetrievalService, type ArchivedMessageRecord, } from "@voidbot/rag"; @@ -27,6 +28,7 @@ import { type ChannelIndexingTarget, type GuildContext, type InteractionMemoryProfile, + type RetrievalResult, type SourceGroundingHint, type SourceMessage, type SituationalSocialRead, @@ -337,6 +339,51 @@ export function renderRateLimitMessage( }); } +export async function searchHistoryWithArchiveFallback(options: { + retrievalService: RetrievalService; + archiveRepository: FileMessageArchiveRepository; + query: string; + limit: number; + guildId?: string; + channelId?: string; + authorId?: string; +}): Promise { + const overfetchLimit = Math.max(options.limit * 3, options.limit + 4); + const vectorResults = await options.retrievalService.searchHistory(options.query, overfetchLimit, { + guildId: options.guildId, + channelId: options.channelId, + authorId: options.authorId, + }); + + const lexicalResults = await options.archiveRepository.searchLexical(options.query, overfetchLimit, { + guildId: options.guildId, + channelId: options.channelId, + authorId: options.authorId, + }); + + if (lexicalResults.length === 0) { + return vectorResults; + } + + const merged = [...vectorResults]; + const seenSourceIds = new Set(merged.map((result) => result.sourceId)); + + for (const result of lexicalResults) { + if (seenSourceIds.has(result.sourceId)) { + continue; + } + + merged.push(result); + seenSourceIds.add(result.sourceId); + + if (merged.length >= overfetchLimit) { + break; + } + } + + return merged; +} + export function formatHistoryResults( results: Awaited>, ): Array> { @@ -373,6 +420,96 @@ export function formatSourceResults( })); } +export function filterPromptEchoHistoryResults< + T extends { + text: string; + }, +>(results: T[], query: string): T[] { + const normalizedQuery = normalizeHistoryEchoText(query); + const queryTokens = buildHistoryEchoTokens(normalizedQuery); + + if (normalizedQuery.length === 0) { + return results; + } + + const filtered = results.filter((result) => { + const normalizedResult = normalizeHistoryEchoText(result.text); + + if (normalizedResult.length === 0) { + return true; + } + + if (normalizedResult === normalizedQuery) { + return false; + } + + if (queryTokens.length >= 4) { + const resultTokens = buildHistoryEchoTokens(normalizedResult); + const sharedTokenCount = countSharedTokens(queryTokens, resultTokens); + const similarity = sharedTokenCount / Math.max(Math.min(queryTokens.length, resultTokens.length), 1); + + if (similarity >= 0.7) { + return false; + } + } + + if ( + normalizedQuery.length >= 48 && + (normalizedResult.includes(normalizedQuery) || normalizedQuery.includes(normalizedResult)) + ) { + return false; + } + + return true; + }); + + return filtered.length > 0 ? filtered : results; +} + +function normalizeHistoryEchoText(value: string): string { + return value + .replace(/<@!?\d+>/g, " ") + .replace(/<@&\d+>/g, " ") + .replace(/\s+/g, " ") + .trim() + .toLowerCase(); +} + +function buildHistoryEchoTokens(value: string): string[] { + return [...new Set( + value + .split(/[^\p{L}\p{N}]+/u) + .map((token) => token.trim()) + .filter((token) => token.length >= 4) + .filter((token) => !HISTORY_ECHO_STOP_WORDS.has(token)), + )]; +} + +function countSharedTokens(left: string[], right: string[]): number { + const rightSet = new Set(right); + return left.reduce((count, token) => count + (rightSet.has(token) ? 1 : 0), 0); +} + +const HISTORY_ECHO_STOP_WORDS = new Set([ + "about", + "after", + "before", + "could", + "from", + "have", + "please", + "pretty", + "tell", + "that", + "them", + "they", + "this", + "what", + "with", + "would", + "your", +]); + export function formatArchivedMessageContext( message: ArchivedMessageRecord, anchorMessageId: string, diff --git a/apps/bot/src/discord-bot.ts b/apps/bot/src/discord-bot.ts index dc7b448..453af3e 100644 --- a/apps/bot/src/discord-bot.ts +++ b/apps/bot/src/discord-bot.ts @@ -65,6 +65,7 @@ import { replyToMessage, rememberAmbientVoidReference, renderSystemMessage, + searchHistoryWithArchiveFallback, stripBotMention, } from "./discord-bot-support"; @@ -235,12 +236,16 @@ export async function startBot(): Promise { }; }, searchHistory: async (input) => { - const rawResults = await retrievalService.searchHistory(input.query, input.limit, { + const rawResults = await searchHistoryWithArchiveFallback({ + retrievalService, + archiveRepository, + query: input.query, + limit: input.limit, guildId: input.guildId, channelId: input.channelId, authorId: input.authorId, }); - const results = filterPromptEchoHistoryResults(rawResults, input.query); + const results = filterPromptEchoHistoryResults(rawResults, input.query).slice(0, input.limit); return { query: input.query, @@ -362,6 +367,7 @@ export async function startBot(): Promise { permissionEngine, contextBuilder, retrievalService, + archiveRepository, sourceArchiveRepository, jobQueue, auditLog, @@ -445,6 +451,7 @@ export async function startBot(): Promise { permissionEngine, contextBuilder, retrievalService, + archiveRepository, sourceArchiveRepository, jobQueue, auditLog, @@ -475,6 +482,7 @@ export async function startBot(): Promise { permissionEngine, contextBuilder, retrievalService, + archiveRepository, sourceArchiveRepository, jobQueue, auditLog, diff --git a/packages/rag/src/message-archive.ts b/packages/rag/src/message-archive.ts index 3620f68..a78c2bc 100644 --- a/packages/rag/src/message-archive.ts +++ b/packages/rag/src/message-archive.ts @@ -1,4 +1,8 @@ -import { type ArchivedMessage } from "@voidbot/shared"; +import { + type ArchivedMessage, + type RetrievalFilters, + type RetrievalResult, +} from "@voidbot/shared"; import { normalizeText } from "./history-ingester"; import { SerializedFileStore } from "./file-store"; @@ -13,6 +17,27 @@ interface MessageArchiveStore { messages: ArchivedMessageRecord[]; } +const MIN_SEARCH_TOKEN_LENGTH = 4; +const SEARCH_STOP_WORDS = new Set([ + "about", + "after", + "before", + "could", + "from", + "have", + "please", + "pretty", + "tell", + "that", + "them", + "they", + "this", + "what", + "with", + "would", + "your", +]); + export interface MessageArchiveMutationResult { created: number; updated: number; @@ -157,6 +182,42 @@ export class FileMessageArchiveRepository { const store = await this.store.snapshot(); return store.messages.length; } + + public async searchLexical( + query: string, + limit = 5, + filters?: Pick, + ): Promise { + const normalizedQuery = normalizeSearchText(query); + const queryTokens = extractSearchTokens(normalizedQuery); + + if (normalizedQuery.length === 0 || queryTokens.length === 0) { + return []; + } + + const store = await this.store.snapshot(); + const scored = store.messages + .filter((message) => !message.deletedAt) + .filter((message) => matchesHistoryFilters(message, filters)) + .map((message) => { + const score = scoreLexicalMatch(message, normalizedQuery, queryTokens); + return { + message, + score, + }; + }) + .filter((entry) => entry.score > 0) + .sort((left, right) => { + if (right.score !== left.score) { + return right.score - left.score; + } + + return right.message.timestamp.localeCompare(left.message.timestamp); + }) + .slice(0, limit); + + return scored.map(({ message, score }) => toRetrievalResult(message, score)); + } } function toArchivedMessageRecord( @@ -201,3 +262,102 @@ function belongsToSameConversation( return candidate.channelId === anchor.channelId; } + +function matchesHistoryFilters( + message: ArchivedMessageRecord, + filters?: Pick, +): boolean { + if (!filters) { + return true; + } + + if (filters.guildId && message.guildId !== filters.guildId) { + return false; + } + + if (filters.channelId && message.channelId !== filters.channelId) { + return false; + } + + if (filters.authorId && message.authorId !== filters.authorId) { + return false; + } + + return true; +} + +function extractSearchTokens(normalizedQuery: string): string[] { + return [...new Set( + normalizedQuery + .split(/[^\p{L}\p{N}]+/u) + .map((token) => token.trim()) + .filter((token) => token.length >= MIN_SEARCH_TOKEN_LENGTH) + .filter((token) => !SEARCH_STOP_WORDS.has(token)), + )]; +} + +function scoreLexicalMatch( + message: ArchivedMessageRecord, + normalizedQuery: string, + queryTokens: string[], +): number { + const searchableContent = normalizeSearchText(message.content); + let score = 0; + + if (searchableContent.includes(normalizedQuery)) { + score += 10; + } + + let overlapCount = 0; + + for (const token of queryTokens) { + if (searchableContent.includes(token)) { + overlapCount += 1; + score += token.length / 10; + } + } + + if (overlapCount === 0) { + return 0; + } + + if (message.content.includes("http://") || message.content.includes("https://")) { + score += 1.5; + } + + if (message.channelId === message.guildId) { + score += 0.25; + } + + return Number(score.toFixed(4)); +} + +function toRetrievalResult(message: ArchivedMessageRecord, score: number): RetrievalResult { + return { + chunkId: `${message.id}:archive`, + score, + text: message.content, + sourceId: message.id, + sourceKind: "discord_message", + metadata: { + sourceId: message.id, + guildId: message.guildId ?? "", + channelId: message.channelId, + channelName: message.metadata?.channelName ?? "", + authorId: message.authorId, + authorName: message.authorName, + timestamp: message.timestamp, + threadId: message.threadId ?? "", + jumpUrl: message.metadata?.jumpUrl ?? "", + corpusKind: "discord_history", + }, + }; +} + +function normalizeSearchText(value: string): string { + return normalizeText(value) + .replace(/<@!?\d+>/g, " ") + .replace(/<@&\d+>/g, " ") + .replace(/\s+/g, " ") + .trim(); +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 0640a12..6933184 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -34,3 +34,4 @@ {"ts":"2026-05-02T22:58:31Z","type":"research","status":"ok","note":"Gemma3 12B Q4_0 on the 4060 Ti box handles the transcript-wide social-read task once the schema sticks to a simpler subset; Ollama chokes on roster-constrained prefixItems/const schema but both Gemma and Qwen work with local roster reconciliation, and Gemma slightly beats Qwen on latency for the tested slice."} {"ts":"2026-05-02T23:40:40Z","type":"implementation","status":"ok","note":"Split room-read sidecar model selection from the main local reply model so Gemma can drive transcript profiling without changing the public/local answer lane."} {"ts":"2026-05-03T07:40:36Z","type":"implementation","status":"ok","note":"Offsite backup was stopping bot and worker, then leaving the stack dead if the remote local-LLM box was asleep; startup now records local-LLM degradation without aborting, and watchdog treats a failed offsite status file as a warning while remote freshness is checked separately."} +{"ts":"2026-05-03T21:23:51Z","type":"implementation","status":"ok","note":"Local history-heavy prompts can time out when vector retrieval fixates on repeated ask-lines or comes back thin; overfetching history, filtering near-duplicate prompt echoes, and backstopping with raw archive lexical search keeps the local lane from burning its tool budget on empty or circular history reads."}