diff --git a/package.json b/package.json index 2a89d74..990bd53 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "qdrant:setup": "powershell -ExecutionPolicy Bypass -File scripts/setup-docker-qdrant.ps1", "qdrant:up": "powershell -ExecutionPolicy Bypass -File scripts/start-qdrant.ps1", "rag:backfill": "tsx scripts/backfill-discord-logs.ts", - "moderation:recent-history": "tsx scripts/export-recent-discord-history.ts", + "moderation:recent-history": "node scripts/export-recent-discord-history.mjs", "rag:scrub-bot-prompts": "tsx scripts/scrub-bot-prompts-from-history.ts", "rag:index-sources": "tsx scripts/index-source-repos.ts", "rag:install-push-hooks": "tsx scripts/install-source-index-post-push-hooks.ts", diff --git a/scripts/export-recent-discord-history.ts b/scripts/export-recent-discord-history.mjs similarity index 52% rename from scripts/export-recent-discord-history.ts rename to scripts/export-recent-discord-history.mjs index bf67457..855f547 100644 --- a/scripts/export-recent-discord-history.ts +++ b/scripts/export-recent-discord-history.mjs @@ -1,44 +1,20 @@ -import "dotenv/config"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; -import { loadConfig } from "@voidbot/config"; -import { FileMessageArchiveRepository } from "@voidbot/rag"; -import { readArchivedMessageKind, type ArchivedMessage } from "@voidbot/shared"; +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -interface CliOptions { - after?: string; - hours?: number; - limit: number; - channelId?: string; - includeBotPrompts: boolean; -} - -interface OutputMessage { - id: string; - timestamp: string; - channelId: string; - threadId?: string; - authorId: string; - authorName: string; - content: string; - attachments: string[]; - messageKind: string; -} - -async function main(): Promise { - if (!process.env.DISCORD_OWNER_ID) { - process.env.DISCORD_OWNER_ID = "__moderation_recent_history__"; - } - - const config = loadConfig(); +function main() { const options = parseArgs(process.argv.slice(2)); - const archiveRepository = new FileMessageArchiveRepository(config.ragArchivePath); - const messages = await archiveRepository.listAllActive(); - const filtered = filterMessages(messages, options); + const archivePath = resolve( + repoRoot, + readConfiguredArchivePath() ?? ".voidbot/rag/messages.json", + ); + const store = readArchiveStore(archivePath); + const filtered = filterMessages(store.messages ?? [], options); const selected = filtered.slice(-options.limit); const outputMessages = selected.map(toOutputMessage); - const transcript = outputMessages - .map((message) => formatTranscriptLine(message)) - .join("\n"); + const transcript = outputMessages.map(formatTranscriptLine).join("\n"); process.stdout.write( `${JSON.stringify( @@ -47,6 +23,7 @@ async function main(): Promise { hours: options.hours ?? null, channelId: options.channelId ?? null, includeBotPrompts: options.includeBotPrompts, + archivePath, totalMatchingMessages: filtered.length, returnedMessages: outputMessages.length, oldestReturnedTimestamp: outputMessages[0]?.timestamp ?? null, @@ -61,8 +38,8 @@ async function main(): Promise { ); } -function parseArgs(args: string[]): CliOptions { - const options: CliOptions = { +function parseArgs(args) { + const options = { limit: 80, includeBotPrompts: false, }; @@ -110,13 +87,77 @@ function parseArgs(args: string[]): CliOptions { return options; } -function validateIsoTimestamp(value: string, flagName: string): void { +function validateIsoTimestamp(value, flagName) { if (Number.isNaN(Date.parse(value))) { throw new Error(`${flagName} must be a valid ISO timestamp.`); } } -function filterMessages(messages: ArchivedMessage[], options: CliOptions): ArchivedMessage[] { +function readConfiguredArchivePath() { + if (process.env.RAG_ARCHIVE_PATH?.trim()) { + return process.env.RAG_ARCHIVE_PATH.trim(); + } + + const envPath = resolve(repoRoot, ".env"); + + try { + const raw = stripLeadingBom(readFileSync(envPath, "utf8")); + const parsed = parseDotEnv(raw); + return parsed.RAG_ARCHIVE_PATH?.trim() || undefined; + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return undefined; + } + + throw error; + } +} + +function parseDotEnv(raw) { + const result = {}; + + for (const line of raw.split(/\r?\n/)) { + const trimmed = line.trim(); + + if (trimmed.length === 0 || trimmed.startsWith("#")) { + continue; + } + + const separatorIndex = trimmed.indexOf("="); + + if (separatorIndex === -1) { + continue; + } + + const key = trimmed.slice(0, separatorIndex).trim(); + let value = trimmed.slice(separatorIndex + 1).trim(); + + if ( + value.length >= 2 && + ((value.startsWith("\"") && value.endsWith("\"")) || + (value.startsWith("'") && value.endsWith("'"))) + ) { + value = value.slice(1, -1); + } + + result[key] = value; + } + + return result; +} + +function readArchiveStore(archivePath) { + const raw = stripLeadingBom(readFileSync(archivePath, "utf8")); + const parsed = JSON.parse(raw); + + if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.messages)) { + throw new Error(`Archive at ${archivePath} does not have a messages array.`); + } + + return parsed; +} + +function filterMessages(messages, options) { const lowerBound = options.after ?? (options.hours !== undefined @@ -125,13 +166,19 @@ function filterMessages(messages: ArchivedMessage[], options: CliOptions): Archi return messages .filter((message) => !message.deletedAt) - .filter((message) => (options.includeBotPrompts ? true : readArchivedMessageKind(message) !== "bot_prompt")) + .filter((message) => + options.includeBotPrompts ? true : readMessageKind(message) !== "bot_prompt", + ) .filter((message) => (options.channelId ? message.channelId === options.channelId : true)) .filter((message) => (lowerBound ? message.timestamp > lowerBound : true)) .sort((left, right) => left.timestamp.localeCompare(right.timestamp)); } -function toOutputMessage(message: ArchivedMessage): OutputMessage { +function readMessageKind(message) { + return message?.metadata?.messageKind === "bot_prompt" ? "bot_prompt" : "default"; +} + +function toOutputMessage(message) { return { id: message.id, timestamp: message.timestamp, @@ -140,12 +187,12 @@ function toOutputMessage(message: ArchivedMessage): OutputMessage { authorId: message.authorId, authorName: message.authorName, content: message.content, - attachments: message.attachments ?? [], - messageKind: readArchivedMessageKind(message), + attachments: Array.isArray(message.attachments) ? message.attachments : [], + messageKind: readMessageKind(message), }; } -function formatTranscriptLine(message: OutputMessage): string { +function formatTranscriptLine(message) { const threadSuffix = message.threadId ? ` thread=${message.threadId}` : ""; const attachmentSuffix = message.attachments.length > 0 @@ -154,7 +201,13 @@ function formatTranscriptLine(message: OutputMessage): string { return `[${message.timestamp}] ${message.authorName} (${message.authorId}) channel=${message.channelId}${threadSuffix}: ${message.content}${attachmentSuffix}`; } -void main().catch((error) => { - console.error(error); +function stripLeadingBom(input) { + return input.charCodeAt(0) === 0xfeff ? input.slice(1) : input; +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; -}); +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl index f164329..09b6cb6 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -38,3 +38,4 @@ {"ts":"2026-05-04T08:53:16.7930326Z","type":"implementation","status":"ok","note":"Bot-directed prompts now stay archived for forensics but are tagged and excluded from default semantic history indexing and lexical fallback; a scrubber can retro-tag old prompt rows from interaction-memory event IDs plus mention heuristics and prune their history vectors without deleting the raw messages."} {"ts":"2026-05-04T17:59:52Z","type":"architecture","status":"ok","note":"Moderation review now lives in a separate hourly Codex automation with tracked rules doctrine, a chronological recent-history helper, and one ignored JSON state file; semantic history search alone is too fuzzy for simple what-changed-since-last-run heartbeat duty."} {"ts":"2026-05-04T18:19:09Z","type":"architecture","status":"ok","note":"The moderation loop should carry an actual participant lens, not only a reviewer posture. Its ignored state file now mirrors a Ghostlight-style split between identity, canonical state, goals, memories, perceived overlays, and runtime moderation bookkeeping so constructive interventions and archive musings accumulate inside one coherent self-model."} +{"ts":"2026-05-04T20:21:40Z","type":"implementation","status":"ok","note":"Sandboxed cron runs should not depend on tsx/esbuild just to read local archive state; the moderation recent-history helper now runs as plain Node so the automation path stops dying on child-process EPERM before it even sees the room."}