Add Discord context purge tool
This commit is contained in:
Meta Vens
parent
aec427d771
commit
9183a885c4
3 changed files with 394 additions and 0 deletions
|
|
@ -42,6 +42,7 @@
|
|||
"qdrant:up": "powershell -ExecutionPolicy Bypass -File scripts/start-qdrant.ps1",
|
||||
"rag:backfill": "tsx scripts/backfill-discord-logs.ts",
|
||||
"moderation:recent-history": "node scripts/export-recent-discord-history.mjs",
|
||||
"moderation:purge-context": "tsx scripts/purge-discord-context.ts",
|
||||
"agents:feed-consensus": "node scripts/feed-codex-chat-consensus.mjs",
|
||||
"moderation:random-history": "node scripts/export-random-discord-history.mjs",
|
||||
"moderation:recent-repo-activity": "node scripts/export-recent-repo-activity.mjs",
|
||||
|
|
|
|||
392
scripts/purge-discord-context.ts
Normal file
392
scripts/purge-discord-context.ts
Normal file
|
|
@ -0,0 +1,392 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import { createTextEmbedder, createVectorStores, FileMessageArchiveRepository, HistoryIngester, RagPipeline } from "@voidbot/rag";
|
||||
import { loadConfig } from "@voidbot/config";
|
||||
|
||||
interface CandidateMessage {
|
||||
id: string;
|
||||
channelId: string;
|
||||
timestamp: string;
|
||||
authorId?: string;
|
||||
authorName?: string;
|
||||
content: string;
|
||||
source: "live" | "archive";
|
||||
}
|
||||
|
||||
interface CliOptions {
|
||||
execute: boolean;
|
||||
archiveOnly: boolean;
|
||||
discordOnly: boolean;
|
||||
channelId?: string;
|
||||
limit: number;
|
||||
hours?: number;
|
||||
after?: string;
|
||||
messageIds: string[];
|
||||
contains: string[];
|
||||
regexes: RegExp[];
|
||||
authorIds: string[];
|
||||
authorNames: string[];
|
||||
reason: string;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (!process.env.DISCORD_OWNER_ID) {
|
||||
process.env.DISCORD_OWNER_ID = "__purge_discord_context__";
|
||||
}
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const config = loadConfig();
|
||||
const channelId = options.channelId ?? config.repoFaceHeartbeats.defaultChannelId;
|
||||
|
||||
if (!channelId && options.messageIds.length === 0) {
|
||||
throw new Error("Provide --channel-id, configure REPO_FACE_HEARTBEAT_DEFAULT_CHANNEL_ID, or select explicit --message-id values.");
|
||||
}
|
||||
|
||||
if (options.discordOnly && options.archiveOnly) {
|
||||
throw new Error("--discord-only and --archive-only cannot both be set.");
|
||||
}
|
||||
|
||||
const archiveRepository = new FileMessageArchiveRepository(config.ragArchivePath);
|
||||
const liveMessages = options.archiveOnly || !channelId
|
||||
? []
|
||||
: await fetchLiveDiscordMessages({
|
||||
token: requireBotToken(config.botToken),
|
||||
channelId,
|
||||
limit: options.limit,
|
||||
});
|
||||
const archiveMessages = options.discordOnly
|
||||
? []
|
||||
: await fetchArchivedMessages({
|
||||
archiveRepository,
|
||||
channelId,
|
||||
limit: Math.max(options.limit, options.messageIds.length),
|
||||
});
|
||||
|
||||
const candidates = selectCandidates([...liveMessages, ...archiveMessages], options);
|
||||
const uniqueCandidates = dedupeCandidates(candidates);
|
||||
|
||||
printPlan(uniqueCandidates, options);
|
||||
|
||||
if (uniqueCandidates.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.execute) {
|
||||
console.log("Dry run only. Re-run with --execute to delete live Discord messages and/or mark archive records deleted.");
|
||||
return;
|
||||
}
|
||||
|
||||
const deletedDiscordIds = options.archiveOnly
|
||||
? []
|
||||
: await deleteDiscordMessages({
|
||||
token: requireBotToken(config.botToken),
|
||||
candidates: uniqueCandidates,
|
||||
reason: options.reason,
|
||||
});
|
||||
|
||||
const archiveDeletedIds = options.discordOnly
|
||||
? []
|
||||
: await markArchiveMessagesDeleted({
|
||||
config,
|
||||
archiveRepository,
|
||||
messageIds: uniqueCandidates.map((candidate) => candidate.id),
|
||||
});
|
||||
|
||||
console.log(`Deleted or already-missing Discord messages: ${deletedDiscordIds.length}`);
|
||||
console.log(`Marked archived messages deleted and removed vectors: ${archiveDeletedIds.length}`);
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): CliOptions {
|
||||
const options: CliOptions = {
|
||||
execute: false,
|
||||
archiveOnly: false,
|
||||
discordOnly: false,
|
||||
limit: 100,
|
||||
messageIds: [],
|
||||
contains: [],
|
||||
regexes: [],
|
||||
authorIds: [],
|
||||
authorNames: [],
|
||||
reason: "VoidBot operator context purge",
|
||||
};
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const argument = args[index];
|
||||
switch (argument) {
|
||||
case "--execute":
|
||||
options.execute = true;
|
||||
break;
|
||||
case "--archive-only":
|
||||
options.archiveOnly = true;
|
||||
break;
|
||||
case "--discord-only":
|
||||
options.discordOnly = true;
|
||||
break;
|
||||
case "--channel-id":
|
||||
options.channelId = readValue(args, ++index, argument);
|
||||
break;
|
||||
case "--limit":
|
||||
options.limit = Number.parseInt(readValue(args, ++index, argument), 10);
|
||||
break;
|
||||
case "--hours":
|
||||
options.hours = Number.parseFloat(readValue(args, ++index, argument));
|
||||
break;
|
||||
case "--after":
|
||||
options.after = readValue(args, ++index, argument);
|
||||
break;
|
||||
case "--message-id":
|
||||
options.messageIds.push(readValue(args, ++index, argument));
|
||||
break;
|
||||
case "--contains":
|
||||
options.contains.push(readValue(args, ++index, argument).toLowerCase());
|
||||
break;
|
||||
case "--regex":
|
||||
options.regexes.push(new RegExp(readValue(args, ++index, argument), "i"));
|
||||
break;
|
||||
case "--author-id":
|
||||
options.authorIds.push(readValue(args, ++index, argument));
|
||||
break;
|
||||
case "--author-name":
|
||||
options.authorNames.push(readValue(args, ++index, argument).toLowerCase());
|
||||
break;
|
||||
case "--reason":
|
||||
options.reason = readValue(args, ++index, argument);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown argument: ${argument}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!Number.isInteger(options.limit) || options.limit <= 0 || options.limit > 100) {
|
||||
throw new Error("--limit must be an integer from 1 to 100. Discord's channel fetch endpoint caps this at 100.");
|
||||
}
|
||||
|
||||
if (options.hours !== undefined && (!Number.isFinite(options.hours) || options.hours <= 0)) {
|
||||
throw new Error("--hours must be a positive number.");
|
||||
}
|
||||
|
||||
if (options.after && Number.isNaN(Date.parse(options.after))) {
|
||||
throw new Error("--after must be a valid timestamp.");
|
||||
}
|
||||
|
||||
if (
|
||||
options.messageIds.length === 0 &&
|
||||
options.contains.length === 0 &&
|
||||
options.regexes.length === 0 &&
|
||||
options.authorIds.length === 0 &&
|
||||
options.authorNames.length === 0
|
||||
) {
|
||||
throw new Error("Refusing unfiltered purge. Provide --message-id, --contains, --regex, --author-id, or --author-name.");
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function readValue(args: string[], index: number, flag: string): string {
|
||||
const value = args[index];
|
||||
if (!value || value.startsWith("--")) {
|
||||
throw new Error(`${flag} requires a value.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function fetchLiveDiscordMessages(input: {
|
||||
token: string;
|
||||
channelId: string;
|
||||
limit: number;
|
||||
}): Promise<CandidateMessage[]> {
|
||||
const response = await fetch(`https://discord.com/api/v10/channels/${input.channelId}/messages?limit=${input.limit}`, {
|
||||
headers: {
|
||||
Authorization: `Bot ${input.token}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Discord fetch failed ${response.status}: ${await response.text()}`);
|
||||
}
|
||||
|
||||
const payload = await response.json();
|
||||
if (!Array.isArray(payload)) {
|
||||
throw new Error("Discord fetch returned a non-array payload.");
|
||||
}
|
||||
|
||||
return payload.map((message) => ({
|
||||
id: readPayloadString(message, "id"),
|
||||
channelId: readPayloadString(message, "channel_id") || input.channelId,
|
||||
timestamp: readPayloadString(message, "timestamp"),
|
||||
authorId: readPayloadString(message?.author, "id"),
|
||||
authorName: readPayloadString(message?.author, "global_name") ||
|
||||
readPayloadString(message?.author, "username"),
|
||||
content: readPayloadString(message, "content"),
|
||||
source: "live",
|
||||
})).filter((message) => message.id && message.channelId);
|
||||
}
|
||||
|
||||
async function fetchArchivedMessages(input: {
|
||||
archiveRepository: FileMessageArchiveRepository;
|
||||
channelId?: string;
|
||||
limit: number;
|
||||
}): Promise<CandidateMessage[]> {
|
||||
const records = input.channelId
|
||||
? await input.archiveRepository.listByChannel(input.channelId, input.limit)
|
||||
: await input.archiveRepository.listAllActive();
|
||||
|
||||
return records.slice(-input.limit).map((message) => ({
|
||||
id: message.id,
|
||||
channelId: message.channelId,
|
||||
timestamp: message.timestamp,
|
||||
authorId: message.authorId,
|
||||
authorName: message.authorName,
|
||||
content: message.content,
|
||||
source: "archive",
|
||||
}));
|
||||
}
|
||||
|
||||
function selectCandidates(messages: CandidateMessage[], options: CliOptions): CandidateMessage[] {
|
||||
const messageIdSet = new Set(options.messageIds);
|
||||
const authorIdSet = new Set(options.authorIds);
|
||||
const authorNameSet = new Set(options.authorNames.map(normalizeText));
|
||||
const afterMs = options.after ? Date.parse(options.after) : undefined;
|
||||
const lowerBoundMs = afterMs ?? (options.hours ? Date.now() - options.hours * 60 * 60 * 1000 : undefined);
|
||||
|
||||
return messages.filter((message) => {
|
||||
if (lowerBoundMs !== undefined) {
|
||||
const timestampMs = Date.parse(message.timestamp);
|
||||
if (Number.isNaN(timestampMs) || timestampMs < lowerBoundMs) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (messageIdSet.has(message.id)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const content = message.content.toLowerCase();
|
||||
const authorName = normalizeText(message.authorName ?? "");
|
||||
return (
|
||||
options.contains.some((needle) => content.includes(needle)) ||
|
||||
options.regexes.some((regex) => regex.test(message.content)) ||
|
||||
(message.authorId !== undefined && authorIdSet.has(message.authorId)) ||
|
||||
(authorName.length > 0 && authorNameSet.has(authorName))
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function dedupeCandidates(candidates: CandidateMessage[]): CandidateMessage[] {
|
||||
const byId = new Map<string, CandidateMessage>();
|
||||
for (const candidate of candidates) {
|
||||
const existing = byId.get(candidate.id);
|
||||
if (!existing || existing.source === "archive") {
|
||||
byId.set(candidate.id, candidate);
|
||||
}
|
||||
}
|
||||
return [...byId.values()].sort((left, right) => left.timestamp.localeCompare(right.timestamp));
|
||||
}
|
||||
|
||||
function printPlan(candidates: CandidateMessage[], options: CliOptions): void {
|
||||
console.log(options.execute ? "EXECUTE purge plan" : "DRY RUN purge plan");
|
||||
console.log(`Matched messages: ${candidates.length}`);
|
||||
console.log(`Targets: ${options.archiveOnly ? "archive/vector only" : options.discordOnly ? "Discord only" : "Discord plus archive/vector"}`);
|
||||
console.log("");
|
||||
|
||||
for (const candidate of candidates) {
|
||||
console.log([
|
||||
candidate.id,
|
||||
candidate.timestamp,
|
||||
`channel=${candidate.channelId}`,
|
||||
`source=${candidate.source}`,
|
||||
`${candidate.authorName ?? "unknown"} (${candidate.authorId ?? "unknown"})`,
|
||||
excerpt(candidate.content),
|
||||
].join(" | "));
|
||||
}
|
||||
|
||||
console.log("");
|
||||
}
|
||||
|
||||
async function deleteDiscordMessages(input: {
|
||||
token: string;
|
||||
candidates: CandidateMessage[];
|
||||
reason: string;
|
||||
}): Promise<string[]> {
|
||||
const deleted: string[] = [];
|
||||
|
||||
for (const candidate of input.candidates) {
|
||||
const response = await fetch(`https://discord.com/api/v10/channels/${candidate.channelId}/messages/${candidate.id}`, {
|
||||
method: "DELETE",
|
||||
headers: {
|
||||
Authorization: `Bot ${input.token}`,
|
||||
"X-Audit-Log-Reason": encodeURIComponent(input.reason),
|
||||
},
|
||||
});
|
||||
|
||||
if (response.status === 204 || response.status === 404) {
|
||||
deleted.push(candidate.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Discord delete failed for ${candidate.id} with ${response.status}: ${await response.text()}`);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
async function markArchiveMessagesDeleted(input: {
|
||||
config: ReturnType<typeof loadConfig>;
|
||||
archiveRepository: FileMessageArchiveRepository;
|
||||
messageIds: string[];
|
||||
}): Promise<string[]> {
|
||||
const embedder = createTextEmbedder({
|
||||
backend: input.config.ragEmbeddingBackend,
|
||||
hashDimensions: input.config.ragEmbeddingDimensions,
|
||||
ollamaBaseUrl: input.config.ragOllamaBaseUrl,
|
||||
ollamaModel: input.config.ragOllamaModel,
|
||||
ollamaTimeoutMs: input.config.ragOllamaTimeoutMs,
|
||||
queryInstruction: input.config.ragQueryInstruction,
|
||||
});
|
||||
const vectorStore = createVectorStores({
|
||||
kind: input.config.vectorStore.kind,
|
||||
historyPath: input.config.vectorStore.path,
|
||||
sourceRoot: input.config.sourceVectorStoreRoot,
|
||||
qdrant: input.config.qdrant,
|
||||
historyEmbedder: embedder,
|
||||
sourceEmbedder: embedder,
|
||||
}).history;
|
||||
const pipeline = new RagPipeline(input.archiveRepository, new HistoryIngester(), vectorStore);
|
||||
const deleted: string[] = [];
|
||||
|
||||
for (const messageId of input.messageIds) {
|
||||
if (await pipeline.markDeleted(messageId)) {
|
||||
deleted.push(messageId);
|
||||
}
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
function requireBotToken(token: string | undefined): string {
|
||||
if (!token) {
|
||||
throw new Error("DISCORD_BOT_TOKEN is required for live Discord purge work.");
|
||||
}
|
||||
return token;
|
||||
}
|
||||
|
||||
function readPayloadString(value: unknown, key: string): string {
|
||||
if (!value || typeof value !== "object") {
|
||||
return "";
|
||||
}
|
||||
const field = (value as Record<string, unknown>)[key];
|
||||
return typeof field === "string" ? field : "";
|
||||
}
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function excerpt(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim().slice(0, 220);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
|
@ -273,3 +273,4 @@
|
|||
{"ts":"2026-05-22T16:02:44Z","type":"implementation","status":"ok","note":"Repo Face character-turn prompt now keeps identity at the top for primacy, leaves repo activity/transcripts/doctrine in the middle, and moves projected state plus live social affordances to the end so recency bias promotes current affect/social pressure. Libby dry-run after the reorder passed with routed state/speech and retained social signal."}
|
||||
{"ts":"2026-05-22T16:19:30Z","type":"implementation","status":"ok","note":"Moved live peer openings into the repo Face state projector input and removed the deterministic social affordances prompt tail, so social reads are narrated in projected state instead of duplicated as transcript-like affordance text. Verified typecheck, assembled Libby prompt, and Libby interpreter dry-run."}
|
||||
{"ts":"2026-05-22T16:21:32Z","type":"implementation","status":"ok","note":"Moved the repo Face response-shape instructions to the final prompt position and made bylined article drafts explicit in that final shape, so article intent can reach the parent Interpreter as natural Face output and be converted to ARTICLE blocks. Verified typecheck, Nibu prompt assembly, and article frontmatter contract smoke."}
|
||||
{"ts":"2026-05-22T16:53:11Z","type":"implementation","status":"ok","note":"Added a dry-run-first Discord context purge script that selects explicit message IDs/text filters, deletes live Discord messages only under --execute, and marks matching archive/vector records deleted through the existing RAG pipeline. Used it to remove two poisoned agent witness-lane messages from Aquarium while leaving the human correction intact; post-purge full swarm dry-run passed with better correction acknowledgement and public speech coverage."}
|
||||
|
|
|
|||
Loading…
Reference in a new issue