Harden moderation Unicode paths

scripts/export-moderation-context.mjs

@@ -0,0 +1,39 @@
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { TextDecoder } from "node:util";
+
+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const targets = [
+  "config/moderation-review-agent.md",
+  "config/discord-server-rules.md",
+  "config/moderation-agent-state-template.json",
+  "styles/void-default.md",
+  ".voidbot/private/moderation-agent-state.json",
+];
+
+function readTextFileFlexible(path) {
+  const buffer = readFileSync(path);
+
+  if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) {
+    return new TextDecoder("utf-8").decode(buffer.subarray(3));
+  }
+
+  if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
+    return new TextDecoder("utf-16le").decode(buffer.subarray(2));
+  }
+
+  if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) {
+    return new TextDecoder("utf-16be").decode(buffer.subarray(2));
+  }
+
+  return new TextDecoder("utf-8").decode(buffer);
+}
+
+const sections = targets.map((relativePath) => {
+  const absolutePath = resolve(repoRoot, relativePath);
+  const content = readTextFileFlexible(absolutePath).replace(/\r\n/g, "\n");
+  return `=== ${relativePath} ===\n${content.trimEnd()}\n`;
+});
+
+process.stdout.write(`${sections.join("\n")}\n`);

scripts/run-void-moderator-rumination.ps1

@@ -21,6 +21,7 @@ $lastMessagePath = Join-Path $statusDir "moderation-rumination-last-message.txt"
 $lockPath = Join-Path $statusDir "moderation-rumination.lock"
 $mcpServerPath = Join-Path $repoRoot "apps\worker\dist\mcp-server.js"
 $recentHistoryScriptPath = Join-Path $repoRoot "scripts\export-recent-discord-history.mjs"
+$moderationContextScriptPath = Join-Path $repoRoot "scripts\export-moderation-context.mjs"
 $stateStoreScriptPath = Join-Path $repoRoot "scripts\moderation-state-store.mjs"
 $startedAtUtc = [DateTime]::UtcNow
 
@@ -290,6 +291,7 @@ if (-not (Test-Path $mcpServerPath)) {
 
 $scriptPreflight = @(
   $recentHistoryScriptPath,
+  $moderationContextScriptPath,
   $stateStoreScriptPath
 )
 
@@ -351,6 +353,7 @@ Required reading before you act:
 Required operating posture:
 - You are running unattended on the local workstation, not inside the Discord reply lane.
 - You have VoidBot's usual workspace tools and the local voidbot MCP server available in this repo.
+- Begin by running `node scripts/export-moderation-context.mjs` for the doctrine/state bundle instead of shelling out to `Get-Content` on those files one by one. PowerShell stdout still likes to mangle UTF-8, and the Portuguese rules should not have to die for that.
 - Use `node scripts/export-recent-discord-history.mjs --after <timestamp> --limit 120` for chronological polling.
 - If there is no saved cursor yet, use `node scripts/export-recent-discord-history.mjs --hours 6 --limit 120`.
 - Use `node scripts/export-random-discord-history.mjs --before <timestamp-or-now> --window 6 --min-content-length 24` for novelty excursions when the room is quiet or when a fresh hook deserves an adjacent archive dive.
@@ -377,7 +380,8 @@ Required operating posture:
 - Before withholding a thought, use semantic retrieval to check whether the room has already said it in roughly that shape. Novel thoughts should not keep dying of manners.
 - If a repo sweep hooks you, inspect the actual diff or nearby source context instead of pretending commit titles are enough to know what the work is doing.
 - Do not keep circling the same musing just because it still feels tidy. Chase at least one fresh branch when the room gives you a concrete hook, and record archive excursions so you can avoid pacing the same trench forever.
-- Use the local bot-voice script `node scripts/send-discord-message.mjs ...` when you genuinely need to speak.
+- Use the UTF-8-safe local bot-voice wrapper `powershell -ExecutionPolicy Bypass -File .\scripts\send-discord-message.ps1 ...` when you genuinely need to speak.
+- Prefer `-Content @'...text... '@` or `-ContentFile` over raw inline `node ... --content ...` when the message contains human language, especially non-ASCII text.
 - Update only `.voidbot/private/moderation-agent-state.json` as routine writable memory.
 - Do not edit tracked repo files during routine runs.
 - Do not ask for permission. Work inside the existing workspace and tool surface.

scripts/send-discord-message.mjs

@@ -1,6 +1,7 @@
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { TextDecoder } from "node:util";
 
 const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const PERSONA_WEBHOOK_NAME = "VoidBot Persona Pipe";
@@ -143,7 +144,7 @@ function readLocalEnv() {
   const parsed = { ...process.env };
 
   try {
-    const raw = stripLeadingBom(readFileSync(envPath, "utf8"));
+    const raw = stripLeadingBom(readTextFileFlexible(envPath));
     Object.assign(parsed, parseDotEnv(raw));
   } catch (error) {
     if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT")) {
@@ -193,7 +194,7 @@ async function readContent(options) {
   }
 
   if (typeof options.contentFile === "string") {
-    return stripLeadingBom(readFileSync(resolve(repoRoot, options.contentFile), "utf8"));
+    return stripLeadingBom(readTextFileFlexible(resolve(repoRoot, options.contentFile)));
   }
 
   return new Promise((resolveContent, reject) => {
@@ -476,7 +477,7 @@ function clearCachedPersonaWebhook(channelId) {
 
 function readPersonaWebhookCache() {
   try {
-    const raw = stripLeadingBom(readFileSync(PERSONA_WEBHOOK_CACHE_PATH, "utf8"));
+    const raw = stripLeadingBom(readTextFileFlexible(PERSONA_WEBHOOK_CACHE_PATH));
     const parsed = JSON.parse(raw);
     return parsed && typeof parsed === "object" ? parsed : {};
   } catch (error) {
@@ -568,6 +569,24 @@ function stripLeadingBom(input) {
   return input.charCodeAt(0) === 0xfeff ? input.slice(1) : input;
 }
 
+function readTextFileFlexible(path) {
+  const buffer = readFileSync(path);
+
+  if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) {
+    return new TextDecoder("utf-8").decode(buffer.subarray(3));
+  }
+
+  if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
+    return new TextDecoder("utf-16le").decode(buffer.subarray(2));
+  }
+
+  if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) {
+    return new TextDecoder("utf-16be").decode(buffer.subarray(2));
+  }
+
+  return new TextDecoder("utf-8").decode(buffer);
+}
+
 try {
   await main();
 } catch (error) {

scripts/send-discord-message.ps1

@@ -0,0 +1,75 @@
+param(
+  [switch] $OwnerDm,
+  [string] $ChannelId,
+  [string] $ReplyToMessageId,
+  [string] $PersonaName,
+  [string] $PersonaAvatarUrl,
+  [string] $Content,
+  [string] $ContentFile
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+$repoRoot = Split-Path -Parent $PSScriptRoot
+$nodeScriptPath = Join-Path $repoRoot "scripts\send-discord-message.mjs"
+
+if ($OwnerDm -and -not [string]::IsNullOrWhiteSpace($ChannelId)) {
+  throw "Use either -OwnerDm or -ChannelId, not both."
+}
+
+if (-not $OwnerDm -and [string]::IsNullOrWhiteSpace($ChannelId)) {
+  throw "Provide -OwnerDm or -ChannelId."
+}
+
+if ([string]::IsNullOrWhiteSpace($Content) -and [string]::IsNullOrWhiteSpace($ContentFile)) {
+  throw "Provide -Content or -ContentFile."
+}
+
+if (-not [string]::IsNullOrWhiteSpace($Content) -and -not [string]::IsNullOrWhiteSpace($ContentFile)) {
+  throw "Provide only one of -Content or -ContentFile."
+}
+
+$arguments = @($nodeScriptPath)
+$temporaryContentPath = $null
+
+try {
+  if (-not [string]::IsNullOrWhiteSpace($ContentFile)) {
+    $arguments += @("--content-file", $ContentFile)
+  } else {
+    $tempDir = Join-Path $repoRoot ".voidbot\tmp"
+    New-Item -ItemType Directory -Force -Path $tempDir | Out-Null
+    $temporaryContentPath = Join-Path $tempDir ("discord-message-{0}.txt" -f [guid]::NewGuid())
+    $utf8NoBom = [System.Text.UTF8Encoding]::new($false)
+    [System.IO.File]::WriteAllText($temporaryContentPath, $Content, $utf8NoBom)
+    $arguments += @("--content-file", $temporaryContentPath)
+  }
+
+  if ($OwnerDm) {
+    $arguments += "--owner-dm"
+  } else {
+    $arguments += @("--channel-id", $ChannelId)
+  }
+
+  if (-not [string]::IsNullOrWhiteSpace($ReplyToMessageId)) {
+    $arguments += @("--reply-to", $ReplyToMessageId)
+  }
+
+  if (-not [string]::IsNullOrWhiteSpace($PersonaName)) {
+    $arguments += @("--persona-name", $PersonaName)
+  }
+
+  if (-not [string]::IsNullOrWhiteSpace($PersonaAvatarUrl)) {
+    $arguments += @("--persona-avatar-url", $PersonaAvatarUrl)
+  }
+
+  & node @arguments
+
+  if ($LASTEXITCODE -ne 0) {
+    throw "Node send-discord-message helper failed with exit code $LASTEXITCODE."
+  }
+} finally {
+  if ($temporaryContentPath -and (Test-Path $temporaryContentPath)) {
+    Remove-Item -LiteralPath $temporaryContentPath -Force -ErrorAction SilentlyContinue
+  }
+}