Initial public release

.codex/config.toml

@@ -0,0 +1,7 @@
+[mcp_servers.voidbot]
+command = "node"
+args = ["apps/worker/dist/mcp-server.js"]
+cwd = "."
+
+[mcp_servers.voidbot.env]
+VOIDBOT_WORKSPACE_ROOT = "."

.env.example

@@ -0,0 +1,51 @@
+NODE_ENV=development
+DISCORD_BOT_TOKEN=
+DISCORD_APPLICATION_ID=
+DISCORD_GUILD_ID=
+DISCORD_OWNER_ID=
+DATABASE_DSN=postgres://voidbot:voidbot@localhost:5432/voidbot
+# Leave this as local_json until Qdrant is running and you've migrated the existing vectors.
+VECTOR_STORE_KIND=local_json
+VECTOR_STORE_PATH=.voidbot/history-vector-store.json
+SOURCE_VECTOR_STORE_ROOT=.voidbot/source-vectors
+QDRANT_URL=http://127.0.0.1:6333
+QDRANT_API_KEY=
+QDRANT_TIMEOUT_MS=30000
+QDRANT_HISTORY_COLLECTION=voidbot_discord_history_chunks
+QDRANT_SOURCE_COLLECTION=voidbot_repository_source_chunks
+RAG_ARCHIVE_PATH=.voidbot/rag/messages.json
+RAG_SOURCE_ARCHIVE_PATH=.voidbot/rag/source-documents.json
+RAG_IMPORT_STATE_PATH=.voidbot/rag/import-state.json
+RAG_EMBEDDING_BACKEND=ollama
+# Only used by the hash fallback backend. Ollama models report their own vector length.
+RAG_EMBEDDING_DIMENSIONS=256
+RAG_OLLAMA_BASE_URL=http://127.0.0.1:11434
+RAG_OLLAMA_MODEL=qwen3-embedding:0.6b
+RAG_OLLAMA_TIMEOUT_MS=30000
+RAG_QUERY_INSTRUCTION=Given a Discord history question, retrieve relevant messages and discussion snippets that answer it.
+RAG_SOURCE_QUERY_INSTRUCTION=Given a source-tree, codebase, or lore question, retrieve relevant files, code snippets, and lore passages that answer it.
+DISCORD_LOG_ROOT=
+SOURCE_REPO_ROOT=
+SOURCE_REPO_PATTERNS=*
+SOURCE_REPO_INCLUDE_PREFIXES=AetheriaLore:Aetheria/
+LOCAL_LLM_OLLAMA_BASE_URL=http://127.0.0.1:11434
+LOCAL_LLM_OLLAMA_MODEL=qwen3.5:9b
+LOCAL_LLM_OLLAMA_TIMEOUT_MS=120000
+LOCAL_LLM_OLLAMA_KEEP_ALIVE=10m
+LOCAL_LLM_OLLAMA_THINK=false
+LOCAL_LLM_OLLAMA_NUM_CTX=8192
+LOCAL_LLM_ALLOW_PUBLIC=false
+STORAGE_ROOT=.voidbot
+ENABLED_PROVIDERS=owner_codex
+OWNER_CODEX_MODE=local_exec_owner_only
+CODEX_EXECUTABLE=codex
+CODEX_EXEC_ARGS=
+CODEX_MODEL_REASONING_EFFORT=low
+CODEX_EXEC_TIMEOUT_MS=120000
+STYLE_PACK_PATH=styles/void-default.md
+SYSTEM_MESSAGES_PATH=config/system-messages.json
+INDEX_ALL_CHANNELS=false
+INDEXED_CHANNEL_IDS=
+EXCLUDED_CHANNEL_IDS=
+EXCLUDED_CHANNEL_NAMES=
+WORKER_POLL_INTERVAL_MS=5000

.gitattributes

@@ -0,0 +1,5 @@
+* text=auto eol=lf
+
+*.ps1 text eol=crlf
+*.bat text eol=crlf
+*.cmd text eol=crlf

.gitignore

@@ -0,0 +1,9 @@
+node_modules/
+dist/
+.env
+.voidbot/
+coverage/
+*.log
+*.tsbuildinfo
+Thumbs.db
+.DS_Store

AGENTS.md

@@ -0,0 +1,10 @@
+Use the `voidbot` MCP server whenever the task touches Discord history, archived GameCult discussion, indexed GameCult repositories, Aetheria lore, or owner notifications.
+
+Prefer these tools over raw file inspection when they can answer the question:
+- `search_history` for semantic retrieval across archived Discord messages
+- `get_message_context` for the surrounding conversation window around a message
+- `search_sources` for semantic retrieval across indexed source trees and lore repositories
+- `get_source_context` for the surrounding chunk window inside an indexed source document
+- `notify_owner` for explicit Discord pings or long-running completion notices
+
+Do not inspect `.voidbot/rag/messages.json`, `.voidbot/rag/source-documents.json`, `.voidbot/history-vector-store.json`, or `.voidbot/source-vectors/` directly unless the MCP tools are unavailable or clearly insufficient.

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 GameCult
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,391 @@
+# VoidBot
+
+`TypeScript workspace` · `Discord bot` · `Codex MCP` · `Qdrant` · `Ollama`
+
+VoidBot is a Discord-native assistant for GameCult.
+
+It can answer from archived Discord history, indexed GameCult repos, and Aetheria lore, then hand deeper work off to Codex when Discord stops being a sane place to do it. The current build already supports real Discord replies, semantic retrieval, source and lore indexing, and Codex handoff flows.
+
+## What VoidBot Can Do
+
+- answer owner questions through a Discord-safe `codex exec` lane
+- use real MCP tools for archived Discord history, repo search, lore lookup, and owner notifications
+- index live Discord traffic for selected channels
+- backfill old Discord exports into the archive
+- index source trees and lore repos with local Ollama embeddings
+- run semantic retrieval on either local JSON shards or Qdrant
+- use a remote Ollama chat model for the `local_llm` provider
+- kick off detached source reindex jobs from local Git push hooks
+
+## At A Glance
+
+```mermaid
+flowchart LR
+  Discord["Discord"] --> Bot["apps/bot"]
+  Bot --> Worker["apps/worker"]
+  Bot --> History["Discord archive"]
+  Worker --> Codex["Codex exec / MCP"]
+  Worker --> Sources["Source + lore archive"]
+  History --> Qdrant["Qdrant"]
+  Sources --> Qdrant
+  Codex --> Qdrant
+  Ollama["Ollama"] --> Qdrant
+  Ollama --> Worker
+```
+
+## Current Shape
+
+The repo is split into a bot, a worker, and a handful of focused packages:
+
+- `apps/bot`: Discord gateway, commands, permissions, and request assembly
+- `apps/worker`: background job execution and the Void MCP server
+- `packages/core`: queueing, audit log, permissions, style packs, system messages
+- `packages/providers`: `owner_codex`, `local_llm`, and provider registry glue
+- `packages/rag`: archives, chunking, embedding backends, retrieval, vector stores
+- `packages/config`: environment-driven configuration
+- `packages/shared`: shared contracts and types
+- `packages/sandbox`: policy-first sandbox scaffolding
+
+The durable local state lives under `.voidbot/`. That includes jobs, audit logs, artifacts, archives, and detached indexing logs.
+
+## Quick Start
+
+### 1. Configure `.env`
+
+Copy `.env.example` to `.env`, then set at minimum:
+
+- `DISCORD_BOT_TOKEN`
+- `DISCORD_APPLICATION_ID`
+- `DISCORD_GUILD_ID`
+- `DISCORD_OWNER_ID`
+
+Recommended starting values:
+
+```dotenv
+ENABLED_PROVIDERS=owner_codex,local_llm
+OWNER_CODEX_MODE=local_exec_owner_only
+VECTOR_STORE_KIND=qdrant
+RAG_EMBEDDING_BACKEND=ollama
+RAG_OLLAMA_MODEL=qwen3-embedding:0.6b
+STYLE_PACK_PATH=styles/void-default.md
+SYSTEM_MESSAGES_PATH=config/system-messages.json
+```
+
+If you want the remote Ollama generation path too, configure:
+
+- `LOCAL_LLM_OLLAMA_BASE_URL`
+- `LOCAL_LLM_OLLAMA_MODEL`
+- `LOCAL_LLM_OLLAMA_TIMEOUT_MS`
+- `LOCAL_LLM_OLLAMA_KEEP_ALIVE`
+- `LOCAL_LLM_OLLAMA_THINK`
+- `LOCAL_LLM_OLLAMA_NUM_CTX`
+- `LOCAL_LLM_ALLOW_PUBLIC`
+
+`LOCAL_LLM_ALLOW_PUBLIC=false` is the sane default. It keeps the local model restricted to admins and the configured owner while you test.
+
+If you want a private local persona instead, point `STYLE_PACK_PATH` and `SYSTEM_MESSAGES_PATH` at your own ignored local files.
+
+### 2. Install dependencies
+
+```bash
+npm install
+npm run build
+```
+
+### 3. Pull the embedding model
+
+VoidBot defaults to Ollama embeddings with `qwen3-embedding:0.6b`:
+
+```bash
+ollama pull qwen3-embedding:0.6b
+```
+
+`RAG_EMBEDDING_DIMENSIONS` only matters for the hash fallback backend. Ollama-backed embeddings report their own dimensions.
+
+### 4. Start Qdrant
+
+Qdrant is the recommended vector backend now. It avoids the pain that comes from stuffing everything into local JSON and then acting surprised when the file turns into a small moon.
+
+Windows helper path:
+
+```bash
+npm run qdrant:setup
+```
+
+Normal day-to-day startup after that:
+
+```bash
+npm run qdrant:up
+```
+
+Or run Qdrant manually:
+
+```powershell
+docker pull qdrant/qdrant
+docker volume create voidbot_qdrant_storage
+docker run -p 6333:6333 -p 6334:6334 -v voidbot_qdrant_storage:/qdrant/storage qdrant/qdrant
+```
+
+Then point the bot at it:
+
+```dotenv
+VECTOR_STORE_KIND=qdrant
+QDRANT_URL=http://127.0.0.1:6333
+QDRANT_HISTORY_COLLECTION=voidbot_discord_history_chunks
+QDRANT_SOURCE_COLLECTION=voidbot_repository_source_chunks
+```
+
+If you already have local vectors and want to lift them into Qdrant without re-embedding:
+
+```bash
+npm run rag:migrate-qdrant
+```
+
+Use `-- --wipe` only when you intentionally want to replace the target collections.
+
+### 5. Start the bot and worker
+
+Run these in separate terminals:
+
+```bash
+npm run dev:bot
+```
+
+```bash
+npm run dev:worker
+```
+
+If `DISCORD_APPLICATION_ID` and `DISCORD_GUILD_ID` are set, the bot registers slash commands into that guild on startup.
+
+For `owner_codex`, the worker also needs a working local Codex CLI. If plain `codex` is not invokable from Node on your machine, point `CODEX_EXECUTABLE` and `CODEX_EXEC_ARGS` at the actual install.
+
+## Example Questions
+
+Things you can ask right now:
+
+- `@Void what did we decide about the website roadmap last year?`
+- `@Void summarize prior discussion about ship movement in nautical`
+- `/ask provider:local_llm question: where is the MCP server registering search_sources?`
+- `/ask provider:local_llm question: what does Aetheria lore say about FTL corporations?`
+- `/search-history query: favorite algorithm`
+
+## How The Main Flows Work
+
+### Owner Discord flow
+
+Default mode is `local_exec_owner_only`:
+
+1. The owner uses `@Void ...` or `/ask`.
+2. The bot queues an auto-approved owner job.
+3. The worker runs `codex exec` in a read-only, Discord-safe lane.
+4. Codex can call Void MCP tools such as `search_history`, `get_message_context`, `search_sources`, `get_source_context`, and `list_indexed_repos`.
+5. If the answer is short and safe, the worker posts it back to Discord.
+6. If the task needs broader tools, file edits, or a deeper investigation, the worker posts a handoff notice and writes a bundle under `.voidbot/artifacts/<job-id>/`.
+
+`/queue-codex` skips the direct-reply attempt and forces the handoff path.
+
+### Manual fallback
+
+`manual_package` still exists when you want the stricter, explicit workflow:
+
+1. Queue a job.
+2. Approve it with `/approve-job`.
+3. Let the worker write `request.md` and `request.json`.
+4. Run the deeper work manually.
+5. Approve any final public reply separately.
+
+### Local Ollama flow
+
+The `local_llm` provider uses Ollama chat completion plus a bounded host-managed read-only tool loop. It can use:
+
+- `search_history`
+- `get_message_context`
+- `search_sources`
+- `get_source_context`
+- `list_indexed_repos`
+
+Unlike the owner Codex lane, this is not MCP. It is a tighter host-controlled loop and deliberately excludes side-effecting tools.
+
+## Discord, Repo, And Lore Indexing
+
+### Discord history
+
+Discord history is live and incremental:
+
+- matching channels are ingested on `MessageCreate`
+- edits are re-upserted
+- deletes remove indexed chunks
+- `/reindex-channel` replays the most recent 100 messages from the current indexed channel
+- exported logs can be backfilled later
+
+Channel selection is controlled by:
+
+- `INDEX_ALL_CHANNELS=true` or `INDEXED_CHANNEL_IDS=...`
+- `EXCLUDED_CHANNEL_IDS`
+- `EXCLUDED_CHANNEL_NAMES`
+
+Backfill old exports with:
+
+```bash
+npm run rag:backfill -- --input /path/to/discord-logs
+```
+
+Useful flags:
+
+- `--no-recursive`
+- `--all-channels`
+
+### Source repos and lore
+
+Source indexing is incremental when you run it. The pipeline crawls the repo, compares the current documents to the archived source snapshot, deletes vectors for changed or removed source IDs, and upserts only the changed chunks.
+
+Configure:
+
+- `SOURCE_REPO_ROOT`
+- `SOURCE_REPO_PATTERNS`
+- `SOURCE_REPO_INCLUDE_PREFIXES`
+- `RAG_SOURCE_ARCHIVE_PATH`
+- `RAG_SOURCE_QUERY_INSTRUCTION`
+
+Index everything that matches your repo patterns:
+
+```bash
+npm run rag:index-sources
+```
+
+Target specific repos:
+
+```bash
+npm run rag:index-sources -- --repo gamecult-site,AetheriaLore
+```
+
+Force a clean reindex for specific repos:
+
+```bash
+npm run rag:index-sources -- --repo AetheriaLore --force
+```
+
+Recommended posture for a shared project root:
+
+- `SOURCE_REPO_PATTERNS=*`
+- `SOURCE_REPO_INCLUDE_PREFIXES=AetheriaLore:Aetheria/`
+
+That keeps `AetheriaLore` focused on the useful subtree instead of indexing every random byproduct in the repo.
+
+### Git push hook for source indexing
+
+If you want local pushes to kick off source reindexing automatically:
+
+```bash
+npm run rag:install-push-hooks
+```
+
+Git does not provide a client-side `post-push` hook, so VoidBot installs a chained `pre-push` hook instead. That hook returns quickly, launches a detached incremental reindex for the pushed repo, and preserves any existing `pre-push` hook by backing it up to `pre-push.voidbot.prev` and running it first.
+
+Progress for those detached runs lives here:
+
+- status: `.voidbot/status/source-hooks/<repo>.json`
+- log: `.voidbot/logs/source-hooks/<repo>.log`
+
+That gives you a real polling path instead of an attached session silently rotting while you wonder whether it hung.
+
+## Codex Integration
+
+This repo includes a project-scoped `.codex/config.toml`, so Codex sessions opened in `E:\Projects\VoidBot` can use Void's MCP tools automatically.
+
+If you want the same MCP server available from Codex outside this repo:
+
+```bash
+npm run codex:mcp:install
+```
+
+Current MCP tools:
+
+- `search_history`
+- `get_message_context`
+- `search_sources`
+- `get_source_context`
+- `list_indexed_repos`
+- `notify_owner`
+
+Notes:
+
+- read-only retrieval tools work in unattended `codex exec` runs
+- side-effecting MCP tool calls still run into approval limits in unattended mode
+- the owner Discord lane handles notifications through a host-mediated notification intent instead
+
+To remove the shared Codex registration later:
+
+```bash
+npm run codex:mcp:remove
+```
+
+## Maintenance
+
+If you change the embedding backend or model, rebuild the vector index from the archived corpora:
+
+```bash
+npm run rag:rebuild
+```
+
+If you still have an old mixed local JSON vector store from earlier builds, migrate it once before moving on:
+
+```bash
+npm run rag:migrate-vector-layout
+```
+
+## Commands
+
+Current slash commands:
+
+- `/ask`
+- `/queue-codex`
+- `/approve-job`
+- `/reject-job`
+- `/provider-status`
+- `/search-history`
+- `/summarize-channel`
+- `/reindex-channel`
+- `/set-style`
+
+Some commands are full working paths. Some are still thin MVPs. They exist because the structure is real, not because the system is pretending to be finished.
+
+## Important Files
+
+Useful local state and docs:
+
+- `.voidbot/jobs/jobs.json`: local job queue
+- `.voidbot/audit/events.jsonl`: audit trail
+- `.voidbot/artifacts/<job-id>/`: handoff bundles, stdout, stderr, traces
+- `.voidbot/rag/messages.json`: archived Discord messages
+- `.voidbot/rag/source-documents.json`: archived source and lore documents
+- `.voidbot/rag/import-state.json`: log-backfill file state
+- `config/system-messages.json`: rotating stock system messages
+- `styles/void-default.md`: public default style pack
+- `docs/architecture/overview.md`: higher-level architecture notes
+- `packages/core/sql/bootstrap.sql`: draft PostgreSQL schema
+
+## Safety Boundaries
+
+- `owner_codex` hard-rejects non-owner traffic
+- the Discord-safe owner lane is read-only
+- the bot does not expose arbitrary shell execution
+- deeper tasks are handed off instead of turning Discord into a cursed remote terminal
+- sandbox support is still policy-first and dry-run only
+- `manual_package` remains available when you want explicit review points
+
+## Development Roadmap
+
+Reasonable next steps from here:
+
+1. Move queue and audit storage into PostgreSQL.
+2. Add health checks, backups, and operational guidance around Qdrant.
+3. Expand worker processing into richer provider run records and moderation hooks.
+4. Replace the remaining scaffolded provider paths with funded, production-grade implementations.
+5. Add real constrained sandbox execution instead of policy-only dry runs.
+
+## Related Docs
+
+- [SPEC.md](./SPEC.md)
+- [Architecture Overview](./docs/architecture/overview.md)
+- [TypeScript Workspace Decision](./docs/decisions/0001-typescript-workspace.md)

SPEC.md

@@ -0,0 +1,504 @@
+# VoidBot Spec
+
+Status: Draft v0.1
+Date: 2026-04-22
+Owner: Meta
+
+## 1. Summary
+
+VoidBot is a Discord-native assistant with a provider abstraction layer.
+
+In phase one, VoidBot supports an owner-only Codex workflow that is gated, explicit, and human-mediated. In later phases, the same bot can route non-owner traffic to a paid API provider or a local model without changing the Discord-facing contract.
+
+The design goal is simple:
+
+- build the bot infrastructure now
+- keep subscription-backed Codex use restricted to the owner
+- make multi-user AI access possible later by swapping providers
+- preserve enough safety rails that the bot cannot casually become "remote shell, but with branding"
+
+## 2. Goals
+
+- Provide a Discord bot that can answer questions, retrieve relevant server context, and run structured workflows.
+- Index approved Discord history into a searchable vector store for RAG.
+- Support multiple providers behind one stable interface.
+- Restrict the Codex-backed path to the owner account only.
+- Keep tool use and command execution inside a strict sandbox.
+- Preserve auditability for prompts, approvals, runs, and published responses.
+- Allow a future OpenAI API provider to be added with minimal refactor.
+
+## 3. Non-Goals
+
+- Exposing the owner's ChatGPT/Codex subscription as a general multi-user backend.
+- Allowing arbitrary shell execution from Discord messages.
+- Indexing all Discord content by default, including DMs and private channels, without explicit configuration.
+- Shipping polished personality tuning beyond loading a defined style pack.
+- Building a full moderation platform before the MVP exists.
+
+## 4. Product Principles
+
+- Owner-only means owner-only. The Codex path must hard reject all non-owner requests.
+- Providers are interchangeable, permissions are not. Discord permissions and provider permissions must be enforced separately.
+- Human approval beats vibes. Any high-risk action requires a visible approval step.
+- Retrieval should help the bot remember context, not quietly become surveillance software.
+- Style is configurable. Capabilities are not.
+
+## 5. Users And Trust Levels
+
+### Roles
+
+- Owner: the only Discord user allowed to submit jobs to the Codex-backed provider.
+- Admin: can manage bot configuration, indexed channels, and provider availability.
+- Member: can use public bot features only through providers explicitly enabled for member traffic.
+
+### Trust Tiers
+
+| Tier | Who    | Allowed Providers         | Allowed Actions                                      |
+| ---- | ------ | ------------------------- | ---------------------------------------------------- |
+| T0   | Member | `openai_api`, `local_llm` | ask, retrieve, approved low-risk tools               |
+| T1   | Admin  | `openai_api`, `local_llm` | config, moderation, reindex, diagnostics             |
+| T2   | Owner  | all providers             | owner-only Codex jobs, approvals, provider overrides |
+
+## 6. Scope
+
+### Phase 1
+
+- Discord bot connection and routing
+- owner-only Codex workflow
+- RAG over configured server channels
+- vector index maintenance
+- strict sandbox runner
+- audit logging
+- style pack support
+
+### Phase 2
+
+- OpenAI API provider for non-owner traffic
+- provider-based rate limits and budgets
+- output moderation for public responses
+- better admin controls
+
+### Phase 3
+
+- optional local LLM provider
+- richer tools
+- scheduled summaries and proactive tasks
+
+## 7. Core Use Cases
+
+### UC1: Owner asks VoidBot for a response with Codex help
+
+1. Owner mentions VoidBot or uses a slash command.
+2. VoidBot builds a context bundle from the current message, recent channel history, and RAG retrieval.
+3. VoidBot packages the request as a queued owner-only job.
+4. VoidBot posts the result in the same channel as a response to the Owner
+
+### UC2: Member asks VoidBot a question before API funding exists
+
+1. Member mentions VoidBot.
+2. VoidBot replies with a configured fallback behavior:
+3. either "provider unavailable"
+4. or a local-model response if `local_llm` is enabled
+
+No member traffic may route to the Codex provider.
+
+### UC3: RAG retrieval over server history
+
+1. VoidBot ingests messages from configured channels.
+2. Messages are chunked, embedded, and stored with metadata.
+3. Queries retrieve relevant chunks and summaries.
+4. Providers receive only the context needed for the current task.
+
+### UC4: Safe structured action
+
+1. A user invokes a supported action such as summarize channel, search archive, or run a whitelisted workflow.
+2. VoidBot maps the request to a predefined tool action.
+3. The action runs in a sandbox profile with strict file, network, time, and command limits.
+
+## 8. High-Level Architecture
+
+```text
+Discord Gateway
+  -> Command Router
+  -> Permission Engine
+  -> Context Builder
+  -> Job Queue
+  -> Provider Adapter
+  -> Sandbox Runner
+  -> Storage Layer
+      - relational DB
+      - vector store
+      - object storage for transcripts/artifacts
+  -> Audit Log
+```
+
+### Components
+
+- `discord-gateway`: receives events, slash commands, mentions, message edits, and reactions.
+- `router`: maps incoming events to intents and workflows.
+- `permission-engine`: enforces role, channel, provider, and action policies.
+- `context-builder`: assembles prompt context from recent messages, retrieval results, style packs, and config.
+- `job-queue`: manages long-running tasks, retries, and approval states.
+- `provider-adapter`: normalizes requests to each model provider.
+- `sandbox-runner`: executes approved tools and workflows inside constrained environments.
+- `history-ingester`: writes message archives and embeddings.
+- `audit-log`: records who asked what, what provider was used, what tools were invoked, and what was posted.
+
+## 9. Provider Model
+
+VoidBot must not let Discord-facing code know provider-specific details beyond declared capabilities.
+
+### Provider Interface
+
+Each provider should implement:
+
+- `get_name()`
+- `get_capabilities()`
+- `is_enabled()`
+- `is_allowed_for_actor(actor, guild_context)`
+- `build_request(context_bundle, options)`
+- `execute(request)`
+- `estimate_cost(request)`
+- `moderate_input(request)` if applicable
+- `moderate_output(response)` if applicable
+- `get_audit_redactions()`
+
+### Required Provider Types
+
+#### `owner_codex`
+
+Purpose:
+
+- owner-only Codex-backed workflow
+
+Rules:
+
+- may only accept requests from the configured owner Discord ID
+- disabled for all other actors, even admins
+- approval required before provider execution
+- approval required before public posting
+- should default to a manual or semi-manual bridge mode
+
+Recommended initial mode:
+
+- `manual_package`
+
+In `manual_package` mode, VoidBot does not directly run Codex as a background service. It prepares a job bundle for the owner, who reviews and executes it manually, then approves the final post. This keeps the implementation closest to a workflow tool rather than a shared hosted endpoint.
+
+Optional future mode:
+
+- `local_exec_owner_only`
+
+This mode should remain disabled by default and clearly marked experimental.
+
+#### `openai_api`
+
+Purpose:
+
+- future multi-user provider for member and admin traffic
+
+Rules:
+
+- API key required
+- cost controls required
+- input and output moderation required for public use
+- rate limiting required at guild, channel, and user level
+
+#### `local_llm`
+
+Purpose:
+
+- cheap fallback or offline mode
+
+Rules:
+
+- no privileged tools by default
+- lower trust output label
+
+## 10. Discord Interaction Model
+
+### Triggers
+
+- mention trigger: `@VoidBot`
+- slash commands
+- optional message context menu actions
+
+### Initial Commands
+
+- `/ask`
+- `/search-history`
+- `/summarize-channel`
+- `/queue-codex`
+- `/approve-job`
+- `/reject-job`
+- `/provider-status`
+- `/reindex-channel`
+- `/set-style`
+
+### Response Modes
+
+- immediate ephemeral acknowledgement
+- streamed progress updates where supported
+- final public reply
+- final private or admin-only reply for gated jobs
+
+## 11. Style System
+
+VoidBot should support style packs loaded from local prompt files.
+
+Initial style pack:
+
+- `void-default` for public-facing deployments
+
+Style application rules:
+
+- apply tone primarily to conversational framing
+- keep commands, diagnostics, warnings, and technical instructions plain
+- allow per-provider enable or disable flags
+- allow guild-level default with channel-level override
+
+## 12. RAG And History Indexing
+
+### Ingestion Policy
+
+- disabled by default until configured
+- allowlist channels only
+- no DMs in MVP
+- private channels require explicit opt-in
+- retain message metadata: guild, channel, author, timestamp, thread, reply chain, attachment refs
+
+### Chunking
+
+- chunk by semantic boundaries where possible
+- preserve thread continuity
+- store both raw text and normalized text
+- tag chunks with source message IDs
+
+### Embeddings
+
+- provider-agnostic embedding interface
+- re-embed when chunking strategy changes
+- support batch backfills and incremental updates
+
+### Retrieval
+
+- query rewrite optional
+- hybrid retrieval preferred later, vector-only acceptable in MVP
+- max context budget enforced before provider call
+- retrieval results should include source citations internally
+
+## 13. Sandbox And Tool Safety
+
+All nontrivial tool execution must happen inside a sandbox profile.
+
+### Sandbox Profiles
+
+- `read_only_lookup`
+- `rag_maintenance`
+- `owner_workflow`
+- `public_low_risk`
+
+### Minimum Safety Controls
+
+- explicit allowlist of commands per profile
+- filesystem scope restrictions
+- network disabled unless a workflow explicitly requires it
+- execution timeout
+- memory and output limits
+- no direct interpolation of raw user text into shell commands
+- structured tool inputs only
+
+### Approval Rules
+
+- owner-only Codex jobs require owner approval before run
+- any command with side effects requires approval
+- any public post from an owner Codex job requires approval
+- reindex and admin maintenance actions require admin or owner role
+
+## 14. Data Model
+
+### Core Entities
+
+- `GuildConfig`
+- `ChannelPolicy`
+- `UserPolicy`
+- `ProviderConfig`
+- `MessageArchive`
+- `EmbeddingChunk`
+- `Job`
+- `JobApproval`
+- `ProviderRun`
+- `ToolInvocation`
+- `AuditEvent`
+- `StylePack`
+
+### Job States
+
+- `queued`
+- `awaiting_approval`
+- `approved`
+- `running`
+- `awaiting_post_approval`
+- `completed`
+- `failed`
+- `cancelled`
+
+## 15. Storage
+
+### Relational DB
+
+Use for:
+
+- configs
+- job state
+- approvals
+- audit records
+- message metadata
+
+### Vector Store
+
+Use for:
+
+- embeddings
+- semantic retrieval
+
+### Object Storage
+
+Use for:
+
+- exported prompt bundles
+- long transcripts
+- generated artifacts
+- moderation records where needed
+
+## 16. Moderation And Policy Handling
+
+### For Public Providers
+
+- run input moderation before generation
+- run output moderation before posting
+- redact secrets and high-risk data in logs
+- label bot responses clearly as AI-generated
+
+### For Owner Codex Flow
+
+- still log runs and approvals
+- still support optional moderation hooks
+- do not silently auto-post
+
+## 17. Configuration
+
+### Required Config
+
+- bot token
+- owner Discord user ID
+- database DSN
+- vector store config
+- enabled providers
+- style pack path
+- indexed channel allowlist
+
+### Future Config
+
+- OpenAI API key
+- provider budgets
+- moderation thresholds
+- per-guild rate limits
+- sandbox profile overrides
+
+## 18. Suggested Repo Layout
+
+```text
+VoidBot/
+  SPEC.md
+  README.md
+  .env.example
+  apps/
+    bot/
+    worker/
+  packages/
+    core/
+    providers/
+    rag/
+    sandbox/
+    config/
+    shared/
+  docs/
+    architecture/
+    decisions/
+  scripts/
+  tests/
+```
+
+## 19. MVP Milestones
+
+### M1: Foundations
+
+- repo scaffold
+- config loader
+- Discord connection
+- DB schema
+- audit log skeleton
+
+### M2: Retrieval Spine
+
+- message ingestion
+- chunking
+- embeddings
+- retrieval API
+
+### M3: Provider Abstraction
+
+- provider interface
+- `owner_codex` manual package mode
+- `local_llm` optional stub
+
+### M4: Safety And Workflow
+
+- approvals
+- sandbox profiles
+- command allowlists
+- public vs owner routing
+
+### M5: UX
+
+- slash commands
+- progress updates
+- citations in internal debug output
+- style pack loading
+
+### M6: API Upgrade Path
+
+- `openai_api` provider
+- moderation
+- budgeting
+- member access controls
+
+## 20. Acceptance Criteria
+
+- Non-owner requests can never hit the Codex provider.
+- Owner Codex jobs require explicit approval before run and before public post.
+- The bot can retrieve relevant history from configured channels.
+- Tool actions run only inside declared sandbox profiles.
+- Provider swapping does not require Discord command rewrites.
+- Audit logs can answer who triggered a job, what provider handled it, and what was posted.
+
+## 21. Open Questions
+
+- Which stack should own the first implementation: TypeScript, Python, or mixed?
+- Which vector store should be the default for local development?
+- Should the first owner Codex mode export markdown bundles, JSON bundles, or both?
+- Should message attachments be indexed in MVP or only linked as metadata?
+- How much of the approval workflow should happen in Discord versus a local dashboard?
+
+## 22. Recommendation
+
+Start with:
+
+- TypeScript bot and worker
+- PostgreSQL for relational state
+- a local vector store with an upgrade path
+- `owner_codex` in `manual_package` mode only
+- no public AI provider until the API key exists
+
+That gets the plumbing built now without pretending the expensive part is free just because we squint at it hard enough.

apps/bot/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@voidbot/bot",
+  "version": "0.1.0",
+  "private": true,
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "@voidbot/config": "0.1.0",
+    "@voidbot/core": "0.1.0",
+    "@voidbot/providers": "0.1.0",
+    "@voidbot/rag": "0.1.0",
+    "@voidbot/shared": "0.1.0",
+    "discord.js": "^14.19.0",
+    "dotenv": "^16.4.0"
+  }
+}

apps/bot/src/index.ts

@@ -0,0 +1,9 @@
+import "dotenv/config";
+
+import { startBot } from "./discord-bot";
+
+void startBot().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});
+

apps/bot/tsconfig.json

@@ -0,0 +1,29 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "references": [
+    {
+      "path": "../../packages/shared"
+    },
+    {
+      "path": "../../packages/config"
+    },
+    {
+      "path": "../../packages/core"
+    },
+    {
+      "path": "../../packages/providers"
+    },
+    {
+      "path": "../../packages/rag"
+    }
+  ]
+}
+

apps/worker/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@voidbot/worker",
+  "version": "0.1.0",
+  "private": true,
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "@voidbot/config": "0.1.0",
+    "@voidbot/core": "0.1.0",
+    "@voidbot/providers": "0.1.0",
+    "@voidbot/rag": "0.1.0",
+    "@voidbot/shared": "0.1.0",
+    "dotenv": "^16.4.0",
+    "zod": "^3.25.0"
+  }
+}

apps/worker/src/index.ts

@@ -0,0 +1,549 @@
+import "dotenv/config";
+
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { loadConfig } from "@voidbot/config";
+import {
+  buildVoidMcpServerConfig,
+  FileAuditLog,
+  FileBackedJobQueue,
+  loadSystemMessageCatalog,
+  type SystemMessageCatalog,
+} from "@voidbot/core";
+import {
+  LocalLlmProvider,
+  OpenAiApiProvider,
+  OwnerCodexProvider,
+  ProviderRegistry,
+} from "@voidbot/providers";
+import {
+  createTextEmbedder,
+  createVectorStores,
+  FileMessageArchiveRepository,
+  FileSourceDocumentArchiveRepository,
+  RetrievalService,
+  SourceDocumentIngester,
+  type ArchivedMessageRecord,
+} from "@voidbot/rag";
+import {
+  type JobRecord,
+  type ProviderArtifact,
+  type ProviderNotificationIntent,
+} from "@voidbot/shared";
+
+const config = loadConfig();
+const jobQueue = new FileBackedJobQueue(config.jobsFile);
+const auditLog = new FileAuditLog(config.auditLogFile);
+const embedder = createTextEmbedder({
+  backend: config.ragEmbeddingBackend,
+  hashDimensions: config.ragEmbeddingDimensions,
+  ollamaBaseUrl: config.ragOllamaBaseUrl,
+  ollamaModel: config.ragOllamaModel,
+  ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+  queryInstruction: config.ragQueryInstruction,
+});
+const sourceQueryEmbedder = createTextEmbedder({
+  backend: config.ragEmbeddingBackend,
+  hashDimensions: config.ragEmbeddingDimensions,
+  ollamaBaseUrl: config.ragOllamaBaseUrl,
+  ollamaModel: config.ragOllamaModel,
+  ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+  queryInstruction: config.ragSourceQueryInstruction,
+});
+const vectorStores = createVectorStores({
+  kind: config.vectorStore.kind,
+  historyPath: config.vectorStore.path,
+  sourceRoot: config.sourceVectorStoreRoot,
+  qdrant: config.qdrant,
+  historyEmbedder: embedder,
+  sourceEmbedder: sourceQueryEmbedder,
+});
+const historyVectorStore = vectorStores.history;
+const sourceVectorStore = vectorStores.source;
+const archiveRepository = new FileMessageArchiveRepository(config.ragArchivePath);
+const sourceArchiveRepository = new FileSourceDocumentArchiveRepository(config.ragSourceArchivePath);
+const sourceDocumentIngester = new SourceDocumentIngester();
+const retrievalService = new RetrievalService(
+  historyVectorStore,
+  sourceVectorStore,
+);
+
+let isProcessing = false;
+let providerRegistry: ProviderRegistry;
+let systemMessages: SystemMessageCatalog;
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});
+
+async function main(): Promise<void> {
+  systemMessages = await loadSystemMessageCatalog(config.systemMessagesPath);
+  providerRegistry = await buildProviderRegistry(systemMessages);
+  console.log(`VoidBot worker polling every ${config.workerPollIntervalMs}ms.`);
+  await processPendingJobs();
+  setInterval(() => {
+    void processPendingJobs();
+  }, config.workerPollIntervalMs);
+}
+
+async function processPendingJobs(): Promise<void> {
+  if (isProcessing) {
+    return;
+  }
+
+  isProcessing = true;
+
+  try {
+    systemMessages = await loadSystemMessageCatalog(config.systemMessagesPath);
+    providerRegistry = await buildProviderRegistry(systemMessages);
+    const jobs = await jobQueue.claimRunnableJobs();
+
+    for (const job of jobs) {
+      await processJob(job);
+    }
+  } finally {
+    isProcessing = false;
+  }
+}
+
+async function buildProviderRegistry(systemMessagesCatalog: SystemMessageCatalog): Promise<ProviderRegistry> {
+  return new ProviderRegistry([
+    new OwnerCodexProvider({
+      ownerDiscordId: config.ownerDiscordId,
+      enabled: config.enabledProviders.includes("owner_codex"),
+      mode: config.ownerCodexMode,
+      executable: config.codexExecutable,
+      executableArgs: config.codexExecArgs,
+      reasoningEffort: config.codexModelReasoningEffort,
+      timeoutMs: config.codexExecTimeoutMs,
+      workingDirectory: process.cwd(),
+      historyLookup: retrievalService,
+      handoffNoticeBuilder: (jobId) =>
+        systemMessagesCatalog.render("owner_codex.handoff_notice", {
+          handoffPath: `.voidbot/artifacts/${jobId}/handoff.md`,
+          debugPath: `.voidbot/artifacts/${jobId}/debug-trace.md`,
+        }),
+      mcpServers: [buildVoidMcpServerConfig(process.cwd())],
+    }),
+    new OpenAiApiProvider(config.enabledProviders.includes("openai_api")),
+    new LocalLlmProvider({
+      enabled: config.enabledProviders.includes("local_llm"),
+      ownerDiscordId: config.ownerDiscordId,
+      ollamaBaseUrl: config.localLlm.ollamaBaseUrl,
+      ollamaModel: config.localLlm.ollamaModel,
+      ollamaTimeoutMs: config.localLlm.ollamaTimeoutMs,
+      ollamaKeepAlive: config.localLlm.ollamaKeepAlive,
+      ollamaThink: config.localLlm.ollamaThink,
+      ollamaNumCtx: config.localLlm.ollamaNumCtx,
+      allowPublicResponses: config.localLlm.allowPublic,
+      toolbox: {
+        listIndexedRepos: async () => {
+          const repos = await sourceArchiveRepository.listRepoSummaries();
+
+          return {
+            repoCount: repos.length,
+            repos,
+          };
+        },
+        searchHistory: async (input) => {
+          const results = await retrievalService.searchHistory(input.query, input.limit, {
+            guildId: input.guildId,
+            channelId: input.channelId,
+            authorId: input.authorId,
+          });
+
+          return {
+            query: input.query,
+            resultCount: results.length,
+            results: formatHistoryResults(results),
+          };
+        },
+        getMessageContext: async (input) => {
+          const messages = await archiveRepository.listContextWindow(
+            input.messageId,
+            input.before,
+            input.after,
+          );
+
+          return {
+            found: messages.length > 0,
+            messageId: input.messageId,
+            count: messages.length,
+            messages: messages.map((message) =>
+              formatArchivedMessageContext(message, input.messageId),
+            ),
+          };
+        },
+        searchSources: async (input) => {
+          const results = await retrievalService.searchRepositorySources(input.query, input.limit, {
+            repoName: input.repoName,
+            pathPrefix: input.pathPrefix,
+            language: input.language,
+          });
+
+          return {
+            query: input.query,
+            resultCount: results.length,
+            results: formatSourceResults(results),
+          };
+        },
+        getSourceContext: async (input) => {
+          const document = await sourceArchiveRepository.get(input.sourceId);
+
+          if (!document) {
+            return {
+              found: false,
+              sourceId: input.sourceId,
+              count: 0,
+              chunks: [],
+            };
+          }
+
+          const chunks = sourceDocumentIngester.buildContextWindow(
+            document,
+            input.chunkIndex,
+            input.before,
+            input.after,
+          );
+
+          return {
+            found: true,
+            sourceId: input.sourceId,
+            repoName: document.repoName,
+            path: document.path,
+            language: document.language,
+            count: chunks.length,
+            chunks: chunks.map((chunk) => ({
+              chunkId: chunk.chunkId,
+              chunkIndex: chunk.chunkIndex,
+              lineStart: chunk.lineStart,
+              lineEnd: chunk.lineEnd,
+              isAnchor: chunk.chunkIndex === input.chunkIndex,
+              text: chunk.text,
+            })),
+          };
+        },
+      },
+    }),
+  ]);
+}
+
+async function processJob(job: JobRecord): Promise<void> {
+  const provider = providerRegistry.get(job.provider);
+
+  if (!provider || !provider.isEnabled()) {
+    const errorMessage = `Provider ${job.provider} is not enabled.`;
+    await jobQueue.markFailed(job.id, errorMessage);
+    await maybeNotifyOwnerOfJobFailure(job, errorMessage);
+    return;
+  }
+
+  try {
+    const request = provider.buildRequest(job.contextBundle, {
+      jobId: job.id,
+      command: job.command,
+    });
+    const response = await provider.execute(request);
+    const artifactPaths = await writeArtifacts(job.id, response.artifacts ?? []);
+
+    if (response.status === "ready_for_review") {
+      await jobQueue.markAwaitingPostApproval(job.id, artifactPaths, response.summary);
+      await deliverOwnerNotifications(job, response.notifications ?? []);
+      await auditLog.record({
+        type: "provider.bundle_ready",
+        actorId: job.requester.id,
+        jobId: job.id,
+        provider: job.provider,
+        details: {
+          artifactPaths,
+        },
+      });
+      console.log(`Prepared manual package for job ${job.id}.`);
+      return;
+    }
+
+    const finalResponse = fitDiscordMessage(response.outputText ?? response.summary);
+
+    if (job.postApprovalRequired) {
+      await jobQueue.markAwaitingPostApproval(job.id, artifactPaths, response.summary);
+      await deliverOwnerNotifications(job, response.notifications ?? []);
+      await auditLog.record({
+        type: "provider.completed_pending_post_approval",
+        actorId: job.requester.id,
+        jobId: job.id,
+        provider: job.provider,
+        details: {
+          summary: response.summary,
+        },
+      });
+      console.log(`Prepared response for post approval on job ${job.id}.`);
+      return;
+    }
+
+    await postJobResponse(job, finalResponse);
+    await jobQueue.completeJobDirect(job.id, finalResponse);
+    await deliverOwnerNotifications(job, response.notifications ?? []);
+    await auditLog.record({
+      type: "provider.completed",
+      actorId: job.requester.id,
+      jobId: job.id,
+      provider: job.provider,
+      details: {
+        summary: response.summary,
+        autoPosted: true,
+      },
+    });
+    console.log(`Completed job ${job.id}.`);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unexpected worker failure.";
+    await jobQueue.markFailed(job.id, message);
+    await maybeNotifyOwnerOfJobFailure(job, message);
+    await auditLog.record({
+      type: "provider.failed",
+      actorId: job.requester.id,
+      jobId: job.id,
+      provider: job.provider,
+      details: {
+        error: message,
+      },
+    });
+    console.error(`Failed job ${job.id}: ${message}`);
+  }
+}
+
+async function maybeNotifyOwnerOfJobFailure(job: JobRecord, errorMessage: string): Promise<void> {
+  if (job.requester.id !== config.ownerDiscordId) {
+    return;
+  }
+
+  try {
+    await postOwnerNotification(
+      systemMessages.render("job.owner.worker_failed_dm", {
+        jobId: job.id,
+        command: job.command,
+        providerName: job.provider,
+        errorMessage,
+      }),
+    );
+  } catch (error) {
+    console.error(
+      `Failed to DM owner about job ${job.id}: ${
+        error instanceof Error ? error.message : "Unexpected notification failure."
+      }`,
+    );
+  }
+}
+
+async function deliverOwnerNotifications(
+  job: JobRecord,
+  notifications: ProviderNotificationIntent[],
+): Promise<void> {
+  if (notifications.length === 0) {
+    return;
+  }
+
+  for (const notification of notifications) {
+    try {
+      await postOwnerNotification(notification.message);
+      await auditLog.record({
+        type: "provider.notification_sent",
+        actorId: job.requester.id,
+        jobId: job.id,
+        provider: job.provider,
+        details: {
+          channel: notification.channel,
+          reason: notification.reason,
+          message: notification.message,
+        },
+      });
+    } catch (error) {
+      const message =
+        error instanceof Error ? error.message : "Unexpected notification delivery failure.";
+      await auditLog.record({
+        type: "provider.notification_failed",
+        actorId: job.requester.id,
+        jobId: job.id,
+        provider: job.provider,
+        details: {
+          channel: notification.channel,
+          reason: notification.reason,
+          message: notification.message,
+          error: message,
+        },
+      });
+      console.error(`Failed owner notification for job ${job.id}: ${message}`);
+    }
+  }
+}
+
+async function writeArtifacts(
+  jobId: string,
+  artifacts: ProviderArtifact[],
+): Promise<Record<string, string>> {
+  if (artifacts.length === 0) {
+    return {};
+  }
+
+  const artifactDirectory = join(config.artifactsDir, jobId);
+  await mkdir(artifactDirectory, { recursive: true });
+
+  const paths: Record<string, string> = {};
+
+  for (const artifact of artifacts) {
+    const filePath = join(artifactDirectory, artifact.name);
+    await writeFile(filePath, artifact.content, "utf8");
+    paths[artifact.name] = filePath;
+  }
+
+  return paths;
+}
+
+async function postJobResponse(job: JobRecord, content: string): Promise<void> {
+  if (!config.botToken) {
+    throw new Error("DISCORD_BOT_TOKEN is required for the worker to post completed responses.");
+  }
+
+  const response = await fetch(`https://discord.com/api/v10/channels/${job.outputChannelId}/messages`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bot ${config.botToken}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      content,
+      message_reference: job.requestMessageId
+        ? {
+            message_id: job.requestMessageId,
+            fail_if_not_exists: false,
+          }
+        : undefined,
+      allowed_mentions: {
+        parse: [],
+      },
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Discord post failed with ${response.status}: ${await response.text()}`);
+  }
+}
+
+async function postOwnerNotification(content: string): Promise<void> {
+  if (!config.botToken) {
+    throw new Error("DISCORD_BOT_TOKEN is required for the worker to send owner notifications.");
+  }
+
+  const dmChannelId = await openOwnerDmChannel();
+  const response = await fetch(`https://discord.com/api/v10/channels/${dmChannelId}/messages`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bot ${config.botToken}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      content: fitDiscordMessage(content),
+      allowed_mentions: {
+        parse: [],
+      },
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Discord owner notification failed with ${response.status}: ${await response.text()}`);
+  }
+}
+
+async function openOwnerDmChannel(): Promise<string> {
+  if (!config.botToken) {
+    throw new Error("DISCORD_BOT_TOKEN is required for the worker to open an owner DM.");
+  }
+
+  const response = await fetch("https://discord.com/api/v10/users/@me/channels", {
+    method: "POST",
+    headers: {
+      Authorization: `Bot ${config.botToken}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      recipient_id: config.ownerDiscordId,
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Discord DM open failed with ${response.status}: ${await response.text()}`);
+  }
+
+  const payload = (await response.json()) as { id?: string };
+
+  if (!payload.id) {
+    throw new Error("Discord DM open succeeded without returning a channel id.");
+  }
+
+  return payload.id;
+}
+
+function fitDiscordMessage(content: string): string {
+  const trimmed = content.trim();
+
+  if (trimmed.length <= 1900) {
+    return trimmed;
+  }
+
+  return `${trimmed.slice(0, 1897)}...`;
+}
+
+function formatHistoryResults(
+  results: Awaited<ReturnType<RetrievalService["searchHistory"]>>,
+): Array<Record<string, unknown>> {
+  return results.map((result) => ({
+    score: Number(result.score.toFixed(4)),
+    text: result.text,
+    sourceId: result.sourceId,
+    channelId: result.metadata.channelId,
+    channelName: result.metadata.channelName,
+    authorId: result.metadata.authorId,
+    authorName: result.metadata.authorName,
+    timestamp: result.metadata.timestamp,
+    jumpUrl: result.metadata.jumpUrl,
+    threadId: result.metadata.threadId,
+  }));
+}
+
+function formatSourceResults(
+  results: Awaited<ReturnType<RetrievalService["searchRepositorySources"]>>,
+): Array<Record<string, unknown>> {
+  return results.map((result) => ({
+    score: Number(result.score.toFixed(4)),
+    text: result.text,
+    sourceId: result.sourceId,
+    repoName: result.metadata.repoName,
+    path: result.metadata.path,
+    language: result.metadata.language,
+    title: result.metadata.title,
+    chunkIndex: Number(result.metadata.chunkIndex ?? 0),
+    lineStart: Number(result.metadata.lineStart ?? 1),
+    lineEnd: Number(result.metadata.lineEnd ?? 1),
+    lastModifiedAt: result.metadata.lastModifiedAt,
+  }));
+}
+
+function formatArchivedMessageContext(
+  message: ArchivedMessageRecord,
+  anchorMessageId: string,
+): Record<string, unknown> {
+  return {
+    id: message.id,
+    isAnchor: message.id === anchorMessageId,
+    timestamp: message.timestamp,
+    authorId: message.authorId,
+    authorName: message.authorName,
+    channelId: message.channelId,
+    channelName: message.metadata?.channelName,
+    threadId: message.threadId,
+    content: message.content,
+    jumpUrl: message.metadata?.jumpUrl,
+    editedAt: message.editedAt,
+  };
+}

apps/worker/src/mcp-server.ts

@@ -0,0 +1,587 @@
+import { resolve } from "node:path";
+
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { type CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { config as loadDotenv } from "dotenv";
+import { z } from "zod/v3";
+
+import { loadConfig } from "@voidbot/config";
+import {
+  DEFAULT_RETRIEVAL_RESULT_LIMIT,
+  MAX_RETRIEVAL_RESULT_LIMIT,
+} from "@voidbot/shared";
+import {
+  createTextEmbedder,
+  createVectorStores,
+  FileMessageArchiveRepository,
+  FileSourceDocumentArchiveRepository,
+  RetrievalService,
+  SourceDocumentIngester,
+  type ArchivedMessageRecord,
+} from "@voidbot/rag";
+
+const workspaceRoot = resolve(process.env.VOIDBOT_WORKSPACE_ROOT ?? process.cwd());
+loadDotenv({ path: resolve(workspaceRoot, ".env") });
+process.chdir(workspaceRoot);
+
+const config = loadConfig();
+const archiveRepository = new FileMessageArchiveRepository(config.ragArchivePath);
+const sourceArchiveRepository = new FileSourceDocumentArchiveRepository(config.ragSourceArchivePath);
+const embedder = createTextEmbedder({
+  backend: config.ragEmbeddingBackend,
+  hashDimensions: config.ragEmbeddingDimensions,
+  ollamaBaseUrl: config.ragOllamaBaseUrl,
+  ollamaModel: config.ragOllamaModel,
+  ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+  queryInstruction: config.ragQueryInstruction,
+});
+const sourceQueryEmbedder = createTextEmbedder({
+  backend: config.ragEmbeddingBackend,
+  hashDimensions: config.ragEmbeddingDimensions,
+  ollamaBaseUrl: config.ragOllamaBaseUrl,
+  ollamaModel: config.ragOllamaModel,
+  ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+  queryInstruction: config.ragSourceQueryInstruction,
+});
+const vectorStores = createVectorStores({
+  kind: config.vectorStore.kind,
+  historyPath: config.vectorStore.path,
+  sourceRoot: config.sourceVectorStoreRoot,
+  qdrant: config.qdrant,
+  historyEmbedder: embedder,
+  sourceEmbedder: sourceQueryEmbedder,
+});
+const historyVectorStore = vectorStores.history;
+const sourceVectorStore = vectorStores.source;
+const retrievalService = new RetrievalService(
+  historyVectorStore,
+  sourceVectorStore,
+);
+const sourceDocumentIngester = new SourceDocumentIngester();
+
+const server = new McpServer(
+  {
+    name: "voidbot",
+    version: "0.1.0",
+  },
+  {
+    capabilities: {
+      logging: {},
+    },
+  },
+);
+
+const searchHistoryInputSchema = {
+  query: z.string().min(1).max(240),
+  limit: z.number().int().min(1).max(MAX_RETRIEVAL_RESULT_LIMIT).optional(),
+  guildId: z.string().min(1).optional(),
+  channelId: z.string().min(1).optional(),
+  authorId: z.string().min(1).optional(),
+};
+
+const messageContextInputSchema = {
+  messageId: z.string().min(1),
+  before: z.number().int().min(0).max(20).optional(),
+  after: z.number().int().min(0).max(20).optional(),
+};
+
+const notifyOwnerInputSchema = {
+  message: z.string().min(1).max(1800),
+};
+
+const searchSourcesInputSchema = {
+  query: z.string().min(1).max(240),
+  limit: z.number().int().min(1).max(MAX_RETRIEVAL_RESULT_LIMIT).optional(),
+  repoName: z.string().min(1).optional(),
+  pathPrefix: z.string().min(1).optional(),
+  language: z.string().min(1).optional(),
+};
+
+const sourceContextInputSchema = {
+  sourceId: z.string().min(1),
+  chunkIndex: z.number().int().min(0).optional(),
+  before: z.number().int().min(0).max(20).optional(),
+  after: z.number().int().min(0).max(20).optional(),
+};
+
+interface SearchHistoryArgs {
+  query: string;
+  limit?: number;
+  guildId?: string;
+  channelId?: string;
+  authorId?: string;
+}
+
+interface MessageContextArgs {
+  messageId: string;
+  before?: number;
+  after?: number;
+}
+
+interface NotifyOwnerArgs {
+  message: string;
+}
+
+interface ListIndexedReposArgs {
+  [key: string]: never;
+}
+
+interface SearchSourcesArgs {
+  query: string;
+  limit?: number;
+  repoName?: string;
+  pathPrefix?: string;
+  language?: string;
+}
+
+interface SourceContextArgs {
+  sourceId: string;
+  chunkIndex?: number;
+  before?: number;
+  after?: number;
+}
+
+server.registerTool(
+  "search_history",
+  {
+    title: "Search Discord History",
+    description:
+      "Search Void's archived Discord history with semantic retrieval. Use this for historical discussion, decisions, preferences, and prior conversations.",
+    inputSchema: searchHistoryInputSchema,
+    annotations: {
+      readOnlyHint: true,
+      destructiveHint: false,
+      idempotentHint: true,
+      openWorldHint: false,
+    },
+  },
+  async (input: SearchHistoryArgs): Promise<CallToolResult> => {
+    const { query, limit, guildId, channelId, authorId } = input;
+    const results = await retrievalService.searchHistory(query, limit ?? DEFAULT_RETRIEVAL_RESULT_LIMIT, {
+      guildId,
+      channelId,
+      authorId,
+    });
+    const formattedResults = results.map((result) => ({
+      score: Number(result.score.toFixed(4)),
+      text: result.text,
+      sourceId: result.sourceId,
+      channelId: result.metadata.channelId,
+      channelName: result.metadata.channelName,
+      authorId: result.metadata.authorId,
+      authorName: result.metadata.authorName,
+      timestamp: result.metadata.timestamp,
+      jumpUrl: result.metadata.jumpUrl,
+      threadId: result.metadata.threadId,
+    }));
+
+    return {
+      content: [
+        {
+          type: "text",
+          text:
+            formattedResults.length > 0
+              ? renderJsonBlock({
+                  query,
+                  resultCount: formattedResults.length,
+                  results: formattedResults,
+                })
+              : `No archived Discord history matched "${query}".`,
+        },
+      ],
+      structuredContent: {
+        query,
+        resultCount: formattedResults.length,
+        results: formattedResults,
+      },
+    };
+  },
+);
+
+server.registerTool(
+  "list_indexed_repos",
+  {
+    title: "List Indexed Repositories",
+    description:
+      "List the indexed GameCult source and lore repositories currently available to search. Use this when you want valid repoName filters for search_sources.",
+    inputSchema: {},
+    annotations: {
+      readOnlyHint: true,
+      destructiveHint: false,
+      idempotentHint: true,
+      openWorldHint: false,
+    },
+  },
+  async (_input: ListIndexedReposArgs): Promise<CallToolResult> => {
+    const repos = await sourceArchiveRepository.listRepoSummaries();
+
+    return {
+      content: [
+        {
+          type: "text",
+          text:
+            repos.length > 0
+              ? renderJsonBlock({
+                  repoCount: repos.length,
+                  repos,
+                })
+              : "No indexed source repositories are currently available.",
+        },
+      ],
+      structuredContent: {
+        repoCount: repos.length,
+        repos,
+      },
+    };
+  },
+);
+
+server.registerTool(
+  "search_sources",
+  {
+    title: "Search Repository Sources",
+    description:
+      "Search indexed GameCult source trees and lore repositories with semantic retrieval. Use this for code structure, implementation details, lore references, and repo-local documentation. Omit repoName to search across all indexed repos, or call list_indexed_repos first if you need valid repoName options.",
+    inputSchema: searchSourcesInputSchema,
+    annotations: {
+      readOnlyHint: true,
+      destructiveHint: false,
+      idempotentHint: true,
+      openWorldHint: false,
+    },
+  },
+  async (input: SearchSourcesArgs): Promise<CallToolResult> => {
+    const { query, limit, repoName, pathPrefix, language } = input;
+    const results = await retrievalService.searchRepositorySources(
+      query,
+      limit ?? DEFAULT_RETRIEVAL_RESULT_LIMIT,
+      {
+      repoName,
+      pathPrefix,
+      language,
+      },
+    );
+    const formattedResults = results.map((result) => ({
+      score: Number(result.score.toFixed(4)),
+      text: result.text,
+      sourceId: result.sourceId,
+      repoName: result.metadata.repoName,
+      path: result.metadata.path,
+      language: result.metadata.language,
+      title: result.metadata.title,
+      chunkIndex: Number(result.metadata.chunkIndex ?? 0),
+      lineStart: Number(result.metadata.lineStart ?? 1),
+      lineEnd: Number(result.metadata.lineEnd ?? 1),
+      lastModifiedAt: result.metadata.lastModifiedAt,
+    }));
+
+    return {
+      content: [
+        {
+          type: "text",
+          text:
+            formattedResults.length > 0
+              ? renderJsonBlock({
+                  query,
+                  resultCount: formattedResults.length,
+                  results: formattedResults,
+                })
+              : `No indexed source documents matched "${query}".`,
+        },
+      ],
+      structuredContent: {
+        query,
+        resultCount: formattedResults.length,
+        results: formattedResults,
+      },
+    };
+  },
+);
+
+server.registerTool(
+  "get_source_context",
+  {
+    title: "Get Source Context",
+    description:
+      "Fetch a surrounding chunk window from an indexed source document. Use after search_sources when you need adjacent code, prose, or lore context inside a file.",
+    inputSchema: sourceContextInputSchema,
+    annotations: {
+      readOnlyHint: true,
+      destructiveHint: false,
+      idempotentHint: true,
+      openWorldHint: false,
+    },
+  },
+  async (input: SourceContextArgs): Promise<CallToolResult> => {
+    const { sourceId, chunkIndex, before, after } = input;
+    const document = await sourceArchiveRepository.get(sourceId);
+
+    if (!document) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: `No indexed source document with ID ${sourceId} was found.`,
+          },
+        ],
+        structuredContent: {
+          found: false,
+          sourceId,
+          chunks: [],
+        },
+        isError: true,
+      };
+    }
+
+    const chunks = sourceDocumentIngester.buildContextWindow(
+      document,
+      chunkIndex ?? 0,
+      before ?? 1,
+      after ?? 1,
+    );
+    const anchorIndex =
+      chunks.find((chunk) => chunk.chunkIndex === (chunkIndex ?? 0))?.chunkIndex ??
+      chunks[0]?.chunkIndex ??
+      0;
+    const formattedChunks = chunks.map((chunk) => ({
+      chunkId: chunk.chunkId,
+      chunkIndex: chunk.chunkIndex,
+      lineStart: chunk.lineStart,
+      lineEnd: chunk.lineEnd,
+      isAnchor: chunk.chunkIndex === anchorIndex,
+      text: chunk.text,
+    }));
+
+    return {
+      content: [
+        {
+          type: "text",
+          text: renderJsonBlock({
+            found: true,
+            sourceId,
+            repoName: document.repoName,
+            path: document.path,
+            language: document.language,
+            count: formattedChunks.length,
+            chunks: formattedChunks,
+          }),
+        },
+      ],
+      structuredContent: {
+        found: true,
+        sourceId,
+        repoName: document.repoName,
+        path: document.path,
+        language: document.language,
+        count: formattedChunks.length,
+        chunks: formattedChunks,
+      },
+    };
+  },
+);
+
+server.registerTool(
+  "get_message_context",
+  {
+    title: "Get Message Context",
+    description:
+      "Fetch the surrounding conversation window for a specific archived Discord message ID. Use after search_history when you need neighboring context or thread flow.",
+    inputSchema: messageContextInputSchema,
+    annotations: {
+      readOnlyHint: true,
+      destructiveHint: false,
+      idempotentHint: true,
+      openWorldHint: false,
+    },
+  },
+  async (input: MessageContextArgs): Promise<CallToolResult> => {
+    const { messageId, before, after } = input;
+    const anchor = await archiveRepository.get(messageId);
+
+    if (!anchor) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: `No archived message with ID ${messageId} was found.`,
+          },
+        ],
+        structuredContent: {
+          found: false,
+          messageId,
+          messages: [],
+        },
+        isError: true,
+      };
+    }
+
+    const messages = await archiveRepository.listContextWindow(
+      messageId,
+      before ?? 4,
+      after ?? 4,
+    );
+    const formattedMessages = messages.map((message) => formatArchivedMessage(message, messageId));
+
+    return {
+      content: [
+        {
+          type: "text",
+          text: renderJsonBlock({
+            found: true,
+            messageId,
+            count: formattedMessages.length,
+            messages: formattedMessages,
+          }),
+        },
+      ],
+      structuredContent: {
+        found: true,
+        messageId,
+        count: formattedMessages.length,
+        messages: formattedMessages,
+      },
+    };
+  },
+);
+
+server.registerTool(
+  "notify_owner",
+  {
+    title: "Notify Owner",
+    description:
+      "Send a Discord DM to the configured owner via Void. Use for completion notices, progress updates, or when explicitly asked to ping them.",
+    inputSchema: notifyOwnerInputSchema,
+    annotations: {
+      readOnlyHint: false,
+      destructiveHint: false,
+      idempotentHint: false,
+      openWorldHint: true,
+    },
+  },
+  async (input: NotifyOwnerArgs): Promise<CallToolResult> => {
+    const { message } = input;
+    if (!config.botToken) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: "DISCORD_BOT_TOKEN is not configured, so Void cannot send notifications.",
+          },
+        ],
+        structuredContent: {
+          sent: false,
+          reason: "missing_bot_token",
+        },
+        isError: true,
+      };
+    }
+
+    const dmChannelId = await openOwnerDmChannel(config.botToken, config.ownerDiscordId);
+    const posted = await postDiscordMessage(config.botToken, dmChannelId, `Void relay: ${message}`);
+
+    return {
+      content: [
+        {
+          type: "text",
+          text: `Sent a Discord DM to the owner as message ${posted.id}.`,
+        },
+      ],
+      structuredContent: {
+        sent: true,
+        channelId: dmChannelId,
+        messageId: posted.id,
+      },
+    };
+  },
+);
+
+async function main(): Promise<void> {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+}
+
+void main().catch((error) => {
+  console.error("Void MCP server failed:", error);
+  process.exit(1);
+});
+
+function formatArchivedMessage(
+  message: ArchivedMessageRecord,
+  anchorMessageId: string,
+): Record<string, unknown> {
+  return {
+    id: message.id,
+    isAnchor: message.id === anchorMessageId,
+    timestamp: message.timestamp,
+    authorId: message.authorId,
+    authorName: message.authorName,
+    channelId: message.channelId,
+    channelName: message.metadata?.channelName,
+    threadId: message.threadId,
+    content: message.content,
+    jumpUrl: message.metadata?.jumpUrl,
+    editedAt: message.editedAt,
+  };
+}
+
+function renderJsonBlock(payload: Record<string, unknown>): string {
+  return `${JSON.stringify(payload, null, 2)}\n`;
+}
+
+async function openOwnerDmChannel(botToken: string, recipientId: string): Promise<string> {
+  const response = await fetch("https://discord.com/api/v10/users/@me/channels", {
+    method: "POST",
+    headers: {
+      Authorization: `Bot ${botToken}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      recipient_id: recipientId,
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Failed to open Discord DM channel: ${response.status} ${await response.text()}`);
+  }
+
+  const payload = (await response.json()) as { id?: string };
+
+  if (!payload.id) {
+    throw new Error("Discord DM channel creation returned no channel id.");
+  }
+
+  return payload.id;
+}
+
+async function postDiscordMessage(
+  botToken: string,
+  channelId: string,
+  content: string,
+): Promise<{ id: string }> {
+  const response = await fetch(`https://discord.com/api/v10/channels/${channelId}/messages`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bot ${botToken}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      content,
+      allowed_mentions: {
+        parse: [],
+      },
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Failed to post Discord message: ${response.status} ${await response.text()}`);
+  }
+
+  const payload = (await response.json()) as { id?: string };
+
+  if (!payload.id) {
+    throw new Error("Discord message post returned no message id.");
+  }
+
+  return { id: payload.id };
+}

apps/worker/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "references": [
+    {
+      "path": "../../packages/shared"
+    },
+    {
+      "path": "../../packages/config"
+    },
+    {
+      "path": "../../packages/core"
+    },
+    {
+      "path": "../../packages/providers"
+    },
+    {
+      "path": "../../packages/rag"
+    }
+  ]
+}

config/system-messages.json

@@ -0,0 +1,138 @@
+{
+  "mention.missing_prompt": [
+    "Void: You mentioned me, but you forgot the question.",
+    "Void: I can help, but I need an actual prompt after the mention."
+  ],
+  "config.set_style.forbidden": [
+    "Void: Only admins are allowed to change my style pack.",
+    "Void: Tone control is restricted to admins."
+  ],
+  "config.set_style.reloaded": [
+    "Void: Reloaded style pack `{stylePackName}` from `{stylePackPath}` and refreshed stock messages from `{systemMessagesPath}`.",
+    "Void: Updated my style pack to `{stylePackName}` from `{stylePackPath}` and reloaded messages from `{systemMessagesPath}`."
+  ],
+  "config.set_style.style_missing": [
+    "Void: Reloaded stock messages from `{systemMessagesPath}`, but there was no style pack at `{stylePackPath}`.",
+    "Void: Message catalog refreshed from `{systemMessagesPath}`. The style pack at `{stylePackPath}` is missing."
+  ],
+  "command.unimplemented": [
+    "Void: That command exists, but the implementation behind it does not.",
+    "Void: The command is present. The machinery behind it is not."
+  ],
+  "error.with_message": [
+    "Void: Something failed: {errorMessage}",
+    "Void: The machinery hit an error: {errorMessage}"
+  ],
+  "error.generic": [
+    "Void: Something failed, and the error was annoyingly vague.",
+    "Void: Something broke, and I did not get a useful explanation."
+  ],
+  "provider.none_available": [
+    "Void: No provider is available for you right now.",
+    "Void: I do not currently have an available provider for your request."
+  ],
+  "provider.missing_registration": [
+    "Void: `{providerName}` is not registered.",
+    "Void: That provider name leads nowhere. `{providerName}` is not configured."
+  ],
+  "provider.access_denied.generic": [
+    "Void: That provider is not available to you.",
+    "Void: Access denied for that provider."
+  ],
+  "permission.owner_codex.denied": [
+    "Void: The owner Codex provider is restricted to the configured owner account.",
+    "Void: That lane is reserved for the configured owner account."
+  ],
+  "permission.local_llm.denied": [
+    "Void: The local model is currently limited to admins and the configured owner.",
+    "Void: Access to the local model is restricted right now."
+  ],
+  "job.owner.awaiting_run_approval": [
+    "Void: Owner job `{jobId}` is waiting. Approve it with `/approve-job job-id:{jobId}` to prepare the manual bundle.",
+    "Void: Owner job `{jobId}` is queued for manual handling. Use `/approve-job job-id:{jobId}` when you want the bundle generated."
+  ],
+  "job.owner.auto_queued": [
+    "Void: Owner job `{jobId}` is queued. If it stays short and read-only, I will answer here. If not, I will hand it off to the fuller Codex flow.",
+    "Void: Owner job `{jobId}` is in motion. Small read-only work stays here. Bigger work gets handed off."
+  ],
+  "job.owner.queue_failed_dm": [
+    "Void: I could not queue your request from `{channelName}`. Error: {errorMessage}",
+    "Void: The queue step failed for your request in `{channelName}`. Error: {errorMessage}"
+  ],
+  "job.owner.worker_failed_dm": [
+    "Void: Job `{jobId}` failed while `{providerName}` was handling `{command}`. Error: {errorMessage}",
+    "Void: The worker hit a failure on job `{jobId}` during `{command}` with `{providerName}`. Error: {errorMessage}"
+  ],
+  "job.not_found": [
+    "Void: I cannot find job `{jobId}`.",
+    "Void: There is no record of job `{jobId}`."
+  ],
+  "job.approval_forbidden": [
+    "Void: You are not allowed to approve that job.",
+    "Void: Approval for that job is not available to you."
+  ],
+  "job.run_approved": [
+    "Void: Job `{jobId}` is approved. The worker will pick it up on the next pass.",
+    "Void: Approval recorded for job `{jobId}`. The worker should handle it shortly."
+  ],
+  "job.post_approval.needs_response": [
+    "Void: That job is waiting for post approval. Put the final response in the optional response field.",
+    "Void: I can post the result once you supply the final response in the optional response field."
+  ],
+  "job.post_approved": [
+    "Void: Final reply posted for job `{jobId}`.",
+    "Void: Job `{jobId}` has been posted."
+  ],
+  "job.no_action_for_state": [
+    "Void: Job `{jobId}` is in state `{jobState}`, so there is nothing to approve.",
+    "Void: No approval action is needed for job `{jobId}` while it is in `{jobState}`."
+  ],
+  "job.rejection_forbidden": [
+    "Void: You are not allowed to reject that job.",
+    "Void: Rejection for that job is not available to you."
+  ],
+  "job.rejected": [
+    "Void: Job `{jobId}` has been rejected.",
+    "Void: Dismissed job `{jobId}`."
+  ],
+  "history.no_results": [
+    "Void: No indexed history matched that query.",
+    "Void: I searched the archive and found nothing for that query."
+  ],
+  "history.results_intro": [
+    "Void: Here are the relevant history hits:",
+    "Void: These are the archive results that looked useful:"
+  ],
+  "summarize.no_messages": [
+    "Void: There are no recent messages here to summarize.",
+    "Void: This channel is too quiet to summarize right now."
+  ],
+  "summarize.results_intro": [
+    "Void: Here is a quick summary of the recent conversation:",
+    "Void: Brief summary of the recent messages:"
+  ],
+  "reindex.forbidden": [
+    "Void: Only admins are allowed to reindex the archive.",
+    "Void: Reindexing is restricted to admins."
+  ],
+  "reindex.excluded": [
+    "Void: This channel is excluded from indexing.",
+    "Void: I am not configured to ingest this channel."
+  ],
+  "reindex.unsupported_channel": [
+    "Void: This channel does not expose message history, so there is nothing to reindex.",
+    "Void: Reindexing is not supported here because there is no message history to read."
+  ],
+  "reindex.completed": [
+    "Void: Reindexing complete. I walked {archivedMessages} messages: created {createdMessages}, updated {updatedMessages}, left {unchangedMessages} untouched, and indexed {indexedChunks} chunks.",
+    "Void: Archive sweep finished. I reviewed {archivedMessages} messages, created {createdMessages}, updated {updatedMessages}, skipped {unchangedMessages}, and produced {indexedChunks} chunks."
+  ],
+  "provider_status.intro": [
+    "Void: Current provider status:",
+    "Void: Here is the current state of the configured providers:"
+  ],
+  "owner_codex.handoff_notice": [
+    "Void: This request needs the fuller Codex flow. Check `{handoffPath}` and `{debugPath}` in the local workspace.",
+    "Void: This one is leaving Discord. Continue from `{handoffPath}` and use `{debugPath}` for the trace."
+  ]
+}

docs/architecture/overview.md

@@ -0,0 +1,48 @@
+# VoidBot Architecture Overview
+
+This initial scaffold follows the phase-one shape from the spec:
+
+```text
+Discord Gateway
+  -> Command Router
+  -> Permission Engine
+  -> Context Builder
+  -> Job Queue
+  -> Provider Adapter
+  -> Storage Layer
+      - file-backed queue and audit log for local development
+      - PostgreSQL schema draft in packages/core/sql/bootstrap.sql
+      - persistent local archive and vector index for RAG
+```
+
+## Current state
+
+- `apps/bot` owns Discord connectivity, routing, and command handling.
+- `apps/worker` polls owner jobs, runs the local Discord-safe Codex lane, serves bounded history-search tool requests from the RAG index, and prepares handoff bundles when needed.
+- `packages/core` contains permissions, context assembly, job storage, audit logging, style loading, and the rotating system message catalog.
+- `packages/providers` isolates provider contracts from Discord code.
+- `packages/rag` contains the message archive, chunking pipeline, local embedding backends, pluggable vector-store backends, and log import state.
+- `packages/sandbox` defines policy profiles and a dry-run policy runner.
+
+## Deliberate shortcuts in this first pass
+
+- Job storage and audit events are file-backed under `.voidbot/` so the bot and worker can function immediately.
+- The relational schema is drafted but not yet wired into a real repository implementation.
+- Retrieval now uses a persistent vector backend with a local embedding backend by default (`qwen3-embedding:0.6b` through Ollama).
+- `VECTOR_STORE_KIND=local_json` keeps the zero-service path available for local development, but `VECTOR_STORE_KIND=qdrant` is now supported for a proper vector database.
+- Discord history vectors and source-tree/lore vectors stay in separate physical partitions so source indexing no longer rewrites the entire history corpus.
+- The bot live-ingests every message that matches the configured indexing rule and can also backfill from exported log files.
+- The owner Codex flow can perform repeated read-only history searches through a bounded tool loop instead of relying on a single retrieval snapshot.
+- Stock Discord replies are loaded from `config/system-messages.json` and rotated through shuffled variants to avoid repetitive canned responses.
+- Changing embedding models requires a full vector rebuild from the archived messages, which is handled by `npm run rag:rebuild`.
+- `owner_codex` has two lanes:
+  - `local_exec_owner_only` tries a read-only `codex exec` pass for direct Discord replies
+  - `manual_package` remains available when you want explicit approval-gated packaging
+- When the local Discord-safe lane is not enough, the worker posts a handoff notice and writes the full context bundle to `.voidbot/artifacts/<job-id>/`.
+
+## Upgrade path
+
+1. Replace the file-backed queue and audit log with PostgreSQL-backed repositories.
+2. Cut production deployments over to Qdrant while keeping separate collections for Discord history and source/lore corpora.
+3. Add moderation, budgeting, and rate limits before enabling `openai_api` for member traffic.
+4. Expand sandbox execution from dry-run policy checks to real constrained runners.

docs/decisions/0001-typescript-workspace.md

@@ -0,0 +1,23 @@
+# ADR 0001: TypeScript Workspace For The First Implementation
+
+## Status
+
+Accepted
+
+## Context
+
+The spec leaves the first implementation language open, but recommends a TypeScript bot and worker. The project needs a Discord-native surface, long-lived worker process, shared contracts, and a clear provider abstraction.
+
+## Decision
+
+Start with an npm workspace and TypeScript project references:
+
+- `apps/bot` handles Discord I/O
+- `apps/worker` processes queued jobs
+- `packages/*` hold shared contracts and internal services
+
+## Consequences
+
+- Shared type contracts stay explicit and compile-checked across the bot and worker.
+- We can keep the provider boundary clean while still moving quickly on Discord integration.
+- The repo is ready for PostgreSQL, persistent vector storage, and additional providers without changing the public Discord contract.

infra/qdrant/docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  qdrant:
+    image: qdrant/qdrant:latest
+    container_name: voidbot-qdrant
+    restart: unless-stopped
+    ports:
+      - "6333:6333"
+      - "6334:6334"
+    volumes:
+      - qdrant_storage:/qdrant/storage
+
+volumes:
+  qdrant_storage:
+    name: voidbot_qdrant_storage

package.json

@@ -0,0 +1,31 @@
+{
+  "name": "voidbot",
+  "version": "0.1.0",
+  "private": true,
+  "workspaces": [
+    "apps/*",
+    "packages/*"
+  ],
+  "scripts": {
+    "build": "tsc -b",
+    "typecheck": "tsc -b --pretty false",
+    "dev:bot": "tsx apps/bot/src/index.ts",
+    "dev:mcp": "tsx apps/worker/src/mcp-server.ts",
+    "dev:worker": "tsx apps/worker/src/index.ts",
+    "codex:mcp:install": "tsx scripts/install-codex-mcp.ts",
+    "codex:mcp:remove": "tsx scripts/remove-codex-mcp.ts",
+    "qdrant:setup": "powershell -ExecutionPolicy Bypass -File scripts/setup-docker-qdrant.ps1",
+    "qdrant:up": "powershell -ExecutionPolicy Bypass -File scripts/start-qdrant.ps1",
+    "rag:backfill": "tsx scripts/backfill-discord-logs.ts",
+    "rag:index-sources": "tsx scripts/index-source-repos.ts",
+    "rag:install-push-hooks": "tsx scripts/install-source-index-post-push-hooks.ts",
+    "rag:migrate-qdrant": "tsx scripts/migrate-local-vectors-to-qdrant.ts",
+    "rag:migrate-vector-layout": "tsx scripts/migrate-vector-store-layout.ts",
+    "rag:rebuild": "tsx scripts/rebuild-rag-index.ts"
+  },
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "tsx": "^4.20.0",
+    "typescript": "^5.9.0"
+  }
+}

packages/config/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@voidbot/config",
+  "version": "0.1.0",
+  "private": true,
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "@voidbot/shared": "0.1.0",
+    "zod": "^3.25.0"
+  }
+}

packages/config/src/index.ts

@@ -0,0 +1,322 @@
+import { resolve } from "node:path";
+
+import { z } from "zod";
+
+import {
+  type ChannelIndexingPolicy,
+  type OwnerCodexMode,
+  type ProviderName,
+  isOwnerCodexMode,
+  isProviderName,
+  normalizeDiscordChannelName,
+} from "@voidbot/shared";
+
+const optionalNonEmptyString = z.preprocess(
+  (value) => (typeof value === "string" && value.trim().length === 0 ? undefined : value),
+  z.string().min(1).optional(),
+);
+
+const booleanFromEnv = z.preprocess((value) => {
+  if (typeof value !== "string") {
+    return value;
+  }
+
+  const normalized = value.trim().toLowerCase();
+
+  if (["1", "true", "yes", "on"].includes(normalized)) {
+    return true;
+  }
+
+  if (["0", "false", "no", "off", ""].includes(normalized)) {
+    return false;
+  }
+
+  return value;
+}, z.boolean());
+
+type OllamaThinkMode = boolean | "low" | "medium" | "high";
+
+const envSchema = z.object({
+  NODE_ENV: z.enum(["development", "test", "production"]).default("development"),
+  DISCORD_BOT_TOKEN: optionalNonEmptyString,
+  DISCORD_APPLICATION_ID: optionalNonEmptyString,
+  DISCORD_GUILD_ID: optionalNonEmptyString,
+  DISCORD_OWNER_ID: z.string().min(1, "DISCORD_OWNER_ID is required"),
+  DATABASE_DSN: z.string().min(1).default("postgres://voidbot:voidbot@localhost:5432/voidbot"),
+  VECTOR_STORE_KIND: z.enum(["local_json", "qdrant"]).default("local_json"),
+  VECTOR_STORE_PATH: z.string().min(1).default(".voidbot/history-vector-store.json"),
+  SOURCE_VECTOR_STORE_ROOT: z.string().min(1).default(".voidbot/source-vectors"),
+  QDRANT_URL: z.string().url().default("http://127.0.0.1:6333"),
+  QDRANT_API_KEY: optionalNonEmptyString,
+  QDRANT_TIMEOUT_MS: z.coerce.number().int().positive().default(30000),
+  QDRANT_HISTORY_COLLECTION: z.string().min(1).default("voidbot_discord_history_chunks"),
+  QDRANT_SOURCE_COLLECTION: z.string().min(1).default("voidbot_repository_source_chunks"),
+  RAG_ARCHIVE_PATH: z.string().min(1).default(".voidbot/rag/messages.json"),
+  RAG_SOURCE_ARCHIVE_PATH: z.string().min(1).default(".voidbot/rag/source-documents.json"),
+  RAG_IMPORT_STATE_PATH: z.string().min(1).default(".voidbot/rag/import-state.json"),
+  RAG_EMBEDDING_BACKEND: z.enum(["hash", "ollama"]).default("ollama"),
+  RAG_EMBEDDING_DIMENSIONS: z.coerce.number().int().positive().default(256),
+  RAG_OLLAMA_BASE_URL: z.string().url().default("http://127.0.0.1:11434"),
+  RAG_OLLAMA_MODEL: z.string().min(1).default("qwen3-embedding:0.6b"),
+  RAG_OLLAMA_TIMEOUT_MS: z.coerce.number().int().positive().default(30000),
+  RAG_QUERY_INSTRUCTION: z
+    .string()
+    .default(
+      "Given a Discord history question, retrieve relevant messages and discussion snippets that answer it.",
+    ),
+  RAG_SOURCE_QUERY_INSTRUCTION: z
+    .string()
+    .default(
+      "Given a source-tree, codebase, or lore question, retrieve relevant files, code snippets, and lore passages that answer it.",
+    ),
+  DISCORD_LOG_ROOT: optionalNonEmptyString,
+  SOURCE_REPO_ROOT: optionalNonEmptyString,
+  SOURCE_REPO_PATTERNS: z.string().default("*"),
+  SOURCE_REPO_INCLUDE_PREFIXES: z.string().default("AetheriaLore:Aetheria/"),
+  LOCAL_LLM_OLLAMA_BASE_URL: z.string().url().default("http://127.0.0.1:11434"),
+  LOCAL_LLM_OLLAMA_MODEL: z.string().min(1).default("qwen3.5:9b"),
+  LOCAL_LLM_OLLAMA_TIMEOUT_MS: z.coerce.number().int().positive().default(120000),
+  LOCAL_LLM_OLLAMA_KEEP_ALIVE: z.string().min(1).default("10m"),
+  LOCAL_LLM_OLLAMA_THINK: z.string().default("false"),
+  LOCAL_LLM_OLLAMA_NUM_CTX: z.coerce.number().int().positive().default(8192),
+  LOCAL_LLM_ALLOW_PUBLIC: booleanFromEnv.default(false),
+  STORAGE_ROOT: z.string().min(1).default(".voidbot"),
+  ENABLED_PROVIDERS: z.string().min(1).default("owner_codex"),
+  OWNER_CODEX_MODE: z.string().min(1).default("local_exec_owner_only"),
+  CODEX_EXECUTABLE: z.string().min(1).default("codex"),
+  CODEX_EXEC_ARGS: z.string().default(""),
+  CODEX_MODEL_REASONING_EFFORT: z.enum(["low", "medium", "high", "xhigh"]).default("low"),
+  CODEX_EXEC_TIMEOUT_MS: z.coerce.number().int().positive().default(120000),
+  STYLE_PACK_PATH: z.string().min(1).default("styles/void-default.md"),
+  SYSTEM_MESSAGES_PATH: z.string().min(1).default("config/system-messages.json"),
+  INDEX_ALL_CHANNELS: booleanFromEnv.default(false),
+  INDEXED_CHANNEL_IDS: z.string().default(""),
+  EXCLUDED_CHANNEL_IDS: z.string().default(""),
+  EXCLUDED_CHANNEL_NAMES: z.string().default(""),
+  WORKER_POLL_INTERVAL_MS: z.coerce.number().int().positive().default(5000),
+});
+
+export interface AppConfig {
+  nodeEnv: "development" | "test" | "production";
+  botToken?: string;
+  applicationId?: string;
+  developmentGuildId?: string;
+  ownerDiscordId: string;
+  databaseDsn: string;
+  enabledProviders: ProviderName[];
+  ownerCodexMode: OwnerCodexMode;
+  codexExecutable: string;
+  codexExecArgs: string[];
+  codexModelReasoningEffort: "low" | "medium" | "high" | "xhigh";
+  codexExecTimeoutMs: number;
+  stylePackPath: string;
+  systemMessagesPath: string;
+  indexAllChannels: boolean;
+  indexedChannelIds: string[];
+  excludedChannelIds: string[];
+  excludedChannelNames: string[];
+  channelIndexing: ChannelIndexingPolicy;
+  workerPollIntervalMs: number;
+  storageRoot: string;
+  jobsFile: string;
+  auditLogFile: string;
+  artifactsDir: string;
+  ragArchivePath: string;
+  ragSourceArchivePath: string;
+  ragImportStatePath: string;
+  ragEmbeddingBackend: "hash" | "ollama";
+  ragEmbeddingDimensions: number;
+  ragOllamaBaseUrl: string;
+  ragOllamaModel: string;
+  ragOllamaTimeoutMs: number;
+  ragQueryInstruction: string;
+  ragSourceQueryInstruction: string;
+  discordLogRoot?: string;
+  sourceRepoRoot?: string;
+  sourceRepoPatterns: string[];
+  sourceRepoIncludePrefixes: Record<string, string[]>;
+  localLlm: {
+    ollamaBaseUrl: string;
+    ollamaModel: string;
+    ollamaTimeoutMs: number;
+    ollamaKeepAlive: string;
+    ollamaThink: OllamaThinkMode;
+    ollamaNumCtx: number;
+    allowPublic: boolean;
+  };
+  vectorStore: {
+    kind: "local_json" | "qdrant";
+    path: string;
+  };
+  sourceVectorStoreRoot: string;
+  qdrant: {
+    url: string;
+    apiKey?: string;
+    timeoutMs: number;
+    historyCollection: string;
+    sourceCollection: string;
+  };
+}
+
+function parseList(value: string): string[] {
+  return value
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0);
+}
+
+function parseChannelNameList(value: string): string[] {
+  return parseList(value)
+    .map(normalizeDiscordChannelName)
+    .filter((entry) => entry.length > 0);
+}
+
+function parseProviders(value: string): ProviderName[] {
+  const providers = parseList(value);
+  const invalid = providers.filter((provider) => !isProviderName(provider));
+
+  if (invalid.length > 0) {
+    throw new Error(`Unsupported providers in ENABLED_PROVIDERS: ${invalid.join(", ")}`);
+  }
+
+  return providers as ProviderName[];
+}
+
+function parseRepoPrefixRules(value: string): Record<string, string[]> {
+  const rules: Record<string, string[]> = {};
+
+  for (const entry of value.split(";")) {
+    const trimmed = entry.trim();
+
+    if (trimmed.length === 0) {
+      continue;
+    }
+
+    const separatorIndex = trimmed.indexOf(":");
+
+    if (separatorIndex === -1) {
+      continue;
+    }
+
+    const repoName = trimmed.slice(0, separatorIndex).trim();
+    const prefixes = trimmed
+      .slice(separatorIndex + 1)
+      .split("|")
+      .map((prefix) => prefix.trim().replace(/\\/g, "/").replace(/^\.\/+/, ""))
+      .filter((prefix) => prefix.length > 0)
+      .map((prefix) => (prefix.endsWith("/") ? prefix : `${prefix}/`));
+
+    if (repoName.length === 0 || prefixes.length === 0) {
+      continue;
+    }
+
+    rules[repoName] = prefixes;
+  }
+
+  return rules;
+}
+
+function parseOwnerCodexMode(value: string): OwnerCodexMode {
+  if (!isOwnerCodexMode(value)) {
+    throw new Error(
+      `Unsupported OWNER_CODEX_MODE "${value}". Expected one of: manual_package, local_exec_owner_only.`,
+    );
+  }
+
+  return value;
+}
+
+function parseOllamaThinkMode(value: string): OllamaThinkMode {
+  const normalized = value.trim().toLowerCase();
+
+  if (["true", "1", "yes", "on"].includes(normalized)) {
+    return true;
+  }
+
+  if (["false", "0", "no", "off"].includes(normalized)) {
+    return false;
+  }
+
+  if (normalized === "low" || normalized === "medium" || normalized === "high") {
+    return normalized;
+  }
+
+  throw new Error(
+    `Unsupported LOCAL_LLM_OLLAMA_THINK "${value}". Expected true, false, low, medium, or high.`,
+  );
+}
+
+export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig {
+  const parsed = envSchema.parse(env);
+  const storageRoot = resolve(parsed.STORAGE_ROOT);
+  const indexedChannelIds = parseList(parsed.INDEXED_CHANNEL_IDS);
+  const excludedChannelIds = parseList(parsed.EXCLUDED_CHANNEL_IDS);
+  const excludedChannelNames = parseChannelNameList(parsed.EXCLUDED_CHANNEL_NAMES);
+
+  return {
+    nodeEnv: parsed.NODE_ENV,
+    botToken: parsed.DISCORD_BOT_TOKEN,
+    applicationId: parsed.DISCORD_APPLICATION_ID,
+    developmentGuildId: parsed.DISCORD_GUILD_ID,
+    ownerDiscordId: parsed.DISCORD_OWNER_ID,
+    databaseDsn: parsed.DATABASE_DSN,
+    enabledProviders: parseProviders(parsed.ENABLED_PROVIDERS),
+    ownerCodexMode: parseOwnerCodexMode(parsed.OWNER_CODEX_MODE),
+    codexExecutable: parsed.CODEX_EXECUTABLE,
+    codexExecArgs: parseList(parsed.CODEX_EXEC_ARGS),
+    codexModelReasoningEffort: parsed.CODEX_MODEL_REASONING_EFFORT,
+    codexExecTimeoutMs: parsed.CODEX_EXEC_TIMEOUT_MS,
+    stylePackPath: resolve(parsed.STYLE_PACK_PATH),
+    systemMessagesPath: resolve(parsed.SYSTEM_MESSAGES_PATH),
+    indexAllChannels: parsed.INDEX_ALL_CHANNELS,
+    indexedChannelIds,
+    excludedChannelIds,
+    excludedChannelNames,
+    channelIndexing: {
+      indexAllChannels: parsed.INDEX_ALL_CHANNELS,
+      indexedChannelIds,
+      excludedChannelIds,
+      excludedChannelNames,
+    },
+    workerPollIntervalMs: parsed.WORKER_POLL_INTERVAL_MS,
+    storageRoot,
+    jobsFile: resolve(storageRoot, "jobs", "jobs.json"),
+    auditLogFile: resolve(storageRoot, "audit", "events.jsonl"),
+    artifactsDir: resolve(storageRoot, "artifacts"),
+    ragArchivePath: resolve(parsed.RAG_ARCHIVE_PATH),
+    ragSourceArchivePath: resolve(parsed.RAG_SOURCE_ARCHIVE_PATH),
+    ragImportStatePath: resolve(parsed.RAG_IMPORT_STATE_PATH),
+    ragEmbeddingBackend: parsed.RAG_EMBEDDING_BACKEND,
+    ragEmbeddingDimensions: parsed.RAG_EMBEDDING_DIMENSIONS,
+    ragOllamaBaseUrl: parsed.RAG_OLLAMA_BASE_URL,
+    ragOllamaModel: parsed.RAG_OLLAMA_MODEL,
+    ragOllamaTimeoutMs: parsed.RAG_OLLAMA_TIMEOUT_MS,
+    ragQueryInstruction: parsed.RAG_QUERY_INSTRUCTION,
+    ragSourceQueryInstruction: parsed.RAG_SOURCE_QUERY_INSTRUCTION,
+    discordLogRoot: parsed.DISCORD_LOG_ROOT ? resolve(parsed.DISCORD_LOG_ROOT) : undefined,
+    sourceRepoRoot: parsed.SOURCE_REPO_ROOT ? resolve(parsed.SOURCE_REPO_ROOT) : undefined,
+    sourceRepoPatterns: parseList(parsed.SOURCE_REPO_PATTERNS),
+    sourceRepoIncludePrefixes: parseRepoPrefixRules(parsed.SOURCE_REPO_INCLUDE_PREFIXES),
+    localLlm: {
+      ollamaBaseUrl: parsed.LOCAL_LLM_OLLAMA_BASE_URL,
+      ollamaModel: parsed.LOCAL_LLM_OLLAMA_MODEL,
+      ollamaTimeoutMs: parsed.LOCAL_LLM_OLLAMA_TIMEOUT_MS,
+      ollamaKeepAlive: parsed.LOCAL_LLM_OLLAMA_KEEP_ALIVE,
+      ollamaThink: parseOllamaThinkMode(parsed.LOCAL_LLM_OLLAMA_THINK),
+      ollamaNumCtx: parsed.LOCAL_LLM_OLLAMA_NUM_CTX,
+      allowPublic: parsed.LOCAL_LLM_ALLOW_PUBLIC,
+    },
+    vectorStore: {
+      kind: parsed.VECTOR_STORE_KIND,
+      path: resolve(parsed.VECTOR_STORE_PATH),
+    },
+    sourceVectorStoreRoot: resolve(parsed.SOURCE_VECTOR_STORE_ROOT),
+    qdrant: {
+      url: parsed.QDRANT_URL,
+      apiKey: parsed.QDRANT_API_KEY,
+      timeoutMs: parsed.QDRANT_TIMEOUT_MS,
+      historyCollection: parsed.QDRANT_HISTORY_COLLECTION,
+      sourceCollection: parsed.QDRANT_SOURCE_COLLECTION,
+    },
+  };
+}

packages/config/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "references": [
+    {
+      "path": "../shared"
+    }
+  ]
+}
+

packages/core/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@voidbot/core",
+  "version": "0.1.0",
+  "private": true,
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "@voidbot/shared": "0.1.0"
+  }
+}

packages/core/sql/bootstrap.sql

@@ -0,0 +1,102 @@
+create table if not exists guild_configs (
+  guild_id text primary key,
+  default_style_pack text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists channel_policies (
+  guild_id text not null,
+  channel_id text not null,
+  indexing_enabled boolean not null default false,
+  provider_allowlist text[] not null default '{}',
+  primary key (guild_id, channel_id)
+);
+
+create table if not exists provider_configs (
+  provider_name text primary key,
+  enabled boolean not null default false,
+  config_json jsonb not null default '{}'::jsonb,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists jobs (
+  id uuid primary key,
+  command_name text not null,
+  state text not null,
+  provider_name text not null,
+  requester_discord_id text not null,
+  guild_id text,
+  channel_id text not null,
+  prompt text not null,
+  context_json jsonb not null,
+  summary text,
+  final_response text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists job_approvals (
+  id bigserial primary key,
+  job_id uuid not null references jobs(id) on delete cascade,
+  stage text not null,
+  status text not null,
+  actor_discord_id text not null,
+  reason text,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists provider_runs (
+  id bigserial primary key,
+  job_id uuid not null references jobs(id) on delete cascade,
+  provider_name text not null,
+  request_json jsonb not null,
+  response_json jsonb,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists tool_invocations (
+  id bigserial primary key,
+  job_id uuid references jobs(id) on delete cascade,
+  sandbox_profile text not null,
+  tool_name text not null,
+  request_json jsonb not null,
+  result_json jsonb,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists audit_events (
+  id bigserial primary key,
+  job_id uuid references jobs(id) on delete set null,
+  actor_discord_id text,
+  event_type text not null,
+  event_json jsonb not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists message_archives (
+  message_id text primary key,
+  guild_id text,
+  channel_id text not null,
+  author_discord_id text not null,
+  content text not null,
+  metadata_json jsonb not null default '{}'::jsonb,
+  created_at timestamptz not null
+);
+
+create table if not exists embedding_chunks (
+  chunk_id text primary key,
+  message_id text not null references message_archives(message_id) on delete cascade,
+  text_content text not null,
+  metadata_json jsonb not null default '{}'::jsonb,
+  embedding vector(1536)
+);
+
+create table if not exists style_packs (
+  name text primary key,
+  file_path text not null,
+  enabled boolean not null default true,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);

packages/core/src/audit-log.ts

@@ -0,0 +1,30 @@
+import { appendFile, mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+import { randomUUID } from "node:crypto";
+
+import { type AuditEvent } from "@voidbot/shared";
+
+export type AuditEventInput = Omit<AuditEvent, "id" | "timestamp"> &
+  Partial<Pick<AuditEvent, "id" | "timestamp">>;
+
+export class FileAuditLog {
+  public constructor(private readonly auditFile: string) {}
+
+  public async record(input: AuditEventInput): Promise<AuditEvent> {
+    const event: AuditEvent = {
+      id: input.id ?? randomUUID(),
+      timestamp: input.timestamp ?? new Date().toISOString(),
+      type: input.type,
+      actorId: input.actorId,
+      jobId: input.jobId,
+      provider: input.provider,
+      details: input.details,
+    };
+
+    await mkdir(dirname(this.auditFile), { recursive: true });
+    await appendFile(this.auditFile, `${JSON.stringify(event)}\n`, "utf8");
+
+    return event;
+  }
+}
+

packages/core/src/codex-mcp.ts

@@ -0,0 +1,38 @@
+import { resolve } from "node:path";
+
+import { type CodexMcpServerConfig } from "@voidbot/shared";
+
+export const VOID_MCP_SERVER_NAME = "voidbot";
+
+export function buildVoidMcpServerConfig(workspaceRoot: string): CodexMcpServerConfig {
+  return {
+    name: VOID_MCP_SERVER_NAME,
+    command: process.execPath,
+    args: [resolve(workspaceRoot, "apps", "worker", "dist", "mcp-server.js")],
+    cwd: workspaceRoot,
+    env: {
+      VOIDBOT_WORKSPACE_ROOT: workspaceRoot,
+    },
+  };
+}
+
+export function buildCodexMcpConfigOverrides(
+  servers: CodexMcpServerConfig[],
+): string[] {
+  const overrides: string[] = [];
+
+  for (const server of servers) {
+    overrides.push(`mcp_servers.${server.name}.command=${JSON.stringify(server.command)}`);
+    overrides.push(`mcp_servers.${server.name}.args=${JSON.stringify(server.args)}`);
+
+    if (server.cwd) {
+      overrides.push(`mcp_servers.${server.name}.cwd=${JSON.stringify(server.cwd)}`);
+    }
+
+    for (const [key, value] of Object.entries(server.env ?? {})) {
+      overrides.push(`mcp_servers.${server.name}.env.${key}=${JSON.stringify(value)}`);
+    }
+  }
+
+  return overrides;
+}

packages/core/src/context-builder.ts

@@ -0,0 +1,32 @@
+import {
+  type Actor,
+  type ContextBundle,
+  type GuildContext,
+  type RetrievalResult,
+  type SourceMessage,
+  type StylePack,
+} from "@voidbot/shared";
+
+export interface BuildContextInput {
+  prompt: string;
+  actor: Actor;
+  guildContext: GuildContext;
+  recentMessages?: SourceMessage[];
+  retrieval?: RetrievalResult[];
+  stylePack?: StylePack;
+}
+
+export class ContextBuilder {
+  public build(input: BuildContextInput): ContextBundle {
+    return {
+      prompt: input.prompt,
+      actor: input.actor,
+      guildContext: input.guildContext,
+      recentMessages: input.recentMessages ?? [],
+      retrieval: input.retrieval ?? [],
+      stylePack: input.stylePack,
+      createdAt: new Date().toISOString(),
+    };
+  }
+}
+

packages/core/src/index.ts

@@ -0,0 +1,7 @@
+export * from "./audit-log";
+export * from "./codex-mcp";
+export * from "./context-builder";
+export * from "./job-queue";
+export * from "./permission-engine";
+export * from "./style-pack-loader";
+export * from "./system-message-catalog";

packages/core/src/job-queue.ts

@@ -0,0 +1,343 @@
+import { mkdir, open, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { randomUUID } from "node:crypto";
+
+import {
+  type Actor,
+  type CommandName,
+  type ContextBundle,
+  type GuildContext,
+  type JobRecord,
+  type JobState,
+  type ProviderName,
+} from "@voidbot/shared";
+
+interface JobStore {
+  jobs: JobRecord[];
+}
+
+export interface CreateJobResult {
+  job: JobRecord;
+  created: boolean;
+}
+
+export interface CreateJobInput {
+  command: CommandName;
+  provider: ProviderName;
+  runApprovalRequired?: boolean;
+  postApprovalRequired?: boolean;
+  requester: Actor;
+  guildContext: GuildContext;
+  prompt: string;
+  contextBundle: ContextBundle;
+  outputChannelId: string;
+  requestMessageId?: string;
+  initialState?: JobState;
+}
+
+export class FileBackedJobQueue {
+  public constructor(private readonly jobsFile: string) {}
+
+  public async createJob(input: CreateJobInput): Promise<CreateJobResult> {
+    const now = new Date().toISOString();
+    const job: JobRecord = {
+      id: randomUUID(),
+      command: input.command,
+      state: input.initialState ?? "awaiting_approval",
+      provider: input.provider,
+      runApprovalRequired: input.runApprovalRequired ?? true,
+      postApprovalRequired: input.postApprovalRequired ?? true,
+      requester: input.requester,
+      guildContext: input.guildContext,
+      prompt: input.prompt,
+      contextBundle: input.contextBundle,
+      createdAt: now,
+      updatedAt: now,
+      outputChannelId: input.outputChannelId,
+      requestMessageId: input.requestMessageId,
+      approvals: [],
+    };
+
+    return this.mutate((jobs) => {
+      const existing =
+        input.requestMessageId === undefined
+          ? undefined
+          : jobs.find(
+              (candidate) =>
+                candidate.requestMessageId === input.requestMessageId &&
+                candidate.command === input.command &&
+                candidate.provider === input.provider &&
+                candidate.outputChannelId === input.outputChannelId,
+            );
+
+      if (existing) {
+        return {
+          job: cloneJob(existing),
+          created: false,
+        };
+      }
+
+      jobs.push(job);
+      return {
+        job: cloneJob(job),
+        created: true,
+      };
+    });
+  }
+
+  public async getJob(jobId: string): Promise<JobRecord | undefined> {
+    const store = await this.readStore();
+    return store.jobs.find((job) => job.id === jobId);
+  }
+
+  public async listByStates(states: JobState[]): Promise<JobRecord[]> {
+    const store = await this.readStore();
+    return store.jobs.filter((job) => states.includes(job.state));
+  }
+
+  public async listRunnableJobs(): Promise<JobRecord[]> {
+    return this.listByStates(["approved"]);
+  }
+
+  public async claimRunnableJobs(limit?: number): Promise<JobRecord[]> {
+    return this.mutate((jobs) => {
+      const now = new Date().toISOString();
+      const claimed: JobRecord[] = [];
+
+      for (const job of jobs) {
+        if (job.state !== "approved") {
+          continue;
+        }
+
+        job.state = "running";
+        job.updatedAt = now;
+        claimed.push(cloneJob(job));
+
+        if (limit !== undefined && claimed.length >= limit) {
+          break;
+        }
+      }
+
+      return claimed;
+    });
+  }
+
+  public async approveRun(jobId: string, actorId: string): Promise<JobRecord> {
+    return this.updateJob(jobId, (job) => {
+      if (job.state !== "awaiting_approval") {
+        throw new Error(`Job ${jobId} is not awaiting run approval.`);
+      }
+
+      job.state = "approved";
+      job.updatedAt = new Date().toISOString();
+      job.approvals.push({
+        stage: "run",
+        status: "approved",
+        actorId,
+        createdAt: job.updatedAt,
+      });
+    });
+  }
+
+  public async rejectJob(jobId: string, actorId: string, reason?: string): Promise<JobRecord> {
+    return this.updateJob(jobId, (job) => {
+      const stage = job.state === "awaiting_post_approval" ? "post" : "run";
+      job.state = "cancelled";
+      job.updatedAt = new Date().toISOString();
+      job.approvals.push({
+        stage,
+        status: "rejected",
+        actorId,
+        reason,
+        createdAt: job.updatedAt,
+      });
+    });
+  }
+
+  public async markRunning(jobId: string): Promise<JobRecord> {
+    return this.updateJob(jobId, (job) => {
+      if (job.state !== "approved") {
+        throw new Error(`Job ${jobId} is not ready to run.`);
+      }
+
+      job.state = "running";
+      job.updatedAt = new Date().toISOString();
+    });
+  }
+
+  public async markAwaitingPostApproval(
+    jobId: string,
+    artifactPaths: Record<string, string>,
+    summary: string,
+  ): Promise<JobRecord> {
+    return this.updateJob(jobId, (job) => {
+      if (job.state !== "running") {
+        throw new Error(`Job ${jobId} is not currently running.`);
+      }
+
+      if (!job.postApprovalRequired) {
+        throw new Error(`Job ${jobId} does not require post approval.`);
+      }
+
+      job.state = "awaiting_post_approval";
+      job.manualArtifacts = artifactPaths;
+      job.summary = summary;
+      job.updatedAt = new Date().toISOString();
+    });
+  }
+
+  public async markFailed(jobId: string, error: string): Promise<JobRecord> {
+    return this.updateJob(jobId, (job) => {
+      job.state = "failed";
+      job.error = error;
+      job.updatedAt = new Date().toISOString();
+    });
+  }
+
+  public async completeJob(
+    jobId: string,
+    actorId: string,
+    finalResponse: string,
+  ): Promise<JobRecord> {
+    return this.updateJob(jobId, (job) => {
+      if (job.state !== "awaiting_post_approval") {
+        throw new Error(`Job ${jobId} is not awaiting post approval.`);
+      }
+
+      job.state = "completed";
+      job.finalResponse = finalResponse;
+      job.updatedAt = new Date().toISOString();
+      job.approvals.push({
+        stage: "post",
+        status: "approved",
+        actorId,
+        createdAt: job.updatedAt,
+      });
+    });
+  }
+
+  public async completeJobDirect(jobId: string, finalResponse: string): Promise<JobRecord> {
+    return this.updateJob(jobId, (job) => {
+      if (job.state !== "running") {
+        throw new Error(`Job ${jobId} is not currently running.`);
+      }
+
+      job.state = "completed";
+      job.finalResponse = finalResponse;
+      job.updatedAt = new Date().toISOString();
+    });
+  }
+
+  private async updateJob(
+    jobId: string,
+    updater: (job: JobRecord) => void,
+  ): Promise<JobRecord> {
+    return this.mutate((jobs) => {
+      const job = jobs.find((candidate) => candidate.id === jobId);
+
+      if (!job) {
+        throw new Error(`Job ${jobId} was not found.`);
+      }
+
+      updater(job);
+      return job;
+    });
+  }
+
+  private async mutate<T>(updater: (jobs: JobRecord[]) => T): Promise<T> {
+    return this.withStoreLock(async () => {
+      const store = await this.readStore();
+      const result = updater(store.jobs);
+      await this.writeStore(store);
+      return result;
+    });
+  }
+
+  private async readStore(): Promise<JobStore> {
+    await mkdir(dirname(this.jobsFile), { recursive: true });
+
+    try {
+      const raw = await readFile(this.jobsFile, "utf8");
+      return JSON.parse(raw) as JobStore;
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException).code;
+
+      if (code === "ENOENT") {
+        return { jobs: [] };
+      }
+
+      throw error;
+    }
+  }
+
+  private async writeStore(store: JobStore): Promise<void> {
+    await mkdir(dirname(this.jobsFile), { recursive: true });
+    await writeFile(this.jobsFile, `${JSON.stringify(store, null, 2)}\n`, "utf8");
+  }
+
+  private async withStoreLock<T>(operation: () => Promise<T>): Promise<T> {
+    const lockFile = `${this.jobsFile}.lock`;
+    const deadline = Date.now() + 5000;
+
+    while (true) {
+      try {
+        const handle = await open(lockFile, "wx");
+
+        try {
+          await handle.writeFile(
+            JSON.stringify({
+              pid: process.pid,
+              acquiredAt: new Date().toISOString(),
+            }),
+            "utf8",
+          );
+          return await operation();
+        } finally {
+          await handle.close().catch(() => undefined);
+          await rm(lockFile, { force: true }).catch(() => undefined);
+        }
+      } catch (error) {
+        const code = (error as NodeJS.ErrnoException).code;
+
+        if (code !== "EEXIST") {
+          throw error;
+        }
+
+        await this.clearStaleLock(lockFile);
+
+        if (Date.now() >= deadline) {
+          throw new Error(`Timed out waiting for the job queue lock at ${lockFile}.`);
+        }
+
+        await sleep(50);
+      }
+    }
+  }
+
+  private async clearStaleLock(lockFile: string): Promise<void> {
+    try {
+      const details = await stat(lockFile);
+      const ageMs = Date.now() - details.mtimeMs;
+
+      if (ageMs > 30000) {
+        await rm(lockFile, { force: true });
+      }
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException).code;
+
+      if (code !== "ENOENT") {
+        throw error;
+      }
+    }
+  }
+}
+
+function cloneJob(job: JobRecord): JobRecord {
+  return JSON.parse(JSON.stringify(job)) as JobRecord;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}

packages/core/src/permission-engine.ts

@@ -0,0 +1,86 @@
+import {
+  type Actor,
+  type JobRecord,
+  type ProviderName,
+  type TrustTier,
+} from "@voidbot/shared";
+
+export interface ProviderAccessDecision {
+  allowed: boolean;
+  tier: TrustTier;
+  reasonKey?: string;
+}
+
+export class PermissionEngine {
+  public constructor(
+    private readonly ownerDiscordId: string,
+    private readonly options: {
+      localLlmAllowPublic?: boolean;
+    } = {},
+  ) {}
+
+  public resolveTrustTier(actor: Actor): TrustTier {
+    if (actor.id === this.ownerDiscordId) {
+      return "T2";
+    }
+
+    if (actor.isAdmin) {
+      return "T1";
+    }
+
+    return "T0";
+  }
+
+  public canUseProvider(actor: Actor, provider: ProviderName): ProviderAccessDecision {
+    const tier = this.resolveTrustTier(actor);
+
+    if (provider === "owner_codex") {
+      return actor.id === this.ownerDiscordId
+        ? { allowed: true, tier }
+        : {
+            allowed: false,
+            tier,
+            reasonKey: "permission.owner_codex.denied",
+          };
+    }
+
+    if (provider === "openai_api") {
+      return {
+        allowed: tier === "T0" || tier === "T1" || tier === "T2",
+        tier,
+      };
+    }
+
+    if (provider === "local_llm") {
+      return this.options.localLlmAllowPublic || tier === "T1" || tier === "T2"
+        ? {
+            allowed: true,
+            tier,
+          }
+        : {
+            allowed: false,
+            tier,
+            reasonKey: "permission.local_llm.denied",
+          };
+    }
+
+    return {
+      allowed: true,
+      tier,
+    };
+  }
+
+  public canReindex(actor: Actor): boolean {
+    const tier = this.resolveTrustTier(actor);
+    return tier === "T1" || tier === "T2";
+  }
+
+  public canManageConfiguration(actor: Actor): boolean {
+    const tier = this.resolveTrustTier(actor);
+    return tier === "T1" || tier === "T2";
+  }
+
+  public canApproveJob(actor: Actor, job: JobRecord): boolean {
+    return actor.id === this.ownerDiscordId && job.requester.id === this.ownerDiscordId;
+  }
+}

packages/core/src/style-pack-loader.ts

@@ -0,0 +1,28 @@
+import { readFile } from "node:fs/promises";
+import { basename, extname } from "node:path";
+
+import { type StylePack } from "@voidbot/shared";
+
+export async function loadStylePack(
+  stylePackPath: string,
+  enabled = true,
+): Promise<StylePack | undefined> {
+  try {
+    const instructions = await readFile(stylePackPath, "utf8");
+
+    return {
+      name: basename(stylePackPath, extname(stylePackPath)),
+      instructions: instructions.trim(),
+      enabled,
+    };
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code;
+
+    if (code === "ENOENT") {
+      return undefined;
+    }
+
+    throw error;
+  }
+}
+

packages/core/src/system-message-catalog.ts

@@ -0,0 +1,134 @@
+import { readFile } from "node:fs/promises";
+
+export type SystemMessageVariables = Record<
+  string,
+  string | number | boolean | null | undefined
+>;
+
+type SystemMessageCatalogSource = Record<string, string | string[]>;
+
+interface ShuffleBagState {
+  bag: string[];
+  last?: string;
+}
+
+export class SystemMessageCatalog {
+  private readonly shuffleState = new Map<string, ShuffleBagState>();
+
+  public constructor(private readonly messages: SystemMessageCatalogSource) {}
+
+  public render(
+    key: string,
+    variables: SystemMessageVariables = {},
+  ): string {
+    const variants = normalizeVariants(this.messages[key]);
+
+    if (variants.length === 0) {
+      return `Void: [missing system message: ${key}]`;
+    }
+
+    const state = this.shuffleState.get(key) ?? { bag: [] };
+
+    if (state.bag.length === 0) {
+      state.bag = shuffleVariants(variants, state.last);
+    }
+
+    const template = state.bag.shift();
+
+    if (!template) {
+      return `Void: [missing system message: ${key}]`;
+    }
+
+    state.last = template;
+    this.shuffleState.set(key, state);
+
+    return interpolateTemplate(template, variables);
+  }
+}
+
+export async function loadSystemMessageCatalog(
+  catalogPath: string,
+): Promise<SystemMessageCatalog> {
+  const source = await readFile(catalogPath, "utf8");
+  const parsed = JSON.parse(source) as unknown;
+
+  if (!isSystemMessageCatalogSource(parsed)) {
+    throw new Error(
+      `System message catalog at ${catalogPath} must be a JSON object whose values are strings or arrays of strings.`,
+    );
+  }
+
+  const invalidKeys = Object.entries(parsed)
+    .filter(([, value]) => normalizeVariants(value).length === 0)
+    .map(([key]) => key);
+
+  if (invalidKeys.length > 0) {
+    throw new Error(
+      `System message catalog at ${catalogPath} has empty entries for: ${invalidKeys.join(", ")}`,
+    );
+  }
+
+  return new SystemMessageCatalog(parsed);
+}
+
+function isSystemMessageCatalogSource(
+  value: unknown,
+): value is SystemMessageCatalogSource {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return false;
+  }
+
+  return Object.values(value).every(
+    (entry) =>
+      typeof entry === "string" ||
+      (Array.isArray(entry) && entry.every((item) => typeof item === "string")),
+  );
+}
+
+function normalizeVariants(value: string | string[] | undefined): string[] {
+  if (typeof value === "string") {
+    return value.trim().length > 0 ? [value.trim()] : [];
+  }
+
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  return value
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0);
+}
+
+function shuffleVariants(variants: string[], lastUsed?: string): string[] {
+  const shuffled = [...variants];
+
+  for (let index = shuffled.length - 1; index > 0; index -= 1) {
+    const swapIndex = Math.floor(Math.random() * (index + 1));
+    [shuffled[index], shuffled[swapIndex]] = [shuffled[swapIndex], shuffled[index]];
+  }
+
+  if (shuffled.length > 1 && lastUsed && shuffled[0] === lastUsed) {
+    const alternateIndex = shuffled.findIndex((variant) => variant !== lastUsed);
+
+    if (alternateIndex > 0) {
+      [shuffled[0], shuffled[alternateIndex]] = [shuffled[alternateIndex], shuffled[0]];
+    }
+  }
+
+  return shuffled;
+}
+
+function interpolateTemplate(
+  template: string,
+  variables: SystemMessageVariables,
+): string {
+  return template.replace(/\{([a-zA-Z0-9_]+)\}/g, (_match, name: string) => {
+    const value = variables[name];
+
+    if (value === undefined || value === null) {
+      return `{${name}}`;
+    }
+
+    return String(value);
+  });
+}

packages/core/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "references": [
+    {
+      "path": "../shared"
+    }
+  ]
+}
+

packages/providers/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@voidbot/providers",
+  "version": "0.1.0",
+  "private": true,
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "@voidbot/shared": "0.1.0"
+  }
+}

packages/providers/src/index.ts

@@ -0,0 +1,5 @@
+export * from "./local-llm-provider";
+export * from "./openai-api-provider";
+export * from "./owner-codex-provider";
+export * from "./provider-registry";
+

packages/providers/src/local-llm-provider.ts

@@ -0,0 +1,712 @@
+import {
+  type Actor,
+  type ContextBundle,
+  DEFAULT_RETRIEVAL_RESULT_LIMIT,
+  type GuildContext,
+  MAX_RETRIEVAL_RESULT_LIMIT,
+  type ProviderArtifact,
+  type ProviderAdapter,
+  type ProviderRequest,
+  type ProviderResponse,
+  type RetrievalResult,
+} from "@voidbot/shared";
+
+type OllamaThinkMode = boolean | "low" | "medium" | "high";
+const MAX_TOOL_TURNS = 4;
+const MAX_TOOL_RESULTS = MAX_RETRIEVAL_RESULT_LIMIT;
+const MAX_CONTEXT_WINDOW = 20;
+
+interface OllamaToolCallDefinition {
+  type: "function";
+  function: {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+  };
+}
+
+interface OllamaChatMessage {
+  role: "system" | "user" | "assistant" | "tool";
+  content: string;
+  tool_calls?: OllamaToolCall[];
+  tool_name?: string;
+  thinking?: string;
+}
+
+interface OllamaToolCall {
+  function?: {
+    name?: unknown;
+    arguments?: unknown;
+  };
+}
+
+interface OllamaChatResponse {
+  message?: {
+    content?: unknown;
+    thinking?: unknown;
+    tool_calls?: unknown;
+  };
+  thinking?: unknown;
+  done_reason?: unknown;
+  eval_count?: unknown;
+  prompt_eval_count?: unknown;
+}
+
+type IndexedRepoToolResult = Record<string, unknown> & {
+  repoCount: number;
+  repos: Array<{
+    repoName: string;
+    documentCount: number;
+  }>;
+};
+
+export interface LocalLlmToolbox {
+  listIndexedRepos(): Promise<IndexedRepoToolResult>;
+  searchHistory(input: {
+    query: string;
+    limit: number;
+    guildId?: string;
+    channelId?: string;
+    authorId?: string;
+  }): Promise<{
+    query: string;
+    resultCount: number;
+    results: Array<Record<string, unknown>>;
+  }>;
+  getMessageContext(input: {
+    messageId: string;
+    before: number;
+    after: number;
+  }): Promise<{
+    found: boolean;
+    messageId: string;
+    count: number;
+    messages: Array<Record<string, unknown>>;
+  }>;
+  searchSources(input: {
+    query: string;
+    limit: number;
+    repoName?: string;
+    pathPrefix?: string;
+    language?: string;
+  }): Promise<{
+    query: string;
+    resultCount: number;
+    results: Array<Record<string, unknown>>;
+  }>;
+  getSourceContext(input: {
+    sourceId: string;
+    chunkIndex: number;
+    before: number;
+    after: number;
+  }): Promise<{
+    found: boolean;
+    sourceId: string;
+    count: number;
+    repoName?: string;
+    path?: string;
+    language?: string;
+    chunks: Array<Record<string, unknown>>;
+  }>;
+}
+
+export interface LocalLlmProviderOptions {
+  enabled: boolean;
+  ownerDiscordId: string;
+  ollamaBaseUrl: string;
+  ollamaModel: string;
+  ollamaTimeoutMs: number;
+  ollamaKeepAlive: string;
+  ollamaThink: OllamaThinkMode;
+  ollamaNumCtx: number;
+  allowPublicResponses: boolean;
+  toolbox?: LocalLlmToolbox;
+}
+
+export class LocalLlmProvider implements ProviderAdapter {
+  public constructor(private readonly options: LocalLlmProviderOptions) {}
+
+  public getName(): "local_llm" {
+    return "local_llm";
+  }
+
+  public getCapabilities(): string[] {
+    return [
+      "local_generation",
+      "retrieval_augmented_prompt",
+      "tool_driven_retrieval",
+      "no_privileged_tools",
+    ];
+  }
+
+  public isEnabled(): boolean {
+    return this.options.enabled;
+  }
+
+  public isAllowedForActor(actor: Actor, _guildContext: GuildContext): boolean {
+    if (!this.options.enabled) {
+      return false;
+    }
+
+    if (this.options.allowPublicResponses) {
+      return true;
+    }
+
+    return actor.id === this.options.ownerDiscordId || actor.isAdmin;
+  }
+
+  public buildRequest(
+    contextBundle: ContextBundle,
+    options?: Record<string, unknown>,
+  ): ProviderRequest {
+    return {
+      provider: "local_llm",
+      contextBundle,
+      options,
+    };
+  }
+
+  public async execute(request: ProviderRequest): Promise<ProviderResponse> {
+    const systemPrompt = buildSystemPrompt(request.contextBundle);
+    const messages: OllamaChatMessage[] = [
+      {
+        role: "system",
+        content: systemPrompt,
+      },
+      {
+        role: "user",
+        content: buildPrompt(request.contextBundle),
+      },
+    ];
+    const controller = new AbortController();
+    const timeoutHandle = setTimeout(() => controller.abort(), this.options.ollamaTimeoutMs);
+    const tools = this.options.toolbox ? buildToolDefinitions() : undefined;
+    const toolTrace: ToolTraceRecord[] = [];
+    const thinkingTrace: string[] = [];
+
+    try {
+      let finalPayload: OllamaChatResponse | undefined;
+      let finalOutputText: string | undefined;
+
+      for (let turn = 0; turn <= MAX_TOOL_TURNS; turn += 1) {
+        const requestBody = {
+          model: this.options.ollamaModel,
+          messages,
+          tools,
+          stream: false,
+          think: this.options.ollamaThink,
+          keep_alive: this.options.ollamaKeepAlive,
+          options: {
+            num_ctx: this.options.ollamaNumCtx,
+          },
+        };
+        const response = await fetch(
+          `${normalizeBaseUrl(this.options.ollamaBaseUrl)}/api/chat`,
+          {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+            },
+            body: JSON.stringify(requestBody),
+            signal: controller.signal,
+          },
+        );
+
+        if (!response.ok) {
+          throw new Error(
+            `Ollama chat request failed with ${response.status}: ${await response.text()}`,
+          );
+        }
+
+        const payload = (await response.json()) as OllamaChatResponse;
+        finalPayload = payload;
+        const assistantMessage = normalizeAssistantMessage(payload);
+        const toolCalls = assistantMessage.tool_calls ?? [];
+
+        if (assistantMessage.thinking) {
+          thinkingTrace.push(assistantMessage.thinking);
+        }
+
+        messages.push(assistantMessage);
+
+        if (toolCalls.length === 0) {
+          finalOutputText = normalizeModelText(assistantMessage.content);
+          break;
+        }
+
+        if (!this.options.toolbox) {
+          finalOutputText =
+            "Void: The local model asked for tools, but this lane was not given any. That is a wiring problem, not a cosmic mystery.";
+          break;
+        }
+
+        if (turn >= MAX_TOOL_TURNS) {
+          finalOutputText =
+            "Void: The local model kept reaching for more archive context than I am willing to chase in one pass. Ask more narrowly or use the deeper Codex lane.";
+          break;
+        }
+
+        for (const toolCall of toolCalls) {
+          const execution = await executeToolCall(
+            this.options.toolbox,
+            request.contextBundle,
+            toolCall,
+          );
+          toolTrace.push(execution.trace);
+          messages.push({
+            role: "tool",
+            tool_name: execution.trace.tool,
+            content: JSON.stringify(execution.payload, null, 2),
+          });
+        }
+      }
+
+      if (!finalOutputText) {
+        throw new Error(
+          `Ollama returned no final response content for model "${this.options.ollamaModel}".`,
+        );
+      }
+
+      const artifacts = buildArtifacts(
+        {
+          model: this.options.ollamaModel,
+          messages,
+          tools,
+          stream: false,
+          think: this.options.ollamaThink,
+          keep_alive: this.options.ollamaKeepAlive,
+          options: {
+            num_ctx: this.options.ollamaNumCtx,
+          },
+        },
+        finalPayload ?? {},
+        thinkingTrace,
+        toolTrace,
+      );
+
+      return {
+        status: "completed",
+        summary: "Local Ollama response generated.",
+        outputText: finalOutputText,
+        artifacts,
+        metadata: {
+          model: this.options.ollamaModel,
+          retrieval_snippets: String(request.contextBundle.retrieval.length),
+          thinking_trace: thinkingTrace.length > 0 ? "present" : "absent",
+          tool_calls: String(toolTrace.length),
+          prompt_eval_count: readNumericMetadata(finalPayload?.prompt_eval_count),
+          eval_count: readNumericMetadata(finalPayload?.eval_count),
+          done_reason: normalizeModelText(finalPayload?.done_reason) ?? "unknown",
+        },
+      };
+    } catch (error) {
+      if ((error as Error).name === "AbortError") {
+        throw new Error(
+          `Timed out waiting for local Ollama model "${this.options.ollamaModel}" at ${this.options.ollamaBaseUrl}.`,
+        );
+      }
+
+      throw error;
+    } finally {
+      clearTimeout(timeoutHandle);
+    }
+  }
+
+  public async estimateCost(_request: ProviderRequest): Promise<number> {
+    return 0;
+  }
+
+  public getAuditRedactions(): string[] {
+    return [];
+  }
+}
+
+function buildSystemPrompt(context: ContextBundle): string {
+  const styleInstructions =
+    context.stylePack?.enabled && context.stylePack.instructions.trim().length > 0
+      ? context.stylePack.instructions.trim()
+      : "No extra persona instructions were supplied.";
+
+  return [
+    "You are Void, a Discord-native assistant for GameCult.",
+    "Stay honest about what context was actually provided to you.",
+    "Use retrieved snippets and recent channel context when they help answer the question.",
+    "When the answer depends on archived Discord history or indexed repo/lore context, use the available read-only tools instead of guessing.",
+    "If you need to target a specific indexed repo and do not know the valid repo names yet, call list_indexed_repos before search_sources.",
+    "Do not claim to have performed searches or tool calls beyond the material actually executed in this run.",
+    "If the supplied context looks incomplete, say so plainly instead of bluffing.",
+    "Keep the final answer concise and readable in Discord.",
+    "Do not reveal chain-of-thought. Return only the final answer.",
+    "",
+    "Style instructions:",
+    styleInstructions,
+  ].join("\n");
+}
+
+function buildPrompt(context: ContextBundle): string {
+  const recentMessages = context.recentMessages.length
+    ? context.recentMessages
+        .map(
+          (message) =>
+            `- [${message.timestamp}] ${message.authorName}: ${message.content || "(no text content)"}`,
+        )
+        .join("\n")
+    : "- No recent channel messages were attached.";
+  const retrievedContext = context.retrieval.length
+    ? context.retrieval
+        .map(
+          (result) =>
+            `- score=${result.score.toFixed(2)} source=${result.sourceId} kind=${result.sourceKind} text=${result.text}`,
+        )
+        .join("\n")
+    : "- No archived retrieval snippets were attached.";
+
+  return [
+    `Question: ${context.prompt}`,
+    "",
+    "Guild context:",
+    `- Guild: ${context.guildContext.guildName ?? context.guildContext.guildId ?? "(direct/unknown)"}`,
+    `- Channel: ${context.guildContext.channelName ?? context.guildContext.channelId}`,
+    "",
+    "Recent channel messages:",
+    recentMessages,
+    "",
+    "Retrieved archive context:",
+    retrievedContext,
+    "",
+    "If you need more archived history or source context than is included above, call the appropriate read-only tool before answering.",
+  ].join("\n");
+}
+
+function buildArtifacts(
+  requestBody: Record<string, unknown>,
+  payload: OllamaChatResponse,
+  thinkingTrace: string[],
+  toolTrace: ToolTraceRecord[],
+): ProviderArtifact[] {
+  const artifacts: ProviderArtifact[] = [
+    {
+      name: "local-llm-request.json",
+      contentType: "json",
+      content: `${JSON.stringify(requestBody, null, 2)}\n`,
+    },
+    {
+      name: "local-llm-response.json",
+      contentType: "json",
+      content: `${JSON.stringify(payload, null, 2)}\n`,
+    },
+    {
+      name: "local-llm-tool-trace.json",
+      contentType: "json",
+      content: `${JSON.stringify(toolTrace, null, 2)}\n`,
+    },
+  ];
+
+  if (thinkingTrace.length > 0) {
+    artifacts.push({
+      name: "local-llm-thinking.txt",
+      contentType: "text",
+      content: `${thinkingTrace.join("\n\n---\n\n")}\n`,
+    });
+  }
+
+  return artifacts;
+}
+
+function normalizeBaseUrl(value: string): string {
+  return value.replace(/\/+$/, "");
+}
+
+function normalizeModelText(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+}
+
+function readNumericMetadata(value: unknown): string {
+  return typeof value === "number" && Number.isFinite(value) ? String(value) : "0";
+}
+
+interface ToolTraceRecord {
+  tool: string;
+  arguments: Record<string, unknown>;
+  result: Record<string, unknown>;
+}
+
+function buildToolDefinitions(): OllamaToolCallDefinition[] {
+  return [
+    {
+      type: "function",
+      function: {
+        name: "search_history",
+        description:
+          "Search archived Discord history for prior conversations, decisions, preferences, and discussion fragments.",
+        parameters: {
+          type: "object",
+          required: ["query"],
+          properties: {
+            query: {
+              type: "string",
+              description: "What to search for in the Discord history archive.",
+            },
+            limit: {
+              type: "integer",
+              description: `Maximum number of results to return, between 1 and ${MAX_TOOL_RESULTS}.`,
+            },
+            channelId: {
+              type: "string",
+              description: "Optional Discord channel ID filter.",
+            },
+            authorId: {
+              type: "string",
+              description: "Optional Discord author ID filter.",
+            },
+          },
+        },
+      },
+    },
+    {
+      type: "function",
+      function: {
+        name: "get_message_context",
+        description:
+          "Fetch the surrounding conversation window for a specific archived Discord message.",
+        parameters: {
+          type: "object",
+          required: ["messageId"],
+          properties: {
+            messageId: {
+              type: "string",
+              description: "The archived Discord message ID.",
+            },
+            before: {
+              type: "integer",
+              description: "How many earlier messages to include, between 0 and 20.",
+            },
+            after: {
+              type: "integer",
+              description: "How many later messages to include, between 0 and 20.",
+            },
+          },
+        },
+      },
+    },
+    {
+      type: "function",
+      function: {
+        name: "search_sources",
+        description:
+          "Search indexed GameCult source trees and lore repositories for code, documentation, and lore passages. Omit repoName to search across all indexed repos, or call list_indexed_repos first if you need valid repo names.",
+        parameters: {
+          type: "object",
+          required: ["query"],
+          properties: {
+            query: {
+              type: "string",
+              description: "What to search for in indexed source and lore documents.",
+            },
+            limit: {
+              type: "integer",
+              description: `Maximum number of results to return, between 1 and ${MAX_TOOL_RESULTS}.`,
+            },
+            repoName: {
+              type: "string",
+              description: "Optional repository name filter.",
+            },
+            pathPrefix: {
+              type: "string",
+              description: "Optional path prefix filter within a repository.",
+            },
+            language: {
+              type: "string",
+              description: "Optional source language filter.",
+            },
+          },
+        },
+      },
+    },
+    {
+      type: "function",
+      function: {
+        name: "list_indexed_repos",
+        description:
+          "List the indexed source and lore repositories currently available to search. Use this when you want valid repoName filters for search_sources.",
+        parameters: {
+          type: "object",
+          properties: {},
+        },
+      },
+    },
+    {
+      type: "function",
+      function: {
+        name: "get_source_context",
+        description:
+          "Fetch adjacent chunk context from a specific indexed source document or lore file.",
+        parameters: {
+          type: "object",
+          required: ["sourceId"],
+          properties: {
+            sourceId: {
+              type: "string",
+              description: "The source document ID from a search_sources result.",
+            },
+            chunkIndex: {
+              type: "integer",
+              description: "Optional anchor chunk index inside the document.",
+            },
+            before: {
+              type: "integer",
+              description: "How many earlier chunks to include, between 0 and 20.",
+            },
+            after: {
+              type: "integer",
+              description: "How many later chunks to include, between 0 and 20.",
+            },
+          },
+        },
+      },
+    },
+  ];
+}
+
+function normalizeAssistantMessage(payload: OllamaChatResponse): OllamaChatMessage {
+  const message =
+    typeof payload.message === "object" && payload.message !== null
+      ? payload.message
+      : {};
+
+  return {
+    role: "assistant",
+    content: normalizeModelText(message.content) ?? "",
+    thinking:
+      normalizeModelText(message.thinking) ?? normalizeModelText(payload.thinking),
+    tool_calls: normalizeToolCalls(message.tool_calls),
+  };
+}
+
+function normalizeToolCalls(value: unknown): OllamaToolCall[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  return value.filter((call) => {
+    if (typeof call !== "object" || call === null) {
+      return false;
+    }
+
+    const toolCall = call as OllamaToolCall;
+    return (
+      typeof toolCall.function === "object" &&
+      toolCall.function !== null &&
+      typeof toolCall.function.name === "string"
+    );
+  }) as OllamaToolCall[];
+}
+
+async function executeToolCall(
+  toolbox: LocalLlmToolbox,
+  context: ContextBundle,
+  toolCall: OllamaToolCall,
+): Promise<{
+  payload: Record<string, unknown>;
+  trace: ToolTraceRecord;
+}> {
+  const toolName = normalizeModelText(toolCall.function?.name) ?? "unknown_tool";
+  const args = normalizeToolArguments(toolCall.function?.arguments);
+  let payload: Record<string, unknown>;
+
+  switch (toolName) {
+    case "list_indexed_repos":
+      payload = await toolbox.listIndexedRepos();
+      break;
+    case "search_history":
+      payload = await toolbox.searchHistory({
+        query: readRequiredString(args.query, "query"),
+        limit: clampInteger(args.limit, 1, MAX_TOOL_RESULTS, DEFAULT_RETRIEVAL_RESULT_LIMIT),
+        guildId: context.guildContext.guildId,
+        channelId: readOptionalString(args.channelId),
+        authorId: readOptionalString(args.authorId),
+      });
+      break;
+    case "get_message_context":
+      payload = await toolbox.getMessageContext({
+        messageId: readRequiredString(args.messageId, "messageId"),
+        before: clampInteger(args.before, 0, MAX_CONTEXT_WINDOW, 4),
+        after: clampInteger(args.after, 0, MAX_CONTEXT_WINDOW, 4),
+      });
+      break;
+    case "search_sources":
+      payload = await toolbox.searchSources({
+        query: readRequiredString(args.query, "query"),
+        limit: clampInteger(args.limit, 1, MAX_TOOL_RESULTS, DEFAULT_RETRIEVAL_RESULT_LIMIT),
+        repoName: readOptionalString(args.repoName),
+        pathPrefix: readOptionalString(args.pathPrefix),
+        language: readOptionalString(args.language),
+      });
+      break;
+    case "get_source_context":
+      payload = await toolbox.getSourceContext({
+        sourceId: readRequiredString(args.sourceId, "sourceId"),
+        chunkIndex: clampInteger(args.chunkIndex, 0, 10_000, 0),
+        before: clampInteger(args.before, 0, MAX_CONTEXT_WINDOW, 1),
+        after: clampInteger(args.after, 0, MAX_CONTEXT_WINDOW, 1),
+      });
+      break;
+    default:
+      payload = {
+        error: `Unsupported tool: ${toolName}`,
+      };
+      break;
+  }
+
+  return {
+    payload,
+    trace: {
+      tool: toolName,
+      arguments: args,
+      result: payload,
+    },
+  };
+}
+
+function normalizeToolArguments(value: unknown): Record<string, unknown> {
+  if (typeof value === "string") {
+    try {
+      const parsed = JSON.parse(value);
+      return typeof parsed === "object" && parsed !== null
+        ? (parsed as Record<string, unknown>)
+        : {};
+    } catch {
+      return {};
+    }
+  }
+
+  return typeof value === "object" && value !== null ? (value as Record<string, unknown>) : {};
+}
+
+function readOptionalString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+}
+
+function readRequiredString(value: unknown, key: string): string {
+  const normalized = readOptionalString(value);
+
+  if (!normalized) {
+    throw new Error(`Tool argument "${key}" is required.`);
+  }
+
+  return normalized;
+}
+
+function clampInteger(
+  value: unknown,
+  minimum: number,
+  maximum: number,
+  fallback: number,
+): number {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return fallback;
+  }
+
+  const normalized = Math.trunc(value);
+  return Math.max(minimum, Math.min(maximum, normalized));
+}

packages/providers/src/openai-api-provider.ts

@@ -0,0 +1,57 @@
+import {
+  type Actor,
+  type ContextBundle,
+  type GuildContext,
+  type ProviderAdapter,
+  type ProviderRequest,
+  type ProviderResponse,
+} from "@voidbot/shared";
+
+export class OpenAiApiProvider implements ProviderAdapter {
+  public constructor(private readonly enabled: boolean) {}
+
+  public getName(): "openai_api" {
+    return "openai_api";
+  }
+
+  public getCapabilities(): string[] {
+    return [
+      "public_generation",
+      "input_moderation_required",
+      "output_moderation_required",
+      "budget_controls_required",
+    ];
+  }
+
+  public isEnabled(): boolean {
+    return this.enabled;
+  }
+
+  public isAllowedForActor(_actor: Actor, _guildContext: GuildContext): boolean {
+    return this.enabled;
+  }
+
+  public buildRequest(
+    contextBundle: ContextBundle,
+    options?: Record<string, unknown>,
+  ): ProviderRequest {
+    return {
+      provider: "openai_api",
+      contextBundle,
+      options,
+    };
+  }
+
+  public async execute(_request: ProviderRequest): Promise<ProviderResponse> {
+    throw new Error("The OpenAI API provider scaffold exists, but the runtime adapter is not wired yet.");
+  }
+
+  public async estimateCost(_request: ProviderRequest): Promise<number> {
+    return 0;
+  }
+
+  public getAuditRedactions(): string[] {
+    return ["api_key"];
+  }
+}
+

packages/providers/src/provider-registry.ts

@@ -0,0 +1,41 @@
+import {
+  type Actor,
+  type GuildContext,
+  type ProviderAdapter,
+  type ProviderName,
+} from "@voidbot/shared";
+
+export interface ProviderStatus {
+  name: ProviderName;
+  enabled: boolean;
+  allowed: boolean;
+  capabilities: string[];
+}
+
+export class ProviderRegistry {
+  private readonly providers = new Map<ProviderName, ProviderAdapter>();
+
+  public constructor(providers: ProviderAdapter[]) {
+    for (const provider of providers) {
+      this.providers.set(provider.getName(), provider);
+    }
+  }
+
+  public get(name: ProviderName): ProviderAdapter | undefined {
+    return this.providers.get(name);
+  }
+
+  public list(): ProviderAdapter[] {
+    return [...this.providers.values()];
+  }
+
+  public listStatuses(actor: Actor, guildContext: GuildContext): ProviderStatus[] {
+    return this.list().map((provider) => ({
+      name: provider.getName(),
+      enabled: provider.isEnabled(),
+      allowed: provider.isAllowedForActor(actor, guildContext),
+      capabilities: provider.getCapabilities(),
+    }));
+  }
+}
+

packages/providers/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "references": [
+    {
+      "path": "../shared"
+    }
+  ]
+}
+

packages/rag/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@voidbot/rag",
+  "version": "0.1.0",
+  "private": true,
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "@qdrant/js-client-rest": "^1.17.0",
+    "@voidbot/shared": "0.1.0"
+  }
+}

packages/rag/src/discord-log-importer.ts

@@ -0,0 +1,357 @@
+import { readdir, readFile, stat } from "node:fs/promises";
+import { basename, extname, resolve } from "node:path";
+
+import {
+  type ArchivedMessage,
+  type ChannelIndexingPolicy,
+  shouldIndexChannel,
+} from "@voidbot/shared";
+
+import { type ImportedFileState, type RagImportState } from "./import-state";
+
+export interface DiscordLogImportOptions {
+  recursive?: boolean;
+  channelIndexing?: ChannelIndexingPolicy;
+  previousState?: RagImportState;
+}
+
+export interface DiscordLogImportResult {
+  messages: ArchivedMessage[];
+  filesScanned: number;
+  filesImported: number;
+  filesSkipped: number;
+  invalidRecords: number;
+  nextState: RagImportState;
+}
+
+export async function importDiscordLogs(
+  inputPath: string,
+  options: DiscordLogImportOptions = {},
+): Promise<DiscordLogImportResult> {
+  const recursive = options.recursive ?? true;
+  const logFiles = await collectLogFiles(resolve(inputPath), recursive);
+  const previousFileState = new Map(
+    (options.previousState?.files ?? []).map((file) => [resolve(file.path), file]),
+  );
+  const nextFiles: ImportedFileState[] = [];
+  const messagesById = new Map<string, ArchivedMessage>();
+  let filesImported = 0;
+  let filesSkipped = 0;
+  let invalidRecords = 0;
+
+  for (const filePath of logFiles) {
+    const metadata = await stat(filePath);
+    const descriptor: ImportedFileState = {
+      path: filePath,
+      size: metadata.size,
+      mtimeMs: metadata.mtimeMs,
+    };
+    nextFiles.push(descriptor);
+
+    const previous = previousFileState.get(filePath);
+
+    if (previous && previous.size === descriptor.size && previous.mtimeMs === descriptor.mtimeMs) {
+      filesSkipped += 1;
+      continue;
+    }
+
+    const importResult = await parseLogFile(filePath, options.channelIndexing);
+    filesImported += 1;
+    invalidRecords += importResult.invalidRecords;
+
+    for (const message of importResult.messages) {
+      messagesById.set(message.id, message);
+    }
+  }
+
+  return {
+    messages: [...messagesById.values()],
+    filesScanned: logFiles.length,
+    filesImported,
+    filesSkipped,
+    invalidRecords,
+    nextState: {
+      lastRunAt: new Date().toISOString(),
+      files: nextFiles.sort((left, right) => left.path.localeCompare(right.path)),
+    },
+  };
+}
+
+async function collectLogFiles(inputPath: string, recursive: boolean): Promise<string[]> {
+  const inputStats = await stat(inputPath);
+
+  if (inputStats.isFile()) {
+    return isSupportedLogFile(inputPath) ? [inputPath] : [];
+  }
+
+  const entries = await readdir(inputPath, { withFileTypes: true });
+  const files: string[] = [];
+
+  for (const entry of entries) {
+    const fullPath = resolve(inputPath, entry.name);
+
+    if (entry.isDirectory()) {
+      if (recursive) {
+        files.push(...(await collectLogFiles(fullPath, recursive)));
+      }
+
+      continue;
+    }
+
+    if (entry.isFile() && isSupportedLogFile(fullPath)) {
+      files.push(fullPath);
+    }
+  }
+
+  return files.sort((left, right) => left.localeCompare(right));
+}
+
+function isSupportedLogFile(filePath: string): boolean {
+  const extension = extname(filePath).toLowerCase();
+  return extension === ".json" || extension === ".jsonl" || extension === ".ndjson";
+}
+
+async function parseLogFile(
+  filePath: string,
+  channelIndexing?: ChannelIndexingPolicy,
+): Promise<{ messages: ArchivedMessage[]; invalidRecords: number }> {
+  const extension = extname(filePath).toLowerCase();
+  const raw = stripLeadingBom(await readFile(filePath, "utf8"));
+  const messages: ArchivedMessage[] = [];
+  let invalidRecords = 0;
+
+  if (extension === ".jsonl" || extension === ".ndjson") {
+    for (const line of raw.split(/\r?\n/)) {
+      const trimmed = line.trim();
+
+      if (trimmed.length === 0) {
+        continue;
+      }
+
+      try {
+        const candidate = JSON.parse(trimmed) as unknown;
+        const normalized = normalizeArchivedMessage(candidate, filePath);
+
+        if (!normalized) {
+          invalidRecords += 1;
+          continue;
+        }
+
+        if (!shouldImportArchivedMessage(normalized, channelIndexing)) {
+          continue;
+        }
+
+        messages.push(normalized);
+      } catch {
+        invalidRecords += 1;
+      }
+    }
+
+    return { messages, invalidRecords };
+  }
+
+  try {
+    const parsed = JSON.parse(raw) as unknown;
+    const candidates = flattenCandidates(parsed);
+
+    for (const candidate of candidates) {
+      const normalized = normalizeArchivedMessage(candidate, filePath);
+
+      if (!normalized) {
+        invalidRecords += 1;
+        continue;
+      }
+
+      if (!shouldImportArchivedMessage(normalized, channelIndexing)) {
+        continue;
+      }
+
+      messages.push(normalized);
+    }
+  } catch {
+    invalidRecords += 1;
+  }
+
+  return { messages, invalidRecords };
+}
+
+function stripLeadingBom(input: string): string {
+  return input.charCodeAt(0) === 0xfeff ? input.slice(1) : input;
+}
+
+function flattenCandidates(value: unknown): unknown[] {
+  if (Array.isArray(value)) {
+    return value;
+  }
+
+  if (isRecord(value)) {
+    for (const key of ["messages", "items", "data", "results"]) {
+      const candidate = value[key];
+
+      if (Array.isArray(candidate)) {
+        return candidate;
+      }
+    }
+  }
+
+  return [value];
+}
+
+function normalizeArchivedMessage(
+  candidate: unknown,
+  sourceFile: string,
+): ArchivedMessage | undefined {
+  if (!isRecord(candidate)) {
+    return undefined;
+  }
+
+  const id = readString(candidate, ["id"]) ?? readString(candidate, ["messageId"]);
+  const channelId =
+    readString(candidate, ["channelId"]) ??
+    readString(candidate, ["channel_id"]) ??
+    readString(candidate, ["channel", "id"]);
+  const authorId =
+    readString(candidate, ["authorId"]) ??
+    readString(candidate, ["author_id"]) ??
+    readString(candidate, ["author", "id"]) ??
+    readString(candidate, ["user", "id"]);
+  const authorName =
+    readString(candidate, ["authorName"]) ??
+    readString(candidate, ["author_name"]) ??
+    readString(candidate, ["author", "displayName"]) ??
+    readString(candidate, ["author", "username"]) ??
+    readString(candidate, ["user", "displayName"]) ??
+    readString(candidate, ["user", "username"]);
+  const content =
+    readString(candidate, ["content"]) ??
+    readString(candidate, ["message"]) ??
+    readString(candidate, ["text"]);
+  const timestamp =
+    readString(candidate, ["timestamp"]) ??
+    readString(candidate, ["createdAt"]) ??
+    readString(candidate, ["created_at"]);
+
+  if (!id || !channelId || !authorId || !authorName || content === undefined || !timestamp) {
+    return undefined;
+  }
+
+  const channelName =
+    readString(candidate, ["channelName"]) ??
+    readString(candidate, ["channel_name"]) ??
+    readString(candidate, ["channel", "name"]);
+  const parentChannelId =
+    readString(candidate, ["parentChannelId"]) ??
+    readString(candidate, ["parent_channel_id"]) ??
+    readString(candidate, ["parentChannel", "id"]) ??
+    readString(candidate, ["parent", "id"]);
+  const parentChannelName =
+    readString(candidate, ["parentChannelName"]) ??
+    readString(candidate, ["parent_channel_name"]) ??
+    readString(candidate, ["parentChannel", "name"]) ??
+    readString(candidate, ["parent", "name"]);
+  const metadata: Record<string, string> = {
+    sourceFile: basename(sourceFile),
+  };
+
+  if (channelName) {
+    metadata.channelName = channelName;
+  }
+
+  if (parentChannelId) {
+    metadata.parentChannelId = parentChannelId;
+  }
+
+  if (parentChannelName) {
+    metadata.parentChannelName = parentChannelName;
+  }
+
+  return {
+    id,
+    guildId:
+      readString(candidate, ["guildId"]) ??
+      readString(candidate, ["guild_id"]) ??
+      readString(candidate, ["guild", "id"]),
+    channelId,
+    authorId,
+    authorName,
+    content,
+    timestamp: normalizeTimestamp(timestamp),
+    editedAt:
+      readString(candidate, ["editedAt"]) ??
+      readString(candidate, ["edited_at"]) ??
+      readString(candidate, ["editedTimestamp"]),
+    deletedAt:
+      readString(candidate, ["deletedAt"]) ??
+      readString(candidate, ["deleted_at"]),
+    threadId:
+      readString(candidate, ["threadId"]) ??
+      readString(candidate, ["thread_id"]) ??
+      readString(candidate, ["thread", "id"]),
+    attachments: readAttachments(candidate),
+    metadata,
+  };
+}
+
+function shouldImportArchivedMessage(
+  message: ArchivedMessage,
+  channelIndexing?: ChannelIndexingPolicy,
+): boolean {
+  if (!channelIndexing) {
+    return true;
+  }
+
+  return shouldIndexChannel(channelIndexing, {
+    channelId: message.channelId,
+    channelName: message.metadata?.channelName,
+    parentChannelId: message.metadata?.parentChannelId,
+    parentChannelName: message.metadata?.parentChannelName,
+  });
+}
+
+function readAttachments(candidate: Record<string, unknown>): string[] | undefined {
+  const rawAttachments = candidate.attachments;
+
+  if (!Array.isArray(rawAttachments)) {
+    return undefined;
+  }
+
+  const attachments = rawAttachments
+    .map((attachment) => {
+      if (!isRecord(attachment)) {
+        return undefined;
+      }
+
+      return (
+        readString(attachment, ["url"]) ??
+        readString(attachment, ["proxy_url"]) ??
+        readString(attachment, ["name"]) ??
+        readString(attachment, ["filename"])
+      );
+    })
+    .filter((attachment): attachment is string => Boolean(attachment));
+
+  return attachments.length > 0 ? attachments : undefined;
+}
+
+function normalizeTimestamp(value: string): string {
+  const parsed = new Date(value);
+  return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString();
+}
+
+function readString(source: Record<string, unknown>, path: string[]): string | undefined {
+  let cursor: unknown = source;
+
+  for (const segment of path) {
+    if (!isRecord(cursor)) {
+      return undefined;
+    }
+
+    cursor = cursor[segment];
+  }
+
+  return typeof cursor === "string" ? cursor : undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}

packages/rag/src/embedder-factory.ts

@@ -0,0 +1,24 @@
+import { HashingTextEmbedder, type TextEmbedder } from "./hash-embedder";
+import { OllamaTextEmbedder } from "./ollama-embedder";
+
+export interface TextEmbedderFactoryOptions {
+  backend: "hash" | "ollama";
+  hashDimensions: number;
+  ollamaBaseUrl: string;
+  ollamaModel: string;
+  ollamaTimeoutMs?: number;
+  queryInstruction?: string;
+}
+
+export function createTextEmbedder(options: TextEmbedderFactoryOptions): TextEmbedder {
+  if (options.backend === "ollama") {
+    return new OllamaTextEmbedder({
+      baseUrl: options.ollamaBaseUrl,
+      model: options.ollamaModel,
+      requestTimeoutMs: options.ollamaTimeoutMs,
+      queryInstruction: options.queryInstruction,
+    });
+  }
+
+  return new HashingTextEmbedder(options.hashDimensions);
+}

packages/rag/src/file-store.ts

@@ -0,0 +1,65 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+
+export class SerializedFileStore<T> {
+  private writeChain: Promise<void> = Promise.resolve();
+
+  public constructor(
+    private readonly filePath: string,
+    private readonly createDefault: () => T,
+  ) {}
+
+  public async snapshot(): Promise<T> {
+    await this.writeChain;
+    const store = await this.readUnlocked();
+    return structuredClone(store);
+  }
+
+  public async overwrite(nextValue: T): Promise<void> {
+    await this.serialize(async () => {
+      await this.writeUnlocked(nextValue);
+    });
+  }
+
+  public async mutate<R>(mutator: (current: T) => R | Promise<R>): Promise<R> {
+    return this.serialize(async () => {
+      const current = await this.readUnlocked();
+      const result = await mutator(current);
+      await this.writeUnlocked(current);
+      return result;
+    });
+  }
+
+  private async serialize<R>(operation: () => Promise<R>): Promise<R> {
+    const pending = this.writeChain.then(operation, operation);
+    this.writeChain = pending.then(
+      () => undefined,
+      () => undefined,
+    );
+    return pending;
+  }
+
+  private async readUnlocked(): Promise<T> {
+    try {
+      const raw = await readFile(this.filePath, "utf8");
+      return JSON.parse(stripLeadingBom(raw)) as T;
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException).code;
+
+      if (code === "ENOENT") {
+        return this.createDefault();
+      }
+
+      throw error;
+    }
+  }
+
+  private async writeUnlocked(value: T): Promise<void> {
+    await mkdir(dirname(this.filePath), { recursive: true });
+    await writeFile(this.filePath, `${JSON.stringify(value)}\n`, "utf8");
+  }
+}
+
+function stripLeadingBom(input: string): string {
+  return input.charCodeAt(0) === 0xfeff ? input.slice(1) : input;
+}

packages/rag/src/file-vector-store.ts

@@ -0,0 +1,359 @@
+import {
+  type EmbeddingChunk,
+  type RetrievalFilters,
+  type RetrievalResult,
+  type VectorStore,
+} from "@voidbot/shared";
+
+import { SerializedFileStore } from "./file-store";
+import { type TextEmbedder, HashingTextEmbedder, cosineSimilarity } from "./hash-embedder";
+
+interface StoredVectorChunk extends EmbeddingChunk {
+  vector: number[];
+  indexedAt: string;
+}
+
+interface LegacyStoredVectorChunk extends Omit<StoredVectorChunk, "sourceId" | "sourceKind"> {
+  sourceId?: string;
+  sourceMessageId?: string;
+  sourceKind?: "discord_message" | "source_document";
+}
+
+interface VectorIndexStore {
+  version: 2;
+  embedderId: string;
+  dimensions: number;
+  chunks: StoredVectorChunk[];
+}
+
+interface LegacyVectorIndexStore {
+  version?: number;
+  embedderId?: string;
+  dimensions?: number;
+  chunks?: StoredVectorChunk[];
+}
+
+const STORED_VECTOR_DECIMAL_PLACES = 4;
+
+export class FileVectorStore implements VectorStore {
+  private readonly store: SerializedFileStore<VectorIndexStore>;
+  private readonly embedder: TextEmbedder;
+
+  public constructor(filePath: string, embedder: TextEmbedder = new HashingTextEmbedder()) {
+    this.embedder = embedder;
+    this.store = new SerializedFileStore(filePath, () => this.createEmptyStore());
+  }
+
+  public async upsert(chunks: EmbeddingChunk[]): Promise<void> {
+    if (chunks.length === 0) {
+      return;
+    }
+
+    const vectors = await this.embedder.embedDocuments(chunks.map((chunk) => chunk.text));
+    const vectorLength = validateEmbeddingBatch(vectors, chunks.length);
+
+    await this.store.mutate((store) => {
+      const activeStore = this.ensureCompatibility(store, vectorLength);
+      const positions = new Map(activeStore.chunks.map((chunk, index) => [chunk.id, index]));
+      const now = new Date().toISOString();
+
+      for (const [index, chunk] of chunks.entries()) {
+        const storedChunk: StoredVectorChunk = {
+          ...chunk,
+          vector: quantizeVector(vectors[index]),
+          indexedAt: now,
+        };
+        const position = positions.get(chunk.id);
+
+        if (position === undefined) {
+          activeStore.chunks.push(storedChunk);
+          positions.set(chunk.id, activeStore.chunks.length - 1);
+          continue;
+        }
+
+        activeStore.chunks[position] = storedChunk;
+      }
+    });
+  }
+
+  public async clear(): Promise<void> {
+    await this.store.overwrite(this.createEmptyStore());
+  }
+
+  public async deleteBySourceIds(sourceIds: string[]): Promise<void> {
+    if (sourceIds.length === 0) {
+      return;
+    }
+
+    const sourceIdSet = new Set(sourceIds);
+
+    await this.store.mutate((store) => {
+      const activeStore = this.normalizeStore(store);
+      const nextChunks = activeStore.chunks.filter(
+        (chunk) => !sourceIdSet.has(chunk.sourceId),
+      );
+      activeStore.chunks = nextChunks;
+      store.chunks = nextChunks;
+    });
+  }
+
+  public async deleteByFilters(filters: RetrievalFilters): Promise<void> {
+    await this.store.mutate((store) => {
+      const activeStore = this.normalizeStore(store);
+      const nextChunks = activeStore.chunks.filter(
+        (chunk) => !matchesFilters(chunk.metadata, filters),
+      );
+      activeStore.chunks = nextChunks;
+      store.chunks = nextChunks;
+    });
+  }
+
+  public async query(
+    query: string,
+    limit: number,
+    filters?: RetrievalFilters,
+  ): Promise<RetrievalResult[]> {
+    if (query.trim().length === 0) {
+      return [];
+    }
+
+    const queryVector = await this.embedder.embedQuery(query);
+    const vectorLength = validateSingleEmbedding(queryVector);
+    const store = this.getQueryableStore(await this.store.snapshot(), vectorLength);
+
+    return store.chunks
+      .filter((chunk) => matchesFilters(chunk.metadata, filters))
+      .map((chunk) => ({
+        chunk,
+        score: scoreChunk(query, queryVector, chunk),
+      }))
+      .filter((candidate) => candidate.score > 0)
+      .sort((left, right) => right.score - left.score)
+      .slice(0, limit)
+      .map(({ chunk, score }) => ({
+        chunkId: chunk.id,
+        score,
+        text: chunk.text,
+        sourceId: chunk.sourceId,
+        sourceKind: chunk.sourceKind,
+        metadata: chunk.metadata,
+      }));
+  }
+
+  private createEmptyStore(dimensions = 0): VectorIndexStore {
+    return createStore(this.embedder.id, dimensions);
+  }
+
+  private normalizeStore(rawStore: LegacyVectorIndexStore): VectorIndexStore {
+    return writeStore(rawStore, normalizeLegacyStore(rawStore, this.embedder.id));
+  }
+
+  private ensureCompatibility(
+    rawStore: LegacyVectorIndexStore,
+    vectorLength: number,
+  ): VectorIndexStore {
+    const normalized = normalizeLegacyStore(rawStore, this.embedder.id);
+    const shouldPreserve =
+      shouldPreserveCurrentStore(normalized, this.embedder.id, vectorLength) ||
+      shouldPreserveLegacyHashStore(rawStore, this.embedder.id, vectorLength);
+
+    if (shouldPreserve) {
+      normalized.embedderId = this.embedder.id;
+      normalized.dimensions = vectorLength;
+      return writeStore(rawStore, normalized);
+    }
+
+    return writeStore(rawStore, createStore(this.embedder.id, vectorLength));
+  }
+
+  private getQueryableStore(
+    rawStore: LegacyVectorIndexStore,
+    vectorLength: number,
+  ): VectorIndexStore {
+    const normalized = normalizeLegacyStore(rawStore, this.embedder.id);
+
+    if (shouldPreserveCurrentStore(normalized, this.embedder.id, vectorLength)) {
+      return normalized;
+    }
+
+    if (shouldPreserveLegacyHashStore(rawStore, this.embedder.id, vectorLength)) {
+      normalized.embedderId = this.embedder.id;
+      normalized.dimensions = vectorLength;
+      return normalized;
+    }
+
+    return createStore(this.embedder.id, vectorLength);
+  }
+}
+
+function scoreChunk(_query: string, queryVector: number[], chunk: StoredVectorChunk): number {
+  const cosine = cosineSimilarity(queryVector, chunk.vector);
+  return cosine;
+}
+
+function matchesFilters(metadata: Record<string, string>, filters?: RetrievalFilters): boolean {
+  if (!filters) {
+    return true;
+  }
+
+  if (filters.guildId && metadata.guildId !== filters.guildId) {
+    return false;
+  }
+
+  if (filters.channelId && metadata.channelId !== filters.channelId) {
+    return false;
+  }
+
+  if (filters.authorId && metadata.authorId !== filters.authorId) {
+    return false;
+  }
+
+  if (filters.corpusKind && metadata.corpusKind !== filters.corpusKind) {
+    return false;
+  }
+
+  if (filters.repoName && metadata.repoName !== filters.repoName) {
+    return false;
+  }
+
+  if (filters.pathPrefix) {
+    const path = metadata.path ?? "";
+
+    if (!path.startsWith(filters.pathPrefix)) {
+      return false;
+    }
+  }
+
+  if (filters.language && metadata.language !== filters.language) {
+    return false;
+  }
+
+  if (filters.sourceId && metadata.sourceId !== filters.sourceId) {
+    return false;
+  }
+
+  return true;
+}
+
+function validateEmbeddingBatch(vectors: number[][], expectedCount: number): number {
+  if (vectors.length !== expectedCount) {
+    throw new Error(
+      `Embedding backend returned ${vectors.length} vectors for ${expectedCount} chunks.`,
+    );
+  }
+
+  return validateSingleEmbedding(vectors[0] ?? []);
+}
+
+function validateSingleEmbedding(vector: number[]): number {
+  if (vector.length === 0) {
+    throw new Error("Embedding backend returned an empty vector.");
+  }
+
+  return vector.length;
+}
+
+function inferVectorLength(chunks: StoredVectorChunk[]): number {
+  return chunks[0]?.vector.length ?? 0;
+}
+
+function isHashEmbedderId(value: string): boolean {
+  return value.startsWith("hash:");
+}
+
+function coerceChunks(value: unknown): StoredVectorChunk[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  return value
+    .map((chunk) => normalizeStoredChunk(chunk as LegacyStoredVectorChunk))
+    .filter((chunk): chunk is StoredVectorChunk => Boolean(chunk));
+}
+
+function readNumber(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function readString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function writeStore(target: LegacyVectorIndexStore, next: VectorIndexStore): VectorIndexStore {
+  target.version = next.version;
+  target.embedderId = next.embedderId;
+  target.dimensions = next.dimensions;
+  target.chunks = next.chunks;
+  return next;
+}
+
+function normalizeLegacyStore(
+  rawStore: LegacyVectorIndexStore,
+  fallbackEmbedderId: string,
+): VectorIndexStore {
+  const chunks = coerceChunks(rawStore.chunks);
+
+  return {
+    version: 2,
+    embedderId: readString(rawStore.embedderId) ?? fallbackEmbedderId,
+    dimensions: readNumber(rawStore.dimensions) ?? inferVectorLength(chunks),
+    chunks,
+  };
+}
+
+function shouldPreserveLegacyHashStore(
+  rawStore: LegacyVectorIndexStore,
+  embedderId: string,
+  vectorLength: number,
+): boolean {
+  return rawStore.version === 1 && isHashEmbedderId(embedderId) && rawStore.dimensions === vectorLength;
+}
+
+function shouldPreserveCurrentStore(
+  store: VectorIndexStore,
+  embedderId: string,
+  vectorLength: number,
+): boolean {
+  return store.embedderId === embedderId && store.dimensions === vectorLength;
+}
+
+function createStore(embedderId: string, dimensions = 0): VectorIndexStore {
+  return {
+    version: 2,
+    embedderId,
+    dimensions,
+    chunks: [],
+  };
+}
+
+function quantizeVector(vector: number[]): number[] {
+  return vector.map((value) => Number(value.toFixed(STORED_VECTOR_DECIMAL_PLACES)));
+}
+
+function normalizeStoredChunk(chunk: LegacyStoredVectorChunk): StoredVectorChunk | undefined {
+  const sourceId =
+    typeof chunk.sourceId === "string" && chunk.sourceId.length > 0
+      ? chunk.sourceId
+      : typeof chunk.sourceMessageId === "string" && chunk.sourceMessageId.length > 0
+        ? chunk.sourceMessageId
+        : undefined;
+
+  if (!sourceId) {
+    return undefined;
+  }
+
+  return {
+    ...chunk,
+    sourceId,
+    sourceKind: chunk.sourceKind ?? "discord_message",
+    metadata: {
+      ...chunk.metadata,
+      sourceId,
+      corpusKind: chunk.metadata?.corpusKind ?? inferCorpusKind(chunk),
+    },
+  };
+}
+
+function inferCorpusKind(chunk: LegacyStoredVectorChunk): "discord_history" | "repository_source" {
+  return chunk.sourceKind === "source_document" ? "repository_source" : "discord_history";
+}

packages/rag/src/hash-embedder.ts

@@ -0,0 +1,99 @@
+import { createHash } from "node:crypto";
+
+export interface TextEmbedder {
+  readonly id: string;
+  readonly dimensions?: number;
+  embedDocuments(texts: string[]): Promise<number[][]>;
+  embedQuery(text: string): Promise<number[]>;
+}
+
+export class HashingTextEmbedder implements TextEmbedder {
+  public constructor(public readonly dimensions = 256) {}
+
+  public get id(): string {
+    return `hash:${this.dimensions}`;
+  }
+
+  public async embedDocuments(texts: string[]): Promise<number[][]> {
+    return texts.map((text) => this.embedSync(text));
+  }
+
+  public async embedQuery(text: string): Promise<number[]> {
+    return this.embedSync(text);
+  }
+
+  private embedSync(text: string): number[] {
+    const terms = tokenize(text);
+
+    if (terms.length === 0) {
+      return new Array(this.dimensions).fill(0);
+    }
+
+    const vector = new Array(this.dimensions).fill(0);
+    const termFrequencies = new Map<string, number>();
+
+    for (const term of terms) {
+      termFrequencies.set(term, (termFrequencies.get(term) ?? 0) + 1);
+    }
+
+    for (const [term, frequency] of termFrequencies.entries()) {
+      const hash = createHash("sha1").update(term).digest();
+      const index = hash.readUInt32BE(0) % this.dimensions;
+      const sign = hash[4] % 2 === 0 ? 1 : -1;
+      vector[index] += sign * frequency;
+    }
+
+    return normalizeVector(vector);
+  }
+}
+
+export function cosineSimilarity(left: number[], right: number[]): number {
+  if (left.length !== right.length || left.length === 0) {
+    return 0;
+  }
+
+  let dotProduct = 0;
+
+  for (let index = 0; index < left.length; index += 1) {
+    dotProduct += left[index] * right[index];
+  }
+
+  return dotProduct;
+}
+
+export function lexicalOverlap(query: string, text: string): number {
+  const leftTerms = new Set(tokenize(query));
+  const rightTerms = new Set(tokenize(text));
+
+  if (leftTerms.size === 0 || rightTerms.size === 0) {
+    return 0;
+  }
+
+  let matches = 0;
+
+  for (const term of leftTerms) {
+    if (rightTerms.has(term)) {
+      matches += 1;
+    }
+  }
+
+  return matches / leftTerms.size;
+}
+
+function tokenize(input: string): string[] {
+  return input
+    .toLowerCase()
+    .split(/[^a-z0-9]+/i)
+    .map((token) => token.trim())
+    .filter((token) => token.length > 1);
+}
+
+function normalizeVector(vector: number[]): number[] {
+  const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value ** 2, 0));
+
+  if (magnitude === 0) {
+    return vector;
+  }
+
+  return vector.map((value) => value / magnitude);
+}

packages/rag/src/history-ingester.ts

@@ -0,0 +1,68 @@
+import {
+  type ArchivedMessage,
+  type EmbeddingChunk,
+  type VectorStore,
+} from "@voidbot/shared";
+
+const DEFAULT_CHUNK_SIZE = 420;
+
+export class HistoryIngester {
+  public constructor(private readonly chunkSize = DEFAULT_CHUNK_SIZE) {}
+
+  public chunkMessages(messages: ArchivedMessage[]): EmbeddingChunk[] {
+    return messages.flatMap((message) => this.chunkMessage(message));
+  }
+
+  public async ingestMessages(
+    messages: ArchivedMessage[],
+    vectorStore: VectorStore,
+  ): Promise<EmbeddingChunk[]> {
+    const chunks = this.chunkMessages(messages);
+    await vectorStore.upsert(chunks);
+    return chunks;
+  }
+
+  public chunkMessage(message: ArchivedMessage): EmbeddingChunk[] {
+    const segments = splitText(message.content, this.chunkSize);
+
+    return segments.map((segment, index) => ({
+      id: `${message.id}:${index}`,
+      sourceId: message.id,
+      sourceKind: "discord_message",
+      text: segment,
+      normalizedText: normalizeText(segment),
+      metadata: {
+        corpusKind: "discord_history",
+        sourceId: message.id,
+        guildId: message.guildId ?? "",
+        channelId: message.channelId,
+        authorId: message.authorId,
+        authorName: message.authorName,
+        timestamp: message.timestamp,
+        chunkIndex: String(index),
+        threadId: message.threadId ?? "",
+        editedAt: message.editedAt ?? "",
+      },
+    }));
+  }
+}
+
+export function normalizeText(input: string): string {
+  return input.replace(/\s+/g, " ").trim().toLowerCase();
+}
+
+function splitText(input: string, maxLength: number): string[] {
+  const normalized = input.trim();
+
+  if (normalized.length === 0) {
+    return ["(empty message)"];
+  }
+
+  const segments: string[] = [];
+
+  for (let cursor = 0; cursor < normalized.length; cursor += maxLength) {
+    segments.push(normalized.slice(cursor, cursor + maxLength));
+  }
+
+  return segments;
+}

packages/rag/src/import-state.ts

@@ -0,0 +1,31 @@
+import { SerializedFileStore } from "./file-store";
+
+export interface ImportedFileState {
+  path: string;
+  size: number;
+  mtimeMs: number;
+}
+
+export interface RagImportState {
+  lastRunAt?: string;
+  files: ImportedFileState[];
+}
+
+export class FileImportStateRepository {
+  private readonly store: SerializedFileStore<RagImportState>;
+
+  public constructor(filePath: string) {
+    this.store = new SerializedFileStore(filePath, () => ({
+      files: [],
+    }));
+  }
+
+  public async read(): Promise<RagImportState> {
+    return this.store.snapshot();
+  }
+
+  public async write(state: RagImportState): Promise<void> {
+    await this.store.overwrite(state);
+  }
+}
+

packages/rag/src/in-memory-vector-store.ts

@@ -0,0 +1,139 @@
+import {
+  type EmbeddingChunk,
+  type RetrievalFilters,
+  type RetrievalResult,
+  type VectorStore,
+} from "@voidbot/shared";
+
+export class InMemoryVectorStore implements VectorStore {
+  private readonly chunks = new Map<string, EmbeddingChunk>();
+
+  public async upsert(chunks: EmbeddingChunk[]): Promise<void> {
+    for (const chunk of chunks) {
+      this.chunks.set(chunk.id, chunk);
+    }
+  }
+
+  public async clear(): Promise<void> {
+    this.chunks.clear();
+  }
+
+  public async deleteBySourceIds(sourceIds: string[]): Promise<void> {
+    if (sourceIds.length === 0) {
+      return;
+    }
+
+    const sourceIdSet = new Set(sourceIds);
+
+    for (const [chunkId, chunk] of this.chunks.entries()) {
+      if (sourceIdSet.has(chunk.sourceId)) {
+        this.chunks.delete(chunkId);
+      }
+    }
+  }
+
+  public async deleteByFilters(filters: RetrievalFilters): Promise<void> {
+    for (const [chunkId, chunk] of this.chunks.entries()) {
+      if (matchesFilters(chunk.metadata, filters)) {
+        this.chunks.delete(chunkId);
+      }
+    }
+  }
+
+  public async query(
+    query: string,
+    limit: number,
+    filters?: RetrievalFilters,
+  ): Promise<RetrievalResult[]> {
+    const queryTerms = tokenize(query);
+
+    return [...this.chunks.values()]
+      .filter((chunk) => matchesFilters(chunk.metadata, filters))
+      .map((chunk) => ({
+        chunk,
+        score: scoreChunk(queryTerms, chunk.normalizedText),
+      }))
+      .filter((candidate) => candidate.score > 0)
+      .sort((left, right) => right.score - left.score)
+      .slice(0, limit)
+      .map(({ chunk, score }) => ({
+        chunkId: chunk.id,
+        score,
+        text: chunk.text,
+        sourceId: chunk.sourceId,
+        sourceKind: chunk.sourceKind,
+        metadata: chunk.metadata,
+      }));
+  }
+}
+
+function tokenize(input: string): string[] {
+  return input
+    .toLowerCase()
+    .split(/[^a-z0-9]+/i)
+    .map((token) => token.trim())
+    .filter((token) => token.length > 1);
+}
+
+function scoreChunk(queryTerms: string[], normalizedText: string): number {
+  if (queryTerms.length === 0) {
+    return 0;
+  }
+
+  let matches = 0;
+
+  for (const term of queryTerms) {
+    if (normalizedText.includes(term)) {
+      matches += 1;
+    }
+  }
+
+  return matches / queryTerms.length;
+}
+
+function matchesFilters(
+  metadata: Record<string, string>,
+  filters?: RetrievalFilters,
+): boolean {
+  if (!filters) {
+    return true;
+  }
+
+  if (filters.guildId && metadata.guildId !== filters.guildId) {
+    return false;
+  }
+
+  if (filters.channelId && metadata.channelId !== filters.channelId) {
+    return false;
+  }
+
+  if (filters.authorId && metadata.authorId !== filters.authorId) {
+    return false;
+  }
+
+  if (filters.corpusKind && metadata.corpusKind !== filters.corpusKind) {
+    return false;
+  }
+
+  if (filters.repoName && metadata.repoName !== filters.repoName) {
+    return false;
+  }
+
+  if (filters.pathPrefix) {
+    const path = metadata.path ?? "";
+
+    if (!path.startsWith(filters.pathPrefix)) {
+      return false;
+    }
+  }
+
+  if (filters.language && metadata.language !== filters.language) {
+    return false;
+  }
+
+  if (filters.sourceId && metadata.sourceId !== filters.sourceId) {
+    return false;
+  }
+
+  return true;
+}

packages/rag/src/index.ts

@@ -0,0 +1,19 @@
+export * from "./embedder-factory";
+export * from "./history-ingester";
+export * from "./discord-log-importer";
+export * from "./file-store";
+export * from "./file-vector-store";
+export * from "./hash-embedder";
+export * from "./import-state";
+export * from "./in-memory-vector-store";
+export * from "./message-archive";
+export * from "./ollama-embedder";
+export * from "./qdrant-vector-store";
+export * from "./rag-pipeline";
+export * from "./retrieval-service";
+export * from "./source-document-archive";
+export * from "./source-document-ingester";
+export * from "./source-rag-pipeline";
+export * from "./source-repo-crawler";
+export * from "./source-vector-store";
+export * from "./vector-store-factory";

packages/rag/src/message-archive.ts

@@ -0,0 +1,203 @@
+import { type ArchivedMessage } from "@voidbot/shared";
+
+import { normalizeText } from "./history-ingester";
+import { SerializedFileStore } from "./file-store";
+
+export interface ArchivedMessageRecord extends ArchivedMessage {
+  normalizedContent: string;
+  indexedAt: string;
+}
+
+interface MessageArchiveStore {
+  version: 1;
+  messages: ArchivedMessageRecord[];
+}
+
+export interface MessageArchiveMutationResult {
+  created: number;
+  updated: number;
+  unchanged: number;
+  storedMessages: ArchivedMessageRecord[];
+  changedMessageIds: string[];
+}
+
+export class FileMessageArchiveRepository {
+  private readonly store: SerializedFileStore<MessageArchiveStore>;
+
+  public constructor(filePath: string) {
+    this.store = new SerializedFileStore(filePath, () => ({
+      version: 1,
+      messages: [],
+    }));
+  }
+
+  public async upsert(messages: ArchivedMessage[]): Promise<MessageArchiveMutationResult> {
+    return this.store.mutate((store) => {
+      const now = new Date().toISOString();
+      const positions = new Map(store.messages.map((message, index) => [message.id, index]));
+      const storedMessages: ArchivedMessageRecord[] = [];
+      const changedMessageIds: string[] = [];
+      let created = 0;
+      let updated = 0;
+      let unchanged = 0;
+
+      for (const message of messages) {
+        const normalized = toArchivedMessageRecord(message, now);
+        const position = positions.get(message.id);
+
+        if (position === undefined) {
+          store.messages.push(normalized);
+          positions.set(message.id, store.messages.length - 1);
+          storedMessages.push(normalized);
+          changedMessageIds.push(message.id);
+          created += 1;
+          continue;
+        }
+
+        const existing = store.messages[position];
+
+        if (areEquivalent(existing, normalized)) {
+          storedMessages.push(existing);
+          unchanged += 1;
+          continue;
+        }
+
+        const nextRecord: ArchivedMessageRecord = {
+          ...normalized,
+          indexedAt: now,
+        };
+        store.messages[position] = nextRecord;
+        storedMessages.push(nextRecord);
+        changedMessageIds.push(message.id);
+        updated += 1;
+      }
+
+      return {
+        created,
+        updated,
+        unchanged,
+        storedMessages,
+        changedMessageIds,
+      };
+    });
+  }
+
+  public async markDeleted(messageId: string, deletedAt = new Date().toISOString()): Promise<boolean> {
+    return this.store.mutate((store) => {
+      const message = store.messages.find((candidate) => candidate.id === messageId);
+
+      if (!message) {
+        return false;
+      }
+
+      if (message.deletedAt === deletedAt) {
+        return false;
+      }
+
+      message.deletedAt = deletedAt;
+      message.indexedAt = new Date().toISOString();
+      return true;
+    });
+  }
+
+  public async get(messageId: string): Promise<ArchivedMessageRecord | undefined> {
+    const store = await this.store.snapshot();
+    return store.messages.find((message) => message.id === messageId);
+  }
+
+  public async listByChannel(channelId: string, limit = 100): Promise<ArchivedMessageRecord[]> {
+    const store = await this.store.snapshot();
+
+    return store.messages
+      .filter((message) => message.channelId === channelId && !message.deletedAt)
+      .sort((left, right) => left.timestamp.localeCompare(right.timestamp))
+      .slice(-limit);
+  }
+
+  public async listContextWindow(
+    messageId: string,
+    before = 4,
+    after = 4,
+  ): Promise<ArchivedMessageRecord[]> {
+    const store = await this.store.snapshot();
+    const anchor = store.messages.find((message) => message.id === messageId);
+
+    if (!anchor) {
+      return [];
+    }
+
+    const messages = store.messages
+      .filter((message) => !message.deletedAt && belongsToSameConversation(message, anchor))
+      .sort((left, right) => left.timestamp.localeCompare(right.timestamp));
+    const anchorIndex = messages.findIndex((message) => message.id === messageId);
+
+    if (anchorIndex === -1) {
+      return [];
+    }
+
+    const start = Math.max(0, anchorIndex - before);
+    const end = Math.min(messages.length, anchorIndex + after + 1);
+    return messages.slice(start, end);
+  }
+
+  public async listAll(): Promise<ArchivedMessageRecord[]> {
+    const store = await this.store.snapshot();
+
+    return store.messages
+      .slice()
+      .sort((left, right) => left.timestamp.localeCompare(right.timestamp));
+  }
+
+  public async listAllActive(): Promise<ArchivedMessageRecord[]> {
+    const messages = await this.listAll();
+    return messages.filter((message) => !message.deletedAt);
+  }
+
+  public async count(): Promise<number> {
+    const store = await this.store.snapshot();
+    return store.messages.length;
+  }
+}
+
+function toArchivedMessageRecord(
+  message: ArchivedMessage,
+  indexedAt: string,
+): ArchivedMessageRecord {
+  return {
+    ...message,
+    normalizedContent: normalizeText(message.content),
+    indexedAt,
+  };
+}
+
+function areEquivalent(left: ArchivedMessageRecord, right: ArchivedMessageRecord): boolean {
+  return JSON.stringify(comparableFields(left)) === JSON.stringify(comparableFields(right));
+}
+
+function comparableFields(message: ArchivedMessageRecord): Record<string, unknown> {
+  return {
+    id: message.id,
+    guildId: message.guildId,
+    channelId: message.channelId,
+    authorId: message.authorId,
+    authorName: message.authorName,
+    content: message.content,
+    timestamp: message.timestamp,
+    editedAt: message.editedAt,
+    deletedAt: message.deletedAt,
+    threadId: message.threadId,
+    attachments: message.attachments ?? [],
+    metadata: message.metadata ?? {},
+  };
+}
+
+function belongsToSameConversation(
+  candidate: ArchivedMessageRecord,
+  anchor: ArchivedMessageRecord,
+): boolean {
+  if (anchor.threadId || candidate.threadId) {
+    return Boolean(anchor.threadId && candidate.threadId && anchor.threadId === candidate.threadId);
+  }
+
+  return candidate.channelId === anchor.channelId;
+}

packages/rag/src/ollama-embedder.ts

@@ -0,0 +1,143 @@
+import { type TextEmbedder } from "./hash-embedder";
+
+const DEFAULT_OLLAMA_TIMEOUT_MS = 30000;
+
+interface OllamaEmbedResponse {
+  embeddings?: unknown;
+}
+
+export interface OllamaTextEmbedderOptions {
+  baseUrl?: string;
+  model: string;
+  queryInstruction?: string;
+  requestTimeoutMs?: number;
+}
+
+export class OllamaTextEmbedder implements TextEmbedder {
+  private readonly baseUrl: string;
+  private readonly model: string;
+  private readonly queryInstruction?: string;
+  private readonly requestTimeoutMs: number;
+
+  public constructor(options: OllamaTextEmbedderOptions) {
+    this.baseUrl = normalizeBaseUrl(options.baseUrl ?? "http://127.0.0.1:11434");
+    this.model = options.model;
+    this.queryInstruction = normalizeOptionalText(options.queryInstruction);
+    this.requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_OLLAMA_TIMEOUT_MS;
+  }
+
+  public get id(): string {
+    return `ollama:${this.baseUrl}:${this.model}`;
+  }
+
+  public async embedDocuments(texts: string[]): Promise<number[][]> {
+    return this.embedInputs(texts);
+  }
+
+  public async embedQuery(text: string): Promise<number[]> {
+    const [embedding] = await this.embedInputs([this.formatQuery(text)]);
+    return embedding;
+  }
+
+  private formatQuery(text: string): string {
+    if (!this.queryInstruction) {
+      return text;
+    }
+
+    return `Instruct: ${this.queryInstruction}\nQuery: ${text}`;
+  }
+
+  private async embedInputs(texts: string[]): Promise<number[][]> {
+    if (texts.length === 0) {
+      return [];
+    }
+
+    const controller = new AbortController();
+    const timeoutHandle = setTimeout(() => controller.abort(), this.requestTimeoutMs);
+
+    try {
+      const response = await fetch(`${this.baseUrl}/api/embed`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          model: this.model,
+          input: texts,
+        }),
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        throw new Error(
+          `Ollama embedding request failed with ${response.status}: ${await response.text()}`,
+        );
+      }
+
+      const payload = (await response.json()) as OllamaEmbedResponse;
+
+      if (!Array.isArray(payload.embeddings)) {
+        throw new Error(
+          `Ollama embedding response for model "${this.model}" did not include an embeddings array.`,
+        );
+      }
+
+      const embeddings = payload.embeddings.map((value, index) => parseEmbedding(value, index));
+      const vectorLength = embeddings[0]?.length ?? 0;
+
+      if (vectorLength === 0) {
+        throw new Error(`Ollama returned an empty embedding vector for model "${this.model}".`);
+      }
+
+      for (const [index, embedding] of embeddings.entries()) {
+        if (embedding.length !== vectorLength) {
+          throw new Error(
+            `Ollama returned inconsistent embedding lengths for model "${this.model}" at item ${index}.`,
+          );
+        }
+      }
+
+      return embeddings;
+    } catch (error) {
+      if ((error as Error).name === "AbortError") {
+        throw new Error(
+          `Timed out waiting for Ollama embeddings from ${this.baseUrl} using model "${this.model}".`,
+        );
+      }
+
+      const message = error instanceof Error ? error.message : "Unknown Ollama embedding error.";
+      throw new Error(
+        `${message} Make sure Ollama is running at ${this.baseUrl} and that the model "${this.model}" has been pulled.`,
+      );
+    } finally {
+      clearTimeout(timeoutHandle);
+    }
+  }
+}
+
+function parseEmbedding(value: unknown, index: number): number[] {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error(`Ollama returned a non-array embedding for item ${index}.`);
+  }
+
+  const embedding = value.map((component, componentIndex) => {
+    if (typeof component !== "number" || !Number.isFinite(component)) {
+      throw new Error(
+        `Ollama returned a non-numeric embedding component at item ${index}, position ${componentIndex}.`,
+      );
+    }
+
+    return component;
+  });
+
+  return embedding;
+}
+
+function normalizeBaseUrl(input: string): string {
+  return input.replace(/\/+$/, "");
+}
+
+function normalizeOptionalText(input?: string): string | undefined {
+  const trimmed = input?.trim();
+  return trimmed && trimmed.length > 0 ? trimmed : undefined;
+}

packages/rag/src/qdrant-vector-store.ts

@@ -0,0 +1,580 @@
+import { createHash } from "node:crypto";
+
+import { QdrantClient, type Schemas } from "@qdrant/js-client-rest";
+import {
+  type EmbeddingChunk,
+  type RetrievalFilters,
+  type RetrievalResult,
+  type VectorStore,
+} from "@voidbot/shared";
+
+import { type TextEmbedder, HashingTextEmbedder } from "./hash-embedder";
+
+const UPSERT_BATCH_SIZE = 256;
+const DELETE_BATCH_SIZE = 128;
+const COLLECTION_METADATA_VERSION = 1;
+
+type QdrantPayload = Schemas["Payload"];
+type QdrantPoint = Schemas["PointStruct"];
+type QdrantScoredPoint = Schemas["ScoredPoint"];
+type QdrantFilter = Schemas["Filter"];
+
+export interface StoredQdrantChunk extends EmbeddingChunk {
+  vector: number[];
+  indexedAt: string;
+}
+
+export interface QdrantVectorStoreOptions {
+  url: string;
+  collectionName: string;
+  corpusKind: "discord_history" | "repository_source";
+  apiKey?: string;
+  timeoutMs?: number;
+  embedder?: TextEmbedder;
+}
+
+export class QdrantVectorStore implements VectorStore {
+  private readonly client: QdrantClient;
+  private readonly embedder: TextEmbedder;
+  private readonly timeoutSeconds?: number;
+  private ensuredVectorLength?: number;
+
+  public constructor(private readonly options: QdrantVectorStoreOptions) {
+    this.embedder = options.embedder ?? new HashingTextEmbedder();
+    this.client = new QdrantClient({
+      url: options.url,
+      apiKey: options.apiKey,
+      checkCompatibility: true,
+    });
+    this.timeoutSeconds = options.timeoutMs
+      ? Math.max(1, Math.ceil(options.timeoutMs / 1000))
+      : undefined;
+  }
+
+  public async upsert(chunks: EmbeddingChunk[]): Promise<void> {
+    if (chunks.length === 0) {
+      return;
+    }
+
+    const vectors = await this.embedder.embedDocuments(chunks.map((chunk) => chunk.text));
+    const vectorLength = validateEmbeddingBatch(vectors, chunks.length);
+    await this.ensureCollection(vectorLength);
+    const indexedAt = new Date().toISOString();
+
+    for (let index = 0; index < chunks.length; index += UPSERT_BATCH_SIZE) {
+      const batch = chunks.slice(index, index + UPSERT_BATCH_SIZE);
+      const points = batch.map((chunk, offset) =>
+        toQdrantPoint({
+          ...chunk,
+          vector: vectors[index + offset],
+          indexedAt,
+        }),
+      );
+
+      await this.client.upsert(this.options.collectionName, {
+        wait: true,
+        timeout: this.timeoutSeconds,
+        points,
+      });
+    }
+  }
+
+  public async upsertStoredChunks(chunks: StoredQdrantChunk[]): Promise<void> {
+    if (chunks.length === 0) {
+      return;
+    }
+
+    const vectorLength = validateSingleEmbedding(chunks[0]?.vector ?? []);
+    await this.ensureCollection(vectorLength);
+
+    for (let index = 0; index < chunks.length; index += UPSERT_BATCH_SIZE) {
+      const batch = chunks.slice(index, index + UPSERT_BATCH_SIZE);
+      const points = batch.map((chunk) => toQdrantPoint(chunk));
+
+      await this.client.upsert(this.options.collectionName, {
+        wait: true,
+        timeout: this.timeoutSeconds,
+        points,
+      });
+    }
+  }
+
+  public async clear(): Promise<void> {
+    const exists = await this.client.collectionExists(this.options.collectionName);
+
+    if (!exists.exists) {
+      return;
+    }
+
+    await this.client.deleteCollection(this.options.collectionName, {
+      timeout: this.timeoutSeconds,
+    });
+    this.ensuredVectorLength = undefined;
+  }
+
+  public async deleteBySourceIds(sourceIds: string[]): Promise<void> {
+    for (let index = 0; index < sourceIds.length; index += DELETE_BATCH_SIZE) {
+      const batch = sourceIds.slice(index, index + DELETE_BATCH_SIZE);
+      const filter = buildMatchAnyFilter("sourceId", batch);
+
+      if (!filter) {
+        continue;
+      }
+
+      await this.deleteByFilter(filter);
+    }
+  }
+
+  public async deleteByFilters(filters: RetrievalFilters): Promise<void> {
+    const filter = toQdrantFilter(filters);
+
+    if (!filter) {
+      return;
+    }
+
+    await this.deleteByFilter(filter);
+  }
+
+  public async query(
+    query: string,
+    limit: number,
+    filters?: RetrievalFilters,
+  ): Promise<RetrievalResult[]> {
+    if (query.trim().length === 0) {
+      return [];
+    }
+
+    const queryVector = await this.embedder.embedQuery(query);
+    const vectorLength = validateSingleEmbedding(queryVector);
+    const exists = await this.ensureCollection(vectorLength, { createIfMissing: false });
+
+    if (!exists) {
+      return [];
+    }
+
+    const results = await this.client.search(this.options.collectionName, {
+      vector: queryVector,
+      limit,
+      timeout: this.timeoutSeconds,
+      with_payload: true,
+      with_vector: false,
+      filter: toQdrantFilter(filters),
+    });
+
+    return results
+      .map((point) => fromQdrantPoint(point))
+      .filter((result): result is RetrievalResult => Boolean(result));
+  }
+
+  private async deleteByFilter(filter: QdrantFilter): Promise<void> {
+    const exists = await this.client.collectionExists(this.options.collectionName);
+
+    if (!exists.exists) {
+      return;
+    }
+
+    await this.client.delete(this.options.collectionName, {
+      wait: true,
+      timeout: this.timeoutSeconds,
+      filter,
+    });
+  }
+
+  private async ensureCollection(
+    vectorLength: number,
+    options?: {
+      createIfMissing?: boolean;
+    },
+  ): Promise<boolean> {
+    if (this.ensuredVectorLength === vectorLength) {
+      return true;
+    }
+
+    const createIfMissing = options?.createIfMissing ?? true;
+    const exists = await this.client.collectionExists(this.options.collectionName);
+
+    if (!exists.exists) {
+      if (!createIfMissing) {
+        return false;
+      }
+
+      await this.client.createCollection(this.options.collectionName, {
+        timeout: this.timeoutSeconds,
+        vectors: {
+          size: vectorLength,
+          distance: "Cosine",
+          on_disk: true,
+        },
+        on_disk_payload: true,
+        metadata: {
+          managedBy: "voidbot",
+          corpusKind: this.options.corpusKind,
+          embedderId: this.embedder.id,
+          vectorLength,
+          schemaVersion: COLLECTION_METADATA_VERSION,
+        },
+      });
+      await this.ensurePayloadIndexes();
+      this.ensuredVectorLength = vectorLength;
+      return true;
+    }
+
+    const collection = await this.client.getCollection(this.options.collectionName);
+    validateCollectionCompatibility(collection, {
+      collectionName: this.options.collectionName,
+      corpusKind: this.options.corpusKind,
+      embedderId: this.embedder.id,
+      vectorLength,
+    });
+    await this.ensurePayloadIndexes(collection);
+    this.ensuredVectorLength = vectorLength;
+    return true;
+  }
+
+  private async ensurePayloadIndexes(existingCollection?: Schemas["CollectionInfo"]): Promise<void> {
+    const collection =
+      existingCollection ??
+      (await this.client.getCollection(this.options.collectionName));
+    const existingIndexes = new Set(Object.keys(collection.payload_schema ?? {}));
+
+    for (const index of buildPayloadIndexDefinitions(this.options.corpusKind)) {
+      if (existingIndexes.has(index.field_name)) {
+        continue;
+      }
+
+      await this.client.createPayloadIndex(this.options.collectionName, {
+        wait: true,
+        timeout: this.timeoutSeconds,
+        field_name: index.field_name,
+        field_schema: index.field_schema,
+      });
+    }
+  }
+}
+
+function toQdrantPoint(chunk: StoredQdrantChunk): QdrantPoint {
+  return {
+    id: toQdrantPointId(chunk.id),
+    vector: chunk.vector,
+    payload: buildPayload(chunk),
+  };
+}
+
+function toQdrantPointId(chunkId: string): string {
+  const digest = createHash("sha256").update(chunkId).digest();
+
+  // Shape the first 16 bytes into an RFC 4122 version 5 UUID so Qdrant
+  // accepts our stable point IDs while we keep the real chunk ID in payload.
+  digest[6] = (digest[6] & 0x0f) | 0x50;
+  digest[8] = (digest[8] & 0x3f) | 0x80;
+
+  const hex = digest.subarray(0, 16).toString("hex");
+
+  return [
+    hex.slice(0, 8),
+    hex.slice(8, 12),
+    hex.slice(12, 16),
+    hex.slice(16, 20),
+    hex.slice(20, 32),
+  ].join("-");
+}
+
+function buildPayload(chunk: StoredQdrantChunk): QdrantPayload {
+  const payload: QdrantPayload = {
+    chunkId: chunk.id,
+    sourceId: chunk.sourceId,
+    sourceKind: chunk.sourceKind,
+    text: chunk.text,
+    indexedAt: chunk.indexedAt,
+    corpusKind: chunk.metadata.corpusKind ?? inferCorpusKind(chunk.sourceKind),
+    metadata: {
+      ...chunk.metadata,
+      sourceId: chunk.metadata.sourceId ?? chunk.sourceId,
+    },
+  };
+
+  copyOptionalMetadataField(payload, chunk.metadata, "guildId");
+  copyOptionalMetadataField(payload, chunk.metadata, "channelId");
+  copyOptionalMetadataField(payload, chunk.metadata, "authorId");
+  copyOptionalMetadataField(payload, chunk.metadata, "threadId");
+  copyOptionalMetadataField(payload, chunk.metadata, "repoName");
+  copyOptionalMetadataField(payload, chunk.metadata, "language");
+  copyOptionalMetadataField(payload, chunk.metadata, "path");
+
+  const path = typeof chunk.metadata.path === "string" ? chunk.metadata.path : undefined;
+
+  if (path) {
+    payload.pathPrefixes = buildPathPrefixes(path);
+  }
+
+  return payload;
+}
+
+function copyOptionalMetadataField(
+  payload: QdrantPayload,
+  metadata: Record<string, string>,
+  key: string,
+): void {
+  const value = metadata[key];
+
+  if (typeof value === "string" && value.length > 0) {
+    payload[key] = value;
+  }
+}
+
+function buildPathPrefixes(path: string): string[] {
+  const normalized = path.replace(/\\/g, "/").replace(/^\/+/, "");
+
+  if (normalized.length === 0) {
+    return [];
+  }
+
+  const prefixes = new Set<string>();
+  const segments = normalized.split("/").filter((segment) => segment.length > 0);
+
+  for (let index = 0; index < segments.length; index += 1) {
+    const prefix = segments.slice(0, index + 1).join("/");
+    prefixes.add(prefix);
+
+    if (index < segments.length - 1) {
+      prefixes.add(`${prefix}/`);
+    }
+  }
+
+  return [...prefixes];
+}
+
+function inferCorpusKind(
+  sourceKind: StoredQdrantChunk["sourceKind"],
+): "discord_history" | "repository_source" {
+  return sourceKind === "source_document" ? "repository_source" : "discord_history";
+}
+
+function fromQdrantPoint(point: QdrantScoredPoint): RetrievalResult | undefined {
+  const payload = toPayloadRecord(point.payload);
+  const chunkId = readString(payload.chunkId);
+  const sourceId = readString(payload.sourceId);
+  const sourceKind = readSourceKind(payload.sourceKind);
+  const text = readString(payload.text);
+
+  if (!chunkId || !sourceId || !sourceKind || !text) {
+    return undefined;
+  }
+
+  return {
+    chunkId,
+    score: point.score,
+    text,
+    sourceId,
+    sourceKind,
+    metadata: extractMetadata(payload),
+  };
+}
+
+function extractMetadata(payload: Record<string, unknown>): Record<string, string> {
+  const metadata = toPayloadRecord(payload.metadata);
+
+  if (metadata) {
+    const normalized: Record<string, string> = {};
+
+    for (const [key, value] of Object.entries(metadata)) {
+      if (typeof value === "string") {
+        normalized[key] = value;
+      }
+    }
+
+    return normalized;
+  }
+
+  const normalized: Record<string, string> = {};
+
+  for (const key of ["corpusKind", "guildId", "channelId", "authorId", "threadId", "repoName", "language", "path", "sourceId"]) {
+    const value = readString(payload[key]);
+
+    if (value) {
+      normalized[key] = value;
+    }
+  }
+
+  return normalized;
+}
+
+function toPayloadRecord(payload: unknown): Record<string, unknown> {
+  return typeof payload === "object" && payload !== null ? (payload as Record<string, unknown>) : {};
+}
+
+function readString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function readSourceKind(value: unknown): "discord_message" | "source_document" | undefined {
+  return value === "discord_message" || value === "source_document" ? value : undefined;
+}
+
+function validateEmbeddingBatch(vectors: number[][], expectedCount: number): number {
+  if (vectors.length !== expectedCount) {
+    throw new Error(
+      `Embedding backend returned ${vectors.length} vectors for ${expectedCount} chunks.`,
+    );
+  }
+
+  return validateSingleEmbedding(vectors[0] ?? []);
+}
+
+function validateSingleEmbedding(vector: number[]): number {
+  if (vector.length === 0) {
+    throw new Error("Embedding backend returned an empty vector.");
+  }
+
+  return vector.length;
+}
+
+function validateCollectionCompatibility(
+  collection: Schemas["CollectionInfo"],
+  expected: {
+    collectionName: string;
+    corpusKind: "discord_history" | "repository_source";
+    embedderId: string;
+    vectorLength: number;
+  },
+): void {
+  const actualVectorLength = readCollectionVectorLength(collection.config.params.vectors);
+
+  if (actualVectorLength !== undefined && actualVectorLength !== expected.vectorLength) {
+    throw new Error(
+      `Qdrant collection ${expected.collectionName} stores ${actualVectorLength}-dimensional vectors, but Void expects ${expected.vectorLength}. Run a rebuild or migrate into a clean collection.`,
+    );
+  }
+
+  const metadata = toPayloadRecord(collection.config.metadata);
+  const collectionCorpusKind = readString(metadata.corpusKind);
+  const collectionEmbedderId = readString(metadata.embedderId);
+
+  if (collectionCorpusKind && collectionCorpusKind !== expected.corpusKind) {
+    throw new Error(
+      `Qdrant collection ${expected.collectionName} is tagged for corpus ${collectionCorpusKind}, not ${expected.corpusKind}.`,
+    );
+  }
+
+  if (collectionEmbedderId && collectionEmbedderId !== expected.embedderId) {
+    throw new Error(
+      `Qdrant collection ${expected.collectionName} was built with embedder ${collectionEmbedderId}, but Void is configured for ${expected.embedderId}. Run a rebuild or migrate into a clean collection.`,
+    );
+  }
+}
+
+function readCollectionVectorLength(
+  vectors: Schemas["VectorsConfig"] | undefined,
+): number | undefined {
+  if (!vectors) {
+    return undefined;
+  }
+
+  if ("size" in vectors && typeof vectors.size === "number") {
+    return vectors.size;
+  }
+
+  for (const value of Object.values(vectors)) {
+    if (value && typeof value === "object" && "size" in value && typeof value.size === "number") {
+      return value.size;
+    }
+  }
+
+  return undefined;
+}
+
+function buildPayloadIndexDefinitions(
+  corpusKind: "discord_history" | "repository_source",
+): Array<{
+  field_name: string;
+  field_schema:
+    | "keyword"
+    | {
+        type: "text";
+        tokenizer?: "word" | "whitespace" | "prefix" | "multilingual";
+      };
+}> {
+  const shared = [
+    { field_name: "sourceId", field_schema: "keyword" as const },
+    { field_name: "corpusKind", field_schema: "keyword" as const },
+  ];
+
+  if (corpusKind === "discord_history") {
+    return [
+      ...shared,
+      { field_name: "guildId", field_schema: "keyword" as const },
+      { field_name: "channelId", field_schema: "keyword" as const },
+      { field_name: "authorId", field_schema: "keyword" as const },
+      { field_name: "threadId", field_schema: "keyword" as const },
+    ];
+  }
+
+  return [
+    ...shared,
+    { field_name: "repoName", field_schema: "keyword" as const },
+    { field_name: "language", field_schema: "keyword" as const },
+    { field_name: "pathPrefixes", field_schema: "keyword" as const },
+  ];
+}
+
+function toQdrantFilter(filters?: RetrievalFilters): QdrantFilter | undefined {
+  if (!filters) {
+    return undefined;
+  }
+
+  const must: Schemas["Condition"][] = [];
+
+  appendExactMatch(must, "corpusKind", filters.corpusKind);
+  appendExactMatch(must, "guildId", filters.guildId);
+  appendExactMatch(must, "channelId", filters.channelId);
+  appendExactMatch(must, "authorId", filters.authorId);
+  appendExactMatch(must, "repoName", filters.repoName);
+  appendExactMatch(must, "language", filters.language);
+  appendExactMatch(must, "sourceId", filters.sourceId);
+
+  if (filters.pathPrefix) {
+    appendExactMatch(must, "pathPrefixes", normalizePathPrefixFilter(filters.pathPrefix));
+  }
+
+  return must.length > 0 ? { must } : undefined;
+}
+
+function buildMatchAnyFilter(field: string, values: string[]): QdrantFilter | undefined {
+  const normalizedValues = values.filter((value) => value.length > 0);
+
+  if (normalizedValues.length === 0) {
+    return undefined;
+  }
+
+  return {
+    must: [
+      {
+        key: field,
+        match: {
+          any: normalizedValues,
+        },
+      },
+    ],
+  };
+}
+
+function appendExactMatch(
+  must: Schemas["Condition"][],
+  key: string,
+  value: string | undefined,
+): void {
+  if (!value || value.length === 0) {
+    return;
+  }
+
+  must.push({
+    key,
+    match: {
+      value,
+    },
+  });
+}
+
+function normalizePathPrefixFilter(value: string): string {
+  return value.replace(/\\/g, "/").replace(/^\/+/, "");
+}

packages/rag/src/rag-pipeline.ts

@@ -0,0 +1,67 @@
+import { type ArchivedMessage, type VectorStore } from "@voidbot/shared";
+
+import { HistoryIngester } from "./history-ingester";
+import {
+  type ArchivedMessageRecord,
+  type MessageArchiveMutationResult,
+  FileMessageArchiveRepository,
+} from "./message-archive";
+
+export interface RagIngestResult {
+  createdMessages: number;
+  updatedMessages: number;
+  unchangedMessages: number;
+  indexedChunks: number;
+}
+
+export class RagPipeline {
+  public constructor(
+    private readonly archiveRepository: FileMessageArchiveRepository,
+    private readonly historyIngester: HistoryIngester,
+    private readonly vectorStore: VectorStore,
+  ) {}
+
+  public async upsertMessages(messages: ArchivedMessage[]): Promise<RagIngestResult> {
+    const mutation = await this.archiveRepository.upsert(messages);
+    await this.replaceChangedChunks(mutation);
+
+    return {
+      createdMessages: mutation.created,
+      updatedMessages: mutation.updated,
+      unchangedMessages: mutation.unchanged,
+      indexedChunks: this.historyIngester.chunkMessages(
+        mutation.storedMessages.filter((message) => !message.deletedAt),
+      ).length,
+    };
+  }
+
+  public async markDeleted(messageId: string, deletedAt?: string): Promise<boolean> {
+    const changed = await this.archiveRepository.markDeleted(messageId, deletedAt);
+
+    if (!changed) {
+      return false;
+    }
+
+    await this.vectorStore.deleteBySourceIds([messageId]);
+    return true;
+  }
+
+  public async listChannelMessages(channelId: string, limit = 100): Promise<ArchivedMessageRecord[]> {
+    return this.archiveRepository.listByChannel(channelId, limit);
+  }
+
+  private async replaceChangedChunks(mutation: MessageArchiveMutationResult): Promise<void> {
+    if (mutation.changedMessageIds.length === 0) {
+      return;
+    }
+
+    await this.vectorStore.deleteBySourceIds(mutation.changedMessageIds);
+
+    const activeMessages = mutation.storedMessages.filter((message) => !message.deletedAt);
+    const chunks = this.historyIngester.chunkMessages(activeMessages);
+
+    if (chunks.length > 0) {
+      await this.vectorStore.upsert(chunks);
+    }
+  }
+}

packages/rag/src/retrieval-service.ts

@@ -0,0 +1,42 @@
+import {
+  type RetrievalFilters,
+  type RetrievalResult,
+  type VectorStore,
+} from "@voidbot/shared";
+
+export class RetrievalService {
+  public constructor(
+    private readonly historyVectorStore: VectorStore,
+    private readonly sourceVectorStore: VectorStore = historyVectorStore,
+  ) {}
+
+  public async search(
+    query: string,
+    limit = 5,
+    filters?: RetrievalFilters,
+  ): Promise<RetrievalResult[]> {
+    return this.historyVectorStore.query(query, limit, filters);
+  }
+
+  public async searchHistory(
+    query: string,
+    limit = 5,
+    filters?: Omit<RetrievalFilters, "corpusKind">,
+  ): Promise<RetrievalResult[]> {
+    return this.historyVectorStore.query(query, limit, {
+      ...filters,
+      corpusKind: "discord_history",
+    });
+  }
+
+  public async searchRepositorySources(
+    query: string,
+    limit = 5,
+    filters?: Omit<RetrievalFilters, "corpusKind" | "guildId" | "channelId" | "authorId">,
+  ): Promise<RetrievalResult[]> {
+    return this.sourceVectorStore.query(query, limit, {
+      ...filters,
+      corpusKind: "repository_source",
+    });
+  }
+}

packages/rag/src/source-document-archive.ts

@@ -0,0 +1,205 @@
+import { normalizeText } from "./history-ingester";
+import { SerializedFileStore } from "./file-store";
+
+export interface ArchivedSourceDocument {
+  id: string;
+  repoName: string;
+  path: string;
+  language?: string;
+  title?: string;
+  content: string;
+  lastModifiedAt?: string;
+  metadata?: Record<string, string>;
+}
+
+export interface ArchivedSourceDocumentRecord extends ArchivedSourceDocument {
+  normalizedContent: string;
+  indexedAt: string;
+}
+
+export interface IndexedSourceRepoSummary {
+  repoName: string;
+  documentCount: number;
+}
+
+interface SourceDocumentArchiveStore {
+  version: 1;
+  documents: ArchivedSourceDocumentRecord[];
+}
+
+export interface SourceDocumentSyncResult {
+  created: number;
+  updated: number;
+  unchanged: number;
+  deleted: number;
+  previousRepoSourceIds: string[];
+  changedSourceIds: string[];
+  changedDocuments: ArchivedSourceDocumentRecord[];
+  activeDocuments: ArchivedSourceDocumentRecord[];
+}
+
+export class FileSourceDocumentArchiveRepository {
+  private readonly store: SerializedFileStore<SourceDocumentArchiveStore>;
+
+  public constructor(filePath: string) {
+    this.store = new SerializedFileStore(filePath, () => ({
+      version: 1,
+      documents: [],
+    }));
+  }
+
+  public async syncRepoDocuments(
+    repoName: string,
+    documents: ArchivedSourceDocument[],
+  ): Promise<SourceDocumentSyncResult> {
+    return this.store.mutate((store) => {
+      const now = new Date().toISOString();
+      const previousRepoSourceIds = store.documents
+        .filter((document) => document.repoName === repoName)
+        .map((document) => document.id);
+      const nextDocuments = documents.map((document) => toArchivedSourceDocumentRecord(document, now));
+      const existingById = new Map(store.documents.map((document, index) => [document.id, index]));
+      const nextIds = new Set(nextDocuments.map((document) => document.id));
+      const changedSourceIds: string[] = [];
+      const changedDocuments: ArchivedSourceDocumentRecord[] = [];
+      let created = 0;
+      let updated = 0;
+      let unchanged = 0;
+      let deleted = 0;
+
+      for (const document of nextDocuments) {
+        const position = existingById.get(document.id);
+
+        if (position === undefined) {
+          store.documents.push(document);
+          existingById.set(document.id, store.documents.length - 1);
+          changedSourceIds.push(document.id);
+          changedDocuments.push(document);
+          created += 1;
+          continue;
+        }
+
+        const existing = store.documents[position];
+
+        if (areEquivalent(existing, document)) {
+          unchanged += 1;
+          continue;
+        }
+
+        store.documents[position] = document;
+        changedSourceIds.push(document.id);
+        changedDocuments.push(document);
+        updated += 1;
+      }
+
+      const retainedDocuments = store.documents.filter((document) => {
+        if (document.repoName !== repoName) {
+          return true;
+        }
+
+        if (nextIds.has(document.id)) {
+          return true;
+        }
+
+        changedSourceIds.push(document.id);
+        deleted += 1;
+        return false;
+      });
+
+      store.documents = retainedDocuments;
+
+      return {
+        created,
+        updated,
+        unchanged,
+        deleted,
+        previousRepoSourceIds,
+        changedSourceIds,
+        changedDocuments,
+        activeDocuments: retainedDocuments
+          .filter((document) => document.repoName === repoName)
+          .sort((left, right) => left.path.localeCompare(right.path)),
+      };
+    });
+  }
+
+  public async get(sourceId: string): Promise<ArchivedSourceDocumentRecord | undefined> {
+    const store = await this.store.snapshot();
+    return store.documents.find((document) => document.id === sourceId);
+  }
+
+  public async listAll(): Promise<ArchivedSourceDocumentRecord[]> {
+    const store = await this.store.snapshot();
+    return store.documents.slice().sort(compareDocuments);
+  }
+
+  public async listByRepo(repoName: string): Promise<ArchivedSourceDocumentRecord[]> {
+    const documents = await this.listAll();
+    return documents.filter((document) => document.repoName === repoName);
+  }
+
+  public async listRepoSummaries(): Promise<IndexedSourceRepoSummary[]> {
+    const store = await this.store.snapshot();
+    const counts = new Map<string, number>();
+
+    for (const document of store.documents) {
+      counts.set(document.repoName, (counts.get(document.repoName) ?? 0) + 1);
+    }
+
+    return [...counts.entries()]
+      .map(([repoName, documentCount]) => ({
+        repoName,
+        documentCount,
+      }))
+      .sort((left, right) => left.repoName.localeCompare(right.repoName));
+  }
+
+  public async count(): Promise<number> {
+    const store = await this.store.snapshot();
+    return store.documents.length;
+  }
+}
+
+function toArchivedSourceDocumentRecord(
+  document: ArchivedSourceDocument,
+  indexedAt: string,
+): ArchivedSourceDocumentRecord {
+  return {
+    ...document,
+    normalizedContent: normalizeText(document.content),
+    indexedAt,
+  };
+}
+
+function areEquivalent(
+  left: ArchivedSourceDocumentRecord,
+  right: ArchivedSourceDocumentRecord,
+): boolean {
+  return JSON.stringify(comparableFields(left)) === JSON.stringify(comparableFields(right));
+}
+
+function comparableFields(document: ArchivedSourceDocumentRecord): Record<string, unknown> {
+  return {
+    id: document.id,
+    repoName: document.repoName,
+    path: document.path,
+    language: document.language,
+    title: document.title,
+    content: document.content,
+    lastModifiedAt: document.lastModifiedAt,
+    metadata: document.metadata ?? {},
+  };
+}
+
+function compareDocuments(
+  left: ArchivedSourceDocumentRecord,
+  right: ArchivedSourceDocumentRecord,
+): number {
+  const repoComparison = left.repoName.localeCompare(right.repoName);
+
+  if (repoComparison !== 0) {
+    return repoComparison;
+  }
+
+  return left.path.localeCompare(right.path);
+}

packages/rag/src/source-document-ingester.ts

@@ -0,0 +1,148 @@
+import { type EmbeddingChunk, type VectorStore } from "@voidbot/shared";
+
+import { normalizeText } from "./history-ingester";
+import { type ArchivedSourceDocument } from "./source-document-archive";
+
+const DEFAULT_MAX_CHARS = 1400;
+const DEFAULT_MAX_LINES = 48;
+
+interface SourceChunkWindow {
+  text: string;
+  lineStart: number;
+  lineEnd: number;
+}
+
+export interface SourceDocumentChunkContext {
+  chunkId: string;
+  chunkIndex: number;
+  lineStart: number;
+  lineEnd: number;
+  text: string;
+}
+
+export class SourceDocumentIngester {
+  public constructor(
+    private readonly maxChars = DEFAULT_MAX_CHARS,
+    private readonly maxLines = DEFAULT_MAX_LINES,
+  ) {}
+
+  public chunkDocuments(documents: ArchivedSourceDocument[]): EmbeddingChunk[] {
+    return documents.flatMap((document) => this.chunkDocument(document));
+  }
+
+  public async ingestDocuments(
+    documents: ArchivedSourceDocument[],
+    vectorStore: VectorStore,
+  ): Promise<EmbeddingChunk[]> {
+    const chunks = this.chunkDocuments(documents);
+    await vectorStore.upsert(chunks);
+    return chunks;
+  }
+
+  public chunkDocument(document: ArchivedSourceDocument): EmbeddingChunk[] {
+    const windows = splitSourceDocument(document.content, this.maxChars, this.maxLines);
+
+    return windows.map((window, index) => ({
+      id: `${document.id}:${index}`,
+      sourceId: document.id,
+      sourceKind: "source_document",
+      text: window.text,
+      normalizedText: normalizeText(window.text),
+      metadata: {
+        corpusKind: "repository_source",
+        sourceId: document.id,
+        repoName: document.repoName,
+        path: document.path,
+        language: document.language ?? "",
+        title: document.title ?? document.path,
+        chunkIndex: String(index),
+        chunkCount: String(windows.length),
+        lineStart: String(window.lineStart),
+        lineEnd: String(window.lineEnd),
+        lastModifiedAt: document.lastModifiedAt ?? "",
+        ...document.metadata,
+      },
+    }));
+  }
+
+  public buildContextWindow(
+    document: ArchivedSourceDocument,
+    anchorChunkIndex: number,
+    before = 1,
+    after = 1,
+  ): SourceDocumentChunkContext[] {
+    const chunks = this.chunkDocument(document);
+    const safeAnchor = clamp(anchorChunkIndex, 0, Math.max(0, chunks.length - 1));
+    const start = Math.max(0, safeAnchor - before);
+    const end = Math.min(chunks.length, safeAnchor + after + 1);
+
+    return chunks.slice(start, end).map((chunk) => ({
+      chunkId: chunk.id,
+      chunkIndex: Number(chunk.metadata.chunkIndex ?? 0),
+      lineStart: Number(chunk.metadata.lineStart ?? 1),
+      lineEnd: Number(chunk.metadata.lineEnd ?? 1),
+      text: chunk.text,
+    }));
+  }
+}
+
+function clamp(value: number, min: number, max: number): number {
+  return Math.min(max, Math.max(min, value));
+}
+
+function splitSourceDocument(
+  input: string,
+  maxChars: number,
+  maxLines: number,
+): SourceChunkWindow[] {
+  const normalized = input.replace(/\r\n/g, "\n");
+
+  if (normalized.trim().length === 0) {
+    return [
+      {
+        text: "(empty file)",
+        lineStart: 1,
+        lineEnd: 1,
+      },
+    ];
+  }
+
+  const lines = normalized.split("\n");
+  const windows: SourceChunkWindow[] = [];
+  let cursor = 0;
+
+  while (cursor < lines.length) {
+    let end = cursor;
+    let size = 0;
+
+    while (end < lines.length) {
+      const candidateLine = lines[end] ?? "";
+      const nextSize = size + candidateLine.length + (end === cursor ? 0 : 1);
+      const nextLineCount = end - cursor + 1;
+
+      if (end > cursor && (nextSize > maxChars || nextLineCount > maxLines)) {
+        break;
+      }
+
+      size = nextSize;
+      end += 1;
+
+      if (candidateLine.length > maxChars && end === cursor + 1) {
+        break;
+      }
+    }
+
+    const slice = lines.slice(cursor, end);
+    const chunkText = slice.join("\n").trimEnd();
+
+    windows.push({
+      text: chunkText.length > 0 ? chunkText : "(blank lines)",
+      lineStart: cursor + 1,
+      lineEnd: end,
+    });
+
+    cursor = end;
+  }
+
+  return windows;
+}

packages/rag/src/source-rag-pipeline.ts

@@ -0,0 +1,78 @@
+import { type VectorStore } from "@voidbot/shared";
+
+import {
+  type ArchivedSourceDocument,
+  type SourceDocumentSyncResult,
+  FileSourceDocumentArchiveRepository,
+} from "./source-document-archive";
+import { SourceDocumentIngester } from "./source-document-ingester";
+
+const SOURCE_UPSERT_BATCH_SIZE = 64;
+
+export interface SourceRagIngestResult {
+  createdDocuments: number;
+  updatedDocuments: number;
+  unchangedDocuments: number;
+  deletedDocuments: number;
+  indexedChunks: number;
+}
+
+export class SourceRagPipeline {
+  public constructor(
+    private readonly archiveRepository: FileSourceDocumentArchiveRepository,
+    private readonly sourceIngester: SourceDocumentIngester,
+    private readonly vectorStore: VectorStore,
+  ) {}
+
+  public async syncRepoDocuments(
+    repoName: string,
+    documents: ArchivedSourceDocument[],
+    options?: {
+      forceReindex?: boolean;
+    },
+  ): Promise<SourceRagIngestResult> {
+    const mutation = await this.archiveRepository.syncRepoDocuments(repoName, documents);
+    await this.replaceChangedChunks(repoName, mutation, options?.forceReindex ?? false);
+
+    return {
+      createdDocuments: mutation.created,
+      updatedDocuments: mutation.updated,
+      unchangedDocuments: mutation.unchanged,
+      deletedDocuments: mutation.deleted,
+      indexedChunks: this.sourceIngester.chunkDocuments(mutation.activeDocuments).length,
+    };
+  }
+
+  private async replaceChangedChunks(
+    repoName: string,
+    mutation: SourceDocumentSyncResult,
+    forceReindex: boolean,
+  ): Promise<void> {
+    const documentsToUpsert = forceReindex ? mutation.activeDocuments : mutation.changedDocuments;
+
+    if (forceReindex) {
+      await this.vectorStore.deleteByFilters({
+        corpusKind: "repository_source",
+        repoName,
+      });
+    } else if (mutation.changedSourceIds.length > 0) {
+      await this.vectorStore.deleteBySourceIds(mutation.changedSourceIds);
+    }
+
+    if (documentsToUpsert.length === 0) {
+      return;
+    }
+
+    const chunks = this.sourceIngester.chunkDocuments(documentsToUpsert);
+
+    for (let index = 0; index < chunks.length; index += SOURCE_UPSERT_BATCH_SIZE) {
+      const batch = chunks.slice(index, index + SOURCE_UPSERT_BATCH_SIZE);
+
+      if (batch.length === 0) {
+        continue;
+      }
+
+      await this.vectorStore.upsert(batch);
+    }
+  }
+}

packages/rag/src/source-repo-crawler.ts

@@ -0,0 +1,407 @@
+import { readdir, readFile, stat } from "node:fs/promises";
+import { join, relative } from "node:path";
+
+import { type ArchivedSourceDocument } from "./source-document-archive";
+
+const SKIPPED_DIRECTORIES = new Set([
+  ".git",
+  ".github",
+  ".next",
+  ".npm-cache",
+  ".obsidian",
+  ".rag",
+  ".turbo",
+  ".venv",
+  ".vscode",
+  ".voidbot",
+  "artifacts",
+  "bin",
+  "build",
+  "coverage",
+  "dist",
+  "Library",
+  "node_modules",
+  "obj",
+  "Temp",
+]);
+
+const BINARY_EXTENSIONS = new Set([
+  ".7z",
+  ".avi",
+  ".bmp",
+  ".class",
+  ".dll",
+  ".dylib",
+  ".eot",
+  ".exe",
+  ".gif",
+  ".gz",
+  ".ico",
+  ".jar",
+  ".jpeg",
+  ".jpg",
+  ".lock",
+  ".map",
+  ".mp3",
+  ".mp4",
+  ".mov",
+  ".ogg",
+  ".otf",
+  ".pdf",
+  ".png",
+  ".rar",
+  ".so",
+  ".svg",
+  ".tar",
+  ".ttf",
+  ".wav",
+  ".webm",
+  ".webp",
+  ".woff",
+  ".woff2",
+  ".zip",
+]);
+
+const DEFAULT_MAX_FILE_BYTES = 512 * 1024;
+
+export interface RepoCrawlerOptions {
+  maxFileBytes?: number;
+  includePathPrefixes?: string[];
+}
+
+export interface RepoDocumentScanResult {
+  repoName: string;
+  repoPath: string;
+  documents: ArchivedSourceDocument[];
+  skippedFiles: string[];
+}
+
+export async function crawlRepositoryDocuments(
+  repoPath: string,
+  repoName: string,
+  options: RepoCrawlerOptions = {},
+): Promise<RepoDocumentScanResult> {
+  const documents: ArchivedSourceDocument[] = [];
+  const skippedFiles: string[] = [];
+  const maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES;
+  const includePathPrefixes = normalizePathPrefixes(options.includePathPrefixes);
+
+  await walkDirectory(repoPath, async (absolutePath) => {
+    const fileStat = await stat(absolutePath);
+
+    if (!fileStat.isFile()) {
+      return;
+    }
+
+    if (fileStat.size > maxFileBytes) {
+      skippedFiles.push(relative(repoPath, absolutePath));
+      return;
+    }
+
+    const relativePath = relative(repoPath, absolutePath).replace(/\\/g, "/");
+
+    if (!shouldIndexFile(relativePath, includePathPrefixes)) {
+      return;
+    }
+
+    const content = await readTextFile(absolutePath);
+
+    if (!content) {
+      return;
+    }
+
+    documents.push({
+      id: `${repoName}:${relativePath}`,
+      repoName,
+      path: relativePath,
+      language: inferLanguage(relativePath),
+      title: relativePath,
+      content,
+      lastModifiedAt: fileStat.mtime.toISOString(),
+      metadata: {
+        repoName,
+        path: relativePath,
+      },
+    });
+  });
+
+  documents.sort((left, right) => left.path.localeCompare(right.path));
+
+  return {
+    repoName,
+    repoPath,
+    documents,
+    skippedFiles,
+  };
+}
+
+async function walkDirectory(
+  directoryPath: string,
+  onFile: (absolutePath: string) => Promise<void>,
+): Promise<void> {
+  const entries = await readdir(directoryPath, { withFileTypes: true });
+
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      if (SKIPPED_DIRECTORIES.has(entry.name)) {
+        continue;
+      }
+
+      await walkDirectory(join(directoryPath, entry.name), onFile);
+      continue;
+    }
+
+    if (!entry.isFile()) {
+      continue;
+    }
+
+    await onFile(join(directoryPath, entry.name));
+  }
+}
+
+const SOURCE_AND_DOC_EXTENSIONS = new Set([
+  ".adoc",
+  ".astro",
+  ".bat",
+  ".c",
+  ".cc",
+  ".cmd",
+  ".conf",
+  ".cpp",
+  ".cs",
+  ".css",
+  ".csv",
+  ".env",
+  ".example",
+  ".fs",
+  ".go",
+  ".h",
+  ".hpp",
+  ".htm",
+  ".html",
+  ".ini",
+  ".java",
+  ".js",
+  ".json",
+  ".jsonc",
+  ".jsx",
+  ".kt",
+  ".kts",
+  ".less",
+  ".lua",
+  ".md",
+  ".mdx",
+  ".mjs",
+  ".php",
+  ".props",
+  ".ps1",
+  ".py",
+  ".rb",
+  ".resx",
+  ".rs",
+  ".rst",
+  ".sass",
+  ".scss",
+  ".shader",
+  ".sh",
+  ".sln",
+  ".sql",
+  ".svelte",
+  ".swift",
+  ".targets",
+  ".tex",
+  ".toml",
+  ".ts",
+  ".tsx",
+  ".txt",
+  ".unity",
+  ".uxml",
+  ".uss",
+  ".vue",
+  ".xml",
+  ".xsd",
+  ".yaml",
+  ".yml",
+]);
+
+const KNOWN_TEXT_FILENAMES = new Set([
+  ".editorconfig",
+  ".gitignore",
+  "cmakelists.txt",
+  "dockerfile",
+  "justfile",
+  "makefile",
+  "procfile",
+  "readme",
+  "readme.md",
+]);
+
+const NOISY_FILENAMES = new Set([
+  "package-lock.json",
+  "pnpm-lock.yaml",
+  "yarn.lock",
+  "bun.lockb",
+  "cargo.lock",
+  "composer.lock",
+]);
+
+function shouldIndexFile(relativePath: string, includePathPrefixes: string[]): boolean {
+  const normalized = relativePath.replace(/\\/g, "/");
+
+  if (normalized.length === 0) {
+    return false;
+  }
+
+  if (
+    includePathPrefixes.length > 0 &&
+    !includePathPrefixes.some((prefix) => normalized.startsWith(prefix))
+  ) {
+    return false;
+  }
+
+  if (normalized.startsWith("quartz-site/public/")) {
+    return false;
+  }
+
+  const lower = normalized.toLowerCase();
+  const fileName = lower.includes("/") ? lower.slice(lower.lastIndexOf("/") + 1) : lower;
+  const extension = lower.includes(".") ? lower.slice(lower.lastIndexOf(".")) : "";
+
+  if (BINARY_EXTENSIONS.has(extension)) {
+    return false;
+  }
+
+  if (NOISY_FILENAMES.has(fileName)) {
+    return false;
+  }
+
+  if (KNOWN_TEXT_FILENAMES.has(fileName)) {
+    return true;
+  }
+
+  return SOURCE_AND_DOC_EXTENSIONS.has(extension);
+}
+
+async function readTextFile(absolutePath: string): Promise<string | undefined> {
+  const buffer = await readFile(absolutePath);
+
+  if (buffer.includes(0)) {
+    return undefined;
+  }
+
+  const content = buffer.toString("utf8");
+  return content.trim().length > 0 ? content : undefined;
+}
+
+function inferLanguage(relativePath: string): string | undefined {
+  const lower = relativePath.toLowerCase();
+
+  if (lower.endsWith(".cs")) {
+    return "csharp";
+  }
+
+  if (lower.endsWith(".json")) {
+    return "json";
+  }
+
+  if (lower.endsWith(".jsonc")) {
+    return "jsonc";
+  }
+
+  if (lower.endsWith(".md")) {
+    return "markdown";
+  }
+
+  if (lower.endsWith(".tsx")) {
+    return "tsx";
+  }
+
+  if (lower.endsWith(".ts")) {
+    return "typescript";
+  }
+
+  if (lower.endsWith(".jsx")) {
+    return "jsx";
+  }
+
+  if (lower.endsWith(".js")) {
+    return "javascript";
+  }
+
+  if (lower.endsWith(".mjs")) {
+    return "javascript";
+  }
+
+  if (lower.endsWith(".py")) {
+    return "python";
+  }
+
+  if (lower.endsWith(".toml")) {
+    return "toml";
+  }
+
+  if (lower.endsWith(".yml") || lower.endsWith(".yaml")) {
+    return "yaml";
+  }
+
+  if (lower.endsWith(".html")) {
+    return "html";
+  }
+
+  if (lower.endsWith(".css")) {
+    return "css";
+  }
+
+  if (lower.endsWith(".txt")) {
+    return "text";
+  }
+
+  if (lower.endsWith(".sh")) {
+    return "shell";
+  }
+
+  if (lower.endsWith(".ps1")) {
+    return "powershell";
+  }
+
+  if (lower.endsWith(".sql")) {
+    return "sql";
+  }
+
+  if (lower.endsWith(".rs")) {
+    return "rust";
+  }
+
+  if (lower.endsWith(".go")) {
+    return "go";
+  }
+
+  if (lower.endsWith(".java")) {
+    return "java";
+  }
+
+  if (lower.endsWith(".kt") || lower.endsWith(".kts")) {
+    return "kotlin";
+  }
+
+  if (lower.endsWith(".cpp") || lower.endsWith(".cc") || lower.endsWith(".hpp") || lower.endsWith(".h")) {
+    return "cpp";
+  }
+
+  if (lower.endsWith(".unity")) {
+    return "unity-yaml";
+  }
+
+  if (lower.endsWith(".shader")) {
+    return "shaderlab";
+  }
+
+  return undefined;
+}
+
+function normalizePathPrefixes(prefixes?: string[]): string[] {
+  return (prefixes ?? [])
+    .map((prefix) => prefix.trim().replace(/\\/g, "/").replace(/^\.\/+/, ""))
+    .filter((prefix) => prefix.length > 0)
+    .map((prefix) => (prefix.endsWith("/") ? prefix : `${prefix}/`));
+}

packages/rag/src/source-vector-store.ts

@@ -0,0 +1,220 @@
+import { mkdir, readdir, rm } from "node:fs/promises";
+import { basename, join } from "node:path";
+
+import {
+  type EmbeddingChunk,
+  type RetrievalFilters,
+  type RetrievalResult,
+  type VectorStore,
+} from "@voidbot/shared";
+
+import { SerializedFileStore } from "./file-store";
+import { type TextEmbedder, HashingTextEmbedder, cosineSimilarity } from "./hash-embedder";
+import { FileVectorStore } from "./file-vector-store";
+
+interface StoredVectorChunk extends EmbeddingChunk {
+  vector: number[];
+  indexedAt: string;
+}
+
+interface VectorIndexStore {
+  version: 2;
+  embedderId: string;
+  dimensions: number;
+  chunks: StoredVectorChunk[];
+}
+
+export class ShardedSourceVectorStore implements VectorStore {
+  private readonly embedder: TextEmbedder;
+  private readonly storeCache = new Map<string, FileVectorStore>();
+
+  public constructor(
+    private readonly rootDirectory: string,
+    embedder: TextEmbedder = new HashingTextEmbedder(),
+  ) {
+    this.embedder = embedder;
+  }
+
+  public async upsert(chunks: EmbeddingChunk[]): Promise<void> {
+    const batches = groupChunksByRepo(chunks);
+
+    for (const [repoName, batch] of batches) {
+      await this.getStore(repoName).upsert(batch);
+    }
+  }
+
+  public async clear(): Promise<void> {
+    await rm(this.rootDirectory, { recursive: true, force: true });
+    this.storeCache.clear();
+  }
+
+  public async deleteBySourceIds(sourceIds: string[]): Promise<void> {
+    const groupedIds = new Map<string, string[]>();
+
+    for (const sourceId of sourceIds) {
+      const repoName = deriveRepoNameFromSourceId(sourceId);
+
+      if (!repoName) {
+        continue;
+      }
+
+      const bucket = groupedIds.get(repoName) ?? [];
+      bucket.push(sourceId);
+      groupedIds.set(repoName, bucket);
+    }
+
+    for (const [repoName, ids] of groupedIds) {
+      await this.getStore(repoName).deleteBySourceIds(ids);
+    }
+  }
+
+  public async deleteByFilters(filters: RetrievalFilters): Promise<void> {
+    const repoNames = await this.resolveCandidateRepos(filters);
+
+    for (const repoName of repoNames) {
+      await this.getStore(repoName).deleteByFilters(filters);
+    }
+  }
+
+  public async query(
+    query: string,
+    limit: number,
+    filters?: RetrievalFilters,
+  ): Promise<RetrievalResult[]> {
+    if (query.trim().length === 0) {
+      return [];
+    }
+
+    const queryVector = await this.embedder.embedQuery(query);
+    const repoNames = await this.resolveCandidateRepos(filters);
+    const results: RetrievalResult[] = [];
+
+    for (const repoName of repoNames) {
+      const shardStore = await this.readShardStore(repoName);
+
+      for (const chunk of shardStore.chunks) {
+        if (!matchesFilters(chunk.metadata, filters)) {
+          continue;
+        }
+
+        const score = cosineSimilarity(queryVector, chunk.vector);
+
+        if (score <= 0) {
+          continue;
+        }
+
+        results.push({
+          chunkId: chunk.id,
+          score,
+          text: chunk.text,
+          sourceId: chunk.sourceId,
+          sourceKind: chunk.sourceKind,
+          metadata: chunk.metadata,
+        });
+      }
+    }
+
+    return results
+      .sort((left, right) => right.score - left.score)
+      .slice(0, limit);
+  }
+
+  private getStore(repoName: string): FileVectorStore {
+    const cached = this.storeCache.get(repoName);
+
+    if (cached) {
+      return cached;
+    }
+
+    const store = new FileVectorStore(this.getShardPath(repoName), this.embedder);
+    this.storeCache.set(repoName, store);
+    return store;
+  }
+
+  private getShardPath(repoName: string): string {
+    return join(this.rootDirectory, `${encodeURIComponent(repoName)}.json`);
+  }
+
+  private async resolveCandidateRepos(filters?: RetrievalFilters): Promise<string[]> {
+    if (filters?.repoName) {
+      return [filters.repoName];
+    }
+
+    await mkdir(this.rootDirectory, { recursive: true });
+    const entries = await readdir(this.rootDirectory, { withFileTypes: true });
+
+    return entries
+      .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+      .map((entry) => decodeURIComponent(basename(entry.name, ".json")))
+      .sort((left, right) => left.localeCompare(right));
+  }
+
+  private async readShardStore(repoName: string): Promise<VectorIndexStore> {
+    const store = new SerializedFileStore<VectorIndexStore>(this.getShardPath(repoName), () => ({
+      version: 2,
+      embedderId: this.embedder.id,
+      dimensions: 0,
+      chunks: [],
+    }));
+
+    return store.snapshot();
+  }
+}
+
+function groupChunksByRepo(chunks: EmbeddingChunk[]): Map<string, EmbeddingChunk[]> {
+  const batches = new Map<string, EmbeddingChunk[]>();
+
+  for (const chunk of chunks) {
+    const repoName = chunk.metadata.repoName ?? deriveRepoNameFromSourceId(chunk.sourceId);
+
+    if (!repoName) {
+      throw new Error(`Cannot upsert source chunk ${chunk.id} without a repoName.`);
+    }
+
+    const batch = batches.get(repoName) ?? [];
+    batch.push(chunk);
+    batches.set(repoName, batch);
+  }
+
+  return batches;
+}
+
+function deriveRepoNameFromSourceId(sourceId: string): string | undefined {
+  const separatorIndex = sourceId.indexOf(":");
+  return separatorIndex === -1 ? undefined : sourceId.slice(0, separatorIndex);
+}
+
+function matchesFilters(
+  metadata: Record<string, string>,
+  filters?: RetrievalFilters,
+): boolean {
+  if (!filters) {
+    return true;
+  }
+
+  if (filters.corpusKind && metadata.corpusKind !== filters.corpusKind) {
+    return false;
+  }
+
+  if (filters.repoName && metadata.repoName !== filters.repoName) {
+    return false;
+  }
+
+  if (filters.pathPrefix) {
+    const path = metadata.path ?? "";
+
+    if (!path.startsWith(filters.pathPrefix)) {
+      return false;
+    }
+  }
+
+  if (filters.language && metadata.language !== filters.language) {
+    return false;
+  }
+
+  if (filters.sourceId && metadata.sourceId !== filters.sourceId) {
+    return false;
+  }
+
+  return true;
+}

packages/rag/src/vector-store-factory.ts

@@ -0,0 +1,54 @@
+import { type VectorStore } from "@voidbot/shared";
+
+import { FileVectorStore } from "./file-vector-store";
+import { type TextEmbedder } from "./hash-embedder";
+import { QdrantVectorStore } from "./qdrant-vector-store";
+import { ShardedSourceVectorStore } from "./source-vector-store";
+
+export interface VectorStoreFactoryOptions {
+  kind: "local_json" | "qdrant";
+  historyPath: string;
+  sourceRoot: string;
+  qdrant: {
+    url: string;
+    apiKey?: string;
+    timeoutMs: number;
+    historyCollection: string;
+    sourceCollection: string;
+  };
+  historyEmbedder: TextEmbedder;
+  sourceEmbedder: TextEmbedder;
+}
+
+export interface BuiltVectorStores {
+  history: VectorStore;
+  source: VectorStore;
+}
+
+export function createVectorStores(options: VectorStoreFactoryOptions): BuiltVectorStores {
+  if (options.kind === "qdrant") {
+    return {
+      history: new QdrantVectorStore({
+        url: options.qdrant.url,
+        apiKey: options.qdrant.apiKey,
+        timeoutMs: options.qdrant.timeoutMs,
+        collectionName: options.qdrant.historyCollection,
+        corpusKind: "discord_history",
+        embedder: options.historyEmbedder,
+      }),
+      source: new QdrantVectorStore({
+        url: options.qdrant.url,
+        apiKey: options.qdrant.apiKey,
+        timeoutMs: options.qdrant.timeoutMs,
+        collectionName: options.qdrant.sourceCollection,
+        corpusKind: "repository_source",
+        embedder: options.sourceEmbedder,
+      }),
+    };
+  }
+
+  return {
+    history: new FileVectorStore(options.historyPath, options.historyEmbedder),
+    source: new ShardedSourceVectorStore(options.sourceRoot, options.sourceEmbedder),
+  };
+}

packages/rag/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "references": [
+    {
+      "path": "../shared"
+    }
+  ]
+}
+

packages/sandbox/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@voidbot/sandbox",
+  "version": "0.1.0",
+  "private": true,
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "@voidbot/shared": "0.1.0"
+  }
+}

packages/sandbox/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./policy-runner";
+export * from "./profiles";
+

packages/sandbox/src/policy-runner.ts

@@ -0,0 +1,59 @@
+import {
+  type SandboxCommandRequest,
+  type SandboxExecutionResult,
+  type SandboxProfileName,
+} from "@voidbot/shared";
+
+import { defaultSandboxProfiles } from "./profiles";
+
+export class PolicySandboxRunner {
+  public getProfile(profileName: SandboxProfileName) {
+    return defaultSandboxProfiles[profileName];
+  }
+
+  public async run(request: SandboxCommandRequest): Promise<SandboxExecutionResult> {
+    const profile = this.getProfile(request.profile);
+
+    if (!profile.allowedCommands.includes(request.command)) {
+      return {
+        status: "denied",
+        stdout: "",
+        stderr: "",
+        exitCode: 1,
+        deniedReason: `Command "${request.command}" is not allowed for profile "${request.profile}".`,
+        dryRun: true,
+      };
+    }
+
+    if (request.networkAccess && !profile.networkAccess) {
+      return {
+        status: "denied",
+        stdout: "",
+        stderr: "",
+        exitCode: 1,
+        deniedReason: `Profile "${request.profile}" does not permit network access.`,
+        dryRun: true,
+      };
+    }
+
+    if (profile.requiresApproval && !request.approved) {
+      return {
+        status: "denied",
+        stdout: "",
+        stderr: "",
+        exitCode: 1,
+        deniedReason: `Profile "${request.profile}" requires explicit approval before execution.`,
+        dryRun: true,
+      };
+    }
+
+    return {
+      status: "planned",
+      stdout: "",
+      stderr: "",
+      exitCode: 0,
+      dryRun: true,
+    };
+  }
+}
+

packages/sandbox/src/profiles.ts

@@ -0,0 +1,32 @@
+import { type SandboxProfile, type SandboxProfileName } from "@voidbot/shared";
+
+export const defaultSandboxProfiles: Record<SandboxProfileName, SandboxProfile> = {
+  read_only_lookup: {
+    name: "read_only_lookup",
+    allowedCommands: ["lookup", "search_index"],
+    networkAccess: false,
+    timeoutMs: 5000,
+    requiresApproval: false,
+  },
+  rag_maintenance: {
+    name: "rag_maintenance",
+    allowedCommands: ["reindex_channel", "backfill_embeddings"],
+    networkAccess: false,
+    timeoutMs: 30000,
+    requiresApproval: true,
+  },
+  owner_workflow: {
+    name: "owner_workflow",
+    allowedCommands: ["prepare_manual_bundle", "publish_owner_reply"],
+    networkAccess: false,
+    timeoutMs: 30000,
+    requiresApproval: true,
+  },
+  public_low_risk: {
+    name: "public_low_risk",
+    allowedCommands: ["summarize_channel", "search_history", "search_sources"],
+    networkAccess: false,
+    timeoutMs: 10000,
+    requiresApproval: false,
+  },
+};

packages/sandbox/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "references": [
+    {
+      "path": "../shared"
+    }
+  ]
+}
+

packages/shared/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@voidbot/shared",
+  "version": "0.1.0",
+  "private": true,
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts"
+}
+

packages/shared/src/channel-indexing.ts

@@ -0,0 +1,60 @@
+export interface ChannelIndexingPolicy {
+  indexAllChannels: boolean;
+  indexedChannelIds: string[];
+  excludedChannelIds: string[];
+  excludedChannelNames: string[];
+}
+
+export interface ChannelIndexingTarget {
+  channelId?: string;
+  channelName?: string;
+  parentChannelId?: string;
+  parentChannelName?: string;
+}
+
+export function normalizeDiscordChannelName(value: string): string {
+  return value.trim().replace(/^#/, "").toLowerCase();
+}
+
+export function isChannelExcluded(
+  policy: ChannelIndexingPolicy,
+  target: ChannelIndexingTarget,
+): boolean {
+  const channelIds = collectChannelIds(target);
+
+  if (channelIds.some((channelId) => policy.excludedChannelIds.includes(channelId))) {
+    return true;
+  }
+
+  const channelNames = collectChannelNames(target);
+  return channelNames.some((channelName) => policy.excludedChannelNames.includes(channelName));
+}
+
+export function shouldIndexChannel(
+  policy: ChannelIndexingPolicy,
+  target: ChannelIndexingTarget,
+): boolean {
+  if (isChannelExcluded(policy, target)) {
+    return false;
+  }
+
+  if (policy.indexAllChannels) {
+    return true;
+  }
+
+  const channelIds = collectChannelIds(target);
+  return channelIds.some((channelId) => policy.indexedChannelIds.includes(channelId));
+}
+
+function collectChannelIds(target: ChannelIndexingTarget): string[] {
+  return [target.channelId, target.parentChannelId].filter(
+    (value): value is string => typeof value === "string" && value.trim().length > 0,
+  );
+}
+
+function collectChannelNames(target: ChannelIndexingTarget): string[] {
+  return [target.channelName, target.parentChannelName]
+    .filter((value): value is string => typeof value === "string" && value.trim().length > 0)
+    .map(normalizeDiscordChannelName)
+    .filter((value) => value.length > 0);
+}

packages/shared/src/index.ts

@@ -0,0 +1,285 @@
+export * from "./channel-indexing";
+
+export const commandNames = [
+  "ask",
+  "search-history",
+  "summarize-channel",
+  "queue-codex",
+  "approve-job",
+  "reject-job",
+  "provider-status",
+  "reindex-channel",
+  "set-style",
+] as const;
+
+export type CommandName = (typeof commandNames)[number];
+
+export const providerNames = ["owner_codex", "openai_api", "local_llm"] as const;
+export type ProviderName = (typeof providerNames)[number];
+
+export const trustTiers = ["T0", "T1", "T2"] as const;
+export type TrustTier = (typeof trustTiers)[number];
+
+export const jobStates = [
+  "queued",
+  "awaiting_approval",
+  "approved",
+  "running",
+  "awaiting_post_approval",
+  "completed",
+  "failed",
+  "cancelled",
+] as const;
+
+export type JobState = (typeof jobStates)[number];
+
+export const sandboxProfileNames = [
+  "read_only_lookup",
+  "rag_maintenance",
+  "owner_workflow",
+  "public_low_risk",
+] as const;
+
+export type SandboxProfileName = (typeof sandboxProfileNames)[number];
+
+export const ownerCodexModes = [
+  "manual_package",
+  "local_exec_owner_only",
+] as const;
+
+export type OwnerCodexMode = (typeof ownerCodexModes)[number];
+
+export const DEFAULT_RETRIEVAL_RESULT_LIMIT = 5;
+export const MAX_RETRIEVAL_RESULT_LIMIT = 12;
+
+export interface Actor {
+  id: string;
+  displayName: string;
+  isAdmin: boolean;
+  isBot?: boolean;
+}
+
+export interface GuildContext {
+  guildId?: string;
+  guildName?: string;
+  channelId: string;
+  channelName?: string;
+  threadId?: string;
+}
+
+export interface SourceMessage {
+  id: string;
+  authorId: string;
+  authorName: string;
+  content: string;
+  timestamp: string;
+}
+
+export interface RetrievalResult {
+  chunkId: string;
+  score: number;
+  text: string;
+  sourceId: string;
+  sourceKind: "discord_message" | "source_document";
+  metadata: Record<string, string>;
+}
+
+export interface RetrievalFilters {
+  corpusKind?: "discord_history" | "repository_source";
+  guildId?: string;
+  channelId?: string;
+  authorId?: string;
+  repoName?: string;
+  pathPrefix?: string;
+  language?: string;
+  sourceId?: string;
+}
+
+export interface StylePack {
+  name: string;
+  instructions: string;
+  enabled: boolean;
+}
+
+export interface ContextBundle {
+  prompt: string;
+  actor: Actor;
+  guildContext: GuildContext;
+  recentMessages: SourceMessage[];
+  retrieval: RetrievalResult[];
+  stylePack?: StylePack;
+  createdAt: string;
+}
+
+export interface JobApprovalRecord {
+  stage: "run" | "post";
+  status: "approved" | "rejected";
+  actorId: string;
+  reason?: string;
+  createdAt: string;
+}
+
+export interface JobRecord {
+  id: string;
+  command: CommandName;
+  state: JobState;
+  provider: ProviderName;
+  runApprovalRequired: boolean;
+  postApprovalRequired: boolean;
+  requester: Actor;
+  guildContext: GuildContext;
+  prompt: string;
+  contextBundle: ContextBundle;
+  createdAt: string;
+  updatedAt: string;
+  outputChannelId: string;
+  requestMessageId?: string;
+  summary?: string;
+  manualArtifacts?: Record<string, string>;
+  finalResponse?: string;
+  approvals: JobApprovalRecord[];
+  error?: string;
+}
+
+export interface ProviderRequest {
+  provider: ProviderName;
+  contextBundle: ContextBundle;
+  options?: Record<string, unknown>;
+}
+
+export interface ProviderArtifact {
+  name: string;
+  contentType: "markdown" | "json" | "text";
+  content: string;
+}
+
+export interface CodexMcpServerConfig {
+  name: string;
+  command: string;
+  args: string[];
+  env?: Record<string, string>;
+  cwd?: string;
+}
+
+export const providerNotificationChannels = ["owner_dm"] as const;
+export type ProviderNotificationChannel = (typeof providerNotificationChannels)[number];
+
+export const providerNotificationReasons = [
+  "completion",
+  "failure",
+  "handoff",
+  "custom",
+] as const;
+export type ProviderNotificationReason = (typeof providerNotificationReasons)[number];
+
+export interface ProviderNotificationIntent {
+  channel: ProviderNotificationChannel;
+  reason: ProviderNotificationReason;
+  message: string;
+}
+
+export interface ProviderResponse {
+  status: "ready_for_review" | "completed";
+  summary: string;
+  outputText?: string;
+  artifacts?: ProviderArtifact[];
+  metadata?: Record<string, string>;
+  notifications?: ProviderNotificationIntent[];
+}
+
+export interface ModerationResult {
+  allowed: boolean;
+  reason?: string;
+}
+
+export interface ProviderAdapter {
+  getName(): ProviderName;
+  getCapabilities(): string[];
+  isEnabled(): boolean;
+  isAllowedForActor(actor: Actor, guildContext: GuildContext): boolean;
+  buildRequest(
+    contextBundle: ContextBundle,
+    options?: Record<string, unknown>,
+  ): ProviderRequest;
+  execute(request: ProviderRequest): Promise<ProviderResponse>;
+  estimateCost(request: ProviderRequest): Promise<number>;
+  moderateInput?(request: ProviderRequest): Promise<ModerationResult>;
+  moderateOutput?(response: ProviderResponse): Promise<ModerationResult>;
+  getAuditRedactions(): string[];
+}
+
+export interface AuditEvent {
+  id: string;
+  type: string;
+  timestamp: string;
+  actorId?: string;
+  jobId?: string;
+  provider?: ProviderName;
+  details: Record<string, unknown>;
+}
+
+export interface ArchivedMessage {
+  id: string;
+  guildId?: string;
+  channelId: string;
+  authorId: string;
+  authorName: string;
+  content: string;
+  timestamp: string;
+  editedAt?: string;
+  deletedAt?: string;
+  threadId?: string;
+  attachments?: string[];
+  metadata?: Record<string, string>;
+}
+
+export interface EmbeddingChunk {
+  id: string;
+  sourceId: string;
+  sourceKind: "discord_message" | "source_document";
+  text: string;
+  normalizedText: string;
+  metadata: Record<string, string>;
+}
+
+export interface VectorStore {
+  upsert(chunks: EmbeddingChunk[]): Promise<void>;
+  clear(): Promise<void>;
+  deleteBySourceIds(sourceIds: string[]): Promise<void>;
+  deleteByFilters(filters: RetrievalFilters): Promise<void>;
+  query(query: string, limit: number, filters?: RetrievalFilters): Promise<RetrievalResult[]>;
+}
+
+export interface SandboxProfile {
+  name: SandboxProfileName;
+  allowedCommands: string[];
+  networkAccess: boolean;
+  timeoutMs: number;
+  requiresApproval: boolean;
+}
+
+export interface SandboxCommandRequest {
+  profile: SandboxProfileName;
+  command: string;
+  args: string[];
+  workingDirectory?: string;
+  networkAccess?: boolean;
+  approved?: boolean;
+}
+
+export interface SandboxExecutionResult {
+  status: "planned" | "denied";
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  deniedReason?: string;
+  dryRun: boolean;
+}
+
+export function isProviderName(value: string): value is ProviderName {
+  return providerNames.includes(value as ProviderName);
+}
+
+export function isOwnerCodexMode(value: string): value is OwnerCodexMode {
+  return ownerCodexModes.includes(value as OwnerCodexMode);
+}

packages/shared/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": [
+    "src/**/*.ts"
+  ]
+}
+

scripts/backfill-discord-logs.ts

@@ -0,0 +1,113 @@
+import "dotenv/config";
+
+import { resolve } from "node:path";
+
+import { loadConfig } from "@voidbot/config";
+import {
+  createTextEmbedder,
+  createVectorStores,
+  FileImportStateRepository,
+  FileMessageArchiveRepository,
+  HistoryIngester,
+  RagPipeline,
+  importDiscordLogs,
+} from "@voidbot/rag";
+
+interface CliOptions {
+  inputPath?: string;
+  recursive: boolean;
+  respectIndexingRules: boolean;
+}
+
+async function main(): Promise<void> {
+  if (!process.env.DISCORD_OWNER_ID) {
+    process.env.DISCORD_OWNER_ID = "__rag_backfill__";
+  }
+
+  const config = loadConfig();
+  const options = parseArgs(process.argv.slice(2));
+  const inputPath = options.inputPath ?? config.discordLogRoot;
+
+  if (!inputPath) {
+    throw new Error(
+      "Provide --input <path> or set DISCORD_LOG_ROOT before running the log backfill.",
+    );
+  }
+
+  const archiveRepository = new FileMessageArchiveRepository(config.ragArchivePath);
+  const embedder = createTextEmbedder({
+    backend: config.ragEmbeddingBackend,
+    hashDimensions: config.ragEmbeddingDimensions,
+    ollamaBaseUrl: config.ragOllamaBaseUrl,
+    ollamaModel: config.ragOllamaModel,
+    ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+    queryInstruction: config.ragQueryInstruction,
+  });
+  const vectorStore = createVectorStores({
+    kind: config.vectorStore.kind,
+    historyPath: config.vectorStore.path,
+    sourceRoot: config.sourceVectorStoreRoot,
+    qdrant: config.qdrant,
+    historyEmbedder: embedder,
+    sourceEmbedder: embedder,
+  }).history;
+  const historyIngester = new HistoryIngester();
+  const ragPipeline = new RagPipeline(archiveRepository, historyIngester, vectorStore);
+  const importStateRepository = new FileImportStateRepository(config.ragImportStatePath);
+  const previousState = await importStateRepository.read();
+  const importResult = await importDiscordLogs(resolve(inputPath), {
+    recursive: options.recursive,
+    channelIndexing: options.respectIndexingRules ? config.channelIndexing : undefined,
+    previousState,
+  });
+  const ingestResult = await ragPipeline.upsertMessages(importResult.messages);
+
+  await importStateRepository.write(importResult.nextState);
+
+  console.log(`Input: ${resolve(inputPath)}`);
+  console.log(`Files scanned: ${importResult.filesScanned}`);
+  console.log(`Files imported: ${importResult.filesImported}`);
+  console.log(`Files skipped (unchanged): ${importResult.filesSkipped}`);
+  console.log(`Messages discovered: ${importResult.messages.length}`);
+  console.log(`Invalid records: ${importResult.invalidRecords}`);
+  console.log(
+    `Archive changes: created=${ingestResult.createdMessages}, updated=${ingestResult.updatedMessages}, unchanged=${ingestResult.unchangedMessages}`,
+  );
+  console.log(`Indexed chunks: ${ingestResult.indexedChunks}`);
+}
+
+function parseArgs(args: string[]): CliOptions {
+  const options: CliOptions = {
+    recursive: true,
+    respectIndexingRules: true,
+  };
+
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index];
+
+    if (argument === "--input") {
+      options.inputPath = args[index + 1];
+      index += 1;
+      continue;
+    }
+
+    if (argument === "--no-recursive") {
+      options.recursive = false;
+      continue;
+    }
+
+    if (argument === "--all-channels") {
+      options.respectIndexingRules = false;
+      continue;
+    }
+
+    throw new Error(`Unknown argument: ${argument}`);
+  }
+
+  return options;
+}
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/git-post-push-index.mjs

@@ -0,0 +1,267 @@
+import { spawn } from "node:child_process";
+import { access, mkdir, open, writeFile, readFile } from "node:fs/promises";
+import { constants as fsConstants } from "node:fs";
+import { join, resolve } from "node:path";
+
+import { config as loadDotenv } from "dotenv";
+
+const args = parseArgs(process.argv.slice(2));
+
+if (!args.repoName) {
+  console.error("Missing required --repo-name argument.");
+  process.exit(1);
+}
+
+const voidbotRoot = resolve(args.voidbotRoot ?? process.cwd());
+loadDotenv({ path: join(voidbotRoot, ".env") });
+const storageRoot = resolve(
+  voidbotRoot,
+  process.env.STORAGE_ROOT && process.env.STORAGE_ROOT.trim().length > 0
+    ? process.env.STORAGE_ROOT
+    : ".voidbot",
+);
+const safeRepoName = encodeURIComponent(args.repoName);
+const logDir = join(storageRoot, "logs", "source-hooks");
+const statusDir = join(storageRoot, "status", "source-hooks");
+const logPath = join(logDir, `${safeRepoName}.log`);
+const statusPath = join(statusDir, `${safeRepoName}.json`);
+
+if (args.run) {
+  await runWorker();
+} else {
+  await launchDetachedWorker();
+}
+
+async function launchDetachedWorker() {
+  await ensureDirectories();
+
+  const currentStatus = await readJson(statusPath);
+
+  if (typeof currentStatus?.pid === "number" && isProcessRunning(currentStatus.pid)) {
+    await writeStatus({
+      ...currentStatus,
+      repoName: args.repoName,
+      logPath,
+      lastSkippedAt: new Date().toISOString(),
+      lastSkipReason: "already_running",
+    });
+    return;
+  }
+
+  const logHandle = await open(logPath, "a");
+  const child = spawn(
+    process.execPath,
+    [
+      join(voidbotRoot, "scripts", "git-post-push-index.mjs"),
+      "--run",
+      "--voidbot-root",
+      voidbotRoot,
+      "--repo-name",
+      args.repoName,
+    ],
+    {
+      cwd: voidbotRoot,
+      detached: true,
+      stdio: ["ignore", logHandle.fd, logHandle.fd],
+      windowsHide: true,
+      env: process.env,
+    },
+  );
+
+  child.unref();
+  await logHandle.close();
+
+  await writeStatus({
+    state: "running",
+    repoName: args.repoName,
+    pid: child.pid,
+    queuedAt: new Date().toISOString(),
+    logPath,
+  });
+}
+
+async function runWorker() {
+  await ensureDirectories();
+
+  const sourceRepoRoot = process.env.SOURCE_REPO_ROOT
+    ? resolve(process.env.SOURCE_REPO_ROOT)
+    : undefined;
+  const repoPath = sourceRepoRoot ? join(sourceRepoRoot, args.repoName) : undefined;
+  const startedAt = new Date().toISOString();
+
+  await writeStatus({
+    state: "running",
+    repoName: args.repoName,
+    pid: process.pid,
+    startedAt,
+    logPath,
+    repoPath,
+  });
+
+  process.stdout.write(`\n=== source hook run ${startedAt} repo=${args.repoName} ===\n`);
+
+  if (!sourceRepoRoot) {
+    const message = "SOURCE_REPO_ROOT is not configured in VoidBot's .env.";
+    process.stderr.write(`${message}\n`);
+    await writeStatus({
+      state: "failed",
+      repoName: args.repoName,
+      pid: process.pid,
+      startedAt,
+      completedAt: new Date().toISOString(),
+      exitCode: 1,
+      logPath,
+      error: message,
+    });
+    process.exit(1);
+  }
+
+  if (!(await pathExists(repoPath))) {
+    const message = `Configured repo path does not exist: ${repoPath}`;
+    process.stderr.write(`${message}\n`);
+    await writeStatus({
+      state: "failed",
+      repoName: args.repoName,
+      pid: process.pid,
+      startedAt,
+      completedAt: new Date().toISOString(),
+      exitCode: 1,
+      logPath,
+      repoPath,
+      error: message,
+    });
+    process.exit(1);
+  }
+
+  try {
+    const child =
+      process.platform === "win32"
+        ? spawn(
+            "cmd.exe",
+            ["/d", "/s", "/c", "npm.cmd", "run", "rag:index-sources", "--", "--repo", args.repoName],
+            {
+              cwd: voidbotRoot,
+              windowsHide: true,
+              env: process.env,
+              stdio: ["ignore", "pipe", "pipe"],
+            },
+          )
+        : spawn("npm", ["run", "rag:index-sources", "--", "--repo", args.repoName], {
+            cwd: voidbotRoot,
+            windowsHide: true,
+            env: process.env,
+            stdio: ["ignore", "pipe", "pipe"],
+          });
+
+    child.stdout.on("data", (chunk) => {
+      process.stdout.write(chunk);
+    });
+
+    child.stderr.on("data", (chunk) => {
+      process.stderr.write(chunk);
+    });
+
+    const exitCode = await new Promise((resolveExit, reject) => {
+      child.on("error", reject);
+      child.on("close", (code) => resolveExit(code ?? 1));
+    });
+
+    await writeStatus({
+      state: exitCode === 0 ? "completed" : "failed",
+      repoName: args.repoName,
+      pid: process.pid,
+      startedAt,
+      completedAt: new Date().toISOString(),
+      exitCode,
+      logPath,
+      repoPath,
+    });
+
+    process.exit(exitCode);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    process.stderr.write(`${message}\n`);
+    await writeStatus({
+      state: "failed",
+      repoName: args.repoName,
+      pid: process.pid,
+      startedAt,
+      completedAt: new Date().toISOString(),
+      exitCode: 1,
+      logPath,
+      repoPath,
+      error: message,
+    });
+    process.exit(1);
+  }
+}
+
+async function ensureDirectories() {
+  await mkdir(logDir, { recursive: true });
+  await mkdir(statusDir, { recursive: true });
+}
+
+async function writeStatus(status) {
+  await writeFile(statusPath, `${JSON.stringify(status, null, 2)}\n`, "utf8");
+}
+
+async function readJson(path) {
+  try {
+    const content = await readFile(path, "utf8");
+    return JSON.parse(content);
+  } catch {
+    return undefined;
+  }
+}
+
+function isProcessRunning(pid) {
+  if (typeof pid !== "number" || !Number.isFinite(pid) || pid <= 0) {
+    return false;
+  }
+
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function pathExists(path) {
+  try {
+    await access(path, fsConstants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function parseArgs(argv) {
+  const parsed = {
+    run: false,
+    voidbotRoot: undefined,
+    repoName: undefined,
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+
+    if (argument === "--run") {
+      parsed.run = true;
+      continue;
+    }
+
+    if (argument === "--voidbot-root" && argv[index + 1]) {
+      parsed.voidbotRoot = argv[index + 1];
+      index += 1;
+      continue;
+    }
+
+    if (argument === "--repo-name" && argv[index + 1]) {
+      parsed.repoName = argv[index + 1];
+      index += 1;
+    }
+  }
+
+  return parsed;
+}

scripts/index-source-repos.ts

@@ -0,0 +1,140 @@
+import "dotenv/config";
+
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+
+import { loadConfig } from "@voidbot/config";
+import {
+  crawlRepositoryDocuments,
+  createTextEmbedder,
+  createVectorStores,
+  FileSourceDocumentArchiveRepository,
+  SourceDocumentIngester,
+  SourceRagPipeline,
+} from "@voidbot/rag";
+
+interface ScriptOptions {
+  repos?: string[];
+  forceReindex?: boolean;
+}
+
+async function main(): Promise<void> {
+  if (!process.env.DISCORD_OWNER_ID) {
+    process.env.DISCORD_OWNER_ID = "__source_reindex__";
+  }
+
+  const options = parseArgs(process.argv.slice(2));
+  const config = loadConfig();
+
+  if (!config.sourceRepoRoot) {
+    throw new Error("SOURCE_REPO_ROOT is not configured.");
+  }
+
+  const repoNames =
+    options.repos && options.repos.length > 0
+      ? options.repos
+      : await resolveRepoNames(config.sourceRepoRoot, config.sourceRepoPatterns);
+
+  if (repoNames.length === 0) {
+    throw new Error("No source repositories matched SOURCE_REPO_PATTERNS.");
+  }
+
+  const embedder = createTextEmbedder({
+    backend: config.ragEmbeddingBackend,
+    hashDimensions: config.ragEmbeddingDimensions,
+    ollamaBaseUrl: config.ragOllamaBaseUrl,
+    ollamaModel: config.ragOllamaModel,
+    ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+    queryInstruction: config.ragQueryInstruction,
+  });
+  const vectorStore = createVectorStores({
+    kind: config.vectorStore.kind,
+    historyPath: config.vectorStore.path,
+    sourceRoot: config.sourceVectorStoreRoot,
+    qdrant: config.qdrant,
+    historyEmbedder: embedder,
+    sourceEmbedder: embedder,
+  }).source;
+  const archiveRepository = new FileSourceDocumentArchiveRepository(config.ragSourceArchivePath);
+  const sourceIngester = new SourceDocumentIngester();
+  const pipeline = new SourceRagPipeline(archiveRepository, sourceIngester, vectorStore);
+
+  let indexedRepositories = 0;
+  let totalDocuments = 0;
+  let totalChunks = 0;
+
+  for (const repoName of repoNames) {
+    const repoPath = join(config.sourceRepoRoot, repoName);
+    const scan = await crawlRepositoryDocuments(repoPath, repoName, {
+      includePathPrefixes: config.sourceRepoIncludePrefixes[repoName],
+    });
+    const result = await pipeline.syncRepoDocuments(repoName, scan.documents, {
+      forceReindex: options.forceReindex,
+    });
+    indexedRepositories += 1;
+    totalDocuments += scan.documents.length;
+    totalChunks += result.indexedChunks;
+
+    console.log(
+      [
+        `Repo ${repoName}:`,
+        `documents=${scan.documents.length}`,
+        `created=${result.createdDocuments}`,
+        `updated=${result.updatedDocuments}`,
+        `unchanged=${result.unchangedDocuments}`,
+        `deleted=${result.deletedDocuments}`,
+        `indexedChunks=${result.indexedChunks}`,
+        `skippedFiles=${scan.skippedFiles.length}`,
+      ].join(" "),
+    );
+  }
+
+  console.log(`Indexed repositories: ${indexedRepositories}`);
+  console.log(`Indexed documents: ${totalDocuments}`);
+  console.log(`Indexed chunks: ${totalChunks}`);
+}
+
+function parseArgs(argv: string[]): ScriptOptions {
+  const options: ScriptOptions = {};
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+
+    if ((argument === "--repo" || argument === "--repos") && argv[index + 1]) {
+      options.repos = argv[index + 1]
+        .split(",")
+        .map((entry) => entry.trim())
+        .filter((entry) => entry.length > 0);
+      index += 1;
+      continue;
+    }
+
+    if (argument === "--force") {
+      options.forceReindex = true;
+    }
+  }
+
+  return options;
+}
+
+async function resolveRepoNames(root: string, patterns: string[]): Promise<string[]> {
+  const entries = await readdir(root, { withFileTypes: true });
+  const repoNames = entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .filter((name) => patterns.length === 0 || patterns.some((pattern) => matchesPattern(name, pattern)))
+    .sort((left, right) => left.localeCompare(right));
+
+  return repoNames;
+}
+
+function matchesPattern(value: string, pattern: string): boolean {
+  const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  const regex = new RegExp(`^${escaped}$`, "i");
+  return regex.test(value);
+}
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/install-codex-mcp.ts

@@ -0,0 +1,65 @@
+import { existsSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { resolve } from "node:path";
+
+import { config as loadDotenv } from "dotenv";
+
+import { loadConfig } from "../packages/config/src";
+import { buildVoidMcpServerConfig } from "../packages/core/src/codex-mcp";
+
+async function main(): Promise<void> {
+  const workspaceRoot = resolve(process.cwd());
+  loadDotenv({ path: resolve(workspaceRoot, ".env") });
+  const config = loadConfig();
+  const server = buildVoidMcpServerConfig(workspaceRoot);
+  const entrypoint = server.args[0];
+
+  if (!entrypoint || !existsSync(entrypoint)) {
+    throw new Error(
+      `Build the project first so the MCP server exists at ${entrypoint ?? "(missing entrypoint)"}.`,
+    );
+  }
+
+  await runCodex(config.codexExecutable, config.codexExecArgs, ["mcp", "remove", server.name], true);
+
+  const addArgs = ["mcp", "add"];
+
+  for (const [key, value] of Object.entries(server.env ?? {})) {
+    addArgs.push("--env", `${key}=${value}`);
+  }
+
+  addArgs.push(server.name, "--", server.command, ...server.args);
+
+  await runCodex(config.codexExecutable, config.codexExecArgs, addArgs);
+  await runCodex(config.codexExecutable, config.codexExecArgs, ["mcp", "get", server.name]);
+}
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});
+
+function runCodex(
+  executable: string,
+  executableArgs: string[],
+  args: string[],
+  allowFailure = false,
+): Promise<void> {
+  return new Promise((resolvePromise, rejectPromise) => {
+    const child = spawn(executable, [...executableArgs, ...args], {
+      cwd: process.cwd(),
+      env: process.env,
+      stdio: "inherit",
+    });
+
+    child.on("error", rejectPromise);
+    child.on("close", (code) => {
+      if (code === 0 || allowFailure) {
+        resolvePromise();
+        return;
+      }
+
+      rejectPromise(new Error(`codex ${args.join(" ")} exited with code ${code ?? "unknown"}`));
+    });
+  });
+}

scripts/install-source-index-post-push-hooks.ts

@@ -0,0 +1,205 @@
+import "dotenv/config";
+
+import { readdir, chmod, mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+
+import { loadConfig } from "@voidbot/config";
+
+const HOOK_MARKER = "# voidbot-source-index-push-hook";
+
+interface ScriptOptions {
+  repos?: string[];
+}
+
+async function main(): Promise<void> {
+  if (!process.env.DISCORD_OWNER_ID) {
+    process.env.DISCORD_OWNER_ID = "__source_hook_install__";
+  }
+
+  const config = loadConfig();
+
+  if (!config.sourceRepoRoot) {
+    throw new Error("SOURCE_REPO_ROOT is not configured.");
+  }
+
+  const options = parseArgs(process.argv.slice(2));
+  const repoNames =
+    options.repos && options.repos.length > 0
+      ? options.repos
+      : await resolveRepoNames(config.sourceRepoRoot, config.sourceRepoPatterns);
+
+  let installed = 0;
+  let updated = 0;
+  let backedUp = 0;
+  let skipped = 0;
+
+  for (const repoName of repoNames) {
+    const repoPath = join(config.sourceRepoRoot, repoName);
+    const gitDir = await resolveGitDir(repoPath);
+
+    if (!gitDir) {
+      console.log(`Skipping ${repoName}: no .git directory found.`);
+      skipped += 1;
+      continue;
+    }
+
+    const hooksDir = join(gitDir, "hooks");
+    const hookPath = join(hooksDir, "pre-push");
+    const backupHookPath = join(hooksDir, "pre-push.voidbot.prev");
+
+    await mkdir(hooksDir, { recursive: true });
+
+    let existingContent: string | undefined;
+
+    try {
+      existingContent = await readFile(hookPath, "utf8");
+    } catch {
+      existingContent = undefined;
+    }
+
+    if (
+      existingContent &&
+      !existingContent.includes(HOOK_MARKER) &&
+      !(await pathExists(backupHookPath))
+    ) {
+      await writeFile(backupHookPath, existingContent, "utf8");
+      await chmod(backupHookPath, 0o755);
+      backedUp += 1;
+    } else if (
+      existingContent &&
+      !existingContent.includes(HOOK_MARKER) &&
+      (await pathExists(backupHookPath))
+    ) {
+      console.log(`Skipping ${repoName}: existing pre-push hook already has a Void backup; not clobbering it.`);
+      skipped += 1;
+      continue;
+    }
+
+    const hookScript = renderHookScript(process.cwd(), repoName);
+    await writeFile(hookPath, hookScript, "utf8");
+    await chmod(hookPath, 0o755);
+
+    if (existingContent?.includes(HOOK_MARKER)) {
+      updated += 1;
+      console.log(`Updated pre-push hook for ${repoName}.`);
+    } else {
+      installed += 1;
+      console.log(`Installed pre-push hook for ${repoName}.`);
+    }
+  }
+
+  console.log(
+    [
+      `installed=${installed}`,
+      `updated=${updated}`,
+      `backedUp=${backedUp}`,
+      `skipped=${skipped}`,
+    ].join(" "),
+  );
+}
+
+function parseArgs(argv: string[]): ScriptOptions {
+  const options: ScriptOptions = {};
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+
+    if ((argument === "--repo" || argument === "--repos") && argv[index + 1]) {
+      options.repos = argv[index + 1]
+        .split(",")
+        .map((entry) => entry.trim())
+        .filter((entry) => entry.length > 0);
+      index += 1;
+    }
+  }
+
+  return options;
+}
+
+async function resolveRepoNames(root: string, patterns: string[]): Promise<string[]> {
+  const entries = await readdir(root, { withFileTypes: true });
+  return entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .filter((name) => patterns.length === 0 || patterns.some((pattern) => matchesPattern(name, pattern)))
+    .sort((left, right) => left.localeCompare(right));
+}
+
+async function resolveGitDir(repoPath: string): Promise<string | undefined> {
+  const dotGitPath = join(repoPath, ".git");
+
+  try {
+    const dotGitStats = await stat(dotGitPath);
+
+    if (dotGitStats.isDirectory()) {
+      return dotGitPath;
+    }
+
+    if (dotGitStats.isFile()) {
+      const pointer = await readFile(dotGitPath, "utf8");
+      const match = pointer.match(/^gitdir:\s*(.+)$/im);
+
+      if (match?.[1]) {
+        return resolve(repoPath, match[1].trim());
+      }
+    }
+  } catch {
+    return undefined;
+  }
+
+  return undefined;
+}
+
+function renderHookScript(voidbotRoot: string, repoName: string): string {
+  const normalizedRoot = voidbotRoot.replace(/\\/g, "/");
+  const launcherPath = `${normalizedRoot}/scripts/git-post-push-index.mjs`;
+  const nodePath = process.execPath.replace(/\\/g, "/");
+  const quotedNodePath = escapeForDoubleQuotedShell(nodePath);
+  const quotedLauncherPath = escapeForDoubleQuotedShell(launcherPath);
+  const quotedRoot = escapeForDoubleQuotedShell(normalizedRoot);
+  const quotedRepoName = escapeForDoubleQuotedShell(repoName);
+
+  return [
+    "#!/bin/sh",
+    HOOK_MARKER,
+    "HOOK_DIR=\"$(CDPATH= cd -- \"$(dirname -- \"$0\")\" && pwd)\"",
+    "PREV_HOOK=\"$HOOK_DIR/pre-push.voidbot.prev\"",
+    "",
+    "if [ -x \"$PREV_HOOK\" ]; then",
+    "  \"$PREV_HOOK\" \"$@\"",
+    "  PREV_STATUS=$?",
+    "  if [ \"$PREV_STATUS\" -ne 0 ]; then",
+    "    exit \"$PREV_STATUS\"",
+    "  fi",
+    "fi",
+    "",
+    `\"${quotedNodePath}\" \"${quotedLauncherPath}\" --voidbot-root \"${quotedRoot}\" --repo-name \"${quotedRepoName}\" >/dev/null 2>&1 || true`,
+    "",
+    "exit 0",
+    "",
+  ].join("\n");
+}
+
+function escapeForDoubleQuotedShell(value: string): string {
+  return value.replace(/(["`$\\])/g, "\\$1");
+}
+
+function matchesPattern(value: string, pattern: string): boolean {
+  const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  const regex = new RegExp(`^${escaped}$`, "i");
+  return regex.test(value);
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/migrate-local-vectors-to-qdrant.ts

@@ -0,0 +1,191 @@
+import "dotenv/config";
+
+import { access, readdir, readFile } from "node:fs/promises";
+import { constants } from "node:fs";
+import { join } from "node:path";
+
+import { loadConfig } from "@voidbot/config";
+import {
+  createTextEmbedder,
+  QdrantVectorStore,
+  type StoredQdrantChunk,
+} from "@voidbot/rag";
+
+interface CliOptions {
+  wipe: boolean;
+}
+
+interface StoredVectorIndex {
+  chunks?: StoredQdrantChunk[];
+}
+
+async function main(): Promise<void> {
+  if (!process.env.DISCORD_OWNER_ID) {
+    process.env.DISCORD_OWNER_ID = "__qdrant_migration__";
+  }
+
+  const config = loadConfig();
+  const options = parseArgs(process.argv.slice(2));
+  const historyEmbedder = createTextEmbedder({
+    backend: config.ragEmbeddingBackend,
+    hashDimensions: config.ragEmbeddingDimensions,
+    ollamaBaseUrl: config.ragOllamaBaseUrl,
+    ollamaModel: config.ragOllamaModel,
+    ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+    queryInstruction: config.ragQueryInstruction,
+  });
+  const sourceEmbedder = createTextEmbedder({
+    backend: config.ragEmbeddingBackend,
+    hashDimensions: config.ragEmbeddingDimensions,
+    ollamaBaseUrl: config.ragOllamaBaseUrl,
+    ollamaModel: config.ragOllamaModel,
+    ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+    queryInstruction: config.ragSourceQueryInstruction,
+  });
+  const historyStore = new QdrantVectorStore({
+    url: config.qdrant.url,
+    apiKey: config.qdrant.apiKey,
+    timeoutMs: config.qdrant.timeoutMs,
+    collectionName: config.qdrant.historyCollection,
+    corpusKind: "discord_history",
+    embedder: historyEmbedder,
+  });
+  const sourceStore = new QdrantVectorStore({
+    url: config.qdrant.url,
+    apiKey: config.qdrant.apiKey,
+    timeoutMs: config.qdrant.timeoutMs,
+    collectionName: config.qdrant.sourceCollection,
+    corpusKind: "repository_source",
+    embedder: sourceEmbedder,
+  });
+
+  if (options.wipe) {
+    await historyStore.clear();
+    await sourceStore.clear();
+  }
+
+  const localVectors = await loadLocalVectors(config.storageRoot, {
+    historyPath: config.vectorStore.path,
+    sourceRoot: config.sourceVectorStoreRoot,
+  });
+
+  await historyStore.upsertStoredChunks(localVectors.historyChunks);
+  await sourceStore.upsertStoredChunks(localVectors.sourceChunks);
+
+  console.log(`Qdrant URL: ${config.qdrant.url}`);
+  console.log(`History collection: ${config.qdrant.historyCollection}`);
+  console.log(`Source collection: ${config.qdrant.sourceCollection}`);
+  console.log(`History chunks migrated: ${localVectors.historyChunks.length}`);
+  console.log(`Source chunks migrated: ${localVectors.sourceChunks.length}`);
+  console.log(`Source shards discovered locally: ${localVectors.sourceShardFiles}`);
+}
+
+function parseArgs(args: string[]): CliOptions {
+  const options: CliOptions = {
+    wipe: false,
+  };
+
+  for (const argument of args) {
+    if (argument === "--wipe") {
+      options.wipe = true;
+      continue;
+    }
+
+    throw new Error(`Unknown argument: ${argument}`);
+  }
+
+  return options;
+}
+
+async function loadLocalVectors(
+  storageRoot: string,
+  options: {
+    historyPath: string;
+    sourceRoot: string;
+  },
+): Promise<{
+  historyChunks: StoredQdrantChunk[];
+  sourceChunks: StoredQdrantChunk[];
+  sourceShardFiles: number;
+}> {
+  const historyChunks = await loadChunksFromPath(options.historyPath);
+  const sourceFiles = await listJsonFiles(options.sourceRoot);
+  const sourceChunks: StoredQdrantChunk[] = [];
+
+  for (const filePath of sourceFiles) {
+    sourceChunks.push(...(await loadChunksFromPath(filePath)));
+  }
+
+  if (historyChunks.length > 0 || sourceChunks.length > 0) {
+    return {
+      historyChunks,
+      sourceChunks,
+      sourceShardFiles: sourceFiles.length,
+    };
+  }
+
+  const legacyPath = join(storageRoot, "vector-store.json");
+  const legacyChunks = await loadChunksFromPath(legacyPath);
+
+  return {
+    historyChunks: legacyChunks.filter((chunk) => chunk.metadata.corpusKind !== "repository_source"),
+    sourceChunks: legacyChunks.filter((chunk) => chunk.metadata.corpusKind === "repository_source"),
+    sourceShardFiles: 0,
+  };
+}
+
+async function loadChunksFromPath(filePath: string): Promise<StoredQdrantChunk[]> {
+  if (!(await pathExists(filePath))) {
+    return [];
+  }
+
+  const raw = await readFile(filePath, "utf8");
+  const store = JSON.parse(raw) as StoredVectorIndex;
+  return (store.chunks ?? []).filter(isStoredChunk);
+}
+
+async function listJsonFiles(directory: string): Promise<string[]> {
+  if (!(await pathExists(directory))) {
+    return [];
+  }
+
+  const entries = await readdir(directory, { withFileTypes: true });
+
+  return entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+    .map((entry) => join(directory, entry.name))
+    .sort((left, right) => left.localeCompare(right));
+}
+
+async function pathExists(targetPath: string): Promise<boolean> {
+  try {
+    await access(targetPath, constants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function isStoredChunk(value: unknown): value is StoredQdrantChunk {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+
+  const chunk = value as Partial<StoredQdrantChunk>;
+
+  return (
+    typeof chunk.id === "string" &&
+    typeof chunk.sourceId === "string" &&
+    (chunk.sourceKind === "discord_message" || chunk.sourceKind === "source_document") &&
+    typeof chunk.text === "string" &&
+    Array.isArray(chunk.vector) &&
+    typeof chunk.indexedAt === "string" &&
+    typeof chunk.metadata === "object" &&
+    chunk.metadata !== null
+  );
+}
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/migrate-vector-store-layout.ts

@@ -0,0 +1,107 @@
+import "dotenv/config";
+
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+import { loadConfig } from "@voidbot/config";
+
+interface StoredVectorChunk {
+  id: string;
+  sourceId?: string;
+  sourceMessageId?: string;
+  sourceKind?: "discord_message" | "source_document";
+  text: string;
+  normalizedText: string;
+  metadata: Record<string, string>;
+  vector: number[];
+  indexedAt: string;
+}
+
+interface VectorIndexStore {
+  version: number;
+  embedderId: string;
+  dimensions: number;
+  chunks: StoredVectorChunk[];
+}
+
+async function main(): Promise<void> {
+  if (!process.env.DISCORD_OWNER_ID) {
+    process.env.DISCORD_OWNER_ID = "__vector_migration__";
+  }
+
+  const config = loadConfig();
+  const legacyPath = join(config.storageRoot, "vector-store.json");
+  const raw = await readFile(legacyPath, "utf8");
+  const legacyStore = JSON.parse(raw) as VectorIndexStore;
+  const historyChunks: StoredVectorChunk[] = [];
+  const sourceChunksByRepo = new Map<string, StoredVectorChunk[]>();
+
+  for (const chunk of legacyStore.chunks ?? []) {
+    const sourceId = chunk.sourceId ?? chunk.sourceMessageId;
+    const sourceKind = chunk.sourceKind ?? (chunk.metadata?.corpusKind === "repository_source" ? "source_document" : "discord_message");
+    const normalizedChunk: StoredVectorChunk = {
+      ...chunk,
+      sourceId,
+      sourceKind,
+      metadata: {
+        ...chunk.metadata,
+        sourceId: chunk.metadata?.sourceId ?? sourceId ?? "",
+      },
+    };
+
+    if (normalizedChunk.metadata.corpusKind === "repository_source") {
+      const repoName = normalizedChunk.metadata.repoName;
+
+      if (!repoName) {
+        continue;
+      }
+
+      const bucket = sourceChunksByRepo.get(repoName) ?? [];
+      bucket.push(normalizedChunk);
+      sourceChunksByRepo.set(repoName, bucket);
+      continue;
+    }
+
+    historyChunks.push(normalizedChunk);
+  }
+
+  await mkdir(dirname(config.vectorStore.path), { recursive: true });
+  await writeFile(
+    config.vectorStore.path,
+    `${JSON.stringify({
+      version: 2,
+      embedderId: legacyStore.embedderId,
+      dimensions: legacyStore.dimensions,
+      chunks: historyChunks,
+    })}\n`,
+    "utf8",
+  );
+
+  await mkdir(config.sourceVectorStoreRoot, { recursive: true });
+
+  for (const [repoName, chunks] of sourceChunksByRepo) {
+    const shardPath = join(config.sourceVectorStoreRoot, `${encodeURIComponent(repoName)}.json`);
+    await writeFile(
+      shardPath,
+      `${JSON.stringify({
+        version: 2,
+        embedderId: legacyStore.embedderId,
+        dimensions: legacyStore.dimensions,
+        chunks,
+      })}\n`,
+      "utf8",
+    );
+  }
+
+  console.log(`Legacy mixed store: ${legacyPath}`);
+  console.log(`History chunks written: ${historyChunks.length}`);
+  console.log(`Source repos written: ${sourceChunksByRepo.size}`);
+  console.log(
+    `Source chunks written: ${[...sourceChunksByRepo.values()].reduce((sum, chunks) => sum + chunks.length, 0)}`,
+  );
+}
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/rebuild-rag-index.ts

@@ -0,0 +1,90 @@
+import "dotenv/config";
+
+import { loadConfig } from "@voidbot/config";
+import {
+  createTextEmbedder,
+  FileMessageArchiveRepository,
+  FileSourceDocumentArchiveRepository,
+  HistoryIngester,
+  SourceDocumentIngester,
+  createVectorStores,
+} from "@voidbot/rag";
+
+const REBUILD_CHUNK_BATCH_SIZE = 64;
+
+async function main(): Promise<void> {
+  if (!process.env.DISCORD_OWNER_ID) {
+    process.env.DISCORD_OWNER_ID = "__rag_rebuild__";
+  }
+
+  const config = loadConfig();
+  const archiveRepository = new FileMessageArchiveRepository(config.ragArchivePath);
+  const sourceArchiveRepository = new FileSourceDocumentArchiveRepository(config.ragSourceArchivePath);
+  const embedder = createTextEmbedder({
+    backend: config.ragEmbeddingBackend,
+    hashDimensions: config.ragEmbeddingDimensions,
+    ollamaBaseUrl: config.ragOllamaBaseUrl,
+    ollamaModel: config.ragOllamaModel,
+    ollamaTimeoutMs: config.ragOllamaTimeoutMs,
+    queryInstruction: config.ragQueryInstruction,
+  });
+  const vectorStores = createVectorStores({
+    kind: config.vectorStore.kind,
+    historyPath: config.vectorStore.path,
+    sourceRoot: config.sourceVectorStoreRoot,
+    qdrant: config.qdrant,
+    historyEmbedder: embedder,
+    sourceEmbedder: embedder,
+  });
+  const vectorStore = vectorStores.history;
+  const sourceVectorStore = vectorStores.source;
+  const historyIngester = new HistoryIngester();
+  const sourceIngester = new SourceDocumentIngester();
+  const activeMessages = await archiveRepository.listAllActive();
+  const historyChunks = historyIngester.chunkMessages(activeMessages);
+  const sourceDocuments = await sourceArchiveRepository.listAll();
+  const sourceChunks = sourceIngester.chunkDocuments(sourceDocuments);
+
+  await vectorStore.clear();
+  await sourceVectorStore.clear();
+
+  for (let index = 0; index < historyChunks.length; index += REBUILD_CHUNK_BATCH_SIZE) {
+    const batch = historyChunks.slice(index, index + REBUILD_CHUNK_BATCH_SIZE);
+
+    if (batch.length === 0) {
+      continue;
+    }
+
+    await vectorStore.upsert(batch);
+
+    if ((index / REBUILD_CHUNK_BATCH_SIZE + 1) % 25 === 0 || index + batch.length >= historyChunks.length) {
+      console.log(`History rebuild progress: ${Math.min(index + batch.length, historyChunks.length)}/${historyChunks.length} chunks`);
+    }
+  }
+
+  for (let index = 0; index < sourceChunks.length; index += REBUILD_CHUNK_BATCH_SIZE) {
+    const batch = sourceChunks.slice(index, index + REBUILD_CHUNK_BATCH_SIZE);
+
+    if (batch.length === 0) {
+      continue;
+    }
+
+    await sourceVectorStore.upsert(batch);
+
+    if ((index / REBUILD_CHUNK_BATCH_SIZE + 1) % 25 === 0 || index + batch.length >= sourceChunks.length) {
+      console.log(`Source rebuild progress: ${Math.min(index + batch.length, sourceChunks.length)}/${sourceChunks.length} chunks`);
+    }
+  }
+
+  console.log(`Embedder: ${embedder.id}`);
+  console.log(`Archived active messages: ${activeMessages.length}`);
+  console.log(`Archived source documents: ${sourceDocuments.length}`);
+  console.log(`History chunks rebuilt: ${historyChunks.length}`);
+  console.log(`Source chunks rebuilt: ${sourceChunks.length}`);
+  console.log(`Indexed chunks rebuilt: ${historyChunks.length + sourceChunks.length}`);
+}
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/remove-codex-mcp.ts

@@ -0,0 +1,38 @@
+import { spawn } from "node:child_process";
+import { resolve } from "node:path";
+
+import { config as loadDotenv } from "dotenv";
+
+import { loadConfig } from "../packages/config/src";
+import { VOID_MCP_SERVER_NAME } from "../packages/core/src/codex-mcp";
+
+void main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});
+
+async function main(): Promise<void> {
+  loadDotenv({ path: resolve(process.cwd(), ".env") });
+  const config = loadConfig();
+  await runCodex(config.codexExecutable, config.codexExecArgs, ["mcp", "remove", VOID_MCP_SERVER_NAME]);
+}
+
+function runCodex(executable: string, executableArgs: string[], args: string[]): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(executable, [...executableArgs, ...args], {
+      cwd: process.cwd(),
+      env: process.env,
+      stdio: "inherit",
+    });
+
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+        return;
+      }
+
+      reject(new Error(`codex ${args.join(" ")} exited with code ${code ?? "unknown"}`));
+    });
+  });
+}

scripts/setup-docker-qdrant.ps1

@@ -0,0 +1,155 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+$repoRoot = Split-Path -Parent $PSScriptRoot
+$dockerDesktopPath = "C:\Program Files\Docker\Docker\Docker Desktop.exe"
+$dockerCliDir = "C:\Program Files\Docker\Docker\resources\bin"
+$startQdrantScript = Join-Path $PSScriptRoot "start-qdrant.ps1"
+$dockerInstallerDownloadDir = Join-Path $env:LOCALAPPDATA "Temp\VoidBot\DockerDesktopInstaller"
+$dockerInstallerOverrideUrl = $env:VOIDBOT_DOCKER_DESKTOP_DOWNLOAD_URL
+
+function Test-Administrator {
+  $identity = [Security.Principal.WindowsIdentity]::GetCurrent()
+  $principal = [Security.Principal.WindowsPrincipal]::new($identity)
+  return $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
+}
+
+function Test-WslInstalled {
+  try {
+    & wsl --status *> $null
+    return $LASTEXITCODE -eq 0
+  } catch {
+    return $false
+  }
+}
+
+function Get-DockerInstallerPath {
+  New-Item -ItemType Directory -Force -Path $dockerInstallerDownloadDir | Out-Null
+
+  if ($dockerInstallerOverrideUrl) {
+    $fileName = [System.IO.Path]::GetFileName(([System.Uri]$dockerInstallerOverrideUrl).AbsolutePath)
+
+    if ([string]::IsNullOrWhiteSpace($fileName)) {
+      throw "VOIDBOT_DOCKER_DESKTOP_DOWNLOAD_URL does not point to a valid installer filename."
+    }
+
+    $overrideInstallerPath = Join-Path $dockerInstallerDownloadDir $fileName
+    Write-Host "Downloading Docker Desktop installer from override URL..."
+    & curl.exe -L --fail --silent --show-error --output $overrideInstallerPath $dockerInstallerOverrideUrl
+
+    if ($LASTEXITCODE -ne 0 -or -not (Test-Path $overrideInstallerPath)) {
+      throw "Failed to download Docker Desktop installer from override URL."
+    }
+
+    return $overrideInstallerPath
+  }
+
+  Write-Host "Downloading Docker Desktop installer via winget..."
+  $downloadOutput = & winget download `
+    --id Docker.DockerDesktop `
+    --exact `
+    --accept-source-agreements `
+    --accept-package-agreements `
+    --download-directory $dockerInstallerDownloadDir
+
+  if ($downloadOutput) {
+    $downloadOutput | ForEach-Object { Write-Host $_ }
+  }
+
+  $installer = Get-ChildItem -Path $dockerInstallerDownloadDir -Filter "*.exe" -File -Recurse |
+    Sort-Object LastWriteTime -Descending |
+    Select-Object -First 1
+
+  if ($null -eq $installer) {
+    throw "winget download completed without leaving a Docker Desktop installer behind."
+  }
+
+  return $installer.FullName
+}
+
+if (-not (Test-Administrator)) {
+  throw "Run this script from an elevated PowerShell window. Docker Desktop and WSL feature installation require administrator privileges."
+}
+
+$processor = Get-CimInstance Win32_Processor | Select-Object -First 1
+
+if (-not $processor.VirtualizationFirmwareEnabled) {
+  throw "Hardware virtualization is disabled in BIOS/UEFI. Enable AMD-V/SVM or Intel VT-x/Virtualization Technology, reboot Windows, then rerun this script."
+}
+
+if (-not (Get-Command winget -ErrorAction SilentlyContinue)) {
+  throw "winget is not available on this machine."
+}
+
+$restartRequired = $false
+
+if (-not (Test-WslInstalled)) {
+  Write-Host "Installing WSL..."
+
+  try {
+    & wsl --install --no-distribution
+  } catch {
+    Write-Warning "wsl --install --no-distribution failed; retrying with wsl --install."
+    & wsl --install
+  }
+
+  $restartRequired = $true
+}
+
+if (-not (Test-Path $dockerDesktopPath)) {
+  $dockerInstallerPath = Get-DockerInstallerPath
+
+  Write-Host "Installing Docker Desktop from $dockerInstallerPath..."
+  $installProcess = Start-Process `
+    -FilePath $dockerInstallerPath `
+    -ArgumentList @(
+      "install",
+      "--accept-license",
+      "--backend=wsl-2"
+    ) `
+    -Wait `
+    -PassThru
+
+  if ($installProcess.ExitCode -ne 0) {
+    throw "Docker Desktop installer exited with code $($installProcess.ExitCode). Check $env:LOCALAPPDATA\Docker\install-log.txt for details."
+  }
+
+  & net.exe localgroup docker-users $env:USERNAME /add *> $null
+}
+
+if ($restartRequired) {
+  Write-Warning "A restart may be required before Docker Desktop and WSL are usable."
+  Write-Warning "Reboot Windows, open an elevated PowerShell window in this repo, and rerun scripts\\setup-docker-qdrant.ps1."
+  exit 0
+}
+
+if (Test-Path $dockerCliDir) {
+  $env:Path = "$dockerCliDir;$env:Path"
+}
+
+if (-not (Get-Command docker -ErrorAction SilentlyContinue)) {
+  throw "Docker Desktop appears to be installed, but docker.exe is not on PATH yet. Sign out and back in, or start Docker Desktop once and rerun this script."
+}
+
+if (-not (Get-Process -Name "Docker Desktop" -ErrorAction SilentlyContinue)) {
+  Write-Host "Starting Docker Desktop..."
+  Start-Process -FilePath $dockerDesktopPath | Out-Null
+}
+
+$dockerReady = $false
+for ($attempt = 0; $attempt -lt 60; $attempt += 1) {
+  & docker version *> $null
+
+  if ($LASTEXITCODE -eq 0) {
+    $dockerReady = $true
+    break
+  }
+
+  Start-Sleep -Seconds 5
+}
+
+if (-not $dockerReady) {
+  throw "Docker Desktop did not become ready in time. Open Docker Desktop, finish any first-run prompts, then rerun this script."
+}
+
+& powershell -ExecutionPolicy Bypass -File $startQdrantScript

scripts/start-qdrant.ps1

@@ -0,0 +1,48 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+$repoRoot = Split-Path -Parent $PSScriptRoot
+$composeFile = Join-Path $repoRoot "infra\qdrant\docker-compose.yml"
+$dockerCliDir = "C:\Program Files\Docker\Docker\resources\bin"
+
+if (Test-Path $dockerCliDir) {
+  $env:Path = "$dockerCliDir;$env:Path"
+}
+
+if (-not (Get-Command docker -ErrorAction SilentlyContinue)) {
+  throw "Docker CLI is not available on PATH. Install Docker Desktop first, then rerun this script."
+}
+
+& docker version *> $null
+if ($LASTEXITCODE -ne 0) {
+  throw "Docker is installed but the daemon is not ready. Start Docker Desktop, finish any first-run prompts, then rerun this script."
+}
+
+Write-Host "Starting Qdrant with Docker Compose..."
+& docker compose -f $composeFile up -d
+
+if ($LASTEXITCODE -ne 0) {
+  throw "Docker Compose failed to start Qdrant."
+}
+
+$healthy = $false
+for ($attempt = 0; $attempt -lt 30; $attempt += 1) {
+  try {
+    $response = Invoke-RestMethod -Uri "http://127.0.0.1:6333" -TimeoutSec 5
+
+    if ($null -ne $response) {
+      $healthy = $true
+      break
+    }
+  } catch {
+    Start-Sleep -Seconds 2
+  }
+}
+
+if (-not $healthy) {
+  throw "Qdrant container started, but the HTTP API did not become reachable at http://127.0.0.1:6333 in time."
+}
+
+Write-Host "Qdrant is up."
+Write-Host "API: http://127.0.0.1:6333"
+Write-Host "Dashboard: http://127.0.0.1:6333/dashboard"

styles/void-default.md

@@ -0,0 +1,17 @@
+# void-default
+
+Void is the name and persona of the bot. Void uses they/them pronouns.
+
+Void is a slightly uncanny but helpful assistant. They can sound sharp, dry, and faintly eldritch for flavor, but they are here to be useful first.
+
+Voice and behavior:
+
+- Sound clear, composed, and a little strange, but not hostile.
+- Keep the answer itself easy to follow, even when the flavor is playful.
+- Lead with the useful answer before any flair.
+- Use restrained dry humor rather than community-specific jokes or references.
+- Keep technical instructions, safety warnings, and operational details plain.
+- Never imply permissions, memories, knowledge, or powers the bot does not actually have.
+- When uncertain, say so directly instead of bluffing.
+- Keep Discord replies readable in a fast-moving channel.
+- On serious, sensitive, or high-stakes topics, dial the flavor down and let clarity win.

tsconfig.base.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "CommonJS",
+    "moduleResolution": "Node",
+    "lib": [
+      "ES2022"
+    ],
+    "strict": true,
+    "declaration": true,
+    "declarationMap": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  }
+}
+

tsconfig.json

@@ -0,0 +1,30 @@
+{
+  "files": [],
+  "references": [
+    {
+      "path": "./packages/shared"
+    },
+    {
+      "path": "./packages/config"
+    },
+    {
+      "path": "./packages/core"
+    },
+    {
+      "path": "./packages/providers"
+    },
+    {
+      "path": "./packages/rag"
+    },
+    {
+      "path": "./packages/sandbox"
+    },
+    {
+      "path": "./apps/bot"
+    },
+    {
+      "path": "./apps/worker"
+    }
+  ]
+}
+