From 1b64fa48433f1e660ab4ca40ae7e380dc7cd86cd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 24 Apr 2026 18:20:58 +0100 Subject: [PATCH] Add watchdog and backup verification --- .env.example | 5 + README.md | 46 +- docs/architecture/overview.md | 3 +- package.json | 3 + scripts/check-voidbot-operations.ps1 | 846 +++++++++++++++++++ scripts/install-operations-watchdog-task.ps1 | 95 +++ scripts/sync-voidbot-backup-offsite.ps1 | 12 + scripts/verify-voidbot-backup.ps1 | 444 ++++++++++ 8 files changed, 1448 insertions(+), 6 deletions(-) create mode 100644 scripts/check-voidbot-operations.ps1 create mode 100644 scripts/install-operations-watchdog-task.ps1 create mode 100644 scripts/verify-voidbot-backup.ps1 diff --git a/.env.example b/.env.example index 90ff9c1..fab9b1c 100644 --- a/.env.example +++ b/.env.example @@ -58,3 +58,8 @@ OFFSITE_BACKUP_REMOTE_KEEP_LATEST=14 OFFSITE_BACKUP_LABEL=offsite-auto OFFSITE_BACKUP_TASK_NAME=VoidBot Offsite Backup OFFSITE_BACKUP_TASK_TIME=04:30 +VOIDBOT_BACKUP_MAX_AGE_HOURS=30 +VOIDBOT_OFFSITE_BACKUP_MAX_AGE_HOURS=36 +VOIDBOT_HEALTHCHECK_NOTIFY_REPEAT_HOURS=12 +VOIDBOT_HEALTHCHECK_TASK_NAME=VoidBot Operations Watchdog +VOIDBOT_HEALTHCHECK_INTERVAL_MINUTES=60 diff --git a/README.md b/README.md index 3a87c59..48969fd 100644 --- a/README.md +++ b/README.md @@ -427,6 +427,7 @@ npm run state:sync-offsite That flow: - runs the normal local backup first +- verifies the fresh backup before it is allowed anywhere near the wire - compresses the resulting backup directory into a zip archive - uploads it over `sftp` - keeps a bounded number of scheduled local backup directories @@ -453,6 +454,40 @@ Defaults come from `.env` / `.env.example`: The scheduled task uses `Interactive` logon on Windows so it can see the current user's SSH key material. That means the backup job is intended for a workstation session that stays logged in; it is not pretending to be a hardened headless backup appliance. +There is also a recurring watchdog now, so the stack can stop being quiet when it drifts: + +```bash +npm run state:check +``` + +That check covers: + +- bot and worker process liveness from `.voidbot/status/runtime-stack.json` +- Qdrant reachability and required collection presence +- Postgres container health and required table presence +- local and remote Ollama model availability +- Discord bot token validity +- latest local backup verification and freshness +- offsite task presence, remote archive freshness, and retention count + +It writes status to `.voidbot/status/operations-health.json` and appends logs to `.voidbot/logs/operations-watchdog.log`. + +Install the recurring watchdog task: + +```bash +npm run state:install-watchdog-task +``` + +Defaults come from `.env` / `.env.example`: + +- `VOIDBOT_BACKUP_MAX_AGE_HOURS` +- `VOIDBOT_OFFSITE_BACKUP_MAX_AGE_HOURS` +- `VOIDBOT_HEALTHCHECK_NOTIFY_REPEAT_HOURS` +- `VOIDBOT_HEALTHCHECK_TASK_NAME` +- `VOIDBOT_HEALTHCHECK_INTERVAL_MINUTES` + +The watchdog task also uses `Interactive` logon because the remote freshness check relies on the same local SSH key material as the offsite sync path. + ## Commands Current slash commands: @@ -478,6 +513,8 @@ Useful local state and docs: - `.voidbot/rag/source-documents.json`: archived source and lore documents - `.voidbot/rag/import-state.json`: log-backfill file state - `.voidbot/status/runtime-stack.json`: live stack status, service reachability, and bot/worker PIDs +- `.voidbot/status/offsite-backup.json`: latest offsite backup sync result +- `.voidbot/status/operations-health.json`: latest watchdog health report - `config/system-messages.json`: rotating stock system messages - `styles/void-default.md`: public default style pack - `docs/architecture/overview.md`: higher-level architecture notes @@ -496,11 +533,10 @@ Useful local state and docs: Reasonable next steps from here: -1. Add health checks, backups, and operational guidance around Postgres and Qdrant. -2. Add periodic restore drills and remote retention monitoring so the offsite backup path stays real instead of ceremonial. -3. Expand worker processing into richer provider run records and moderation hooks. -4. Replace the remaining scaffolded provider paths with funded, production-grade implementations. -5. Add real constrained sandbox execution instead of policy-only dry runs. +1. Add a real scheduled restore drill instead of stopping at backup verification and freshness checks. +2. Expand worker processing into richer provider run records, moderation hooks, and admin inspection tools. +3. Replace the remaining scaffolded provider paths with funded, production-grade implementations. +4. Add real constrained sandbox execution instead of policy-only dry runs. ## Related Docs diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index dff9e8f..a2c017f 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -35,6 +35,7 @@ Discord Gateway - The owner Codex flow can perform repeated read-only history searches through a bounded tool loop instead of relying on a single retrieval snapshot. - Stock Discord replies are loaded from `config/system-messages.json` and rotated through shuffled variants to avoid repetitive canned responses. - Changing embedding models requires a full vector rebuild from the archived messages, which is handled by `npm run rag:rebuild`. +- Local and offsite backups now have a verifier plus a recurring watchdog, so backup freshness and remote retention stop relying on operator vibes. - `owner_codex` has two lanes: - `local_exec_owner_only` tries a read-only `codex exec` pass for direct Discord replies - `manual_package` remains available when you want explicit approval-gated packaging @@ -42,7 +43,7 @@ Discord Gateway ## Upgrade path -1. Add restore drills and retention monitoring around the new local-plus-offsite backup path. +1. Add a true restore drill path instead of stopping at backup verification plus freshness monitoring. 2. Add moderation, budgeting, and rate limits before enabling `openai_api` for member traffic. 3. Expand worker-side run records and admin tooling around the interaction memory/event stores. 4. Expand sandbox execution from dry-run policy checks to real constrained runners. diff --git a/package.json b/package.json index 4c9c60c..9b7c1a0 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,11 @@ "stack:start": "powershell -ExecutionPolicy Bypass -File scripts/start-voidbot-stack.ps1", "state:backup": "powershell -ExecutionPolicy Bypass -File scripts/backup-voidbot-state.ps1", "state:restore": "powershell -ExecutionPolicy Bypass -File scripts/restore-voidbot-state.ps1", + "state:verify-backup": "powershell -ExecutionPolicy Bypass -File scripts/verify-voidbot-backup.ps1", + "state:check": "powershell -ExecutionPolicy Bypass -File scripts/check-voidbot-operations.ps1", "state:sync-offsite": "powershell -ExecutionPolicy Bypass -File scripts/sync-voidbot-backup-offsite.ps1", "state:install-offsite-task": "powershell -ExecutionPolicy Bypass -File scripts/install-offsite-backup-task.ps1", + "state:install-watchdog-task": "powershell -ExecutionPolicy Bypass -File scripts/install-operations-watchdog-task.ps1", "postgres:up": "docker compose -f infra/postgres/docker-compose.yml up -d", "codex:mcp:install": "tsx scripts/install-codex-mcp.ts", "codex:mcp:remove": "tsx scripts/remove-codex-mcp.ts", diff --git a/scripts/check-voidbot-operations.ps1 b/scripts/check-voidbot-operations.ps1 new file mode 100644 index 0000000..d4f8bc6 --- /dev/null +++ b/scripts/check-voidbot-operations.ps1 @@ -0,0 +1,846 @@ +param( + [switch] $NotifyOwner, + [switch] $FailOnIssues +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$repoRoot = Split-Path -Parent $PSScriptRoot +$envFile = Join-Path $repoRoot ".env" +$verifyScript = Join-Path $PSScriptRoot "verify-voidbot-backup.ps1" + +function Read-DotEnv { + param( + [Parameter(Mandatory = $true)] + [string] $Path + ) + + if (-not (Test-Path $Path)) { + throw "Missing .env at $Path" + } + + $values = @{} + + foreach ($line in Get-Content -Path $Path) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + if ($line.TrimStart().StartsWith("#")) { + continue + } + + $separatorIndex = $line.IndexOf("=") + + if ($separatorIndex -lt 1) { + continue + } + + $key = $line.Substring(0, $separatorIndex).Trim() + $value = $line.Substring($separatorIndex + 1) + $values[$key] = $value + } + + return $values +} + +function Resolve-ConfigPath { + param( + [Parameter(Mandatory = $true)] + [string] $RepoRoot, + [Parameter(Mandatory = $false)] + [AllowEmptyString()] + [string] $Value, + [Parameter(Mandatory = $true)] + [string] $Fallback + ) + + $raw = if ([string]::IsNullOrWhiteSpace($Value)) { $Fallback } else { $Value } + return [System.IO.Path]::GetFullPath((Join-Path $RepoRoot $raw)) +} + +function Write-StatusFile { + param( + [Parameter(Mandatory = $true)] + [string] $Path, + [Parameter(Mandatory = $true)] + [object] $Status + ) + + $directory = Split-Path -Parent $Path + New-Item -ItemType Directory -Force -Path $directory | Out-Null + $Status | ConvertTo-Json -Depth 10 | Set-Content -Path $Path -Encoding utf8 +} + +function Start-LogCapture { + param( + [Parameter(Mandatory = $true)] + [string] $LogPath + ) + + $directory = Split-Path -Parent $LogPath + New-Item -ItemType Directory -Force -Path $directory | Out-Null + + try { + Start-Transcript -Path $LogPath -Append | Out-Null + return $true + } catch { + return $false + } +} + +function Stop-LogCapture { + param( + [Parameter(Mandatory = $true)] + [bool] $Started + ) + + if (-not $Started) { + return + } + + try { + Stop-Transcript | Out-Null + } catch { + } +} + +function Add-Check { + param( + [Parameter(Mandatory = $true)] + [string] $Name, + [Parameter(Mandatory = $true)] + [ValidateSet("passed", "warning", "failed")] + [string] $Status, + [Parameter(Mandatory = $true)] + [string] $Detail, + [hashtable] $Data = @{} + ) + + $script:checks += [ordered]@{ + name = $Name + status = $Status + detail = $Detail + data = $Data + } +} + +function Finalize-Report { + $failedCount = @($script:checks | Where-Object { $_.status -eq "failed" }).Count + $warningCount = @($script:checks | Where-Object { $_.status -eq "warning" }).Count + $passedCount = @($script:checks | Where-Object { $_.status -eq "passed" }).Count + + $status = if ($failedCount -gt 0) { + "failed" + } elseif ($warningCount -gt 0) { + "warning" + } else { + "healthy" + } + + return [ordered]@{ + checkedAt = (Get-Date).ToString("o") + repoRoot = $repoRoot + status = $status + summary = @{ + passed = $passedCount + warning = $warningCount + failed = $failedCount + } + checks = $script:checks + } +} + +function Get-Json { + param( + [Parameter(Mandatory = $true)] + [string] $Url, + [string[]] $Headers = @() + ) + + $args = @( + "-fsS", + "--connect-timeout", "5", + "--max-time", "20", + $Url + ) + + foreach ($header in $Headers) { + $args = @("-H", $header) + $args + } + + $response = & curl.exe @args + + if ($LASTEXITCODE -ne 0) { + throw "Request failed for $Url" + } + + return $response | ConvertFrom-Json +} + +function Test-DockerAvailable { + try { + & docker version *> $null + return $LASTEXITCODE -eq 0 + } catch { + return $false + } +} + +function Get-DockerContainerHealth { + param( + [Parameter(Mandatory = $true)] + [string] $ContainerName + ) + + try { + $status = & docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}" $ContainerName 2>$null + + if ($LASTEXITCODE -ne 0) { + return $null + } + + return $status.Trim() + } catch { + return $null + } +} + +function Parse-DatabaseDsn { + param( + [Parameter(Mandatory = $true)] + [string] $Dsn + ) + + $uri = [Uri]$Dsn + $userInfo = if ([string]::IsNullOrWhiteSpace($uri.UserInfo)) { @() } else { $uri.UserInfo.Split(":", 2) } + + return @{ + host = $uri.Host + port = if ($uri.Port -gt 0) { $uri.Port } else { 5432 } + database = $uri.AbsolutePath.TrimStart("/") + username = if ($userInfo.Length -ge 1) { [Uri]::UnescapeDataString($userInfo[0]) } else { "voidbot" } + } +} + +function Test-ProcessAlive { + param( + [Parameter(Mandatory = $true)] + [int] $ProcessId + ) + + try { + $process = Get-Process -Id $ProcessId -ErrorAction Stop + return $null -ne $process + } catch { + return $false + } +} + +function Escape-SingleQuotedPowerShell { + param( + [Parameter(Mandatory = $true)] + [string] $Value + ) + + return $Value.Replace("'", "''") +} + +function Get-OptionalPropertyValue { + param( + [Parameter(Mandatory = $true)] + [object] $InputObject, + [Parameter(Mandatory = $true)] + [string] $PropertyName + ) + + if ($null -eq $InputObject) { + return $null + } + + $property = $InputObject.PSObject.Properties[$PropertyName] + + if ($null -eq $property) { + return $null + } + + return $property.Value +} + +function Invoke-RemotePowerShellJson { + param( + [Parameter(Mandatory = $true)] + [string] $Target, + [Parameter(Mandatory = $true)] + [string] $ScriptText + ) + + $bytes = [System.Text.Encoding]::Unicode.GetBytes($ScriptText) + $encoded = [Convert]::ToBase64String($bytes) + $response = & ssh.exe -o BatchMode=yes -o ConnectTimeout=10 $Target powershell -NoProfile -EncodedCommand $encoded + + if ($LASTEXITCODE -ne 0) { + throw "SSH command failed for $Target." + } + + if ([string]::IsNullOrWhiteSpace($response)) { + return $null + } + + return $response | ConvertFrom-Json +} + +function Open-OwnerDmChannel { + param( + [Parameter(Mandatory = $true)] + [string] $BotToken, + [Parameter(Mandatory = $true)] + [string] $OwnerDiscordId + ) + + $payload = @{ recipient_id = $OwnerDiscordId } | ConvertTo-Json -Compress + $response = & curl.exe -fsS -X POST "https://discord.com/api/v10/users/@me/channels" ` + -H "Authorization: Bot $BotToken" ` + -H "Content-Type: application/json" ` + --data-raw $payload + + if ($LASTEXITCODE -ne 0) { + throw "Failed to open the owner DM channel." + } + + $parsed = $response | ConvertFrom-Json + + if (-not $parsed.id) { + throw "Discord DM open returned no channel id." + } + + return [string]$parsed.id +} + +function Send-OwnerDm { + param( + [Parameter(Mandatory = $true)] + [string] $BotToken, + [Parameter(Mandatory = $true)] + [string] $OwnerDiscordId, + [Parameter(Mandatory = $true)] + [string] $Content + ) + + $dmChannelId = Open-OwnerDmChannel -BotToken $BotToken -OwnerDiscordId $OwnerDiscordId + $payload = @{ + content = if ($Content.Length -le 1900) { $Content } else { "$($Content.Substring(0, 1897))..." } + allowed_mentions = @{ + parse = @() + } + } | ConvertTo-Json -Depth 4 -Compress + + & curl.exe -fsS -X POST "https://discord.com/api/v10/channels/$dmChannelId/messages" ` + -H "Authorization: Bot $BotToken" ` + -H "Content-Type: application/json" ` + --data-raw $payload *> $null + + if ($LASTEXITCODE -ne 0) { + throw "Failed to send the owner DM." + } +} + +function Build-OwnerNotificationMessage { + param( + [Parameter(Mandatory = $true)] + [string] $Status, + [Parameter(Mandatory = $true)] + [string[]] $IssueLines, + [Parameter(Mandatory = $true)] + [string] $HealthStatusPath + ) + + if ($Status -eq "healthy") { + return "VoidBot watchdog checked again. The stack looks normal now. Health report: $HealthStatusPath" + } + + $lines = @( + "VoidBot watchdog smelled smoke." + "Health report: $HealthStatusPath" + "" + ) + ($IssueLines | ForEach-Object { "- $_" }) + + return ($lines -join [Environment]::NewLine).Trim() +} + +$checks = @() +$config = Read-DotEnv -Path $envFile +$storageRoot = Resolve-ConfigPath -RepoRoot $repoRoot -Value $config["STORAGE_ROOT"] -Fallback ".voidbot" +$statusPath = Join-Path $storageRoot "status\operations-health.json" +$watchdogStatePath = Join-Path $storageRoot "status\operations-watchdog.json" +$logPath = Join-Path $storageRoot "logs\operations-watchdog.log" +$runtimeStatusPath = Join-Path $storageRoot "status\runtime-stack.json" +$offsiteStatusPath = Join-Path $storageRoot "status\offsite-backup.json" +$backupRoot = Join-Path $storageRoot "backups" +$backupMaxAgeHours = if ($config.ContainsKey("VOIDBOT_BACKUP_MAX_AGE_HOURS")) { [double]$config["VOIDBOT_BACKUP_MAX_AGE_HOURS"] } else { 30.0 } +$offsiteMaxAgeHours = if ($config.ContainsKey("VOIDBOT_OFFSITE_BACKUP_MAX_AGE_HOURS")) { [double]$config["VOIDBOT_OFFSITE_BACKUP_MAX_AGE_HOURS"] } else { 36.0 } +$notifyRepeatHours = if ($config.ContainsKey("VOIDBOT_HEALTHCHECK_NOTIFY_REPEAT_HOURS")) { [double]$config["VOIDBOT_HEALTHCHECK_NOTIFY_REPEAT_HOURS"] } else { 12.0 } +$watchdogTaskName = if ($config.ContainsKey("VOIDBOT_HEALTHCHECK_TASK_NAME") -and -not [string]::IsNullOrWhiteSpace($config["VOIDBOT_HEALTHCHECK_TASK_NAME"])) { $config["VOIDBOT_HEALTHCHECK_TASK_NAME"] } else { "VoidBot Operations Watchdog" } +$offsiteTaskName = if ($config.ContainsKey("OFFSITE_BACKUP_TASK_NAME") -and -not [string]::IsNullOrWhiteSpace($config["OFFSITE_BACKUP_TASK_NAME"])) { $config["OFFSITE_BACKUP_TASK_NAME"] } else { "VoidBot Offsite Backup" } +$offsiteRemoteWindowsDir = if ($config.ContainsKey("OFFSITE_BACKUP_REMOTE_WINDOWS_DIR") -and -not [string]::IsNullOrWhiteSpace($config["OFFSITE_BACKUP_REMOTE_WINDOWS_DIR"])) { $config["OFFSITE_BACKUP_REMOTE_WINDOWS_DIR"] } else { "C:\Meta\voidbot-backups" } +$offsiteTarget = if ($config.ContainsKey("OFFSITE_BACKUP_SSH_TARGET")) { $config["OFFSITE_BACKUP_SSH_TARGET"] } else { "" } +$offsiteKeepLatest = if ($config.ContainsKey("OFFSITE_BACKUP_REMOTE_KEEP_LATEST")) { [int]$config["OFFSITE_BACKUP_REMOTE_KEEP_LATEST"] } else { 14 } +$offsiteLabel = if ($config.ContainsKey("OFFSITE_BACKUP_LABEL") -and -not [string]::IsNullOrWhiteSpace($config["OFFSITE_BACKUP_LABEL"])) { $config["OFFSITE_BACKUP_LABEL"] } else { "offsite-auto" } +$logCaptureStarted = Start-LogCapture -LogPath $logPath +$watchdogState = if (Test-Path -LiteralPath $watchdogStatePath) { + try { + Get-Content -LiteralPath $watchdogStatePath -Raw | ConvertFrom-Json + } catch { + $null + } +} else { + $null +} + +try { + if (-not (Test-Path -LiteralPath $runtimeStatusPath)) { + Add-Check -Name "runtime.status_file" -Status "failed" -Detail "runtime-stack.json is missing at $runtimeStatusPath." + } else { + $runtimeStatus = Get-Content -LiteralPath $runtimeStatusPath -Raw | ConvertFrom-Json + Add-Check -Name "runtime.status_file" -Status "passed" -Detail "Found runtime-stack.json." -Data @{ + path = $runtimeStatusPath + } + + foreach ($component in @("bot", "worker")) { + $processIdValue = [int]$runtimeStatus.$component.pid + + if ($processIdValue -le 0) { + Add-Check -Name "runtime.$component" -Status "failed" -Detail "$component PID is missing from runtime-stack.json." + continue + } + + if (Test-ProcessAlive -ProcessId $processIdValue) { + Add-Check -Name "runtime.$component" -Status "passed" -Detail "$component process $processIdValue is running." -Data @{ + pid = $processIdValue + } + } else { + Add-Check -Name "runtime.$component" -Status "failed" -Detail "$component process $processIdValue is not running anymore." -Data @{ + pid = $processIdValue + } + } + } + } + + $vectorStoreKind = if ($config.ContainsKey("VECTOR_STORE_KIND")) { $config["VECTOR_STORE_KIND"] } else { "local_json" } + $historyCollection = if ($config.ContainsKey("QDRANT_HISTORY_COLLECTION")) { $config["QDRANT_HISTORY_COLLECTION"] } else { "voidbot_discord_history_chunks" } + $sourceCollection = if ($config.ContainsKey("QDRANT_SOURCE_COLLECTION")) { $config["QDRANT_SOURCE_COLLECTION"] } else { "voidbot_repository_source_chunks" } + + if ($vectorStoreKind -eq "qdrant") { + $qdrantUrl = if ($config.ContainsKey("QDRANT_URL")) { $config["QDRANT_URL"] } else { "http://127.0.0.1:6333" } + try { + $collectionsResponse = Get-Json -Url "$($qdrantUrl.TrimEnd('/'))/collections" + $collectionNames = @($collectionsResponse.result.collections | ForEach-Object { [string]$_.name }) + Add-Check -Name "qdrant.reachable" -Status "passed" -Detail "Qdrant is reachable at $qdrantUrl." -Data @{ + url = $qdrantUrl + collections = $collectionNames + } + + foreach ($requiredCollection in @($historyCollection, $sourceCollection)) { + if ($collectionNames -contains $requiredCollection) { + Add-Check -Name "qdrant.collection.$requiredCollection" -Status "passed" -Detail "Qdrant collection $requiredCollection exists." + } else { + Add-Check -Name "qdrant.collection.$requiredCollection" -Status "failed" -Detail "Qdrant collection $requiredCollection is missing." + } + } + } catch { + Add-Check -Name "qdrant.reachable" -Status "failed" -Detail "Qdrant check failed: $($_.Exception.Message)" -Data @{ + url = $qdrantUrl + } + } + } else { + Add-Check -Name "qdrant.reachable" -Status "passed" -Detail "Qdrant check skipped because VECTOR_STORE_KIND=$vectorStoreKind." -Data @{ + vectorStoreKind = $vectorStoreKind + } + } + + $stateStorageBackend = if ($config.ContainsKey("STATE_STORAGE_BACKEND")) { $config["STATE_STORAGE_BACKEND"] } else { "postgres" } + + if ($stateStorageBackend -eq "postgres") { + $databaseDsn = if ($config.ContainsKey("DATABASE_DSN")) { $config["DATABASE_DSN"] } else { "" } + $dsn = Parse-DatabaseDsn -Dsn $databaseDsn + + if (-not (Test-DockerAvailable)) { + Add-Check -Name "postgres.health" -Status "warning" -Detail "Skipped Postgres container health check because Docker is unavailable." -Data @{ + dsn = $databaseDsn + } + } else { + $containerHealth = Get-DockerContainerHealth -ContainerName "voidbot-postgres" + + if ($containerHealth -eq "healthy") { + Add-Check -Name "postgres.health" -Status "passed" -Detail "Postgres container is healthy." -Data @{ + containerName = "voidbot-postgres" + database = $dsn.database + } + + try { + $tableList = & docker exec voidbot-postgres psql -U $dsn.username -d $dsn.database -tAc "select table_name from information_schema.tables where table_schema = 'public';" + + if ($LASTEXITCODE -ne 0) { + throw "psql table listing failed." + } + + $tables = @($tableList -split "`r?`n" | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + foreach ($requiredTable in @("jobs", "audit_events", "interaction_memory_events")) { + if ($tables -contains $requiredTable) { + Add-Check -Name "postgres.table.$requiredTable" -Status "passed" -Detail "Postgres table $requiredTable exists." + } else { + Add-Check -Name "postgres.table.$requiredTable" -Status "failed" -Detail "Postgres table $requiredTable is missing." + } + } + } catch { + Add-Check -Name "postgres.tables" -Status "warning" -Detail "Could not verify Postgres tables: $($_.Exception.Message)" + } + } elseif ($containerHealth) { + Add-Check -Name "postgres.health" -Status "failed" -Detail "Postgres container health is '$containerHealth' instead of healthy." -Data @{ + containerName = "voidbot-postgres" + } + } else { + Add-Check -Name "postgres.health" -Status "failed" -Detail "Postgres container voidbot-postgres is not available." + } + } + } else { + Add-Check -Name "postgres.health" -Status "passed" -Detail "Postgres check skipped because STATE_STORAGE_BACKEND=$stateStorageBackend." -Data @{ + stateStorageBackend = $stateStorageBackend + } + } + + $ragOllamaUrl = if ($config.ContainsKey("RAG_OLLAMA_BASE_URL")) { $config["RAG_OLLAMA_BASE_URL"] } else { "http://127.0.0.1:11434" } + $ragOllamaModel = if ($config.ContainsKey("RAG_OLLAMA_MODEL")) { $config["RAG_OLLAMA_MODEL"] } else { "qwen3-embedding:0.6b" } + + try { + $ragTags = Get-Json -Url "$($ragOllamaUrl.TrimEnd('/'))/api/tags" + $ragModels = @($ragTags.models | ForEach-Object { [string]$_.name }) + + if ($ragModels -contains $ragOllamaModel) { + Add-Check -Name "rag_ollama.model" -Status "passed" -Detail "Embedding Ollama is reachable and has $ragOllamaModel." -Data @{ + url = $ragOllamaUrl + model = $ragOllamaModel + } + } else { + Add-Check -Name "rag_ollama.model" -Status "failed" -Detail "Embedding Ollama is reachable but missing $ragOllamaModel." -Data @{ + url = $ragOllamaUrl + models = $ragModels + } + } + } catch { + Add-Check -Name "rag_ollama.model" -Status "failed" -Detail "Embedding Ollama check failed: $($_.Exception.Message)" -Data @{ + url = $ragOllamaUrl + model = $ragOllamaModel + } + } + + $enabledProviders = if ($config.ContainsKey("ENABLED_PROVIDERS") -and -not [string]::IsNullOrWhiteSpace($config["ENABLED_PROVIDERS"])) { + @($config["ENABLED_PROVIDERS"].Split(",") | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + } else { + @() + } + $localLlmUrl = if ($config.ContainsKey("LOCAL_LLM_OLLAMA_BASE_URL")) { $config["LOCAL_LLM_OLLAMA_BASE_URL"] } else { "" } + $localLlmModel = if ($config.ContainsKey("LOCAL_LLM_OLLAMA_MODEL")) { $config["LOCAL_LLM_OLLAMA_MODEL"] } else { "" } + $localLlmEnabled = ($enabledProviders -contains "local_llm") -or ( + $config.ContainsKey("LOCAL_LLM_ALLOW_PUBLIC") -and [string]$config["LOCAL_LLM_ALLOW_PUBLIC"] -eq "true" + ) + + if ($localLlmEnabled -and -not [string]::IsNullOrWhiteSpace($localLlmUrl) -and -not [string]::IsNullOrWhiteSpace($localLlmModel)) { + try { + $localLlmTags = Get-Json -Url "$($localLlmUrl.TrimEnd('/'))/api/tags" + $localLlmModels = @($localLlmTags.models | ForEach-Object { [string]$_.name }) + + if ($localLlmModels -contains $localLlmModel) { + Add-Check -Name "local_llm.model" -Status "passed" -Detail "Local LLM Ollama is reachable and has $localLlmModel." -Data @{ + url = $localLlmUrl + model = $localLlmModel + } + } else { + Add-Check -Name "local_llm.model" -Status "failed" -Detail "Local LLM Ollama is reachable but missing $localLlmModel." -Data @{ + url = $localLlmUrl + models = $localLlmModels + } + } + } catch { + Add-Check -Name "local_llm.model" -Status "failed" -Detail "Local LLM Ollama check failed: $($_.Exception.Message)" -Data @{ + url = $localLlmUrl + model = $localLlmModel + } + } + } else { + Add-Check -Name "local_llm.model" -Status "passed" -Detail "Local LLM check skipped because the provider is not enabled for this runtime." -Data @{ + enabledProviders = $enabledProviders + } + } + + if ($config.ContainsKey("DISCORD_BOT_TOKEN") -and -not [string]::IsNullOrWhiteSpace($config["DISCORD_BOT_TOKEN"])) { + try { + $botIdentity = Get-Json -Url "https://discord.com/api/v10/users/@me" -Headers @("Authorization: Bot $($config["DISCORD_BOT_TOKEN"])") + Add-Check -Name "discord.bot_token" -Status "passed" -Detail "Discord bot token is valid for $([string]$botIdentity.username)." -Data @{ + id = [string]$botIdentity.id + username = [string]$botIdentity.username + } + } catch { + Add-Check -Name "discord.bot_token" -Status "failed" -Detail "Discord bot token check failed: $($_.Exception.Message)" + } + } else { + Add-Check -Name "discord.bot_token" -Status "warning" -Detail "DISCORD_BOT_TOKEN is blank; bot identity could not be checked." + } + + $latestBackup = Get-ChildItem -LiteralPath $backupRoot -Directory -ErrorAction SilentlyContinue | + Sort-Object Name -Descending | + Select-Object -First 1 + + if ($null -eq $latestBackup) { + Add-Check -Name "backup.latest" -Status "failed" -Detail "No local backup directory exists under $backupRoot." + } else { + try { + $verificationJson = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $verifyScript -BackupPath $latestBackup.FullName -AsJson -Quiet + $verificationReport = $verificationJson | ConvertFrom-Json + $createdAtCheck = $verificationReport.checks | Where-Object { $_.name -eq "backup.created_at" } | Select-Object -First 1 + $backupAgeHours = if ($createdAtCheck -and $createdAtCheck.data -and $createdAtCheck.data.ageHours) { [double]$createdAtCheck.data.ageHours } else { [Math]::Round(((Get-Date) - $latestBackup.LastWriteTime).TotalHours, 2) } + + if ([string]$verificationReport.status -eq "failed") { + Add-Check -Name "backup.latest" -Status "failed" -Detail "Latest local backup failed verification." -Data @{ + path = $latestBackup.FullName + verification = $verificationReport + } + } else { + Add-Check -Name "backup.latest" -Status "passed" -Detail "Latest local backup passed verification." -Data @{ + path = $latestBackup.FullName + verificationStatus = [string]$verificationReport.status + ageHours = $backupAgeHours + } + } + + if ($backupAgeHours -le $backupMaxAgeHours) { + Add-Check -Name "backup.freshness" -Status "passed" -Detail "Latest local backup is $backupAgeHours hours old." -Data @{ + thresholdHours = $backupMaxAgeHours + path = $latestBackup.FullName + } + } else { + Add-Check -Name "backup.freshness" -Status "failed" -Detail "Latest local backup is $backupAgeHours hours old, which exceeds the $backupMaxAgeHours hour threshold." -Data @{ + thresholdHours = $backupMaxAgeHours + path = $latestBackup.FullName + } + } + } catch { + Add-Check -Name "backup.latest" -Status "failed" -Detail "Failed to verify the latest local backup: $($_.Exception.Message)" -Data @{ + path = $latestBackup.FullName + } + } + } + + try { + $offsiteTask = Get-ScheduledTask -TaskName $offsiteTaskName -ErrorAction Stop + $offsiteEnabled = [bool]$offsiteTask.Settings.Enabled + $offsiteTaskStatus = if ($offsiteEnabled) { "passed" } else { "failed" } + $offsiteTaskDetail = if ($offsiteEnabled) { "Offsite backup task is installed and enabled." } else { "Offsite backup task is installed but disabled." } + Add-Check -Name "scheduled_task.offsite_backup" -Status $offsiteTaskStatus -Detail $offsiteTaskDetail -Data @{ + taskName = $offsiteTaskName + state = [string]$offsiteTask.State + } + } catch { + Add-Check -Name "scheduled_task.offsite_backup" -Status "failed" -Detail "Offsite backup task $offsiteTaskName is not installed." + } + + try { + $watchdogTask = Get-ScheduledTask -TaskName $watchdogTaskName -ErrorAction Stop + $watchdogEnabled = [bool]$watchdogTask.Settings.Enabled + $watchdogTaskStatus = if ($watchdogEnabled) { "passed" } else { "failed" } + $watchdogTaskDetail = if ($watchdogEnabled) { "Operations watchdog task is installed and enabled." } else { "Operations watchdog task is installed but disabled." } + Add-Check -Name "scheduled_task.watchdog" -Status $watchdogTaskStatus -Detail $watchdogTaskDetail -Data @{ + taskName = $watchdogTaskName + state = [string]$watchdogTask.State + } + } catch { + Add-Check -Name "scheduled_task.watchdog" -Status "warning" -Detail "Operations watchdog task $watchdogTaskName is not installed yet." + } + + if (Test-Path -LiteralPath $offsiteStatusPath) { + try { + $offsiteStatus = Get-Content -LiteralPath $offsiteStatusPath -Raw | ConvertFrom-Json + + if ([string]$offsiteStatus.status -eq "completed") { + Add-Check -Name "offsite.status_file" -Status "passed" -Detail "Latest offsite sync status is completed." -Data @{ + completedAt = [string]$offsiteStatus.completedAt + remoteFile = $offsiteStatus.remoteFile + } + } else { + Add-Check -Name "offsite.status_file" -Status "failed" -Detail "Latest offsite sync status is '$([string]$offsiteStatus.status)' instead of completed." -Data @{ + error = [string](Get-OptionalPropertyValue -InputObject $offsiteStatus -PropertyName "error") + } + } + } catch { + Add-Check -Name "offsite.status_file" -Status "failed" -Detail "Failed to parse offsite-backup.json: $($_.Exception.Message)" + } + } else { + Add-Check -Name "offsite.status_file" -Status "failed" -Detail "offsite-backup.json is missing at $offsiteStatusPath." + } + + if ([string]::IsNullOrWhiteSpace($offsiteTarget)) { + Add-Check -Name "offsite.remote" -Status "warning" -Detail "Remote offsite backup check skipped because OFFSITE_BACKUP_SSH_TARGET is blank." + } else { + try { + $escapedRemoteDir = Escape-SingleQuotedPowerShell -Value $offsiteRemoteWindowsDir + $escapedPattern = Escape-SingleQuotedPowerShell -Value "*-$offsiteLabel.zip" + $remoteScript = @' +$dir = '{0}' +$pattern = '{1}' +if (-not (Test-Path -LiteralPath $dir)) {{ + throw "Missing remote backup directory $dir" +}} +$files = Get-ChildItem -LiteralPath $dir -Filter $pattern -File | Sort-Object LastWriteTimeUtc -Descending +[pscustomobject]@{{ + count = @($files).Count + newest = if (@($files).Count -gt 0) {{ + [pscustomobject]@{{ + name = $files[0].Name + size = $files[0].Length + lastWriteTimeUtc = $files[0].LastWriteTimeUtc.ToString('o') + }} + }} else {{ + $null + }} +}} | ConvertTo-Json -Compress -Depth 4 +'@ -f $escapedRemoteDir, $escapedPattern + $remoteListing = Invoke-RemotePowerShellJson -Target $offsiteTarget -ScriptText $remoteScript + + if ($null -eq $remoteListing -or [int]$remoteListing.count -lt 1) { + Add-Check -Name "offsite.remote" -Status "failed" -Detail "Remote offsite backup directory has no archives matching *-$offsiteLabel.zip." -Data @{ + target = $offsiteTarget + remoteWindowsDir = $offsiteRemoteWindowsDir + } + } else { + $remoteNewest = $remoteListing.newest + $remoteAgeHours = [Math]::Round(([DateTimeOffset]::UtcNow - [DateTimeOffset]::Parse([string]$remoteNewest.lastWriteTimeUtc)).TotalHours, 2) + + if ($remoteAgeHours -le $offsiteMaxAgeHours) { + Add-Check -Name "offsite.remote" -Status "passed" -Detail "Newest remote offsite archive is $remoteAgeHours hours old." -Data @{ + target = $offsiteTarget + remoteWindowsDir = $offsiteRemoteWindowsDir + newest = $remoteNewest + archiveCount = [int]$remoteListing.count + thresholdHours = $offsiteMaxAgeHours + } + } else { + Add-Check -Name "offsite.remote" -Status "failed" -Detail "Newest remote offsite archive is $remoteAgeHours hours old, which exceeds the $offsiteMaxAgeHours hour threshold." -Data @{ + target = $offsiteTarget + remoteWindowsDir = $offsiteRemoteWindowsDir + newest = $remoteNewest + archiveCount = [int]$remoteListing.count + thresholdHours = $offsiteMaxAgeHours + } + } + + if ([int]$remoteListing.count -le $offsiteKeepLatest) { + Add-Check -Name "offsite.retention" -Status "passed" -Detail "Remote offsite archive count $([int]$remoteListing.count) is within the configured keep limit of $offsiteKeepLatest." -Data @{ + archiveCount = [int]$remoteListing.count + keepLatest = $offsiteKeepLatest + } + } else { + Add-Check -Name "offsite.retention" -Status "warning" -Detail "Remote offsite archive count $([int]$remoteListing.count) exceeds the configured keep limit of $offsiteKeepLatest." -Data @{ + archiveCount = [int]$remoteListing.count + keepLatest = $offsiteKeepLatest + } + } + } + } catch { + Add-Check -Name "offsite.remote" -Status "failed" -Detail "Remote offsite backup check failed: $($_.Exception.Message)" -Data @{ + target = $offsiteTarget + remoteWindowsDir = $offsiteRemoteWindowsDir + } + } + } + + $preNotificationReport = Finalize-Report + $issueLines = @($preNotificationReport.checks | Where-Object { $_.status -ne "passed" } | ForEach-Object { "$($_.name): $($_.detail)" }) + $fingerprintSource = if ($issueLines.Count -eq 0) { "healthy" } else { $issueLines -join "`n" } + $fingerprint = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($fingerprintSource)) + + $notification = @{ + attempted = $false + sent = $false + reason = $null + } + + if ($NotifyOwner -and $config.ContainsKey("DISCORD_BOT_TOKEN") -and -not [string]::IsNullOrWhiteSpace($config["DISCORD_BOT_TOKEN"])) { + $lastFingerprint = if ($watchdogState -and $watchdogState.lastFingerprint) { [string]$watchdogState.lastFingerprint } else { "" } + $lastNotifiedAt = if ($watchdogState -and $watchdogState.lastNotifiedAt) { [DateTimeOffset]::Parse([string]$watchdogState.lastNotifiedAt) } else { $null } + $lastStatus = if ($watchdogState -and $watchdogState.lastStatus) { [string]$watchdogState.lastStatus } else { "" } + $shouldNotify = $false + $notificationReason = $null + + if ($preNotificationReport.status -eq "healthy") { + if ($lastStatus -and $lastStatus -ne "healthy") { + $shouldNotify = $true + $notificationReason = "recovered" + } + } else { + $elapsedHours = if ($lastNotifiedAt) { ([DateTimeOffset]::Now - $lastNotifiedAt).TotalHours } else { $null } + if ($fingerprint -ne $lastFingerprint) { + $shouldNotify = $true + $notificationReason = "issue_changed" + } elseif ($null -eq $elapsedHours -or $elapsedHours -ge $notifyRepeatHours) { + $shouldNotify = $true + $notificationReason = "repeat" + } + } + + if ($shouldNotify) { + $notification.attempted = $true + try { + $message = Build-OwnerNotificationMessage -Status $preNotificationReport.status -IssueLines $issueLines -HealthStatusPath $statusPath + Send-OwnerDm -BotToken $config["DISCORD_BOT_TOKEN"] -OwnerDiscordId $config["DISCORD_OWNER_ID"] -Content $message + $notification.sent = $true + $notification.reason = $notificationReason + } catch { + $notification.sent = $false + $notification.reason = "failed" + $notification.error = $_.Exception.Message + Add-Check -Name "watchdog.notification" -Status "warning" -Detail "Failed to DM the owner about watchdog status: $($_.Exception.Message)" + } + } + } + + $report = Finalize-Report + $report.notification = $notification + Write-StatusFile -Path $statusPath -Status $report + + $watchdogStatePayload = @{ + checkedAt = $report.checkedAt + lastStatus = $report.status + lastFingerprint = $fingerprint + lastNotifiedAt = if ($notification.sent) { (Get-Date).ToString("o") } elseif ($watchdogState -and $watchdogState.lastNotifiedAt) { [string]$watchdogState.lastNotifiedAt } else { $null } + lastNotificationReason = $notification.reason + } + Write-StatusFile -Path $watchdogStatePath -Status $watchdogStatePayload + + Write-Host "VoidBot operations status: $($report.status)" + foreach ($check in $report.checks) { + Write-Host ("[{0}] {1}: {2}" -f $check.status.ToUpperInvariant(), $check.name, $check.detail) + } + + if ($FailOnIssues -and $report.status -eq "failed") { + exit 1 + } + + exit 0 +} catch { + Add-Check -Name "watchdog.internal" -Status "failed" -Detail "Operations watchdog crashed: $($_.Exception.Message)" + $report = Finalize-Report + $report.notification = @{ + attempted = $false + sent = $false + reason = "internal_error" + error = $_.Exception.Message + } + Write-StatusFile -Path $statusPath -Status $report + Write-Host "VoidBot operations status: failed" + Write-Host "[FAILED] watchdog.internal: Operations watchdog crashed: $($_.Exception.Message)" + exit 1 +} finally { + Stop-LogCapture -Started $logCaptureStarted +} diff --git a/scripts/install-operations-watchdog-task.ps1 b/scripts/install-operations-watchdog-task.ps1 new file mode 100644 index 0000000..0c45d6b --- /dev/null +++ b/scripts/install-operations-watchdog-task.ps1 @@ -0,0 +1,95 @@ +param( + [string] $TaskName, + [int] $IntervalMinutes, + [switch] $RunNow +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$repoRoot = Split-Path -Parent $PSScriptRoot +$envFile = Join-Path $repoRoot ".env" +$checkScript = Join-Path $PSScriptRoot "check-voidbot-operations.ps1" + +function Read-DotEnv { + param( + [Parameter(Mandatory = $true)] + [string] $Path + ) + + if (-not (Test-Path $Path)) { + throw "Missing .env at $Path" + } + + $values = @{} + + foreach ($line in Get-Content -Path $Path) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + if ($line.TrimStart().StartsWith("#")) { + continue + } + + $separatorIndex = $line.IndexOf("=") + + if ($separatorIndex -lt 1) { + continue + } + + $key = $line.Substring(0, $separatorIndex).Trim() + $value = $line.Substring($separatorIndex + 1) + $values[$key] = $value + } + + return $values +} + +$config = Read-DotEnv -Path $envFile +$taskNameValue = if (-not [string]::IsNullOrWhiteSpace($TaskName)) { + $TaskName +} elseif ($config.ContainsKey("VOIDBOT_HEALTHCHECK_TASK_NAME") -and -not [string]::IsNullOrWhiteSpace($config["VOIDBOT_HEALTHCHECK_TASK_NAME"])) { + $config["VOIDBOT_HEALTHCHECK_TASK_NAME"] +} else { + "VoidBot Operations Watchdog" +} +$intervalMinutesValue = if ($PSBoundParameters.ContainsKey("IntervalMinutes")) { + $IntervalMinutes +} elseif ($config.ContainsKey("VOIDBOT_HEALTHCHECK_INTERVAL_MINUTES") -and -not [string]::IsNullOrWhiteSpace($config["VOIDBOT_HEALTHCHECK_INTERVAL_MINUTES"])) { + [int]$config["VOIDBOT_HEALTHCHECK_INTERVAL_MINUTES"] +} else { + 60 +} + +if ($intervalMinutesValue -lt 15) { + throw "VOIDBOT_HEALTHCHECK_INTERVAL_MINUTES must be at least 15." +} + +if (-not ($config.ContainsKey("DISCORD_BOT_TOKEN")) -or [string]::IsNullOrWhiteSpace($config["DISCORD_BOT_TOKEN"])) { + throw "DISCORD_BOT_TOKEN must be configured in .env before installing the watchdog task." +} + +if (-not ($config.ContainsKey("DISCORD_OWNER_ID")) -or [string]::IsNullOrWhiteSpace($config["DISCORD_OWNER_ID"])) { + throw "DISCORD_OWNER_ID must be configured in .env before installing the watchdog task." +} + +$startAt = (Get-Date).AddMinutes(5) +$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$checkScript`" -NotifyOwner -FailOnIssues" +$trigger = New-ScheduledTaskTrigger -Once -At $startAt -RepetitionInterval (New-TimeSpan -Minutes $intervalMinutesValue) -RepetitionDuration (New-TimeSpan -Days 3650) +$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries +$principal = New-ScheduledTaskPrincipal -UserId "$env:USERDOMAIN\$env:USERNAME" -LogonType Interactive -RunLevel Limited +$description = "Runs the VoidBot operations watchdog on a repeating interval and DMs the owner when the stack or backup path drifts into the swamp." + +Register-ScheduledTask -TaskName $taskNameValue -Action $action -Trigger $trigger -Settings $settings -Principal $principal -Description $description -Force | Out-Null + +Write-Host "Installed scheduled task: $taskNameValue" +Write-Host "Schedule: every $intervalMinutesValue minute(s), starting at $($startAt.ToString('yyyy-MM-dd HH:mm'))" +Write-Host "Task runs as: $($env:USERDOMAIN)\$($env:USERNAME)" +Write-Host "Logon mode: Interactive" + +if ($RunNow) { + Start-ScheduledTask -TaskName $taskNameValue + Write-Host "Started task immediately." +} diff --git a/scripts/sync-voidbot-backup-offsite.ps1 b/scripts/sync-voidbot-backup-offsite.ps1 index c517c74..98c9f96 100644 --- a/scripts/sync-voidbot-backup-offsite.ps1 +++ b/scripts/sync-voidbot-backup-offsite.ps1 @@ -14,6 +14,7 @@ $ErrorActionPreference = "Stop" $repoRoot = Split-Path -Parent $PSScriptRoot $envFile = Join-Path $repoRoot ".env" $backupScript = Join-Path $PSScriptRoot "backup-voidbot-state.ps1" +$verifyScript = Join-Path $PSScriptRoot "verify-voidbot-backup.ps1" function Read-DotEnv { param( @@ -422,9 +423,19 @@ try { $backupName = $backupDir.Name $archivePath = Join-Path $backupRoot "$backupName.zip" + Write-Host "Verifying $backupName before offsite upload..." + $verificationJson = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $verifyScript -BackupPath $backupDir.FullName -AsJson -Quiet + + if ($LASTEXITCODE -ne 0) { + throw "Latest local backup failed verification and will not be uploaded." + } + + $verificationReport = $verificationJson | ConvertFrom-Json + $status.status = "compressing" $status.backupDirectory = $backupDir.FullName $status.archivePath = $archivePath + $status.backupVerification = $verificationReport Write-StatusFile -Path $statusPath -Status $status Write-Host "Compressing $backupName for offsite transfer..." @@ -460,6 +471,7 @@ try { Remove-Item -LiteralPath $archivePath -Force $archivePath = $null + $status.archivePath = $null $status.status = "pruning" Write-StatusFile -Path $statusPath -Status $status diff --git a/scripts/verify-voidbot-backup.ps1 b/scripts/verify-voidbot-backup.ps1 new file mode 100644 index 0000000..63f2f3b --- /dev/null +++ b/scripts/verify-voidbot-backup.ps1 @@ -0,0 +1,444 @@ +param( + [string] $BackupPath, + [switch] $AsJson, + [switch] $Quiet +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$repoRoot = Split-Path -Parent $PSScriptRoot +$envFile = Join-Path $repoRoot ".env" + +function Read-DotEnv { + param( + [Parameter(Mandatory = $true)] + [string] $Path + ) + + if (-not (Test-Path $Path)) { + throw "Missing .env at $Path" + } + + $values = @{} + + foreach ($line in Get-Content -Path $Path) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + if ($line.TrimStart().StartsWith("#")) { + continue + } + + $separatorIndex = $line.IndexOf("=") + + if ($separatorIndex -lt 1) { + continue + } + + $key = $line.Substring(0, $separatorIndex).Trim() + $value = $line.Substring($separatorIndex + 1) + $values[$key] = $value + } + + return $values +} + +function Resolve-ConfigPath { + param( + [Parameter(Mandatory = $true)] + [string] $RepoRoot, + [Parameter(Mandatory = $false)] + [AllowEmptyString()] + [string] $Value, + [Parameter(Mandatory = $true)] + [string] $Fallback + ) + + $raw = if ([string]::IsNullOrWhiteSpace($Value)) { $Fallback } else { $Value } + return [System.IO.Path]::GetFullPath((Join-Path $RepoRoot $raw)) +} + +function Add-Check { + param( + [Parameter(Mandatory = $true)] + [string] $Name, + [Parameter(Mandatory = $true)] + [ValidateSet("passed", "warning", "failed")] + [string] $Status, + [Parameter(Mandatory = $true)] + [string] $Detail, + [hashtable] $Data = @{} + ) + + $script:checks += [ordered]@{ + name = $Name + status = $Status + detail = $Detail + data = $Data + } +} + +function Get-LatestBackupDirectory { + param( + [Parameter(Mandatory = $true)] + [string] $BackupRoot + ) + + $latest = Get-ChildItem -LiteralPath $BackupRoot -Directory -ErrorAction SilentlyContinue | + Sort-Object Name -Descending | + Select-Object -First 1 + + if ($null -eq $latest) { + return $null + } + + return $latest +} + +function Test-DockerAvailable { + try { + & docker version *> $null + return $LASTEXITCODE -eq 0 + } catch { + return $false + } +} + +function Test-DockerContainerExists { + param( + [Parameter(Mandatory = $true)] + [string] $ContainerName + ) + + try { + & docker inspect $ContainerName *> $null + return $LASTEXITCODE -eq 0 + } catch { + return $false + } +} + +function Test-PostgresDumpReadable { + param( + [Parameter(Mandatory = $true)] + [string] $ContainerName, + [Parameter(Mandatory = $true)] + [string] $DumpFile + ) + + $tempContainerPath = "/tmp/voidbot-verify-$([Guid]::NewGuid().ToString('N')).dump" + + try { + & docker cp $DumpFile "${ContainerName}:$tempContainerPath" *> $null + + if ($LASTEXITCODE -ne 0) { + throw "docker cp failed for $DumpFile." + } + + & docker exec $ContainerName pg_restore --list $tempContainerPath *> $null + + if ($LASTEXITCODE -ne 0) { + throw "pg_restore --list failed for the copied dump." + } + + return @{ + status = "passed" + detail = "Postgres dump passed pg_restore --list inside $ContainerName." + } + } catch { + return @{ + status = "failed" + detail = $_.Exception.Message + } + } finally { + try { + & docker exec $ContainerName rm -f $tempContainerPath *> $null + } catch { + } + } +} + +function Finalize-Report { + $failedCount = @($script:checks | Where-Object { $_.status -eq "failed" }).Count + $warningCount = @($script:checks | Where-Object { $_.status -eq "warning" }).Count + $passedCount = @($script:checks | Where-Object { $_.status -eq "passed" }).Count + + $status = if ($failedCount -gt 0) { + "failed" + } elseif ($warningCount -gt 0) { + "warning" + } else { + "passed" + } + + return [ordered]@{ + checkedAt = (Get-Date).ToString("o") + repoRoot = $repoRoot + backupDirectory = if ($script:backupDirectory) { $script:backupDirectory.FullName } else { $null } + status = $status + summary = @{ + passed = $passedCount + warning = $warningCount + failed = $failedCount + } + checks = $script:checks + } +} + +$checks = @() +$backupDirectory = $null +$config = Read-DotEnv -Path $envFile +$storageRoot = Resolve-ConfigPath -RepoRoot $repoRoot -Value $config["STORAGE_ROOT"] -Fallback ".voidbot" +$backupRoot = Join-Path $storageRoot "backups" + +if ([string]::IsNullOrWhiteSpace($BackupPath)) { + $backupDirectory = Get-LatestBackupDirectory -BackupRoot $backupRoot +} elseif (Test-Path -LiteralPath $BackupPath) { + $backupDirectory = Get-Item -LiteralPath $BackupPath +} + +if ($null -eq $backupDirectory) { + Add-Check -Name "backup.directory" -Status "failed" -Detail "No backup directory was found to verify." -Data @{ + backupRoot = $backupRoot + requestedPath = $BackupPath + } + $report = Finalize-Report + $json = $report | ConvertTo-Json -Depth 8 + + if ($AsJson) { + Write-Output $json + } elseif (-not $Quiet) { + Write-Host "Backup verification failed." + Write-Host $json + } + + exit 1 +} + +Add-Check -Name "backup.directory" -Status "passed" -Detail "Found backup directory $($backupDirectory.FullName)." -Data @{ + path = $backupDirectory.FullName +} + +$manifestPath = Join-Path $backupDirectory.FullName "manifest.json" +$manifest = $null + +if (-not (Test-Path -LiteralPath $manifestPath)) { + Add-Check -Name "backup.manifest" -Status "failed" -Detail "Backup manifest is missing at $manifestPath." -Data @{ + manifestPath = $manifestPath + } + $report = Finalize-Report + $json = $report | ConvertTo-Json -Depth 8 + + if ($AsJson) { + Write-Output $json + } elseif (-not $Quiet) { + Write-Host "Backup verification failed." + Write-Host $json + } + + exit 1 +} + +try { + $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json + Add-Check -Name "backup.manifest" -Status "passed" -Detail "Parsed manifest.json successfully." -Data @{ + manifestPath = $manifestPath + } +} catch { + Add-Check -Name "backup.manifest" -Status "failed" -Detail "Failed to parse manifest.json: $($_.Exception.Message)" -Data @{ + manifestPath = $manifestPath + } + $report = Finalize-Report + $json = $report | ConvertTo-Json -Depth 8 + + if ($AsJson) { + Write-Output $json + } elseif (-not $Quiet) { + Write-Host "Backup verification failed." + Write-Host $json + } + + exit 1 +} + +if ([string]$manifest.backupFormat -eq "voidbot_state_backup_v1") { + Add-Check -Name "backup.format" -Status "passed" -Detail "Manifest format matches voidbot_state_backup_v1." +} else { + Add-Check -Name "backup.format" -Status "failed" -Detail "Unexpected backupFormat '$([string]$manifest.backupFormat)'." -Data @{ + expected = "voidbot_state_backup_v1" + actual = [string]$manifest.backupFormat + } +} + +try { + $createdAt = [DateTimeOffset]::Parse([string]$manifest.createdAt) + Add-Check -Name "backup.created_at" -Status "passed" -Detail "Backup timestamp parsed successfully." -Data @{ + createdAt = $createdAt.ToString("o") + ageHours = [Math]::Round(([DateTimeOffset]::Now - $createdAt).TotalHours, 2) + } +} catch { + Add-Check -Name "backup.created_at" -Status "failed" -Detail "Manifest createdAt could not be parsed." -Data @{ + value = [string]$manifest.createdAt + } +} + +$postgresSection = $manifest.postgres + +if ($null -eq $postgresSection) { + Add-Check -Name "postgres.section" -Status "warning" -Detail "Manifest has no postgres section." +} elseif ($postgresSection.skipped) { + Add-Check -Name "postgres.section" -Status "passed" -Detail "Postgres backup was intentionally skipped." -Data @{ + reason = [string]$postgresSection.reason + } +} else { + $dumpRelativePath = [string]$postgresSection.dumpFile + $dumpPath = Join-Path $backupDirectory.FullName $dumpRelativePath + + if (-not (Test-Path -LiteralPath $dumpPath)) { + Add-Check -Name "postgres.dump" -Status "failed" -Detail "Postgres dump file is missing at $dumpPath." -Data @{ + dumpFile = $dumpRelativePath + } + } else { + $dumpItem = Get-Item -LiteralPath $dumpPath + Add-Check -Name "postgres.dump" -Status "passed" -Detail "Found Postgres dump file." -Data @{ + path = $dumpPath + size = $dumpItem.Length + containerName = [string]$postgresSection.containerName + } + + $containerName = [string]$postgresSection.containerName + + if ([string]::IsNullOrWhiteSpace($containerName)) { + Add-Check -Name "postgres.dump_readable" -Status "warning" -Detail "Skipped pg_restore validation because the manifest has no containerName." + } elseif (-not (Test-DockerAvailable)) { + Add-Check -Name "postgres.dump_readable" -Status "warning" -Detail "Skipped pg_restore validation because Docker is unavailable." + } elseif (-not (Test-DockerContainerExists -ContainerName $containerName)) { + Add-Check -Name "postgres.dump_readable" -Status "warning" -Detail "Skipped pg_restore validation because container $containerName is not available." + } else { + $pgRestoreCheck = Test-PostgresDumpReadable -ContainerName $containerName -DumpFile $dumpPath + Add-Check -Name "postgres.dump_readable" -Status $pgRestoreCheck.status -Detail $pgRestoreCheck.detail -Data @{ + containerName = $containerName + } + } + } +} + +$qdrantSection = $manifest.qdrant + +if ($null -eq $qdrantSection) { + Add-Check -Name "qdrant.section" -Status "warning" -Detail "Manifest has no qdrant section." +} elseif ($qdrantSection.skipped) { + Add-Check -Name "qdrant.section" -Status "passed" -Detail "Qdrant snapshot backup was intentionally skipped." -Data @{ + reason = [string]$qdrantSection.reason + } +} else { + $collectionCount = @($qdrantSection.collections).Count + + if ($collectionCount -eq 0) { + Add-Check -Name "qdrant.snapshots" -Status "failed" -Detail "Manifest lists zero Qdrant collection snapshots." + } else { + Add-Check -Name "qdrant.snapshots" -Status "passed" -Detail "Manifest lists $collectionCount Qdrant collection snapshots." -Data @{ + collectionCount = $collectionCount + version = [string]$qdrantSection.version + } + } + + foreach ($collection in @($qdrantSection.collections)) { + $snapshotRelativePath = [string]$collection.snapshotFile + $snapshotPath = Join-Path $backupDirectory.FullName $snapshotRelativePath + $checkName = "qdrant.snapshot.$([string]$collection.name)" + + if (-not (Test-Path -LiteralPath $snapshotPath)) { + Add-Check -Name $checkName -Status "failed" -Detail "Snapshot file is missing at $snapshotPath." -Data @{ + collection = [string]$collection.name + snapshotFile = $snapshotRelativePath + } + continue + } + + $snapshotItem = Get-Item -LiteralPath $snapshotPath + + if ($snapshotItem.Length -le 0) { + Add-Check -Name $checkName -Status "failed" -Detail "Snapshot file exists but is empty." -Data @{ + collection = [string]$collection.name + snapshotFile = $snapshotRelativePath + } + continue + } + + Add-Check -Name $checkName -Status "passed" -Detail "Snapshot file exists for $([string]$collection.name)." -Data @{ + collection = [string]$collection.name + path = $snapshotPath + size = $snapshotItem.Length + } + } +} + +$archivesSection = $manifest.archives + +if ($null -eq $archivesSection) { + Add-Check -Name "archives.section" -Status "failed" -Detail "Manifest has no archives section." +} elseif ($archivesSection.skipped) { + Add-Check -Name "archives.section" -Status "warning" -Detail "Archive copy was skipped." -Data @{ + reason = [string]$archivesSection.reason + } +} else { + $archiveFiles = @($archivesSection.files) + + if ($archiveFiles.Count -eq 0) { + Add-Check -Name "archives.files" -Status "failed" -Detail "Manifest lists zero archive files." + } else { + Add-Check -Name "archives.files" -Status "passed" -Detail "Manifest lists $($archiveFiles.Count) archive files." -Data @{ + sourceRoot = [string]$archivesSection.sourceRoot + } + } + + foreach ($archive in $archiveFiles) { + $relativePath = [string]$archive.relativePath + $filePath = Join-Path (Join-Path $backupDirectory.FullName "archives") $relativePath + $checkName = "archives.file.$relativePath" + + if (-not (Test-Path -LiteralPath $filePath)) { + Add-Check -Name $checkName -Status "failed" -Detail "Archive file is missing at $filePath." -Data @{ + relativePath = $relativePath + } + continue + } + + $fileItem = Get-Item -LiteralPath $filePath + + if ($fileItem.Length -le 0) { + Add-Check -Name $checkName -Status "failed" -Detail "Archive file exists but is empty." -Data @{ + relativePath = $relativePath + path = $filePath + } + continue + } + + Add-Check -Name $checkName -Status "passed" -Detail "Archive file exists for $relativePath." -Data @{ + relativePath = $relativePath + path = $filePath + size = $fileItem.Length + } + } +} + +$report = Finalize-Report +$json = $report | ConvertTo-Json -Depth 8 + +if ($AsJson) { + Write-Output $json +} elseif (-not $Quiet) { + Write-Host "Backup verification status: $($report.status)" + foreach ($check in $report.checks) { + Write-Host ("[{0}] {1}: {2}" -f $check.status.ToUpperInvariant(), $check.name, $check.detail) + } +} + +if ($report.status -eq "failed") { + exit 1 +} + +exit 0