From b5c71d322e19b573f6a61d3b078b36824fa7af5e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 24 Apr 2026 20:11:18 +0100 Subject: [PATCH] Add local operations dashboard --- README.md | 10 + package.json | 2 + scripts/check-voidbot-operations.ps1 | 102 ++- .../render-voidbot-operations-dashboard.ps1 | 71 ++ scripts/start-voidbot-stack.ps1 | 28 +- scripts/sync-voidbot-backup-offsite.ps1 | 7 + scripts/voidbot-operations-dashboard-lib.ps1 | 725 ++++++++++++++++++ 7 files changed, 938 insertions(+), 7 deletions(-) create mode 100644 scripts/render-voidbot-operations-dashboard.ps1 create mode 100644 scripts/voidbot-operations-dashboard-lib.ps1 diff --git a/README.md b/README.md index 48969fd..490abd4 100644 --- a/README.md +++ b/README.md @@ -472,6 +472,14 @@ That check covers: It writes status to `.voidbot/status/operations-health.json` and appends logs to `.voidbot/logs/operations-watchdog.log`. +If you want an actual local dashboard instead of spelunking JSON by candlelight: + +```bash +npm run state:dashboard +``` + +That renders `.voidbot/status/operations-dashboard.html`, opens it in your default browser, and auto-refreshes every 10 seconds. The watchdog now also writes a live run-state file while it is executing, so a hidden task that wedges should stay visibly stuck on its last known step instead of failing in perfect silence. + Install the recurring watchdog task: ```bash @@ -515,6 +523,8 @@ Useful local state and docs: - `.voidbot/status/runtime-stack.json`: live stack status, service reachability, and bot/worker PIDs - `.voidbot/status/offsite-backup.json`: latest offsite backup sync result - `.voidbot/status/operations-health.json`: latest watchdog health report +- `.voidbot/status/operations-watchdog.json`: live watchdog run-state plus last notification metadata +- `.voidbot/status/operations-dashboard.html`: auto-refreshing local operations dashboard - `config/system-messages.json`: rotating stock system messages - `styles/void-default.md`: public default style pack - `docs/architecture/overview.md`: higher-level architecture notes diff --git a/package.json b/package.json index 9b7c1a0..b66a6ff 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "state:restore": "powershell -ExecutionPolicy Bypass -File scripts/restore-voidbot-state.ps1", "state:verify-backup": "powershell -ExecutionPolicy Bypass -File scripts/verify-voidbot-backup.ps1", "state:check": "powershell -ExecutionPolicy Bypass -File scripts/check-voidbot-operations.ps1", + "state:dashboard": "powershell -ExecutionPolicy Bypass -File scripts/render-voidbot-operations-dashboard.ps1 -Open", + "state:render-dashboard": "powershell -ExecutionPolicy Bypass -File scripts/render-voidbot-operations-dashboard.ps1", "state:sync-offsite": "powershell -ExecutionPolicy Bypass -File scripts/sync-voidbot-backup-offsite.ps1", "state:install-offsite-task": "powershell -ExecutionPolicy Bypass -File scripts/install-offsite-backup-task.ps1", "state:install-watchdog-task": "powershell -ExecutionPolicy Bypass -File scripts/install-operations-watchdog-task.ps1", diff --git a/scripts/check-voidbot-operations.ps1 b/scripts/check-voidbot-operations.ps1 index 5783639..15cd14a 100644 --- a/scripts/check-voidbot-operations.ps1 +++ b/scripts/check-voidbot-operations.ps1 @@ -10,6 +10,7 @@ $ProgressPreference = "SilentlyContinue" $repoRoot = Split-Path -Parent $PSScriptRoot $envFile = Join-Path $repoRoot ".env" $verifyScript = Join-Path $PSScriptRoot "verify-voidbot-backup.ps1" +. (Join-Path $PSScriptRoot "voidbot-operations-dashboard-lib.ps1") function Read-DotEnv { param( @@ -72,6 +73,10 @@ function Write-StatusFile { $directory = Split-Path -Parent $Path New-Item -ItemType Directory -Force -Path $directory | Out-Null $Status | ConvertTo-Json -Depth 10 | Set-Content -Path $Path -Encoding utf8 + + if ($script:storageRoot) { + [void](Update-VoidBotOperationsDashboard -RepoRoot $repoRoot -StorageRoot $script:storageRoot) + } } function Start-LogCapture { @@ -417,6 +422,20 @@ function Build-OwnerNotificationMessage { return ($lines -join [Environment]::NewLine).Trim() } +function Set-WatchdogStep { + param( + [Parameter(Mandatory = $true)] + [string] $CurrentStep, + [Parameter(Mandatory = $false)] + [string] $Detail = "" + ) + + $script:watchdogState.currentStep = $CurrentStep + $script:watchdogState.currentDetail = $Detail + $script:watchdogState.stepUpdatedAt = (Get-Date).ToString("o") + Write-StatusFile -Path $script:watchdogStatePath -Status $script:watchdogState +} + $checks = @() $config = Read-DotEnv -Path $envFile $storageRoot = Resolve-ConfigPath -RepoRoot $repoRoot -Value $config["STORAGE_ROOT"] -Fallback ".voidbot" @@ -436,7 +455,7 @@ $offsiteTarget = if ($config.ContainsKey("OFFSITE_BACKUP_SSH_TARGET")) { $config $offsiteKeepLatest = if ($config.ContainsKey("OFFSITE_BACKUP_REMOTE_KEEP_LATEST")) { [int]$config["OFFSITE_BACKUP_REMOTE_KEEP_LATEST"] } else { 14 } $offsiteLabel = if ($config.ContainsKey("OFFSITE_BACKUP_LABEL") -and -not [string]::IsNullOrWhiteSpace($config["OFFSITE_BACKUP_LABEL"])) { $config["OFFSITE_BACKUP_LABEL"] } else { "offsite-auto" } $logCaptureStarted = Start-LogCapture -LogPath $logPath -$watchdogState = if (Test-Path -LiteralPath $watchdogStatePath) { +$previousWatchdogState = if (Test-Path -LiteralPath $watchdogStatePath) { try { Get-Content -LiteralPath $watchdogStatePath -Raw | ConvertFrom-Json } catch { @@ -446,7 +465,32 @@ $watchdogState = if (Test-Path -LiteralPath $watchdogStatePath) { $null } +$script:watchdogStatePath = $watchdogStatePath +$watchdogStartedAt = (Get-Date).ToString("o") +$watchdogState = @{ + taskName = $watchdogTaskName + logPath = $logPath + reportPath = $statusPath + startedAt = $watchdogStartedAt + completedAt = $null + runStatus = "running" + reportStatus = $null + currentStep = "initializing" + currentDetail = "Starting watchdog checks." + stepUpdatedAt = $watchdogStartedAt + executionTimeLimitMinutes = 15 + lastStatus = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastStatus")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastStatus") } else { $null } + lastFingerprint = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastFingerprint")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastFingerprint") } else { $null } + lastNotifiedAt = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotifiedAt")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotifiedAt") } else { $null } + lastNotificationReason = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotificationReason")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotificationReason") } else { $null } + lastCompletedAt = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastCompletedAt")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastCompletedAt") } elseif ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "checkedAt")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "checkedAt") } else { $null } + lastCompletedRunStatus = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastCompletedRunStatus")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastCompletedRunStatus") } else { $null } + lastRunError = $null +} +Write-StatusFile -Path $watchdogStatePath -Status $watchdogState + try { + Set-WatchdogStep -CurrentStep "runtime_checks" -Detail "Inspecting runtime stack status and bot/worker liveness." if (-not (Test-Path -LiteralPath $runtimeStatusPath)) { Add-Check -Name "runtime.status_file" -Status "failed" -Detail "runtime-stack.json is missing at $runtimeStatusPath." } else { @@ -479,6 +523,7 @@ try { $historyCollection = if ($config.ContainsKey("QDRANT_HISTORY_COLLECTION")) { $config["QDRANT_HISTORY_COLLECTION"] } else { "voidbot_discord_history_chunks" } $sourceCollection = if ($config.ContainsKey("QDRANT_SOURCE_COLLECTION")) { $config["QDRANT_SOURCE_COLLECTION"] } else { "voidbot_repository_source_chunks" } + Set-WatchdogStep -CurrentStep "qdrant_checks" -Detail "Verifying Qdrant reachability and required collections." if ($vectorStoreKind -eq "qdrant") { $qdrantUrl = if ($config.ContainsKey("QDRANT_URL")) { $config["QDRANT_URL"] } else { "http://127.0.0.1:6333" } try { @@ -509,6 +554,7 @@ try { $stateStorageBackend = if ($config.ContainsKey("STATE_STORAGE_BACKEND")) { $config["STATE_STORAGE_BACKEND"] } else { "postgres" } + Set-WatchdogStep -CurrentStep "postgres_checks" -Detail "Checking Postgres container health and required tables." if ($stateStorageBackend -eq "postgres") { $databaseDsn = if ($config.ContainsKey("DATABASE_DSN")) { $config["DATABASE_DSN"] } else { "" } $dsn = Parse-DatabaseDsn -Dsn $databaseDsn @@ -566,6 +612,7 @@ try { $ragOllamaUrl = if ($config.ContainsKey("RAG_OLLAMA_BASE_URL")) { $config["RAG_OLLAMA_BASE_URL"] } else { "http://127.0.0.1:11434" } $ragOllamaModel = if ($config.ContainsKey("RAG_OLLAMA_MODEL")) { $config["RAG_OLLAMA_MODEL"] } else { "qwen3-embedding:0.6b" } + Set-WatchdogStep -CurrentStep "ollama_checks" -Detail "Checking embedding and local LLM Ollama endpoints." try { $ragTags = Get-Json -Url "$($ragOllamaUrl.TrimEnd('/'))/api/tags" $ragModels = @($ragTags.models | ForEach-Object { [string]$_.name }) @@ -628,6 +675,7 @@ try { } if ($config.ContainsKey("DISCORD_BOT_TOKEN") -and -not [string]::IsNullOrWhiteSpace($config["DISCORD_BOT_TOKEN"])) { + Set-WatchdogStep -CurrentStep "discord_checks" -Detail "Validating Discord bot credentials." try { $botIdentity = Get-Json -Url "https://discord.com/api/v10/users/@me" -Headers @("Authorization: Bot $($config["DISCORD_BOT_TOKEN"])") Add-Check -Name "discord.bot_token" -Status "passed" -Detail "Discord bot token is valid for $([string]$botIdentity.username)." -Data @{ @@ -645,6 +693,7 @@ try { Sort-Object Name -Descending | Select-Object -First 1 + Set-WatchdogStep -CurrentStep "backup_checks" -Detail "Verifying the latest local backup and freshness thresholds." if ($null -eq $latestBackup) { Add-Check -Name "backup.latest" -Status "failed" -Detail "No local backup directory exists under $backupRoot." } else { @@ -685,6 +734,7 @@ try { } } + Set-WatchdogStep -CurrentStep "scheduled_task_checks" -Detail "Inspecting scheduled task installation and status files." try { $offsiteTask = Get-ScheduledTask -TaskName $offsiteTaskName -ErrorAction Stop $offsiteEnabled = [bool]$offsiteTask.Settings.Enabled @@ -732,6 +782,7 @@ try { Add-Check -Name "offsite.status_file" -Status "failed" -Detail "offsite-backup.json is missing at $offsiteStatusPath." } + Set-WatchdogStep -CurrentStep "offsite_remote_checks" -Detail "Checking remote offsite backup freshness and retention." if ([string]::IsNullOrWhiteSpace($offsiteTarget)) { Add-Check -Name "offsite.remote" -Status "warning" -Detail "Remote offsite backup check skipped because OFFSITE_BACKUP_SSH_TARGET is blank." } else { @@ -807,6 +858,7 @@ $files = Get-ChildItem -LiteralPath $dir -Filter $pattern -File | Sort-Object La } } + Set-WatchdogStep -CurrentStep "notification_decision" -Detail "Computing report fingerprint and notification rules." $preNotificationReport = Finalize-Report $issueLines = @($preNotificationReport.checks | Where-Object { $_.status -ne "passed" } | ForEach-Object { "$($_.name): $($_.detail)" }) $fingerprintSource = if ($issueLines.Count -eq 0) { "healthy" } else { $issueLines -join "`n" } @@ -819,9 +871,9 @@ $files = Get-ChildItem -LiteralPath $dir -Filter $pattern -File | Sort-Object La } if ($NotifyOwner -and $config.ContainsKey("DISCORD_BOT_TOKEN") -and -not [string]::IsNullOrWhiteSpace($config["DISCORD_BOT_TOKEN"])) { - $lastFingerprint = if ($watchdogState -and $watchdogState.lastFingerprint) { [string]$watchdogState.lastFingerprint } else { "" } - $lastNotifiedAt = if ($watchdogState -and $watchdogState.lastNotifiedAt) { [DateTimeOffset]::Parse([string]$watchdogState.lastNotifiedAt) } else { $null } - $lastStatus = if ($watchdogState -and $watchdogState.lastStatus) { [string]$watchdogState.lastStatus } else { "" } + $lastFingerprint = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastFingerprint")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastFingerprint") } else { "" } + $lastNotifiedAt = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotifiedAt")) { [DateTimeOffset]::Parse([string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotifiedAt")) } else { $null } + $lastStatus = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastStatus")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastStatus") } else { "" } $shouldNotify = $false $notificationReason = $null @@ -842,7 +894,8 @@ $files = Get-ChildItem -LiteralPath $dir -Filter $pattern -File | Sort-Object La } if ($shouldNotify) { - $notification.attempted = $true + Set-WatchdogStep -CurrentStep "notification_send" -Detail "Sending owner notification." + $notification.attempted = $true try { $message = Build-OwnerNotificationMessage -Status $preNotificationReport.status -IssueLines $issueLines -HealthStatusPath $statusPath Send-OwnerDm -BotToken $config["DISCORD_BOT_TOKEN"] -OwnerDiscordId $config["DISCORD_OWNER_ID"] -Content $message @@ -857,16 +910,31 @@ $files = Get-ChildItem -LiteralPath $dir -Filter $pattern -File | Sort-Object La } } + Set-WatchdogStep -CurrentStep "finalizing" -Detail "Writing final health report." $report = Finalize-Report $report.notification = $notification Write-StatusFile -Path $statusPath -Status $report $watchdogStatePayload = @{ + taskName = $watchdogTaskName + logPath = $logPath + reportPath = $statusPath + startedAt = $watchdogStartedAt + completedAt = (Get-Date).ToString("o") checkedAt = $report.checkedAt + runStatus = "completed" + reportStatus = $report.status + currentStep = "finished" + currentDetail = "Watchdog run completed." + stepUpdatedAt = (Get-Date).ToString("o") + executionTimeLimitMinutes = 15 lastStatus = $report.status lastFingerprint = $fingerprint - lastNotifiedAt = if ($notification.sent) { (Get-Date).ToString("o") } elseif ($watchdogState -and $watchdogState.lastNotifiedAt) { [string]$watchdogState.lastNotifiedAt } else { $null } + lastNotifiedAt = if ($notification.sent) { (Get-Date).ToString("o") } elseif ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotifiedAt")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotifiedAt") } else { $null } lastNotificationReason = $notification.reason + lastCompletedAt = (Get-Date).ToString("o") + lastCompletedRunStatus = "completed" + lastRunError = $null } Write-StatusFile -Path $watchdogStatePath -Status $watchdogStatePayload @@ -890,6 +958,28 @@ $files = Get-ChildItem -LiteralPath $dir -Filter $pattern -File | Sort-Object La error = $_.Exception.Message } Write-StatusFile -Path $statusPath -Status $report + $watchdogFailureState = @{ + taskName = $watchdogTaskName + logPath = $logPath + reportPath = $statusPath + startedAt = $watchdogStartedAt + completedAt = (Get-Date).ToString("o") + checkedAt = $report.checkedAt + runStatus = "failed" + reportStatus = $report.status + currentStep = "crashed" + currentDetail = "Watchdog crashed before finishing." + stepUpdatedAt = (Get-Date).ToString("o") + executionTimeLimitMinutes = 15 + lastStatus = $report.status + lastFingerprint = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastFingerprint")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastFingerprint") } else { $null } + lastNotifiedAt = if ($previousWatchdogState -and (Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotifiedAt")) { [string](Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName "lastNotifiedAt") } else { $null } + lastNotificationReason = "internal_error" + lastCompletedAt = (Get-Date).ToString("o") + lastCompletedRunStatus = "failed" + lastRunError = $_.Exception.Message + } + Write-StatusFile -Path $watchdogStatePath -Status $watchdogFailureState Write-Host "VoidBot operations status: failed" Write-Host "[FAILED] watchdog.internal: Operations watchdog crashed: $($_.Exception.Message)" exit 1 diff --git a/scripts/render-voidbot-operations-dashboard.ps1 b/scripts/render-voidbot-operations-dashboard.ps1 new file mode 100644 index 0000000..adb3549 --- /dev/null +++ b/scripts/render-voidbot-operations-dashboard.ps1 @@ -0,0 +1,71 @@ +param( + [switch] $Open +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$repoRoot = Split-Path -Parent $PSScriptRoot +$envFile = Join-Path $repoRoot ".env" +. (Join-Path $PSScriptRoot "voidbot-operations-dashboard-lib.ps1") + +function Read-DotEnv { + param( + [Parameter(Mandatory = $true)] + [string] $Path + ) + + if (-not (Test-Path -LiteralPath $Path)) { + throw "Missing .env at $Path" + } + + $values = @{} + + foreach ($line in Get-Content -LiteralPath $Path) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + if ($line.TrimStart().StartsWith("#")) { + continue + } + + $separatorIndex = $line.IndexOf("=") + + if ($separatorIndex -lt 1) { + continue + } + + $key = $line.Substring(0, $separatorIndex).Trim() + $value = $line.Substring($separatorIndex + 1) + $values[$key] = $value + } + + return $values +} + +function Resolve-ConfigPath { + param( + [Parameter(Mandatory = $true)] + [string] $RepoRoot, + [Parameter(Mandatory = $false)] + [AllowEmptyString()] + [string] $Value, + [Parameter(Mandatory = $true)] + [string] $Fallback + ) + + $raw = if ([string]::IsNullOrWhiteSpace($Value)) { $Fallback } else { $Value } + return [System.IO.Path]::GetFullPath((Join-Path $RepoRoot $raw)) +} + +$config = Read-DotEnv -Path $envFile +$storageRoot = Resolve-ConfigPath -RepoRoot $repoRoot -Value $config["STORAGE_ROOT"] -Fallback ".voidbot" +$result = Update-VoidBotOperationsDashboard -RepoRoot $repoRoot -StorageRoot $storageRoot + +Write-Host "Dashboard HTML: $($result.DashboardPath)" +Write-Host "Dashboard snapshot: $($result.SnapshotPath)" + +if ($Open) { + Invoke-Item -LiteralPath $result.DashboardPath +} diff --git a/scripts/start-voidbot-stack.ps1 b/scripts/start-voidbot-stack.ps1 index eb9a448..0ff79e2 100644 --- a/scripts/start-voidbot-stack.ps1 +++ b/scripts/start-voidbot-stack.ps1 @@ -3,6 +3,7 @@ $ErrorActionPreference = "Stop" $repoRoot = Split-Path -Parent $PSScriptRoot $envFile = Join-Path $repoRoot ".env" +. (Join-Path $PSScriptRoot "voidbot-operations-dashboard-lib.ps1") function Read-DotEnv { param( @@ -65,6 +66,10 @@ function Write-StatusFile { $directory = Split-Path -Parent $Path New-Item -ItemType Directory -Force -Path $directory | Out-Null $Status | ConvertTo-Json -Depth 8 | Set-Content -Path $Path -Encoding utf8 + + if ($script:storageRoot) { + [void](Update-VoidBotOperationsDashboard -RepoRoot $repoRoot -StorageRoot $script:storageRoot) + } } function Invoke-JsonGet { @@ -457,6 +462,7 @@ $status = @{ startedAt = (Get-Date).ToString("o") repoRoot = $repoRoot statusPath = $statusPath + stage = "bootstrapping" } Write-StatusFile -Path $statusPath -Status $status @@ -473,6 +479,8 @@ $localLlmBaseUrl = if ($config.ContainsKey("LOCAL_LLM_OLLAMA_BASE_URL")) { $conf $localLlmModel = if ($config.ContainsKey("LOCAL_LLM_OLLAMA_MODEL")) { $config["LOCAL_LLM_OLLAMA_MODEL"] } else { "qwen3.5:9b" } if ($stateStorageBackend -eq "postgres") { + $status.stage = "postgres" + Write-StatusFile -Path $statusPath -Status $status $status.postgres = Ensure-Postgres -RepoRoot $repoRoot -DatabaseDsn $databaseDsn Write-StatusFile -Path $statusPath -Status $status } else { @@ -483,6 +491,8 @@ if ($stateStorageBackend -eq "postgres") { } if ($vectorStoreKind -eq "qdrant") { + $status.stage = "qdrant" + Write-StatusFile -Path $statusPath -Status $status $status.qdrant = Ensure-Qdrant -RepoRoot $repoRoot -QdrantUrl $qdrantUrl Write-StatusFile -Path $statusPath -Status $status } else { @@ -493,11 +503,15 @@ if ($vectorStoreKind -eq "qdrant") { } if ($ragBackend -eq "ollama") { + $status.stage = "rag_ollama" + Write-StatusFile -Path $statusPath -Status $status $status.ragOllama = Ensure-OllamaEndpoint -BaseUrl $ragOllamaBaseUrl -Model $ragOllamaModel -Label "Local embedding Ollama" -LogDir $logDir Write-StatusFile -Path $statusPath -Status $status } if ($localLlmEnabled) { + $status.stage = "local_llm" + Write-StatusFile -Path $statusPath -Status $status $status.localLlmOllama = Ensure-OllamaEndpoint -BaseUrl $localLlmBaseUrl -Model $localLlmModel -Label "Local LLM Ollama" -LogDir $logDir Write-StatusFile -Path $statusPath -Status $status } @@ -505,6 +519,9 @@ if ($localLlmEnabled) { $npmCommand = Get-Command npm.cmd -ErrorAction Stop $nodeCommand = Get-Command node.exe -ErrorAction Stop +$status.stage = "build" +Write-StatusFile -Path $statusPath -Status $status + Write-Host "Building VoidBot..." & $npmCommand.Source run build @@ -519,10 +536,18 @@ $botErr = Join-Path $logDir "bot.err.log" $workerOut = Join-Path $logDir "worker.log" $workerErr = Join-Path $logDir "worker.err.log" +$status.stage = "bot_start" +Write-StatusFile -Path $statusPath -Status $status + $botProcess = Start-Process -FilePath $nodeCommand.Source -ArgumentList (Join-Path $repoRoot "apps\bot\dist\index.js") -WorkingDirectory $repoRoot -RedirectStandardOutput $botOut -RedirectStandardError $botErr -WindowStyle Hidden -PassThru -$workerProcess = Start-Process -FilePath $nodeCommand.Source -ArgumentList (Join-Path $repoRoot "apps\worker\dist\index.js") -WorkingDirectory $repoRoot -RedirectStandardOutput $workerOut -RedirectStandardError $workerErr -WindowStyle Hidden -PassThru Wait-ForProcessReady -Process $botProcess -Name "VoidBot bot" -LogPath $botOut -ErrPath $botErr -ReadyText "VoidBot connected as" + +$status.stage = "worker_start" +Write-StatusFile -Path $statusPath -Status $status + +$workerProcess = Start-Process -FilePath $nodeCommand.Source -ArgumentList (Join-Path $repoRoot "apps\worker\dist\index.js") -WorkingDirectory $repoRoot -RedirectStandardOutput $workerOut -RedirectStandardError $workerErr -WindowStyle Hidden -PassThru + Wait-ForProcessReady -Process $workerProcess -Name "VoidBot worker" -LogPath $workerOut -ErrPath $workerErr -ReadyText "VoidBot worker polling" $status.bot = @{ @@ -537,6 +562,7 @@ $status.worker = @{ errLog = $workerErr } +$status.stage = "ready" $status.completedAt = (Get-Date).ToString("o") $status.ready = $true Write-StatusFile -Path $statusPath -Status $status diff --git a/scripts/sync-voidbot-backup-offsite.ps1 b/scripts/sync-voidbot-backup-offsite.ps1 index 98c9f96..49cec64 100644 --- a/scripts/sync-voidbot-backup-offsite.ps1 +++ b/scripts/sync-voidbot-backup-offsite.ps1 @@ -15,6 +15,7 @@ $repoRoot = Split-Path -Parent $PSScriptRoot $envFile = Join-Path $repoRoot ".env" $backupScript = Join-Path $PSScriptRoot "backup-voidbot-state.ps1" $verifyScript = Join-Path $PSScriptRoot "verify-voidbot-backup.ps1" +. (Join-Path $PSScriptRoot "voidbot-operations-dashboard-lib.ps1") function Read-DotEnv { param( @@ -77,6 +78,10 @@ function Write-StatusFile { $directory = Split-Path -Parent $Path New-Item -ItemType Directory -Force -Path $directory | Out-Null $Status | ConvertTo-Json -Depth 8 | Set-Content -Path $Path -Encoding utf8 + + if ($script:storageRoot) { + [void](Update-VoidBotOperationsDashboard -RepoRoot $repoRoot -StorageRoot $script:storageRoot) + } } function Start-LogCapture { @@ -385,6 +390,8 @@ $status = @{ startedAt = (Get-Date).ToString("o") repoRoot = $repoRoot status = "starting" + statusPath = $statusPath + logPath = $logPath sshTarget = $sshTargetValue remoteWindowsDir = $remoteWindowsDirValue remoteSftpDir = $remoteSftpDirValue diff --git a/scripts/voidbot-operations-dashboard-lib.ps1 b/scripts/voidbot-operations-dashboard-lib.ps1 new file mode 100644 index 0000000..80cc368 --- /dev/null +++ b/scripts/voidbot-operations-dashboard-lib.ps1 @@ -0,0 +1,725 @@ +function Get-VoidBotDashboardProperty { + param( + [Parameter(Mandatory = $false)] + [AllowNull()] + $InputObject, + [Parameter(Mandatory = $true)] + [string] $PropertyName + ) + + if ($null -eq $InputObject) { + return $null + } + + $property = $InputObject.PSObject.Properties[$PropertyName] + + if ($null -eq $property) { + return $null + } + + return $property.Value +} + +function Read-VoidBotDashboardJson { + param( + [Parameter(Mandatory = $true)] + [string] $Path + ) + + if (-not (Test-Path -LiteralPath $Path)) { + return [PSCustomObject]@{ + _path = $Path + _missing = $true + } + } + + try { + $content = Get-Content -LiteralPath $Path -Raw + + if ([string]::IsNullOrWhiteSpace($content)) { + return [PSCustomObject]@{ + _path = $Path + _missing = $true + } + } + + $parsed = $content | ConvertFrom-Json + Add-Member -InputObject $parsed -NotePropertyName "_path" -NotePropertyValue $Path -Force + return $parsed + } catch { + return [PSCustomObject]@{ + _path = $Path + _error = $_.Exception.Message + } + } +} + +function ConvertTo-VoidBotDashboardText { + param( + [Parameter(Mandatory = $false)] + [AllowNull()] + $Value + ) + + if ($null -eq $Value) { + return "missing" + } + + if ($Value -is [bool]) { + return $(if ($Value) { "yes" } else { "no" }) + } + + if ($Value -is [DateTime]) { + return $Value.ToString("yyyy-MM-dd HH:mm:ss") + } + + if ($Value -is [DateTimeOffset]) { + return $Value.ToString("yyyy-MM-dd HH:mm:ss zzz") + } + + if ($Value -is [System.Collections.IEnumerable] -and -not ($Value -is [string])) { + $items = @($Value) + if ($items.Count -eq 0) { + return "none" + } + + if ($items | Where-Object { $_ -is [string] -or $_ -is [ValueType] }) { + return ($items | ForEach-Object { [string]$_ }) -join ", " + } + + return ($Value | ConvertTo-Json -Depth 6 -Compress) + } + + return [string]$Value +} + +function Escape-VoidBotDashboardHtml { + param( + [Parameter(Mandatory = $false)] + [AllowNull()] + $Value + ) + + return [System.Net.WebUtility]::HtmlEncode((ConvertTo-VoidBotDashboardText -Value $Value)) +} + +function ConvertTo-VoidBotDashboardFileUri { + param( + [Parameter(Mandatory = $true)] + [string] $Path + ) + + return ([System.Uri]::new($Path)).AbsoluteUri +} + +function New-VoidBotDashboardFileLink { + param( + [Parameter(Mandatory = $true)] + [string] $Path, + [Parameter(Mandatory = $false)] + [string] $Label + ) + + $text = if ([string]::IsNullOrWhiteSpace($Label)) { $Path } else { $Label } + return "" + (Escape-VoidBotDashboardHtml $text) + "" +} + +function Get-VoidBotDashboardTone { + param( + [Parameter(Mandatory = $false)] + [AllowNull()] + [string] $State + ) + + if ([string]::IsNullOrWhiteSpace($State)) { + return "neutral" + } + + switch ($State.ToLowerInvariant()) { + "healthy" { return "good" } + "passed" { return "good" } + "completed" { return "good" } + "ready" { return "good" } + "enabled" { return "good" } + "ok" { return "good" } + "warning" { return "warn" } + "running" { return "info" } + "starting" { return "info" } + "compressing" { return "info" } + "uploading" { return "info" } + "pruning" { return "info" } + "bootstrapping" { return "info" } + "build" { return "info" } + "bot_start" { return "info" } + "worker_start" { return "info" } + "already_running" { return "warn" } + "failed" { return "bad" } + "error" { return "bad" } + "disabled" { return "bad" } + "stalled" { return "bad" } + "missing" { return "bad" } + default { return "neutral" } + } +} + +function New-VoidBotDashboardBadge { + param( + [Parameter(Mandatory = $false)] + [AllowNull()] + [string] $State, + [Parameter(Mandatory = $false)] + [string] $Label + ) + + $text = if ([string]::IsNullOrWhiteSpace($Label)) { + if ([string]::IsNullOrWhiteSpace($State)) { "unknown" } else { $State } + } else { + $Label + } + $tone = Get-VoidBotDashboardTone -State $State + return "" + (Escape-VoidBotDashboardHtml $text) + "" +} + +function Get-VoidBotDashboardRelativeTime { + param( + [Parameter(Mandatory = $false)] + [AllowNull()] + [string] $Timestamp + ) + + if ([string]::IsNullOrWhiteSpace($Timestamp)) { + return "unknown" + } + + try { + $target = [DateTimeOffset]::Parse($Timestamp) + $delta = [DateTimeOffset]::Now - $target + + if ($delta.TotalSeconds -lt 60) { + return "{0}s ago" -f [Math]::Max([int][Math]::Round($delta.TotalSeconds), 0) + } + + if ($delta.TotalMinutes -lt 60) { + return "{0}m ago" -f [int][Math]::Round($delta.TotalMinutes) + } + + if ($delta.TotalHours -lt 48) { + return "{0}h ago" -f [int][Math]::Round($delta.TotalHours) + } + + return "{0}d ago" -f [int][Math]::Round($delta.TotalDays) + } catch { + return "unknown" + } +} + +function Get-VoidBotDashboardElapsedTime { + param( + [Parameter(Mandatory = $false)] + [AllowNull()] + [string] $StartedAt + ) + + if ([string]::IsNullOrWhiteSpace($StartedAt)) { + return "unknown" + } + + try { + $delta = [DateTimeOffset]::Now - [DateTimeOffset]::Parse($StartedAt) + + if ($delta.TotalMinutes -lt 1) { + return "{0}s" -f [Math]::Max([int][Math]::Round($delta.TotalSeconds), 0) + } + + if ($delta.TotalHours -lt 1) { + return "{0}m" -f [int][Math]::Round($delta.TotalMinutes) + } + + return "{0}h {1}m" -f [int][Math]::Floor($delta.TotalHours), $delta.Minutes + } catch { + return "unknown" + } +} + +function New-VoidBotDashboardFactRow { + param( + [Parameter(Mandatory = $true)] + [string] $Label, + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string] $ValueHtml + ) + + return "" + (Escape-VoidBotDashboardHtml $Label) + "" + $ValueHtml + "" +} + +function New-VoidBotDashboardFactSection { + param( + [Parameter(Mandatory = $true)] + [string] $Title, + [Parameter(Mandatory = $true)] + [string[]] $Rows + ) + + $body = if ($Rows.Count -gt 0) { + ($Rows -join [Environment]::NewLine) + } else { + "Nothing here yet." + } + + return @" +
+

$(Escape-VoidBotDashboardHtml $Title)

+ + +$body + +
+
+"@ +} + +function New-VoidBotDashboardCard { + param( + [Parameter(Mandatory = $true)] + [string] $Title, + [Parameter(Mandatory = $false)] + [AllowNull()] + [string] $State, + [Parameter(Mandatory = $true)] + [string] $Subtitle, + [Parameter(Mandatory = $false)] + [string] $Meta = "" + ) + + $metaHtml = if ([string]::IsNullOrWhiteSpace($Meta)) { + "" + } else { + "
" + (Escape-VoidBotDashboardHtml $Meta) + "
" + } + + return @" +
+
+

$(Escape-VoidBotDashboardHtml $Title)

+ $(New-VoidBotDashboardBadge -State $State) +
+

$(Escape-VoidBotDashboardHtml $Subtitle)

+ $metaHtml +
+"@ +} + +function Update-VoidBotOperationsDashboard { + param( + [Parameter(Mandatory = $true)] + [string] $RepoRoot, + [Parameter(Mandatory = $true)] + [string] $StorageRoot + ) + + $statusDir = Join-Path $StorageRoot "status" + $logsDir = Join-Path $StorageRoot "logs" + $dashboardPath = Join-Path $statusDir "operations-dashboard.html" + $snapshotPath = Join-Path $statusDir "operations-dashboard.json" + New-Item -ItemType Directory -Force -Path $statusDir | Out-Null + + $runtimePath = Join-Path $statusDir "runtime-stack.json" + $operationsPath = Join-Path $statusDir "operations-health.json" + $watchdogPath = Join-Path $statusDir "operations-watchdog.json" + $offsitePath = Join-Path $statusDir "offsite-backup.json" + + $runtime = Read-VoidBotDashboardJson -Path $runtimePath + $operations = Read-VoidBotDashboardJson -Path $operationsPath + $watchdog = Read-VoidBotDashboardJson -Path $watchdogPath + $offsite = Read-VoidBotDashboardJson -Path $offsitePath + + $operationsState = [string](Get-VoidBotDashboardProperty -InputObject $operations -PropertyName "status") + if ([string]::IsNullOrWhiteSpace($operationsState)) { + $operationsState = if (Get-VoidBotDashboardProperty -InputObject $operations -PropertyName "_missing") { "missing" } else { "unknown" } + } + + $runtimeState = if (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "_missing") { + "missing" + } elseif ([bool](Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "ready")) { + "ready" + } else { + $stageValue = [string](Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "stage") + if ([string]::IsNullOrWhiteSpace($stageValue)) { "starting" } else { $stageValue } + } + + $watchdogRunStatus = [string](Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "runStatus") + $watchdogStartedAt = [string](Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "startedAt") + $watchdogStep = [string](Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "currentStep") + $watchdogLimitMinutes = Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "executionTimeLimitMinutes" + $watchdogReportStatus = [string](Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "reportStatus") + $watchdogState = if ([string]::IsNullOrWhiteSpace($watchdogRunStatus)) { + if (Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "_missing") { "missing" } else { $operationsState } + } elseif ($watchdogRunStatus -eq "running" -and -not [string]::IsNullOrWhiteSpace($watchdogStartedAt)) { + try { + $elapsedMinutes = ([DateTimeOffset]::Now - [DateTimeOffset]::Parse($watchdogStartedAt)).TotalMinutes + $limitValue = if ($watchdogLimitMinutes -is [ValueType]) { [double]$watchdogLimitMinutes } else { 15.0 } + if ($elapsedMinutes -gt $limitValue) { "stalled" } else { "running" } + } catch { + "running" + } + } elseif ($watchdogRunStatus -eq "completed" -and -not [string]::IsNullOrWhiteSpace($watchdogReportStatus)) { + $watchdogReportStatus + } else { + $watchdogRunStatus + } + + $offsiteState = [string](Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "status") + if ([string]::IsNullOrWhiteSpace($offsiteState)) { + $offsiteState = if (Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "_missing") { "missing" } else { "unknown" } + } + + $operationsSummary = Get-VoidBotDashboardProperty -InputObject $operations -PropertyName "summary" + $operationsSubtitle = if ($null -ne $operationsSummary) { + "{0} failed, {1} warning, {2} passed" -f ` + (Get-VoidBotDashboardProperty -InputObject $operationsSummary -PropertyName "failed"), ` + (Get-VoidBotDashboardProperty -InputObject $operationsSummary -PropertyName "warning"), ` + (Get-VoidBotDashboardProperty -InputObject $operationsSummary -PropertyName "passed") + } else { + "No health report yet." + } + + $watchdogSubtitle = if ($watchdogRunStatus -eq "running") { + "Current step: {0}" -f $(if ([string]::IsNullOrWhiteSpace($watchdogStep)) { "working" } else { $watchdogStep }) + } elseif ($watchdogState -eq "stalled") { + "Still marked running after {0}" -f (Get-VoidBotDashboardElapsedTime -StartedAt $watchdogStartedAt) + } elseif ($watchdogRunStatus -eq "failed") { + "Watchdog run crashed before finishing." + } elseif ($watchdogState -eq "healthy") { + "Watchdog run completed cleanly." + } elseif ($watchdogState -eq "warning") { + "Watchdog run finished with warnings." + } elseif ($watchdogState -eq "failed") { + "Watchdog run finished with failed checks." + } else { + "No active watchdog run." + } + + $runtimeSubtitle = if ($runtimeState -eq "ready") { + "Bot and worker reported ready." + } elseif ($runtimeState -eq "missing") { + "Runtime stack has not written status yet." + } else { + "Current stage: {0}" -f $runtimeState + } + + $offsiteSubtitle = if ($offsiteState -eq "completed") { + "Latest offsite sync finished." + } elseif ($offsiteState -eq "missing") { + "No offsite sync status yet." + } else { + "Current phase: {0}" -f $offsiteState + } + + $cardsHtml = @( + New-VoidBotDashboardCard -Title "Operations Health" -State $operationsState -Subtitle $operationsSubtitle -Meta ("Checked " + (Get-VoidBotDashboardRelativeTime -Timestamp ([string](Get-VoidBotDashboardProperty -InputObject $operations -PropertyName "checkedAt")))) + New-VoidBotDashboardCard -Title "Watchdog Run" -State $watchdogState -Subtitle $watchdogSubtitle -Meta ("Started " + (Get-VoidBotDashboardRelativeTime -Timestamp $watchdogStartedAt)) + New-VoidBotDashboardCard -Title "Runtime Stack" -State $runtimeState -Subtitle $runtimeSubtitle -Meta ("Updated " + (Get-VoidBotDashboardRelativeTime -Timestamp ([string](Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "completedAt")))) + New-VoidBotDashboardCard -Title "Offsite Backup" -State $offsiteState -Subtitle $offsiteSubtitle -Meta ("Updated " + (Get-VoidBotDashboardRelativeTime -Timestamp ([string](Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "completedAt")))) + ) -join [Environment]::NewLine + + $runtimeRows = @( + New-VoidBotDashboardFactRow -Label "Status file" -ValueHtml (New-VoidBotDashboardFileLink -Path $runtimePath) + New-VoidBotDashboardFactRow -Label "Ready" -ValueHtml (New-VoidBotDashboardBadge -State $runtimeState -Label ([string](Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "ready"))) + New-VoidBotDashboardFactRow -Label "Stage" -ValueHtml ("" + (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "stage")) + "") + New-VoidBotDashboardFactRow -Label "Bot PID" -ValueHtml ("" + (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "bot") -PropertyName "pid")) + "") + New-VoidBotDashboardFactRow -Label "Bot log" -ValueHtml $(if (Get-VoidBotDashboardProperty -InputObject (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "bot") -PropertyName "log") { New-VoidBotDashboardFileLink -Path ([string](Get-VoidBotDashboardProperty -InputObject (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "bot") -PropertyName "log")) } else { "missing" }) + New-VoidBotDashboardFactRow -Label "Worker PID" -ValueHtml ("" + (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "worker") -PropertyName "pid")) + "") + New-VoidBotDashboardFactRow -Label "Worker log" -ValueHtml $(if (Get-VoidBotDashboardProperty -InputObject (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "worker") -PropertyName "log") { New-VoidBotDashboardFileLink -Path ([string](Get-VoidBotDashboardProperty -InputObject (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "worker") -PropertyName "log")) } else { "missing" }) + New-VoidBotDashboardFactRow -Label "Completed" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $runtime -PropertyName "completedAt")) + ) + + $watchdogRows = @( + New-VoidBotDashboardFactRow -Label "Status file" -ValueHtml (New-VoidBotDashboardFileLink -Path $watchdogPath) + New-VoidBotDashboardFactRow -Label "Run status" -ValueHtml (New-VoidBotDashboardBadge -State $watchdogState -Label $watchdogState) + New-VoidBotDashboardFactRow -Label "Current step" -ValueHtml ("" + (Escape-VoidBotDashboardHtml $watchdogStep) + "") + New-VoidBotDashboardFactRow -Label "Started" -ValueHtml (Escape-VoidBotDashboardHtml $watchdogStartedAt) + New-VoidBotDashboardFactRow -Label "Elapsed" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardElapsedTime -StartedAt $watchdogStartedAt)) + New-VoidBotDashboardFactRow -Label "Completed" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "completedAt")) + New-VoidBotDashboardFactRow -Label "Last report status" -ValueHtml (New-VoidBotDashboardBadge -State ([string](Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "reportStatus")) -Label ([string](Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "reportStatus"))) + New-VoidBotDashboardFactRow -Label "Notification" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "lastNotificationReason")) + New-VoidBotDashboardFactRow -Label "Last run error" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "lastRunError")) + New-VoidBotDashboardFactRow -Label "Watchdog log" -ValueHtml $(if (Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "logPath") { New-VoidBotDashboardFileLink -Path ([string](Get-VoidBotDashboardProperty -InputObject $watchdog -PropertyName "logPath")) } else { "missing" }) + ) + + $offsiteRows = @( + New-VoidBotDashboardFactRow -Label "Status file" -ValueHtml (New-VoidBotDashboardFileLink -Path $offsitePath) + New-VoidBotDashboardFactRow -Label "Status" -ValueHtml (New-VoidBotDashboardBadge -State $offsiteState -Label $offsiteState) + New-VoidBotDashboardFactRow -Label "Started" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "startedAt")) + New-VoidBotDashboardFactRow -Label "Completed" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "completedAt")) + New-VoidBotDashboardFactRow -Label "Backup directory" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "backupDirectory")) + New-VoidBotDashboardFactRow -Label "Archive path" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "archivePath")) + New-VoidBotDashboardFactRow -Label "Remote file" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject (Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "remoteFile") -PropertyName "path")) + New-VoidBotDashboardFactRow -Label "Error" -ValueHtml (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "error")) + New-VoidBotDashboardFactRow -Label "Offsite log" -ValueHtml $(if (Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "logPath") { New-VoidBotDashboardFileLink -Path ([string](Get-VoidBotDashboardProperty -InputObject $offsite -PropertyName "logPath")) } else { New-VoidBotDashboardFileLink -Path (Join-Path $logsDir "offsite-backup.log") }) + ) + + $fileRows = @( + New-VoidBotDashboardFactRow -Label "Dashboard HTML" -ValueHtml (New-VoidBotDashboardFileLink -Path $dashboardPath) + New-VoidBotDashboardFactRow -Label "Dashboard snapshot" -ValueHtml (New-VoidBotDashboardFileLink -Path $snapshotPath) + New-VoidBotDashboardFactRow -Label "Operations report" -ValueHtml (New-VoidBotDashboardFileLink -Path $operationsPath) + New-VoidBotDashboardFactRow -Label "Operations log" -ValueHtml (New-VoidBotDashboardFileLink -Path (Join-Path $logsDir "operations-watchdog.log")) + New-VoidBotDashboardFactRow -Label "Repo root" -ValueHtml ("" + (Escape-VoidBotDashboardHtml $RepoRoot) + "") + New-VoidBotDashboardFactRow -Label "Storage root" -ValueHtml ("" + (Escape-VoidBotDashboardHtml $StorageRoot) + "") + ) + + $checkRows = @() + foreach ($check in @((Get-VoidBotDashboardProperty -InputObject $operations -PropertyName "checks"))) { + $checkRows += "" + (New-VoidBotDashboardBadge -State ([string](Get-VoidBotDashboardProperty -InputObject $check -PropertyName "status")) -Label ([string](Get-VoidBotDashboardProperty -InputObject $check -PropertyName "status"))) + "" + (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $check -PropertyName "name")) + "" + (Escape-VoidBotDashboardHtml (Get-VoidBotDashboardProperty -InputObject $check -PropertyName "detail")) + "" + } + + if ($checkRows.Count -eq 0) { + $checkRows = @("No watchdog checks have been written yet.") + } + + $checksSection = @" +
+

Watchdog Checks

+ + + + + +$($checkRows -join [Environment]::NewLine) + +
StatusCheckDetail
+
+"@ + + $snapshot = [ordered]@{ + generatedAt = (Get-Date).ToString("o") + dashboardPath = $dashboardPath + repoRoot = $RepoRoot + storageRoot = $StorageRoot + states = [ordered]@{ + operations = $operationsState + runtime = $runtimeState + watchdog = $watchdogState + offsite = $offsiteState + } + files = [ordered]@{ + runtime = $runtimePath + operations = $operationsPath + watchdog = $watchdogPath + offsite = $offsitePath + operationsLog = Join-Path $logsDir "operations-watchdog.log" + offsiteLog = Join-Path $logsDir "offsite-backup.log" + } + } + $snapshot | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $snapshotPath -Encoding utf8 + + $html = @" + + + + + + VoidBot Operations Dashboard + + + +
+
+
VoidBot Operations Dashboard
+

Status files, but with fewer lies.

+

This page is regenerated from the live JSON status files under $(Escape-VoidBotDashboardHtml $statusDir) and refreshes itself every 10 seconds. If a hidden task hangs, it should stay frozen here with the last step it managed to confess.

+
Generated $(Escape-VoidBotDashboardHtml (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz"))
+
+ +
+$cardsHtml +
+ +
+$(New-VoidBotDashboardFactSection -Title "Runtime Stack" -Rows $runtimeRows) +$(New-VoidBotDashboardFactSection -Title "Watchdog Run" -Rows $watchdogRows) +$(New-VoidBotDashboardFactSection -Title "Offsite Backup" -Rows $offsiteRows) +$(New-VoidBotDashboardFactSection -Title "Useful Files" -Rows $fileRows) +
+ +$checksSection +
+ + +"@ + + Set-Content -LiteralPath $dashboardPath -Value $html -Encoding utf8 + + return [PSCustomObject]@{ + DashboardPath = $dashboardPath + SnapshotPath = $snapshotPath + } +}