2026-04-24 18:20:58 +01:00
param (
[ switch ] $NotifyOwner ,
[ switch ] $FailOnIssues
)
Set-StrictMode -Version Latest
$ErrorActionPreference = " Stop "
$ProgressPreference = " SilentlyContinue "
$repoRoot = Split-Path -Parent $PSScriptRoot
$envFile = Join-Path $repoRoot " .env "
$verifyScript = Join-Path $PSScriptRoot " verify-voidbot-backup.ps1 "
2026-04-24 20:11:18 +01:00
. ( Join-Path $PSScriptRoot " voidbot-operations-dashboard-lib.ps1 " )
2026-04-24 18:20:58 +01:00
function Read-DotEnv {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Path
)
if ( -not ( Test-Path $Path ) ) {
throw " Missing .env at $Path "
}
$values = @ { }
foreach ( $line in Get-Content -Path $Path ) {
if ( [ string ] :: IsNullOrWhiteSpace ( $line ) ) {
continue
}
if ( $line . TrimStart ( ) . StartsWith ( " # " ) ) {
continue
}
$separatorIndex = $line . IndexOf ( " = " )
if ( $separatorIndex -lt 1 ) {
continue
}
$key = $line . Substring ( 0 , $separatorIndex ) . Trim ( )
$value = $line . Substring ( $separatorIndex + 1 )
$values [ $key ] = $value
}
return $values
}
function Resolve-ConfigPath {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $RepoRoot ,
[ Parameter ( Mandatory = $false ) ]
[ AllowEmptyString ( ) ]
[ string ] $Value ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $Fallback
)
$raw = if ( [ string ] :: IsNullOrWhiteSpace ( $Value ) ) { $Fallback } else { $Value }
return [ System.IO.Path ] :: GetFullPath ( ( Join-Path $RepoRoot $raw ) )
}
function Write-StatusFile {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Path ,
[ Parameter ( Mandatory = $true ) ]
[ object ] $Status
)
$directory = Split-Path -Parent $Path
New-Item -ItemType Directory -Force -Path $directory | Out-Null
$Status | ConvertTo-Json -Depth 10 | Set-Content -Path $Path -Encoding utf8
2026-04-24 20:11:18 +01:00
if ( $script:storageRoot ) {
[ void ] ( Update-VoidBotOperationsDashboard -RepoRoot $repoRoot -StorageRoot $script:storageRoot )
}
2026-04-24 18:20:58 +01:00
}
function Start-LogCapture {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $LogPath
)
$directory = Split-Path -Parent $LogPath
New-Item -ItemType Directory -Force -Path $directory | Out-Null
try {
Start-Transcript -Path $LogPath -Append | Out-Null
return $true
} catch {
return $false
}
}
function Stop-LogCapture {
param (
[ Parameter ( Mandatory = $true ) ]
[ bool ] $Started
)
if ( -not $Started ) {
return
}
try {
Stop-Transcript | Out-Null
} catch {
}
}
function Add-Check {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Name ,
[ Parameter ( Mandatory = $true ) ]
[ ValidateSet ( " passed " , " warning " , " failed " ) ]
[ string ] $Status ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $Detail ,
[ hashtable ] $Data = @ { }
)
$script:checks + = [ ordered ] @ {
name = $Name
status = $Status
detail = $Detail
data = $Data
}
}
function Finalize-Report {
$failedCount = @ ( $script:checks | Where-Object { $_ . status -eq " failed " } ) . Count
$warningCount = @ ( $script:checks | Where-Object { $_ . status -eq " warning " } ) . Count
$passedCount = @ ( $script:checks | Where-Object { $_ . status -eq " passed " } ) . Count
$status = if ( $failedCount -gt 0 ) {
" failed "
} elseif ( $warningCount -gt 0 ) {
" warning "
} else {
" healthy "
}
return [ ordered ] @ {
checkedAt = ( Get-Date ) . ToString ( " o " )
repoRoot = $repoRoot
status = $status
summary = @ {
passed = $passedCount
warning = $warningCount
failed = $failedCount
}
checks = $script:checks
}
}
function Get-Json {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Url ,
[ string[] ] $Headers = @ ( )
)
$args = @ (
" -fsS " ,
" --connect-timeout " , " 5 " ,
" --max-time " , " 20 " ,
$Url
)
foreach ( $header in $Headers ) {
$args = @ ( " -H " , $header ) + $args
}
$response = & curl . exe @args
if ( $LASTEXITCODE -ne 0 ) {
throw " Request failed for $Url "
}
return $response | ConvertFrom-Json
}
2026-04-24 19:39:38 +01:00
function Invoke-CommandWithTimeout {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $FilePath ,
[ Parameter ( Mandatory = $false ) ]
[ string[] ] $ArgumentList = @ ( ) ,
[ Parameter ( Mandatory = $false ) ]
[ int ] $TimeoutSeconds = 15
)
$stdoutPath = Join-Path ( [ System.IO.Path ] :: GetTempPath ( ) ) ( [ System.IO.Path ] :: GetRandomFileName ( ) )
$stderrPath = Join-Path ( [ System.IO.Path ] :: GetTempPath ( ) ) ( [ System.IO.Path ] :: GetRandomFileName ( ) )
try {
$process = Start-Process -FilePath $FilePath -ArgumentList $ArgumentList -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
if ( -not $process . WaitForExit ( $TimeoutSeconds * 1000 ) ) {
try {
$process . Kill ( )
} catch {
}
throw " Command timed out after $TimeoutSeconds second(s): $FilePath $( $ArgumentList -join ' ' ) "
}
2026-04-25 06:29:20 +01:00
$process . Refresh ( )
$exitCode = $process . ExitCode
2026-04-24 19:39:38 +01:00
$stdout = if ( Test-Path -LiteralPath $stdoutPath ) {
Get-Content -LiteralPath $stdoutPath -Raw
} else {
" "
}
$stderr = if ( Test-Path -LiteralPath $stderrPath ) {
Get-Content -LiteralPath $stderrPath -Raw
} else {
" "
}
return [ PSCustomObject ] @ {
2026-04-25 06:29:20 +01:00
ExitCode = $exitCode
2026-04-24 19:39:38 +01:00
StdOut = $stdout
StdErr = $stderr
}
} finally {
Remove-Item -LiteralPath $stdoutPath , $stderrPath -Force -ErrorAction SilentlyContinue
}
}
2026-05-02 12:49:23 +01:00
function Invoke-SourceRepoReconcile {
param (
[ switch ] $CheckOnly ,
[ switch ] $Detached
)
$tsxCommand = Join-Path $repoRoot " node_modules\.bin\tsx.cmd "
2026-05-02 15:06:13 +01:00
$reconcileScript = Join-Path $repoRoot " scripts\reconcile-source-repos.ts "
2026-05-02 12:49:23 +01:00
if ( -not ( Test-Path -LiteralPath $tsxCommand ) ) {
throw " Local tsx launcher is missing at $tsxCommand . "
}
2026-05-02 15:06:13 +01:00
if ( -not ( Test-Path -LiteralPath $reconcileScript ) ) {
throw " Source repo reconcile script is missing at $reconcileScript . "
}
$arguments = @ ( $reconcileScript , " --json " )
2026-05-02 12:49:23 +01:00
if ( $CheckOnly ) {
$arguments + = " --check "
}
if ( $Detached ) {
$arguments + = " --detached "
}
$stdout = ( & $tsxCommand @arguments 2 > & 1 | Out-String ) . Trim ( )
$exitCode = $LASTEXITCODE
if ( $exitCode -ne 0 ) {
if ( [ string ] :: IsNullOrWhiteSpace ( $stdout ) ) {
$stdout = " Source repo reconcile exited with code $exitCode . "
}
throw $stdout
}
if ( [ string ] :: IsNullOrWhiteSpace ( $stdout ) ) {
throw " Source repo reconcile returned no JSON output. "
}
return $stdout | ConvertFrom-Json
}
2026-05-06 08:48:45 +01:00
function Get-LatestCompleteBackupDirectory {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $BackupRoot
)
$directories = @ ( Get-ChildItem -LiteralPath $BackupRoot -Directory -ErrorAction SilentlyContinue |
Sort-Object Name -Descending )
$latestDirectory = $directories | Select-Object -First 1
$latestCompleteDirectory = $directories |
Where-Object { Test-Path -LiteralPath ( Join-Path $_ . FullName " manifest.json " ) } |
Select-Object -First 1
return @ {
latest = $latestDirectory
latestComplete = $latestCompleteDirectory
}
}
2026-04-25 06:29:20 +01:00
function Ensure-DockerCliOnPath {
$dockerCliDir = " C:\Program Files\Docker\Docker\resources\bin "
if ( Test-Path -LiteralPath $dockerCliDir ) {
$pathEntries = @ ( $env:Path -split " ; " | Where-Object { -not [ string ] :: IsNullOrWhiteSpace ( $_ ) } )
if ( $pathEntries -notcontains $dockerCliDir ) {
$env:Path = " $dockerCliDir ; $env:Path "
}
}
}
function Get-DockerExecutable {
Ensure-DockerCliOnPath
$command = Get-Command docker . exe -ErrorAction SilentlyContinue
if ( $null -eq $command ) {
$command = Get-Command docker -ErrorAction SilentlyContinue
}
if ( $null -eq $command ) {
throw " Docker CLI is not available on PATH. "
}
return $command . Source
}
2026-04-24 18:20:58 +01:00
function Test-DockerAvailable {
try {
2026-04-25 06:29:20 +01:00
$dockerExe = Get-DockerExecutable
& $dockerExe version * > $null
return $LASTEXITCODE -eq 0
2026-04-24 18:20:58 +01:00
} catch {
return $false
}
}
function Get-DockerContainerHealth {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $ContainerName
)
try {
2026-04-25 06:29:20 +01:00
$dockerExe = Get-DockerExecutable
$result = & $dockerExe inspect - -format " {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} " $ContainerName 2 > $null
2026-04-24 18:20:58 +01:00
2026-04-25 06:29:20 +01:00
if ( $LASTEXITCODE -ne 0 ) {
2026-04-24 18:20:58 +01:00
return $null
}
2026-04-25 06:29:20 +01:00
return $result . Trim ( )
2026-04-24 18:20:58 +01:00
} catch {
return $null
}
}
function Parse-DatabaseDsn {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Dsn
)
$uri = [ Uri ] $Dsn
$userInfo = if ( [ string ] :: IsNullOrWhiteSpace ( $uri . UserInfo ) ) { @ ( ) } else { $uri . UserInfo . Split ( " : " , 2 ) }
return @ {
host = $uri . Host
port = if ( $uri . Port -gt 0 ) { $uri . Port } else { 5432 }
database = $uri . AbsolutePath . TrimStart ( " / " )
username = if ( $userInfo . Length -ge 1 ) { [ Uri ] :: UnescapeDataString ( $userInfo [ 0 ] ) } else { " voidbot " }
}
}
function Test-ProcessAlive {
param (
[ Parameter ( Mandatory = $true ) ]
[ int ] $ProcessId
)
try {
$process = Get-Process -Id $ProcessId -ErrorAction Stop
return $null -ne $process
} catch {
return $false
}
}
function Escape-SingleQuotedPowerShell {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Value
)
return $Value . Replace ( " ' " , " '' " )
}
function Get-OptionalPropertyValue {
param (
2026-04-25 06:29:20 +01:00
[ Parameter ( Mandatory = $false ) ]
[ AllowNull ( ) ]
2026-04-24 18:20:58 +01:00
[ object ] $InputObject ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $PropertyName
)
if ( $null -eq $InputObject ) {
return $null
}
$property = $InputObject . PSObject . Properties [ $PropertyName ]
if ( $null -eq $property ) {
return $null
}
return $property . Value
}
function Invoke-RemotePowerShellJson {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Target ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $ScriptText
)
$bytes = [ System.Text.Encoding ] :: Unicode . GetBytes ( $ScriptText )
$encoded = [ Convert ] :: ToBase64String ( $bytes )
$response = & ssh . exe -o BatchMode = yes -o ConnectTimeout = 10 $Target powershell -NoProfile -EncodedCommand $encoded
if ( $LASTEXITCODE -ne 0 ) {
throw " SSH command failed for $Target . "
}
if ( [ string ] :: IsNullOrWhiteSpace ( $response ) ) {
return $null
}
return $response | ConvertFrom-Json
}
2026-04-29 18:37:04 +01:00
function Invoke-RemoteLinuxCommand {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Target ,
[ Parameter ( Mandatory = $false ) ]
[ AllowEmptyString ( ) ]
[ string ] $IdentityFile = " " ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $Command
)
$args = @ ( " -o " , " BatchMode=yes " , " -o " , " ConnectTimeout=10 " )
if ( -not [ string ] :: IsNullOrWhiteSpace ( $IdentityFile ) ) {
$args + = @ ( " -i " , $IdentityFile )
}
$args + = @ ( $Target , $Command )
$response = & ssh . exe @args
if ( $LASTEXITCODE -ne 0 ) {
throw " SSH command failed for $Target . "
}
return [ string ] ( $response -join " `n " )
}
2026-04-29 18:50:19 +01:00
function Invoke-WatchdogExtension {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Path ,
[ Parameter ( Mandatory = $true ) ]
[ hashtable ] $Context
)
. $Path
$extensionCommand = Get-Command -Name " Invoke-VoidBotWatchdogExtension " -ErrorAction SilentlyContinue
if ( $null -eq $extensionCommand ) {
throw " Watchdog extension at $Path did not define Invoke-VoidBotWatchdogExtension. "
}
& $extensionCommand -Context $Context
}
2026-04-29 19:06:20 +01:00
function Invoke-DiscordJsonRequest {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Method ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $Url ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $BotToken ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $JsonBody
)
$bodyPath = Join-Path ( [ System.IO.Path ] :: GetTempPath ( ) ) ( [ System.IO.Path ] :: GetRandomFileName ( ) + " .json " )
try {
$utf8NoBom = New-Object System . Text . UTF8Encoding ( $false )
[ System.IO.File ] :: WriteAllText ( $bodyPath , $JsonBody , $utf8NoBom )
$response = & curl . exe -fsS -X $Method $Url `
-H " Authorization: Bot $BotToken " `
-H " Content-Type: application/json " `
- -data -binary " @ $bodyPath "
if ( $LASTEXITCODE -ne 0 ) {
throw " Request failed for $Url "
}
return $response
} finally {
Remove-Item -LiteralPath $bodyPath -Force -ErrorAction SilentlyContinue
}
}
2026-04-24 18:20:58 +01:00
function Open-OwnerDmChannel {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $BotToken ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $OwnerDiscordId
)
$payload = @ { recipient_id = $OwnerDiscordId } | ConvertTo-Json -Compress
2026-04-29 19:06:20 +01:00
$response = Invoke-DiscordJsonRequest -Method " POST " -Url " https://discord.com/api/v10/users/@me/channels " -BotToken $BotToken -JsonBody $payload
2026-04-24 18:20:58 +01:00
$parsed = $response | ConvertFrom-Json
if ( -not $parsed . id ) {
throw " Discord DM open returned no channel id. "
}
return [ string ] $parsed . id
}
function Send-OwnerDm {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $BotToken ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $OwnerDiscordId ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $Content
)
$dmChannelId = Open-OwnerDmChannel -BotToken $BotToken -OwnerDiscordId $OwnerDiscordId
$payload = @ {
content = if ( $Content . Length -le 1900 ) { $Content } else { " $( $Content . Substring ( 0 , 1897 ) ) ... " }
allowed_mentions = @ {
parse = @ ( )
}
} | ConvertTo-Json -Depth 4 -Compress
2026-04-29 19:06:20 +01:00
[ void ] ( Invoke-DiscordJsonRequest -Method " POST " -Url " https://discord.com/api/v10/channels/ $dmChannelId /messages " -BotToken $BotToken -JsonBody $payload )
2026-04-24 18:20:58 +01:00
}
function Build-OwnerNotificationMessage {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $Status ,
[ Parameter ( Mandatory = $true ) ]
[ string[] ] $IssueLines ,
[ Parameter ( Mandatory = $true ) ]
[ string ] $HealthStatusPath
)
if ( $Status -eq " healthy " ) {
return " VoidBot watchdog checked again. The stack looks normal now. Health report: $HealthStatusPath "
}
$lines = @ (
" VoidBot watchdog smelled smoke. "
" Health report: $HealthStatusPath "
" "
) + ( $IssueLines | ForEach-Object { " - $_ " } )
return ( $lines -join [ Environment ] :: NewLine ) . Trim ( )
}
2026-04-24 20:11:18 +01:00
function Set-WatchdogStep {
param (
[ Parameter ( Mandatory = $true ) ]
[ string ] $CurrentStep ,
[ Parameter ( Mandatory = $false ) ]
[ string ] $Detail = " "
)
$script:watchdogState . currentStep = $CurrentStep
$script:watchdogState . currentDetail = $Detail
$script:watchdogState . stepUpdatedAt = ( Get-Date ) . ToString ( " o " )
Write-StatusFile -Path $script:watchdogStatePath -Status $script:watchdogState
}
2026-04-24 18:20:58 +01:00
$checks = @ ( )
$config = Read-DotEnv -Path $envFile
$storageRoot = Resolve-ConfigPath -RepoRoot $repoRoot -Value $config [ " STORAGE_ROOT " ] -Fallback " .voidbot "
$statusPath = Join-Path $storageRoot " status\operations-health.json "
$watchdogStatePath = Join-Path $storageRoot " status\operations-watchdog.json "
$logPath = Join-Path $storageRoot " logs\operations-watchdog.log "
$runtimeStatusPath = Join-Path $storageRoot " status\runtime-stack.json "
$offsiteStatusPath = Join-Path $storageRoot " status\offsite-backup.json "
$backupRoot = Join-Path $storageRoot " backups "
$backupMaxAgeHours = if ( $config . ContainsKey ( " VOIDBOT_BACKUP_MAX_AGE_HOURS " ) ) { [ double ] $config [ " VOIDBOT_BACKUP_MAX_AGE_HOURS " ] } else { 30.0 }
$offsiteMaxAgeHours = if ( $config . ContainsKey ( " VOIDBOT_OFFSITE_BACKUP_MAX_AGE_HOURS " ) ) { [ double ] $config [ " VOIDBOT_OFFSITE_BACKUP_MAX_AGE_HOURS " ] } else { 36.0 }
$notifyRepeatHours = if ( $config . ContainsKey ( " VOIDBOT_HEALTHCHECK_NOTIFY_REPEAT_HOURS " ) ) { [ double ] $config [ " VOIDBOT_HEALTHCHECK_NOTIFY_REPEAT_HOURS " ] } else { 12.0 }
2026-04-30 16:41:44 +01:00
$stackStartupTaskName = if ( $config . ContainsKey ( " VOIDBOT_STACK_STARTUP_TASK_NAME " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " VOIDBOT_STACK_STARTUP_TASK_NAME " ] ) ) { $config [ " VOIDBOT_STACK_STARTUP_TASK_NAME " ] } else { " VoidBot Stack Startup " }
2026-04-24 18:20:58 +01:00
$watchdogTaskName = if ( $config . ContainsKey ( " VOIDBOT_HEALTHCHECK_TASK_NAME " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " VOIDBOT_HEALTHCHECK_TASK_NAME " ] ) ) { $config [ " VOIDBOT_HEALTHCHECK_TASK_NAME " ] } else { " VoidBot Operations Watchdog " }
$offsiteTaskName = if ( $config . ContainsKey ( " OFFSITE_BACKUP_TASK_NAME " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " OFFSITE_BACKUP_TASK_NAME " ] ) ) { $config [ " OFFSITE_BACKUP_TASK_NAME " ] } else { " VoidBot Offsite Backup " }
$offsiteRemoteWindowsDir = if ( $config . ContainsKey ( " OFFSITE_BACKUP_REMOTE_WINDOWS_DIR " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " OFFSITE_BACKUP_REMOTE_WINDOWS_DIR " ] ) ) { $config [ " OFFSITE_BACKUP_REMOTE_WINDOWS_DIR " ] } else { " C:\Meta\voidbot-backups " }
$offsiteTarget = if ( $config . ContainsKey ( " OFFSITE_BACKUP_SSH_TARGET " ) ) { $config [ " OFFSITE_BACKUP_SSH_TARGET " ] } else { " " }
$offsiteKeepLatest = if ( $config . ContainsKey ( " OFFSITE_BACKUP_REMOTE_KEEP_LATEST " ) ) { [ int ] $config [ " OFFSITE_BACKUP_REMOTE_KEEP_LATEST " ] } else { 14 }
$offsiteLabel = if ( $config . ContainsKey ( " OFFSITE_BACKUP_LABEL " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " OFFSITE_BACKUP_LABEL " ] ) ) { $config [ " OFFSITE_BACKUP_LABEL " ] } else { " offsite-auto " }
2026-04-29 18:50:19 +01:00
$watchdogExtensionScriptRaw = if ( $config . ContainsKey ( " VOIDBOT_HEALTHCHECK_EXTENSION_SCRIPT " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " VOIDBOT_HEALTHCHECK_EXTENSION_SCRIPT " ] ) ) { $config [ " VOIDBOT_HEALTHCHECK_EXTENSION_SCRIPT " ] } else { " .voidbot/private/check-voidbot-operations.local.ps1 " }
$watchdogExtensionScript = Resolve-ConfigPath -RepoRoot $repoRoot -Value $watchdogExtensionScriptRaw -Fallback " .voidbot/private/check-voidbot-operations.local.ps1 "
$watchdogExtensionExplicit = $config . ContainsKey ( " VOIDBOT_HEALTHCHECK_EXTENSION_SCRIPT " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " VOIDBOT_HEALTHCHECK_EXTENSION_SCRIPT " ] )
2026-04-24 18:20:58 +01:00
$logCaptureStarted = Start-LogCapture -LogPath $logPath
2026-04-24 20:11:18 +01:00
$previousWatchdogState = if ( Test-Path -LiteralPath $watchdogStatePath ) {
2026-04-24 18:20:58 +01:00
try {
Get-Content -LiteralPath $watchdogStatePath -Raw | ConvertFrom-Json
} catch {
$null
}
} else {
$null
}
2026-04-24 20:11:18 +01:00
$script:watchdogStatePath = $watchdogStatePath
$watchdogStartedAt = ( Get-Date ) . ToString ( " o " )
$watchdogState = @ {
taskName = $watchdogTaskName
logPath = $logPath
reportPath = $statusPath
startedAt = $watchdogStartedAt
completedAt = $null
runStatus = " running "
reportStatus = $null
currentStep = " initializing "
currentDetail = " Starting watchdog checks. "
stepUpdatedAt = $watchdogStartedAt
executionTimeLimitMinutes = 15
lastStatus = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastStatus " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastStatus " ) } else { $null }
lastFingerprint = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastFingerprint " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastFingerprint " ) } else { $null }
lastNotifiedAt = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotifiedAt " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotifiedAt " ) } else { $null }
lastNotificationReason = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotificationReason " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotificationReason " ) } else { $null }
lastCompletedAt = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastCompletedAt " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastCompletedAt " ) } elseif ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " checkedAt " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " checkedAt " ) } else { $null }
lastCompletedRunStatus = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastCompletedRunStatus " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastCompletedRunStatus " ) } else { $null }
lastRunError = $null
}
Write-StatusFile -Path $watchdogStatePath -Status $watchdogState
2026-04-24 18:20:58 +01:00
try {
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " runtime_checks " -Detail " Inspecting runtime stack status and bot/worker liveness. "
2026-04-24 18:20:58 +01:00
if ( -not ( Test-Path -LiteralPath $runtimeStatusPath ) ) {
Add-Check -Name " runtime.status_file " -Status " failed " -Detail " runtime-stack.json is missing at $runtimeStatusPath . "
} else {
2026-04-25 06:29:20 +01:00
try {
$runtimeStatus = Get-Content -LiteralPath $runtimeStatusPath -Raw | ConvertFrom-Json
} catch {
Add-Check -Name " runtime.status_file " -Status " failed " -Detail " runtime-stack.json could not be parsed: $( $_ . Exception . Message ) " -Data @ {
path = $runtimeStatusPath
}
$runtimeStatus = $null
2026-04-24 18:20:58 +01:00
}
2026-04-25 06:29:20 +01:00
if ( $null -ne $runtimeStatus ) {
Add-Check -Name " runtime.status_file " -Status " passed " -Detail " Found runtime-stack.json. " -Data @ {
path = $runtimeStatusPath
2026-04-24 18:20:58 +01:00
}
2026-04-25 06:29:20 +01:00
$runtimeStage = [ string ] ( Get-OptionalPropertyValue -InputObject $runtimeStatus -PropertyName " stage " )
$runtimeReady = Get-OptionalPropertyValue -InputObject $runtimeStatus -PropertyName " ready "
if ( $runtimeReady -eq $true -and $runtimeStage -eq " ready " ) {
Add-Check -Name " runtime.lifecycle " -Status " passed " -Detail " Runtime status reports ready. " -Data @ {
stage = $runtimeStage
ready = $runtimeReady
2026-04-24 18:20:58 +01:00
}
} else {
2026-04-25 06:29:20 +01:00
$stageLabel = if ( [ string ] :: IsNullOrWhiteSpace ( $runtimeStage ) ) { " (missing) " } else { $runtimeStage }
Add-Check -Name " runtime.lifecycle " -Status " failed " -Detail " Runtime status reports stage ' $stageLabel ' with ready= $runtimeReady . " -Data @ {
stage = $runtimeStage
ready = $runtimeReady
}
}
foreach ( $component in @ ( " bot " , " worker " ) ) {
$componentStatus = Get-OptionalPropertyValue -InputObject $runtimeStatus -PropertyName $component
$processIdValueRaw = Get-OptionalPropertyValue -InputObject $componentStatus -PropertyName " pid "
$processIdValue = if ( $null -ne $processIdValueRaw ) { ( $processIdValueRaw -as [ int ] ) } else { $null }
if ( $null -eq $processIdValue -or $processIdValue -le 0 ) {
Add-Check -Name " runtime. $component " -Status " failed " -Detail " $component PID is missing from runtime-stack.json. "
continue
}
if ( Test-ProcessAlive -ProcessId $processIdValue ) {
Add-Check -Name " runtime. $component " -Status " passed " -Detail " $component process $processIdValue is running. " -Data @ {
pid = $processIdValue
}
} else {
Add-Check -Name " runtime. $component " -Status " failed " -Detail " $component process $processIdValue is not running anymore. " -Data @ {
pid = $processIdValue
}
2026-04-24 18:20:58 +01:00
}
}
}
}
$vectorStoreKind = if ( $config . ContainsKey ( " VECTOR_STORE_KIND " ) ) { $config [ " VECTOR_STORE_KIND " ] } else { " local_json " }
$historyCollection = if ( $config . ContainsKey ( " QDRANT_HISTORY_COLLECTION " ) ) { $config [ " QDRANT_HISTORY_COLLECTION " ] } else { " voidbot_discord_history_chunks " }
$sourceCollection = if ( $config . ContainsKey ( " QDRANT_SOURCE_COLLECTION " ) ) { $config [ " QDRANT_SOURCE_COLLECTION " ] } else { " voidbot_repository_source_chunks " }
2026-05-02 12:49:23 +01:00
$qdrantReady = $vectorStoreKind -ne " qdrant "
2026-04-24 18:20:58 +01:00
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " qdrant_checks " -Detail " Verifying Qdrant reachability and required collections. "
2026-04-24 18:20:58 +01:00
if ( $vectorStoreKind -eq " qdrant " ) {
$qdrantUrl = if ( $config . ContainsKey ( " QDRANT_URL " ) ) { $config [ " QDRANT_URL " ] } else { " http://127.0.0.1:6333 " }
try {
$collectionsResponse = Get-Json -Url " $( $qdrantUrl . TrimEnd ( '/' ) ) /collections "
$collectionNames = @ ( $collectionsResponse . result . collections | ForEach-Object { [ string ] $_ . name } )
2026-05-02 12:49:23 +01:00
$qdrantReady = $true
2026-04-24 18:20:58 +01:00
Add-Check -Name " qdrant.reachable " -Status " passed " -Detail " Qdrant is reachable at $qdrantUrl . " -Data @ {
url = $qdrantUrl
collections = $collectionNames
}
foreach ( $requiredCollection in @ ( $historyCollection , $sourceCollection ) ) {
if ( $collectionNames -contains $requiredCollection ) {
Add-Check -Name " qdrant.collection. $requiredCollection " -Status " passed " -Detail " Qdrant collection $requiredCollection exists. "
} else {
Add-Check -Name " qdrant.collection. $requiredCollection " -Status " failed " -Detail " Qdrant collection $requiredCollection is missing. "
}
}
} catch {
Add-Check -Name " qdrant.reachable " -Status " failed " -Detail " Qdrant check failed: $( $_ . Exception . Message ) " -Data @ {
url = $qdrantUrl
}
}
} else {
Add-Check -Name " qdrant.reachable " -Status " passed " -Detail " Qdrant check skipped because VECTOR_STORE_KIND= $vectorStoreKind . " -Data @ {
vectorStoreKind = $vectorStoreKind
}
}
$stateStorageBackend = if ( $config . ContainsKey ( " STATE_STORAGE_BACKEND " ) ) { $config [ " STATE_STORAGE_BACKEND " ] } else { " postgres " }
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " postgres_checks " -Detail " Checking Postgres container health and required tables. "
2026-04-24 18:20:58 +01:00
if ( $stateStorageBackend -eq " postgres " ) {
$databaseDsn = if ( $config . ContainsKey ( " DATABASE_DSN " ) ) { $config [ " DATABASE_DSN " ] } else { " " }
$dsn = Parse-DatabaseDsn -Dsn $databaseDsn
if ( -not ( Test-DockerAvailable ) ) {
Add-Check -Name " postgres.health " -Status " warning " -Detail " Skipped Postgres container health check because Docker is unavailable. " -Data @ {
dsn = $databaseDsn
}
} else {
2026-04-25 06:29:20 +01:00
$dockerExe = Get-DockerExecutable
2026-04-24 18:20:58 +01:00
$containerHealth = Get-DockerContainerHealth -ContainerName " voidbot-postgres "
if ( $containerHealth -eq " healthy " ) {
Add-Check -Name " postgres.health " -Status " passed " -Detail " Postgres container is healthy. " -Data @ {
containerName = " voidbot-postgres "
database = $dsn . database
}
try {
2026-04-25 06:29:20 +01:00
$tableResult = & $dockerExe exec voidbot-postgres psql -U $dsn . username -d $dsn . database -tAc " select table_name from information_schema.tables where table_schema = 'public'; "
2026-04-24 19:39:38 +01:00
2026-04-25 06:29:20 +01:00
if ( $LASTEXITCODE -ne 0 ) {
$stderr = $Error [ 0 ] . ToString ( ) . Trim ( )
2026-04-24 19:39:38 +01:00
if ( [ string ] :: IsNullOrWhiteSpace ( $stderr ) ) {
throw " psql table listing failed. "
}
2026-04-24 18:20:58 +01:00
2026-04-24 19:39:38 +01:00
throw " psql table listing failed: $stderr "
2026-04-24 18:20:58 +01:00
}
2026-04-25 06:29:20 +01:00
$tables = @ ( $tableResult -split " `r ? `n " | ForEach-Object { $_ . Trim ( ) } | Where-Object { $_ } )
2026-04-24 18:20:58 +01:00
foreach ( $requiredTable in @ ( " jobs " , " audit_events " , " interaction_memory_events " ) ) {
if ( $tables -contains $requiredTable ) {
Add-Check -Name " postgres.table. $requiredTable " -Status " passed " -Detail " Postgres table $requiredTable exists. "
} else {
Add-Check -Name " postgres.table. $requiredTable " -Status " failed " -Detail " Postgres table $requiredTable is missing. "
}
}
} catch {
Add-Check -Name " postgres.tables " -Status " warning " -Detail " Could not verify Postgres tables: $( $_ . Exception . Message ) "
}
} elseif ( $containerHealth ) {
Add-Check -Name " postgres.health " -Status " failed " -Detail " Postgres container health is ' $containerHealth ' instead of healthy. " -Data @ {
containerName = " voidbot-postgres "
}
} else {
Add-Check -Name " postgres.health " -Status " failed " -Detail " Postgres container voidbot-postgres is not available. "
}
}
} else {
Add-Check -Name " postgres.health " -Status " passed " -Detail " Postgres check skipped because STATE_STORAGE_BACKEND= $stateStorageBackend . " -Data @ {
stateStorageBackend = $stateStorageBackend
}
}
$ragOllamaUrl = if ( $config . ContainsKey ( " RAG_OLLAMA_BASE_URL " ) ) { $config [ " RAG_OLLAMA_BASE_URL " ] } else { " http://127.0.0.1:11434 " }
$ragOllamaModel = if ( $config . ContainsKey ( " RAG_OLLAMA_MODEL " ) ) { $config [ " RAG_OLLAMA_MODEL " ] } else { " qwen3-embedding:0.6b " }
2026-05-02 12:49:23 +01:00
$ragOllamaReady = $false
2026-04-24 18:20:58 +01:00
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " ollama_checks " -Detail " Checking embedding and local LLM Ollama endpoints. "
2026-04-24 18:20:58 +01:00
try {
$ragTags = Get-Json -Url " $( $ragOllamaUrl . TrimEnd ( '/' ) ) /api/tags "
$ragModels = @ ( $ragTags . models | ForEach-Object { [ string ] $_ . name } )
if ( $ragModels -contains $ragOllamaModel ) {
2026-05-02 12:49:23 +01:00
$ragOllamaReady = $true
2026-04-24 18:20:58 +01:00
Add-Check -Name " rag_ollama.model " -Status " passed " -Detail " Embedding Ollama is reachable and has $ragOllamaModel . " -Data @ {
url = $ragOllamaUrl
model = $ragOllamaModel
}
} else {
Add-Check -Name " rag_ollama.model " -Status " failed " -Detail " Embedding Ollama is reachable but missing $ragOllamaModel . " -Data @ {
url = $ragOllamaUrl
models = $ragModels
}
}
} catch {
Add-Check -Name " rag_ollama.model " -Status " failed " -Detail " Embedding Ollama check failed: $( $_ . Exception . Message ) " -Data @ {
url = $ragOllamaUrl
model = $ragOllamaModel
}
}
$enabledProviders = if ( $config . ContainsKey ( " ENABLED_PROVIDERS " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " ENABLED_PROVIDERS " ] ) ) {
@ ( $config [ " ENABLED_PROVIDERS " ] . Split ( " , " ) | ForEach-Object { $_ . Trim ( ) } | Where-Object { $_ } )
} else {
@ ( )
}
$localLlmUrl = if ( $config . ContainsKey ( " LOCAL_LLM_OLLAMA_BASE_URL " ) ) { $config [ " LOCAL_LLM_OLLAMA_BASE_URL " ] } else { " " }
$localLlmModel = if ( $config . ContainsKey ( " LOCAL_LLM_OLLAMA_MODEL " ) ) { $config [ " LOCAL_LLM_OLLAMA_MODEL " ] } else { " " }
2026-05-03 00:46:05 +01:00
$localLlmSocialReadModel = if (
$config . ContainsKey ( " LOCAL_LLM_SOCIAL_READ_OLLAMA_MODEL " ) -and
-not [ string ] :: IsNullOrWhiteSpace ( $config [ " LOCAL_LLM_SOCIAL_READ_OLLAMA_MODEL " ] )
) {
$config [ " LOCAL_LLM_SOCIAL_READ_OLLAMA_MODEL " ]
} else {
$localLlmModel
}
2026-04-24 18:20:58 +01:00
$localLlmEnabled = ( $enabledProviders -contains " local_llm " ) -or (
$config . ContainsKey ( " LOCAL_LLM_ALLOW_PUBLIC " ) -and [ string ] $config [ " LOCAL_LLM_ALLOW_PUBLIC " ] -eq " true "
)
if ( $localLlmEnabled -and -not [ string ] :: IsNullOrWhiteSpace ( $localLlmUrl ) -and -not [ string ] :: IsNullOrWhiteSpace ( $localLlmModel ) ) {
try {
$localLlmTags = Get-Json -Url " $( $localLlmUrl . TrimEnd ( '/' ) ) /api/tags "
$localLlmModels = @ ( $localLlmTags . models | ForEach-Object { [ string ] $_ . name } )
if ( $localLlmModels -contains $localLlmModel ) {
Add-Check -Name " local_llm.model " -Status " passed " -Detail " Local LLM Ollama is reachable and has $localLlmModel . " -Data @ {
url = $localLlmUrl
model = $localLlmModel
}
} else {
Add-Check -Name " local_llm.model " -Status " failed " -Detail " Local LLM Ollama is reachable but missing $localLlmModel . " -Data @ {
url = $localLlmUrl
models = $localLlmModels
}
}
2026-05-03 00:46:05 +01:00
if ( [ string ] :: IsNullOrWhiteSpace ( $localLlmSocialReadModel ) ) {
Add-Check -Name " local_llm.social_read_model " -Status " warning " -Detail " Social-read sidecar model is blank, so it will fall back to the main local LLM model. " -Data @ {
url = $localLlmUrl
model = $localLlmModel
}
} elseif ( $localLlmModels -contains $localLlmSocialReadModel ) {
$socialReadDetail = if ( $localLlmSocialReadModel -eq $localLlmModel ) {
" Social-read sidecar reuses the main local LLM model $localLlmSocialReadModel . "
} else {
" Social-read sidecar is reachable and has $localLlmSocialReadModel . "
}
Add-Check -Name " local_llm.social_read_model " -Status " passed " -Detail $socialReadDetail -Data @ {
url = $localLlmUrl
model = $localLlmSocialReadModel
}
} else {
Add-Check -Name " local_llm.social_read_model " -Status " failed " -Detail " Local LLM Ollama is reachable but missing social-read model $localLlmSocialReadModel . " -Data @ {
url = $localLlmUrl
models = $localLlmModels
}
}
2026-04-24 18:20:58 +01:00
} catch {
Add-Check -Name " local_llm.model " -Status " failed " -Detail " Local LLM Ollama check failed: $( $_ . Exception . Message ) " -Data @ {
url = $localLlmUrl
model = $localLlmModel
}
2026-05-03 00:46:05 +01:00
Add-Check -Name " local_llm.social_read_model " -Status " failed " -Detail " Social-read Ollama check failed: $( $_ . Exception . Message ) " -Data @ {
url = $localLlmUrl
model = $localLlmSocialReadModel
}
2026-04-24 18:20:58 +01:00
}
} else {
Add-Check -Name " local_llm.model " -Status " passed " -Detail " Local LLM check skipped because the provider is not enabled for this runtime. " -Data @ {
enabledProviders = $enabledProviders
}
2026-05-03 00:46:05 +01:00
Add-Check -Name " local_llm.social_read_model " -Status " passed " -Detail " Social-read model check skipped because the local LLM provider is not enabled for this runtime. " -Data @ {
enabledProviders = $enabledProviders
}
2026-04-24 18:20:58 +01:00
}
2026-05-02 12:49:23 +01:00
$sourceRepoRoot = if ( $config . ContainsKey ( " SOURCE_REPO_ROOT " ) ) { $config [ " SOURCE_REPO_ROOT " ] } else { " " }
if ( [ string ] :: IsNullOrWhiteSpace ( $sourceRepoRoot ) ) {
Add-Check -Name " source.repos " -Status " passed " -Detail " Source repo reconciliation check skipped because SOURCE_REPO_ROOT is blank. "
} else {
Set-WatchdogStep -CurrentStep " source_repo_checks " -Detail " Reconciling discovered source repos against the indexed source archive. "
try {
$sourceRepoPreview = Invoke-SourceRepoReconcile -CheckOnly
$missingRepos = @ ( $sourceRepoPreview . missingRepos | ForEach-Object { [ string ] $_ } )
$staleIndexedRepos = @ ( $sourceRepoPreview . staleIndexedRepos | ForEach-Object { [ string ] $_ } )
if ( $missingRepos . Count -eq 0 ) {
Add-Check -Name " source.repos " -Status " passed " -Detail " All discovered source repos are represented in the indexed source archive. " -Data @ {
discoveredRepoCount = [ int ] $sourceRepoPreview . discoveredRepoCount
indexedRepoCount = [ int ] $sourceRepoPreview . indexedRepoCount
selectedRepoCount = [ int ] $sourceRepoPreview . selectedRepoCount
}
} elseif ( $qdrantReady -and $ragOllamaReady ) {
$sourceRepoRepair = Invoke-SourceRepoReconcile -Detached
$launchedRepos = @ ( $sourceRepoRepair . launchedRepos | ForEach-Object { [ string ] $_ } )
Add-Check -Name " source.repos " -Status " warning " -Detail " Detected $( $missingRepos . Count ) unindexed source repos and launched detached reconciliation. " -Data @ {
missingRepos = $missingRepos
launchedRepos = $launchedRepos
discoveredRepoCount = [ int ] $sourceRepoPreview . discoveredRepoCount
indexedRepoCount = [ int ] $sourceRepoPreview . indexedRepoCount
}
} else {
Add-Check -Name " source.repos " -Status " failed " -Detail " Detected $( $missingRepos . Count ) unindexed source repos, but source indexing dependencies are unhealthy. " -Data @ {
missingRepos = $missingRepos
qdrantReady = $qdrantReady
ragOllamaReady = $ragOllamaReady
}
}
if ( $staleIndexedRepos . Count -eq 0 ) {
Add-Check -Name " source.stale_index " -Status " passed " -Detail " Indexed source archive has no repo summaries for repos that are no longer present. "
} else {
Add-Check -Name " source.stale_index " -Status " warning " -Detail " Indexed source archive still carries repo summaries for repos that are no longer present. " -Data @ {
staleIndexedRepos = $staleIndexedRepos
}
}
} catch {
Add-Check -Name " source.repos " -Status " failed " -Detail " Source repo reconciliation check failed: $( $_ . Exception . Message ) " -Data @ {
sourceRepoRoot = $sourceRepoRoot
}
}
}
2026-04-24 18:20:58 +01:00
if ( $config . ContainsKey ( " DISCORD_BOT_TOKEN " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " DISCORD_BOT_TOKEN " ] ) ) {
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " discord_checks " -Detail " Validating Discord bot credentials. "
2026-04-24 18:20:58 +01:00
try {
$botIdentity = Get-Json -Url " https://discord.com/api/v10/users/@me " -Headers @ ( " Authorization: Bot $( $config [ " DISCORD_BOT_TOKEN " ] ) " )
Add-Check -Name " discord.bot_token " -Status " passed " -Detail " Discord bot token is valid for $( [ string ] $botIdentity . username ) . " -Data @ {
id = [ string ] $botIdentity . id
username = [ string ] $botIdentity . username
}
} catch {
Add-Check -Name " discord.bot_token " -Status " failed " -Detail " Discord bot token check failed: $( $_ . Exception . Message ) "
}
} else {
Add-Check -Name " discord.bot_token " -Status " warning " -Detail " DISCORD_BOT_TOKEN is blank; bot identity could not be checked. "
}
2026-05-06 08:48:45 +01:00
$backupSelection = Get-LatestCompleteBackupDirectory -BackupRoot $backupRoot
$latestBackup = $backupSelection . latest
$latestCompleteBackup = $backupSelection . latestComplete
2026-04-24 18:20:58 +01:00
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " backup_checks " -Detail " Verifying the latest local backup and freshness thresholds. "
2026-04-24 18:20:58 +01:00
if ( $null -eq $latestBackup ) {
Add-Check -Name " backup.latest " -Status " failed " -Detail " No local backup directory exists under $backupRoot . "
2026-05-06 08:48:45 +01:00
} elseif ( $null -eq $latestCompleteBackup ) {
Add-Check -Name " backup.latest " -Status " failed " -Detail " No complete local backup with a manifest exists under $backupRoot . " -Data @ {
latestPath = $latestBackup . FullName
}
2026-04-24 18:20:58 +01:00
} else {
2026-05-06 08:48:45 +01:00
if ( $latestBackup . FullName -ne $latestCompleteBackup . FullName ) {
Add-Check -Name " backup.incomplete_latest " -Status " warning " -Detail " Newest backup directory is incomplete and was ignored until a manifest exists. " -Data @ {
incompletePath = $latestBackup . FullName
latestCompletePath = $latestCompleteBackup . FullName
}
}
2026-04-24 18:20:58 +01:00
try {
2026-05-06 08:48:45 +01:00
$verificationJson = & powershell . exe -NoProfile -ExecutionPolicy Bypass -File $verifyScript -BackupPath $latestCompleteBackup . FullName -AsJson -Quiet
2026-04-24 18:20:58 +01:00
$verificationReport = $verificationJson | ConvertFrom-Json
$createdAtCheck = $verificationReport . checks | Where-Object { $_ . name -eq " backup.created_at " } | Select-Object -First 1
2026-05-06 08:48:45 +01:00
$backupAgeHours = if ( $createdAtCheck -and $createdAtCheck . data -and $createdAtCheck . data . ageHours ) { [ double ] $createdAtCheck . data . ageHours } else { [ Math ] :: Round ( ( ( Get-Date ) - $latestCompleteBackup . LastWriteTime ) . TotalHours , 2 ) }
2026-04-24 18:20:58 +01:00
if ( [ string ] $verificationReport . status -eq " failed " ) {
Add-Check -Name " backup.latest " -Status " failed " -Detail " Latest local backup failed verification. " -Data @ {
2026-05-06 08:48:45 +01:00
path = $latestCompleteBackup . FullName
2026-04-24 18:20:58 +01:00
verification = $verificationReport
}
} else {
Add-Check -Name " backup.latest " -Status " passed " -Detail " Latest local backup passed verification. " -Data @ {
2026-05-06 08:48:45 +01:00
path = $latestCompleteBackup . FullName
2026-04-24 18:20:58 +01:00
verificationStatus = [ string ] $verificationReport . status
ageHours = $backupAgeHours
}
}
if ( $backupAgeHours -le $backupMaxAgeHours ) {
Add-Check -Name " backup.freshness " -Status " passed " -Detail " Latest local backup is $backupAgeHours hours old. " -Data @ {
thresholdHours = $backupMaxAgeHours
2026-05-06 08:48:45 +01:00
path = $latestCompleteBackup . FullName
2026-04-24 18:20:58 +01:00
}
} else {
Add-Check -Name " backup.freshness " -Status " failed " -Detail " Latest local backup is $backupAgeHours hours old, which exceeds the $backupMaxAgeHours hour threshold. " -Data @ {
thresholdHours = $backupMaxAgeHours
2026-05-06 08:48:45 +01:00
path = $latestCompleteBackup . FullName
2026-04-24 18:20:58 +01:00
}
}
} catch {
Add-Check -Name " backup.latest " -Status " failed " -Detail " Failed to verify the latest local backup: $( $_ . Exception . Message ) " -Data @ {
2026-05-06 08:48:45 +01:00
path = $latestCompleteBackup . FullName
2026-04-24 18:20:58 +01:00
}
}
}
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " scheduled_task_checks " -Detail " Inspecting scheduled task installation and status files. "
2026-04-30 16:41:44 +01:00
try {
$startupTask = Get-ScheduledTask -TaskName $stackStartupTaskName -ErrorAction Stop
$startupEnabled = [ bool ] $startupTask . Settings . Enabled
$startupTaskStatus = if ( $startupEnabled ) { " passed " } else { " failed " }
$startupTaskDetail = if ( $startupEnabled ) { " Stack startup task is installed and enabled. " } else { " Stack startup task is installed but disabled. " }
Add-Check -Name " scheduled_task.stack_startup " -Status $startupTaskStatus -Detail $startupTaskDetail -Data @ {
taskName = $stackStartupTaskName
state = [ string ] $startupTask . State
}
} catch {
Add-Check -Name " scheduled_task.stack_startup " -Status " warning " -Detail " Stack startup task $stackStartupTaskName is not installed yet. "
}
2026-04-24 18:20:58 +01:00
try {
$offsiteTask = Get-ScheduledTask -TaskName $offsiteTaskName -ErrorAction Stop
$offsiteEnabled = [ bool ] $offsiteTask . Settings . Enabled
$offsiteTaskStatus = if ( $offsiteEnabled ) { " passed " } else { " failed " }
$offsiteTaskDetail = if ( $offsiteEnabled ) { " Offsite backup task is installed and enabled. " } else { " Offsite backup task is installed but disabled. " }
Add-Check -Name " scheduled_task.offsite_backup " -Status $offsiteTaskStatus -Detail $offsiteTaskDetail -Data @ {
taskName = $offsiteTaskName
state = [ string ] $offsiteTask . State
}
} catch {
Add-Check -Name " scheduled_task.offsite_backup " -Status " failed " -Detail " Offsite backup task $offsiteTaskName is not installed. "
}
try {
$watchdogTask = Get-ScheduledTask -TaskName $watchdogTaskName -ErrorAction Stop
$watchdogEnabled = [ bool ] $watchdogTask . Settings . Enabled
$watchdogTaskStatus = if ( $watchdogEnabled ) { " passed " } else { " failed " }
$watchdogTaskDetail = if ( $watchdogEnabled ) { " Operations watchdog task is installed and enabled. " } else { " Operations watchdog task is installed but disabled. " }
Add-Check -Name " scheduled_task.watchdog " -Status $watchdogTaskStatus -Detail $watchdogTaskDetail -Data @ {
taskName = $watchdogTaskName
state = [ string ] $watchdogTask . State
}
} catch {
Add-Check -Name " scheduled_task.watchdog " -Status " warning " -Detail " Operations watchdog task $watchdogTaskName is not installed yet. "
}
if ( Test-Path -LiteralPath $offsiteStatusPath ) {
try {
$offsiteStatus = Get-Content -LiteralPath $offsiteStatusPath -Raw | ConvertFrom-Json
if ( [ string ] $offsiteStatus . status -eq " completed " ) {
Add-Check -Name " offsite.status_file " -Status " passed " -Detail " Latest offsite sync status is completed. " -Data @ {
completedAt = [ string ] $offsiteStatus . completedAt
remoteFile = $offsiteStatus . remoteFile
}
} else {
2026-05-03 08:44:38 +01:00
Add-Check -Name " offsite.status_file " -Status " warning " -Detail " Latest offsite sync status is ' $( [ string ] $offsiteStatus . status ) ' instead of completed. " -Data @ {
2026-04-24 18:20:58 +01:00
error = [ string ] ( Get-OptionalPropertyValue -InputObject $offsiteStatus -PropertyName " error " )
}
}
} catch {
Add-Check -Name " offsite.status_file " -Status " failed " -Detail " Failed to parse offsite-backup.json: $( $_ . Exception . Message ) "
}
} else {
Add-Check -Name " offsite.status_file " -Status " failed " -Detail " offsite-backup.json is missing at $offsiteStatusPath . "
}
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " offsite_remote_checks " -Detail " Checking remote offsite backup freshness and retention. "
2026-04-24 18:20:58 +01:00
if ( [ string ] :: IsNullOrWhiteSpace ( $offsiteTarget ) ) {
Add-Check -Name " offsite.remote " -Status " warning " -Detail " Remote offsite backup check skipped because OFFSITE_BACKUP_SSH_TARGET is blank. "
} else {
try {
$escapedRemoteDir = Escape-SingleQuotedPowerShell -Value $offsiteRemoteWindowsDir
$escapedPattern = Escape-SingleQuotedPowerShell -Value " *- $offsiteLabel .zip "
$remoteScript = @ '
$dir = '{0}'
$pattern = '{1}'
if ( -not ( Test-Path -LiteralPath $dir ) ) { {
throw " Missing remote backup directory $dir "
} }
$files = Get-ChildItem -LiteralPath $dir -Filter $pattern -File | Sort-Object LastWriteTimeUtc -Descending
[ pscustomobject ] @ { {
count = @ ( $files ) . Count
newest = if ( @ ( $files ) . Count -gt 0 ) { {
[ pscustomobject ] @ { {
name = $files [ 0 ] . Name
size = $files [ 0 ] . Length
lastWriteTimeUtc = $files [ 0 ] . LastWriteTimeUtc . ToString ( 'o' )
} }
} } else { {
$null
} }
} } | ConvertTo-Json -Compress -Depth 4
' @ -f $escapedRemoteDir , $escapedPattern
$remoteListing = Invoke-RemotePowerShellJson -Target $offsiteTarget -ScriptText $remoteScript
if ( $null -eq $remoteListing -or [ int ] $remoteListing . count -lt 1 ) {
Add-Check -Name " offsite.remote " -Status " failed " -Detail " Remote offsite backup directory has no archives matching *- $offsiteLabel .zip. " -Data @ {
target = $offsiteTarget
remoteWindowsDir = $offsiteRemoteWindowsDir
}
} else {
$remoteNewest = $remoteListing . newest
$remoteAgeHours = [ Math ] :: Round ( ( [ DateTimeOffset ] :: UtcNow - [ DateTimeOffset ] :: Parse ( [ string ] $remoteNewest . lastWriteTimeUtc ) ) . TotalHours , 2 )
if ( $remoteAgeHours -le $offsiteMaxAgeHours ) {
Add-Check -Name " offsite.remote " -Status " passed " -Detail " Newest remote offsite archive is $remoteAgeHours hours old. " -Data @ {
target = $offsiteTarget
remoteWindowsDir = $offsiteRemoteWindowsDir
newest = $remoteNewest
archiveCount = [ int ] $remoteListing . count
thresholdHours = $offsiteMaxAgeHours
}
} else {
Add-Check -Name " offsite.remote " -Status " failed " -Detail " Newest remote offsite archive is $remoteAgeHours hours old, which exceeds the $offsiteMaxAgeHours hour threshold. " -Data @ {
target = $offsiteTarget
remoteWindowsDir = $offsiteRemoteWindowsDir
newest = $remoteNewest
archiveCount = [ int ] $remoteListing . count
thresholdHours = $offsiteMaxAgeHours
}
}
if ( [ int ] $remoteListing . count -le $offsiteKeepLatest ) {
Add-Check -Name " offsite.retention " -Status " passed " -Detail " Remote offsite archive count $( [ int ] $remoteListing . count ) is within the configured keep limit of $offsiteKeepLatest . " -Data @ {
archiveCount = [ int ] $remoteListing . count
keepLatest = $offsiteKeepLatest
}
} else {
Add-Check -Name " offsite.retention " -Status " warning " -Detail " Remote offsite archive count $( [ int ] $remoteListing . count ) exceeds the configured keep limit of $offsiteKeepLatest . " -Data @ {
archiveCount = [ int ] $remoteListing . count
keepLatest = $offsiteKeepLatest
}
}
}
} catch {
Add-Check -Name " offsite.remote " -Status " failed " -Detail " Remote offsite backup check failed: $( $_ . Exception . Message ) " -Data @ {
target = $offsiteTarget
remoteWindowsDir = $offsiteRemoteWindowsDir
}
}
}
2026-04-29 18:50:19 +01:00
Set-WatchdogStep -CurrentStep " extension_checks " -Detail " Running local watchdog extension checks. "
if ( Test-Path -LiteralPath $watchdogExtensionScript ) {
2026-04-29 18:37:04 +01:00
try {
2026-04-29 18:50:19 +01:00
Invoke-WatchdogExtension -Path $watchdogExtensionScript -Context @ {
RepoRoot = $repoRoot
StorageRoot = $storageRoot
Config = $config
RuntimeStatusPath = $runtimeStatusPath
HealthStatusPath = $statusPath
OffsiteStatusPath = $offsiteStatusPath
2026-05-01 16:07:15 +01:00
NotifyOwner = [ bool ] $NotifyOwner
2026-04-29 18:50:19 +01:00
}
Add-Check -Name " watchdog.extension " -Status " passed " -Detail " Loaded local watchdog extension checks. " -Data @ {
path = $watchdogExtensionScript
2026-04-29 18:37:04 +01:00
}
} catch {
2026-04-29 18:50:19 +01:00
Add-Check -Name " watchdog.extension " -Status " failed " -Detail " Local watchdog extension failed: $( $_ . Exception . Message ) " -Data @ {
path = $watchdogExtensionScript
2026-04-29 18:37:04 +01:00
}
}
2026-04-29 18:50:19 +01:00
} elseif ( $watchdogExtensionExplicit ) {
Add-Check -Name " watchdog.extension " -Status " warning " -Detail " Configured watchdog extension script is missing. " -Data @ {
path = $watchdogExtensionScript
2026-04-29 18:37:04 +01:00
}
}
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " notification_decision " -Detail " Computing report fingerprint and notification rules. "
2026-04-24 18:20:58 +01:00
$preNotificationReport = Finalize-Report
$issueLines = @ ( $preNotificationReport . checks | Where-Object { $_ . status -ne " passed " } | ForEach-Object { " $( $_ . name ) : $( $_ . detail ) " } )
$fingerprintSource = if ( $issueLines . Count -eq 0 ) { " healthy " } else { $issueLines -join " `n " }
$fingerprint = [ Convert ] :: ToBase64String ( [ System.Text.Encoding ] :: UTF8 . GetBytes ( $fingerprintSource ) )
$notification = @ {
attempted = $false
sent = $false
reason = $null
}
if ( $NotifyOwner -and $config . ContainsKey ( " DISCORD_BOT_TOKEN " ) -and -not [ string ] :: IsNullOrWhiteSpace ( $config [ " DISCORD_BOT_TOKEN " ] ) ) {
2026-04-24 20:11:18 +01:00
$lastFingerprint = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastFingerprint " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastFingerprint " ) } else { " " }
$lastNotifiedAt = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotifiedAt " ) ) { [ DateTimeOffset ] :: Parse ( [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotifiedAt " ) ) } else { $null }
$lastStatus = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastStatus " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastStatus " ) } else { " " }
2026-04-24 18:20:58 +01:00
$shouldNotify = $false
$notificationReason = $null
if ( $preNotificationReport . status -eq " healthy " ) {
if ( $lastStatus -and $lastStatus -ne " healthy " ) {
$shouldNotify = $true
$notificationReason = " recovered "
}
} else {
$elapsedHours = if ( $lastNotifiedAt ) { ( [ DateTimeOffset ] :: Now - $lastNotifiedAt ) . TotalHours } else { $null }
if ( $fingerprint -ne $lastFingerprint ) {
$shouldNotify = $true
$notificationReason = " issue_changed "
} elseif ( $null -eq $elapsedHours -or $elapsedHours -ge $notifyRepeatHours ) {
$shouldNotify = $true
$notificationReason = " repeat "
}
}
if ( $shouldNotify ) {
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " notification_send " -Detail " Sending owner notification. "
$notification . attempted = $true
2026-04-24 18:20:58 +01:00
try {
$message = Build-OwnerNotificationMessage -Status $preNotificationReport . status -IssueLines $issueLines -HealthStatusPath $statusPath
Send-OwnerDm -BotToken $config [ " DISCORD_BOT_TOKEN " ] -OwnerDiscordId $config [ " DISCORD_OWNER_ID " ] -Content $message
$notification . sent = $true
$notification . reason = $notificationReason
} catch {
$notification . sent = $false
$notification . reason = " failed "
$notification . error = $_ . Exception . Message
Add-Check -Name " watchdog.notification " -Status " warning " -Detail " Failed to DM the owner about watchdog status: $( $_ . Exception . Message ) "
}
}
}
2026-04-24 20:11:18 +01:00
Set-WatchdogStep -CurrentStep " finalizing " -Detail " Writing final health report. "
2026-04-24 18:20:58 +01:00
$report = Finalize-Report
$report . notification = $notification
Write-StatusFile -Path $statusPath -Status $report
$watchdogStatePayload = @ {
2026-04-24 20:11:18 +01:00
taskName = $watchdogTaskName
logPath = $logPath
reportPath = $statusPath
startedAt = $watchdogStartedAt
completedAt = ( Get-Date ) . ToString ( " o " )
2026-04-24 18:20:58 +01:00
checkedAt = $report . checkedAt
2026-04-24 20:11:18 +01:00
runStatus = " completed "
reportStatus = $report . status
currentStep = " finished "
currentDetail = " Watchdog run completed. "
stepUpdatedAt = ( Get-Date ) . ToString ( " o " )
executionTimeLimitMinutes = 15
2026-04-24 18:20:58 +01:00
lastStatus = $report . status
lastFingerprint = $fingerprint
2026-04-24 20:11:18 +01:00
lastNotifiedAt = if ( $notification . sent ) { ( Get-Date ) . ToString ( " o " ) } elseif ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotifiedAt " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotifiedAt " ) } else { $null }
2026-04-24 18:20:58 +01:00
lastNotificationReason = $notification . reason
2026-04-24 20:11:18 +01:00
lastCompletedAt = ( Get-Date ) . ToString ( " o " )
lastCompletedRunStatus = " completed "
lastRunError = $null
2026-04-24 18:20:58 +01:00
}
Write-StatusFile -Path $watchdogStatePath -Status $watchdogStatePayload
Write-Host " VoidBot operations status: $( $report . status ) "
foreach ( $check in $report . checks ) {
Write-Host ( " [{0}] {1}: {2} " -f $check . status . ToUpperInvariant ( ) , $check . name , $check . detail )
}
if ( $FailOnIssues -and $report . status -eq " failed " ) {
exit 1
}
exit 0
} catch {
Add-Check -Name " watchdog.internal " -Status " failed " -Detail " Operations watchdog crashed: $( $_ . Exception . Message ) "
$report = Finalize-Report
$report . notification = @ {
attempted = $false
sent = $false
reason = " internal_error "
error = $_ . Exception . Message
}
Write-StatusFile -Path $statusPath -Status $report
2026-04-24 20:11:18 +01:00
$watchdogFailureState = @ {
taskName = $watchdogTaskName
logPath = $logPath
reportPath = $statusPath
startedAt = $watchdogStartedAt
completedAt = ( Get-Date ) . ToString ( " o " )
checkedAt = $report . checkedAt
runStatus = " failed "
reportStatus = $report . status
currentStep = " crashed "
currentDetail = " Watchdog crashed before finishing. "
stepUpdatedAt = ( Get-Date ) . ToString ( " o " )
executionTimeLimitMinutes = 15
lastStatus = $report . status
lastFingerprint = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastFingerprint " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastFingerprint " ) } else { $null }
lastNotifiedAt = if ( $previousWatchdogState -and ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotifiedAt " ) ) { [ string ] ( Get-OptionalPropertyValue -InputObject $previousWatchdogState -PropertyName " lastNotifiedAt " ) } else { $null }
lastNotificationReason = " internal_error "
lastCompletedAt = ( Get-Date ) . ToString ( " o " )
lastCompletedRunStatus = " failed "
lastRunError = $_ . Exception . Message
}
Write-StatusFile -Path $watchdogStatePath -Status $watchdogFailureState
2026-04-24 18:20:58 +01:00
Write-Host " VoidBot operations status: failed "
Write-Host " [FAILED] watchdog.internal: Operations watchdog crashed: $( $_ . Exception . Message ) "
exit 1
} finally {
Stop-LogCapture -Started $logCaptureStarted
}