From 3352c7c901fb74a0dfa2c0760ea9446370c235fc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 29 Apr 2026 18:37:04 +0100 Subject: [PATCH] Monitor Repixelizer fire signals --- .env.example | 10 ++ scripts/check-voidbot-operations.ps1 | 142 +++++++++++++++++++++++++++ state/evidence.jsonl | 1 + 3 files changed, 153 insertions(+) diff --git a/.env.example b/.env.example index 05908c2..e387a44 100644 --- a/.env.example +++ b/.env.example @@ -70,3 +70,13 @@ VOIDBOT_OFFSITE_BACKUP_MAX_AGE_HOURS=36 VOIDBOT_HEALTHCHECK_NOTIFY_REPEAT_HOURS=12 VOIDBOT_HEALTHCHECK_TASK_NAME=VoidBot Operations Watchdog VOIDBOT_HEALTHCHECK_INTERVAL_MINUTES=60 +REPIXELIZER_MONITOR_ENABLED=true +REPIXELIZER_MONITOR_PUBLIC_BASE_URL=https://repixelizer.gamecult.org +REPIXELIZER_MONITOR_SSH_TARGET= +REPIXELIZER_MONITOR_SSH_IDENTITY_FILE= +REPIXELIZER_MONITOR_RECENT_MINUTES=15 +REPIXELIZER_MONITOR_QUEUE_FULL_FAIL_COUNT=1 +REPIXELIZER_MONITOR_JOB_FAILED_FAIL_COUNT=1 +REPIXELIZER_MONITOR_STALE_CANCEL_WARN_COUNT=3 +REPIXELIZER_MONITOR_SLOW_JOB_SECONDS=45 +REPIXELIZER_MONITOR_SLOW_JOB_WARN_COUNT=3 diff --git a/scripts/check-voidbot-operations.ps1 b/scripts/check-voidbot-operations.ps1 index 72f4969..f461f17 100644 --- a/scripts/check-voidbot-operations.ps1 +++ b/scripts/check-voidbot-operations.ps1 @@ -378,6 +378,33 @@ function Invoke-RemotePowerShellJson { return $response | ConvertFrom-Json } +function Invoke-RemoteLinuxCommand { + param( + [Parameter(Mandatory = $true)] + [string] $Target, + [Parameter(Mandatory = $false)] + [AllowEmptyString()] + [string] $IdentityFile = "", + [Parameter(Mandatory = $true)] + [string] $Command + ) + + $args = @("-o", "BatchMode=yes", "-o", "ConnectTimeout=10") + + if (-not [string]::IsNullOrWhiteSpace($IdentityFile)) { + $args += @("-i", $IdentityFile) + } + + $args += @($Target, $Command) + $response = & ssh.exe @args + + if ($LASTEXITCODE -ne 0) { + throw "SSH command failed for $Target." + } + + return [string]($response -join "`n") +} + function Open-OwnerDmChannel { param( [Parameter(Mandatory = $true)] @@ -488,6 +515,16 @@ $offsiteRemoteWindowsDir = if ($config.ContainsKey("OFFSITE_BACKUP_REMOTE_WINDOW $offsiteTarget = if ($config.ContainsKey("OFFSITE_BACKUP_SSH_TARGET")) { $config["OFFSITE_BACKUP_SSH_TARGET"] } else { "" } $offsiteKeepLatest = if ($config.ContainsKey("OFFSITE_BACKUP_REMOTE_KEEP_LATEST")) { [int]$config["OFFSITE_BACKUP_REMOTE_KEEP_LATEST"] } else { 14 } $offsiteLabel = if ($config.ContainsKey("OFFSITE_BACKUP_LABEL") -and -not [string]::IsNullOrWhiteSpace($config["OFFSITE_BACKUP_LABEL"])) { $config["OFFSITE_BACKUP_LABEL"] } else { "offsite-auto" } +$repixelizerMonitorEnabled = if ($config.ContainsKey("REPIXELIZER_MONITOR_ENABLED") -and -not [string]::IsNullOrWhiteSpace($config["REPIXELIZER_MONITOR_ENABLED"])) { [System.Convert]::ToBoolean($config["REPIXELIZER_MONITOR_ENABLED"]) } else { $true } +$repixelizerPublicBaseUrl = if ($config.ContainsKey("REPIXELIZER_MONITOR_PUBLIC_BASE_URL") -and -not [string]::IsNullOrWhiteSpace($config["REPIXELIZER_MONITOR_PUBLIC_BASE_URL"])) { $config["REPIXELIZER_MONITOR_PUBLIC_BASE_URL"].TrimEnd("/") } else { "https://repixelizer.gamecult.org" } +$repixelizerSshTarget = if ($config.ContainsKey("REPIXELIZER_MONITOR_SSH_TARGET")) { $config["REPIXELIZER_MONITOR_SSH_TARGET"] } else { "" } +$repixelizerSshIdentityFile = if ($config.ContainsKey("REPIXELIZER_MONITOR_SSH_IDENTITY_FILE")) { $config["REPIXELIZER_MONITOR_SSH_IDENTITY_FILE"] } else { "" } +$repixelizerRecentMinutes = if ($config.ContainsKey("REPIXELIZER_MONITOR_RECENT_MINUTES")) { [int]$config["REPIXELIZER_MONITOR_RECENT_MINUTES"] } else { 15 } +$repixelizerQueueFullFailCount = if ($config.ContainsKey("REPIXELIZER_MONITOR_QUEUE_FULL_FAIL_COUNT")) { [int]$config["REPIXELIZER_MONITOR_QUEUE_FULL_FAIL_COUNT"] } else { 1 } +$repixelizerJobFailedFailCount = if ($config.ContainsKey("REPIXELIZER_MONITOR_JOB_FAILED_FAIL_COUNT")) { [int]$config["REPIXELIZER_MONITOR_JOB_FAILED_FAIL_COUNT"] } else { 1 } +$repixelizerStaleCancelWarnCount = if ($config.ContainsKey("REPIXELIZER_MONITOR_STALE_CANCEL_WARN_COUNT")) { [int]$config["REPIXELIZER_MONITOR_STALE_CANCEL_WARN_COUNT"] } else { 3 } +$repixelizerSlowJobSeconds = if ($config.ContainsKey("REPIXELIZER_MONITOR_SLOW_JOB_SECONDS")) { [double]$config["REPIXELIZER_MONITOR_SLOW_JOB_SECONDS"] } else { 45.0 } +$repixelizerSlowJobWarnCount = if ($config.ContainsKey("REPIXELIZER_MONITOR_SLOW_JOB_WARN_COUNT")) { [int]$config["REPIXELIZER_MONITOR_SLOW_JOB_WARN_COUNT"] } else { 3 } $logCaptureStarted = Start-LogCapture -LogPath $logPath $previousWatchdogState = if (Test-Path -LiteralPath $watchdogStatePath) { try { @@ -921,6 +958,111 @@ $files = Get-ChildItem -LiteralPath $dir -Filter $pattern -File | Sort-Object La } } + Set-WatchdogStep -CurrentStep "repixelizer_checks" -Detail "Checking Repixelizer public health and recent service pressure." + if (-not $repixelizerMonitorEnabled) { + Add-Check -Name "repixelizer.monitor" -Status "passed" -Detail "Repixelizer monitor is disabled by REPIXELIZER_MONITOR_ENABLED." + } else { + try { + $health = Get-Json -Url "$repixelizerPublicBaseUrl/api/health" + if ([string]$health.status -eq "ok") { + Add-Check -Name "repixelizer.health" -Status "passed" -Detail "Repixelizer public health endpoint is ok." -Data @{ + url = "$repixelizerPublicBaseUrl/api/health" + } + } else { + Add-Check -Name "repixelizer.health" -Status "failed" -Detail "Repixelizer public health returned status '$([string]$health.status)'." -Data @{ + url = "$repixelizerPublicBaseUrl/api/health" + } + } + } catch { + Add-Check -Name "repixelizer.health" -Status "failed" -Detail "Repixelizer public health check failed: $($_.Exception.Message)" -Data @{ + url = "$repixelizerPublicBaseUrl/api/health" + } + } + + if ([string]::IsNullOrWhiteSpace($repixelizerSshTarget)) { + Add-Check -Name "repixelizer.journal" -Status "warning" -Detail "Repixelizer journal pressure check skipped because REPIXELIZER_MONITOR_SSH_TARGET is blank." + } else { + try { + $recentWindow = [Math]::Max(1, $repixelizerRecentMinutes) + $since = "$recentWindow minutes ago" + $escapedSince = $since.Replace("'", "'\''") + $journalCommand = "journalctl -u repixelizer-gui --since '$escapedSince' --no-pager -o cat" + $journalText = Invoke-RemoteLinuxCommand -Target $repixelizerSshTarget -IdentityFile $repixelizerSshIdentityFile -Command $journalCommand + $journalLines = @($journalText -split "`r?`n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + $queueFullLines = @($journalLines | Where-Object { $_ -match "repixelizer_queue_full" }) + $failedLines = @($journalLines | Where-Object { $_ -match "repixelizer_job_failed" }) + $staleCancelLines = @($journalLines | Where-Object { $_ -match "repixelizer_job_canceled" -and $_ -match "stopped heartbeating" }) + $slowRunSeconds = @( + $journalLines | ForEach-Object { + if ($_ -match "repixelizer_job_completed" -and $_ -match "run_seconds=([0-9]+(?:\.[0-9]+)?)") { + [double]$Matches[1] + } + } | Where-Object { $_ -ge $repixelizerSlowJobSeconds } + ) + + if ($queueFullLines.Count -ge $repixelizerQueueFullFailCount) { + Add-Check -Name "repixelizer.queue_full" -Status "failed" -Detail "Repixelizer queue filled $($queueFullLines.Count) time(s) in the last $recentWindow minute(s)." -Data @{ + threshold = $repixelizerQueueFullFailCount + recentMinutes = $recentWindow + } + } else { + Add-Check -Name "repixelizer.queue_full" -Status "passed" -Detail "No Repixelizer queue saturation above threshold in the last $recentWindow minute(s)." -Data @{ + count = $queueFullLines.Count + threshold = $repixelizerQueueFullFailCount + recentMinutes = $recentWindow + } + } + + if ($failedLines.Count -ge $repixelizerJobFailedFailCount) { + Add-Check -Name "repixelizer.job_failed" -Status "failed" -Detail "Repixelizer logged $($failedLines.Count) failed job(s) in the last $recentWindow minute(s)." -Data @{ + threshold = $repixelizerJobFailedFailCount + recentMinutes = $recentWindow + } + } else { + Add-Check -Name "repixelizer.job_failed" -Status "passed" -Detail "No Repixelizer job failures above threshold in the last $recentWindow minute(s)." -Data @{ + count = $failedLines.Count + threshold = $repixelizerJobFailedFailCount + recentMinutes = $recentWindow + } + } + + if ($staleCancelLines.Count -ge $repixelizerStaleCancelWarnCount) { + Add-Check -Name "repixelizer.stale_cancellations" -Status "warning" -Detail "Repixelizer canceled $($staleCancelLines.Count) stale browser job(s) in the last $recentWindow minute(s)." -Data @{ + threshold = $repixelizerStaleCancelWarnCount + recentMinutes = $recentWindow + } + } else { + Add-Check -Name "repixelizer.stale_cancellations" -Status "passed" -Detail "Repixelizer stale-browser cancellations are below threshold." -Data @{ + count = $staleCancelLines.Count + threshold = $repixelizerStaleCancelWarnCount + recentMinutes = $recentWindow + } + } + + if ($slowRunSeconds.Count -ge $repixelizerSlowJobWarnCount) { + Add-Check -Name "repixelizer.slow_jobs" -Status "warning" -Detail "Repixelizer had $($slowRunSeconds.Count) job(s) at or above $repixelizerSlowJobSeconds run seconds in the last $recentWindow minute(s)." -Data @{ + thresholdCount = $repixelizerSlowJobWarnCount + thresholdSeconds = $repixelizerSlowJobSeconds + maxRunSeconds = [Math]::Round(($slowRunSeconds | Measure-Object -Maximum).Maximum, 3) + recentMinutes = $recentWindow + } + } else { + Add-Check -Name "repixelizer.slow_jobs" -Status "passed" -Detail "Repixelizer slow-job count is below threshold." -Data @{ + count = $slowRunSeconds.Count + thresholdCount = $repixelizerSlowJobWarnCount + thresholdSeconds = $repixelizerSlowJobSeconds + recentMinutes = $recentWindow + } + } + } catch { + Add-Check -Name "repixelizer.journal" -Status "failed" -Detail "Repixelizer journal pressure check failed: $($_.Exception.Message)" -Data @{ + target = $repixelizerSshTarget + recentMinutes = $repixelizerRecentMinutes + } + } + } + } + Set-WatchdogStep -CurrentStep "notification_decision" -Detail "Computing report fingerprint and notification rules." $preNotificationReport = Finalize-Report $issueLines = @($preNotificationReport.checks | Where-Object { $_.status -ne "passed" } | ForEach-Object { "$($_.name): $($_.detail)" }) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index c906594..56261a6 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -5,3 +5,4 @@ {"ts":"2026-04-26T15:20:00+01:00","type":"architecture","status":"ok","note":"The Discord bot lane was split into a gateway shell plus dedicated command-handler and support-helper modules; behavior stayed stable, but line-count monarchy had become the likelier future failure surface here too."} {"ts":"2026-04-26T15:55:00+01:00","type":"architecture","status":"ok","note":"Interaction-memory logic was split into event/tone analysis, shared constants, and profile-synthesis modules; behavior stayed stable, but one file trying to be both the ear and the therapist had become the likelier future failure surface."} {"ts":"2026-04-26T16:25:00+01:00","type":"architecture","status":"ok","note":"State storage was split into a thin factory, bootstrap/migration helper, per-domain Postgres stores, and dedicated rate-limit stores; behavior stayed stable, but one persistence omnibus had become the likelier future failure surface."} +{"ts":"2026-04-29T17:36:43Z","type":"implementation","status":"ok","note":"Operations watchdog now checks Repixelizer public health plus recent Yggdrasil journald pressure signals and routes queue-full, job-failed, stale-cancel, and slow-job issues through the existing owner DM notification path."}