From ccb2b978277a8e49d811793495596fa784591ee6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 6 May 2026 08:48:45 +0100 Subject: [PATCH] Harden backup handling after incomplete runs --- scripts/backup-voidbot-state.ps1 | 19 +++++++++-- scripts/check-voidbot-operations.ps1 | 51 ++++++++++++++++++++++------ state/evidence.jsonl | 1 + 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/scripts/backup-voidbot-state.ps1 b/scripts/backup-voidbot-state.ps1 index 1f4ff7b..262a029 100644 --- a/scripts/backup-voidbot-state.ps1 +++ b/scripts/backup-voidbot-state.ps1 @@ -82,10 +82,12 @@ function Parse-DatabaseDsn { function Invoke-CurlJson { param( [Parameter(Mandatory = $true)] - [string[]] $Arguments + [string[]] $Arguments, + [int] $ConnectTimeoutSeconds = 5, + [int] $MaxTimeSeconds = 30 ) - $response = & curl.exe --connect-timeout 5 --max-time 30 @Arguments + $response = & curl.exe --connect-timeout $ConnectTimeoutSeconds --max-time $MaxTimeSeconds @Arguments if ($LASTEXITCODE -ne 0) { throw "curl.exe failed: $($Arguments -join ' ')" @@ -473,7 +475,7 @@ function Backup-QdrantCollections { "-X", "POST", "$($QdrantUrl.TrimEnd('/'))/collections/$escapedCollection/snapshots" - ) + ) -MaxTimeSeconds 180 $snapshotName = [string]$createResponse.result.name Wait-Until -Condition { @@ -602,6 +604,7 @@ $manifestPath = Join-Path $backupDir "manifest.json" $stoppedRuntime = @() $shouldRestart = $false $pendingError = $null +$backupCompleted = $false try { New-Item -ItemType Directory -Force -Path $backupDir | Out-Null @@ -646,6 +649,7 @@ try { Write-Host "Copying RAG archives..." $manifest.archives = Backup-RagArchives -StorageRoot $storageRoot -OutputDir $archiveDir $manifest | ConvertTo-Json -Depth 8 | Set-Content -Path $manifestPath -Encoding utf8 + $backupCompleted = $true Write-Host "VoidBot state backup complete." Write-Host "Backup: $backupDir" @@ -671,6 +675,15 @@ try { } if ($null -ne $pendingError) { + if (-not $backupCompleted -and (Test-Path -LiteralPath $backupDir)) { + try { + Remove-Item -LiteralPath $backupDir -Recurse -Force -ErrorAction Stop + Write-Warning "Removed incomplete backup directory $backupDir after failure." + } catch { + Write-Warning ("Failed to remove incomplete backup directory {0}: {1}" -f $backupDir, $_.Exception.Message) + } + } + if ($pendingError.Exception) { throw $pendingError.Exception } diff --git a/scripts/check-voidbot-operations.ps1 b/scripts/check-voidbot-operations.ps1 index 3666e15..4640d25 100644 --- a/scripts/check-voidbot-operations.ps1 +++ b/scripts/check-voidbot-operations.ps1 @@ -280,6 +280,26 @@ function Invoke-SourceRepoReconcile { return $stdout | ConvertFrom-Json } +function Get-LatestCompleteBackupDirectory { + param( + [Parameter(Mandatory = $true)] + [string] $BackupRoot + ) + + $directories = @(Get-ChildItem -LiteralPath $BackupRoot -Directory -ErrorAction SilentlyContinue | + Sort-Object Name -Descending) + + $latestDirectory = $directories | Select-Object -First 1 + $latestCompleteDirectory = $directories | + Where-Object { Test-Path -LiteralPath (Join-Path $_.FullName "manifest.json") } | + Select-Object -First 1 + + return @{ + latest = $latestDirectory + latestComplete = $latestCompleteDirectory + } +} + function Ensure-DockerCliOnPath { $dockerCliDir = "C:\Program Files\Docker\Docker\resources\bin" @@ -955,28 +975,39 @@ try { Add-Check -Name "discord.bot_token" -Status "warning" -Detail "DISCORD_BOT_TOKEN is blank; bot identity could not be checked." } - $latestBackup = Get-ChildItem -LiteralPath $backupRoot -Directory -ErrorAction SilentlyContinue | - Sort-Object Name -Descending | - Select-Object -First 1 + $backupSelection = Get-LatestCompleteBackupDirectory -BackupRoot $backupRoot + $latestBackup = $backupSelection.latest + $latestCompleteBackup = $backupSelection.latestComplete Set-WatchdogStep -CurrentStep "backup_checks" -Detail "Verifying the latest local backup and freshness thresholds." if ($null -eq $latestBackup) { Add-Check -Name "backup.latest" -Status "failed" -Detail "No local backup directory exists under $backupRoot." + } elseif ($null -eq $latestCompleteBackup) { + Add-Check -Name "backup.latest" -Status "failed" -Detail "No complete local backup with a manifest exists under $backupRoot." -Data @{ + latestPath = $latestBackup.FullName + } } else { + if ($latestBackup.FullName -ne $latestCompleteBackup.FullName) { + Add-Check -Name "backup.incomplete_latest" -Status "warning" -Detail "Newest backup directory is incomplete and was ignored until a manifest exists." -Data @{ + incompletePath = $latestBackup.FullName + latestCompletePath = $latestCompleteBackup.FullName + } + } + try { - $verificationJson = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $verifyScript -BackupPath $latestBackup.FullName -AsJson -Quiet + $verificationJson = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $verifyScript -BackupPath $latestCompleteBackup.FullName -AsJson -Quiet $verificationReport = $verificationJson | ConvertFrom-Json $createdAtCheck = $verificationReport.checks | Where-Object { $_.name -eq "backup.created_at" } | Select-Object -First 1 - $backupAgeHours = if ($createdAtCheck -and $createdAtCheck.data -and $createdAtCheck.data.ageHours) { [double]$createdAtCheck.data.ageHours } else { [Math]::Round(((Get-Date) - $latestBackup.LastWriteTime).TotalHours, 2) } + $backupAgeHours = if ($createdAtCheck -and $createdAtCheck.data -and $createdAtCheck.data.ageHours) { [double]$createdAtCheck.data.ageHours } else { [Math]::Round(((Get-Date) - $latestCompleteBackup.LastWriteTime).TotalHours, 2) } if ([string]$verificationReport.status -eq "failed") { Add-Check -Name "backup.latest" -Status "failed" -Detail "Latest local backup failed verification." -Data @{ - path = $latestBackup.FullName + path = $latestCompleteBackup.FullName verification = $verificationReport } } else { Add-Check -Name "backup.latest" -Status "passed" -Detail "Latest local backup passed verification." -Data @{ - path = $latestBackup.FullName + path = $latestCompleteBackup.FullName verificationStatus = [string]$verificationReport.status ageHours = $backupAgeHours } @@ -985,17 +1016,17 @@ try { if ($backupAgeHours -le $backupMaxAgeHours) { Add-Check -Name "backup.freshness" -Status "passed" -Detail "Latest local backup is $backupAgeHours hours old." -Data @{ thresholdHours = $backupMaxAgeHours - path = $latestBackup.FullName + path = $latestCompleteBackup.FullName } } else { Add-Check -Name "backup.freshness" -Status "failed" -Detail "Latest local backup is $backupAgeHours hours old, which exceeds the $backupMaxAgeHours hour threshold." -Data @{ thresholdHours = $backupMaxAgeHours - path = $latestBackup.FullName + path = $latestCompleteBackup.FullName } } } catch { Add-Check -Name "backup.latest" -Status "failed" -Detail "Failed to verify the latest local backup: $($_.Exception.Message)" -Data @{ - path = $latestBackup.FullName + path = $latestCompleteBackup.FullName } } } diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 851d19d..a2fc887 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -56,3 +56,4 @@ {"ts":"2026-05-06T01:12:30Z","type":"implementation","status":"ok","note":"Void's unattended rumination now uses hybrid symbolic-plus-vector memory: a mood-driven helper assigns compact semantic vectors to persistent memories and runtime seeds, builds resonance clusters, keeps an incubation queue alive across turns, and turns the strongest recurring seam into dream residue during nap windows so deeper thoughts can mature before they either surface or cool off."} {"ts":"2026-05-06T02:24:53Z","type":"architecture","status":"ok","note":"Void's shared moderation self-state now rides through a standalone CultCacheTS submodule: canonical schema-validated MessagePack in .msgpack, editable unattended JSON working projection in .json, and consumer code talking to the typed persistence layer instead of pawing raw state files directly."} {"ts":"2026-05-06T04:25:53Z","type":"architecture","status":"ok","note":"Discord outbound now supports a shared webhook persona pipe for guild channels and threads: agents can override personaName and avatar per message through one reusable per-channel webhook while DMs stay on the base bot identity."} +{"ts":"2026-05-06T07:48:31Z","type":"implementation","status":"ok","note":"Backup watchdog now ignores manifest-less backup carcasses, backup cleanup removes incomplete directories on failure, and Qdrant snapshot creation uses a longer curl timeout so larger source collections stop timing out the offsite backup run."}