param( [string]$SpecPath, [string]$OutputRoot, [string]$ProjectPath ) $ErrorActionPreference = "Stop" $scriptRoot = Split-Path -Parent $MyInvocation.MyCommand.Path $repoRoot = (Resolve-Path (Join-Path $scriptRoot "..")).Path if ([string]::IsNullOrWhiteSpace($SpecPath)) { $SpecPath = Join-Path $repoRoot "benchmarks\aetheria-economy\source-tree-map.benchmark.json" } if ([string]::IsNullOrWhiteSpace($OutputRoot)) { $OutputRoot = Join-Path $repoRoot "out\benchmarks" } if ([string]::IsNullOrWhiteSpace($ProjectPath)) { $ProjectPath = Join-Path $repoRoot "Norn.csproj" } function Get-SvgRasterizerPath { $candidates = @( "C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe", "C:\Program Files\Microsoft\Edge\Application\msedge.exe", "C:\Program Files\Google\Chrome\Application\chrome.exe" ) foreach ($candidate in $candidates) { if (Test-Path $candidate) { return $candidate } } return $null } function Get-SvgCanvasSize { param( [string]$SvgPath ) $header = [string]::Join("`n", (Get-Content -LiteralPath $SvgPath -TotalCount 4)) $width = 2048 $height = 2048 $widthMatch = [regex]::Match($header, 'width="([0-9]+(?:\.[0-9]+)?)"') if ($widthMatch.Success) { $width = [math]::Ceiling([double]$widthMatch.Groups[1].Value) } $heightMatch = [regex]::Match($header, 'height="([0-9]+(?:\.[0-9]+)?)"') if ($heightMatch.Success) { $height = [math]::Ceiling([double]$heightMatch.Groups[1].Value) } $padding = 32 return [pscustomobject]@{ Width = [math]::Min([math]::Max($width + $padding, 512), 8192) Height = [math]::Min([math]::Max($height + $padding, 512), 8192) } } function Convert-SvgToPng { param( [string]$BrowserPath, [string]$SvgPath, [string]$PngPath ) $canvas = Get-SvgCanvasSize -SvgPath $SvgPath $svgUri = [System.Uri]::new((Resolve-Path $SvgPath).Path).AbsoluteUri Remove-Item -LiteralPath $PngPath -Force -ErrorAction SilentlyContinue $browserArgs = @( "--headless", "--disable-gpu", "--hide-scrollbars", "--force-device-scale-factor=1", "--window-size=$($canvas.Width),$($canvas.Height)", "--screenshot=$PngPath", $svgUri ) $stdoutPath = [System.IO.Path]::GetTempFileName() $stderrPath = [System.IO.Path]::GetTempFileName() try { $process = Start-Process ` -FilePath $BrowserPath ` -ArgumentList $browserArgs ` -Wait ` -NoNewWindow ` -PassThru ` -RedirectStandardOutput $stdoutPath ` -RedirectStandardError $stderrPath for ($attempt = 0; $attempt -lt 20; $attempt++) { if (Test-Path $PngPath) { break } Start-Sleep -Milliseconds 250 } } finally { Remove-Item -LiteralPath $stdoutPath -Force -ErrorAction SilentlyContinue Remove-Item -LiteralPath $stderrPath -Force -ErrorAction SilentlyContinue } if (-not (Test-Path $PngPath)) { throw "Failed to rasterize '$SvgPath' to '$PngPath'." } } function Resolve-BenchmarkPath { param( [string]$BaseDirectory, [string]$PathValue ) if ([System.IO.Path]::IsPathRooted($PathValue)) { return (Resolve-Path $PathValue).Path } $fromSpec = Join-Path $BaseDirectory $PathValue if (Test-Path $fromSpec) { return (Resolve-Path $fromSpec).Path } return (Resolve-Path (Join-Path $repoRoot $PathValue)).Path } $resolvedSpecPath = (Resolve-Path $SpecPath).Path $specDirectory = Split-Path -Parent $resolvedSpecPath $spec = Get-Content -LiteralPath $resolvedSpecPath -Raw | ConvertFrom-Json $questionsPath = Resolve-BenchmarkPath -BaseDirectory $specDirectory -PathValue $spec.questionsPath $goldPath = Resolve-BenchmarkPath -BaseDirectory $specDirectory -PathValue $spec.goldPath $questions = Get-Content -LiteralPath $questionsPath -Raw | ConvertFrom-Json $svgRasterizerPath = Get-SvgRasterizerPath $runId = (Get-Date).ToUniversalTime().ToString("yyyyMMddTHHmmssZ") $runRoot = Join-Path $OutputRoot $spec.name $runDir = Join-Path $runRoot $runId $candidateRoot = Join-Path $runDir "candidates" New-Item -ItemType Directory -Path $candidateRoot -Force | Out-Null Copy-Item -LiteralPath $resolvedSpecPath -Destination (Join-Path $runDir "benchmark.json") Copy-Item -LiteralPath $questionsPath -Destination (Join-Path $runDir "questions.json") Copy-Item -LiteralPath $goldPath -Destination (Join-Path $runDir "gold.json") $candidateResults = New-Object System.Collections.Generic.List[object] foreach ($candidate in $spec.candidates) { $candidateId = $candidate.id $candidateDir = Join-Path $candidateRoot $candidateId $renderDir = Join-Path $candidateDir "render" $logPath = Join-Path $candidateDir "generate.log" New-Item -ItemType Directory -Path $renderDir -Force | Out-Null $dotnetArgs = @( "run", "--project", $ProjectPath, "--", "--vault-root", $spec.vaultRoot, "--entry-note", $spec.entryNote, "--output-dir", $renderDir, "--layout", $candidate.layout ) if ($candidate.PSObject.Properties.Name -contains "renderer" -and -not [string]::IsNullOrWhiteSpace($candidate.renderer)) { $dotnetArgs += @( "--renderer", $candidate.renderer ) } if ($spec.includeUnreachable -eq $true) { $dotnetArgs += "--include-unreachable" } Write-Host "Rendering candidate '$candidateId'..." & dotnet @dotnetArgs 2>&1 | Tee-Object -FilePath $logPath if ($LASTEXITCODE -ne 0) { throw "Candidate '$candidateId' failed. See $logPath" } $pngPreviewPaths = New-Object System.Collections.Generic.List[object] if ($svgRasterizerPath) { foreach ($graphKey in @("source-tree", "control-flow")) { $svgPath = Join-Path $renderDir "$graphKey.svg" if (-not (Test-Path $svgPath)) { continue } $pngPath = Join-Path $renderDir "$graphKey.png" Convert-SvgToPng -BrowserPath $svgRasterizerPath -SvgPath $svgPath -PngPath $pngPath $pngPreviewPaths.Add([ordered]@{ graph = $graphKey pngPath = $pngPath svgPath = $svgPath }) } } Copy-Item -LiteralPath $questionsPath -Destination (Join-Path $candidateDir "questions.json") $answerSheetPath = Join-Path $candidateDir "answer-sheet.md" $scoreTemplatePath = Join-Path $candidateDir "scores.template.json" $renderRelative = "render" $answerLines = New-Object System.Collections.Generic.List[string] $answerLines.Add("# Benchmark Answer Sheet") $answerLines.Add("") $answerLines.Add("Candidate: ``$candidateId``") $answerLines.Add("") $answerLines.Add("Render directory: ``$renderRelative``") $answerLines.Add("") $answerLines.Add("Important: answer these probes before opening the run-level ``gold.json``. The whole point is not to grade a contaminated oracle. Tiny basement science, yes, but still science.") $answerLines.Add("") $answerLines.Add("Open as needed:") $answerLines.Add("") if ($svgRasterizerPath) { $answerLines.Add("- ``render\source-tree.png`` for source-tree image probes") $answerLines.Add("- ``render\control-flow.png`` for control-flow image probes") $answerLines.Add("- ``render\source-tree.svg`` and ``render\control-flow.svg`` only as vector reference for the candidate under test, not as the primary image input") } else { $answerLines.Add("- ``render\source-tree.svg``") $answerLines.Add("- ``render\control-flow.svg``") $answerLines.Add("- Warning: no local browser rasterizer was found, so PNG previews were not generated.") } $answerLines.Add("- Ignore ``*.msagl.svg`` and ``*.rich.svg`` during blind review unless you are doing post-hoc renderer autopsy work") $answerLines.Add("- ``render\source-tree.json``") $answerLines.Add("- ``render\control-flow.json``") $answerLines.Add("- ``render\cross-links.json``") $answerLines.Add("") $answerLines.Add("## Probe Answers") foreach ($probe in $questions.probes) { $allowedInputs = [string]::Join(", ", @($probe.allowedInputs)) $answerLines.Add("") $answerLines.Add("### $($probe.id)") $answerLines.Add("") $answerLines.Add("- Graph: ``$($probe.graph)``") $answerLines.Add("- Allowed inputs: ``$allowedInputs``") $answerLines.Add("- Max score: ``$($probe.maxScore)``") $answerLines.Add("") $answerLines.Add("Question: $($probe.question)") $answerLines.Add("") $answerLines.Add("Expected answer format: $($probe.answerFormat)") $answerLines.Add("") $answerLines.Add("Answer:") $answerLines.Add("") $answerLines.Add("> ") } $answerLines | Set-Content -LiteralPath $answerSheetPath $probeScores = @( foreach ($probe in $questions.probes) { [ordered]@{ probeId = $probe.id maxScore = $probe.maxScore score = $null notes = "" } } ) $scoreTemplate = [ordered]@{ candidateId = $candidateId scoredBy = "" scoredAtUtc = "" readability = [ordered]@{ labelLegibility = $null edgeTraceability = $null clusterReadability = $null flowReadability = $null clutterControl = $null notes = "" } probeScores = $probeScores } $scoreTemplate | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $scoreTemplatePath $candidateResults.Add([ordered]@{ id = $candidateId description = $candidate.description layout = $candidate.layout renderer = if ($candidate.PSObject.Properties.Name -contains "renderer" -and -not [string]::IsNullOrWhiteSpace($candidate.renderer)) { $candidate.renderer } else { "msagl" } directory = $candidateDir renderDirectory = $renderDir rasterizer = $svgRasterizerPath imagePreviews = @($pngPreviewPaths.ToArray()) answerSheet = $answerSheetPath scoreTemplate = $scoreTemplatePath log = $logPath }) } $candidateArray = @($candidateResults.ToArray()) $runManifest = [ordered]@{ benchmark = $spec.name runId = $runId createdAtUtc = (Get-Date).ToUniversalTime().ToString("O") specPath = $resolvedSpecPath questionsPath = (Join-Path $runDir "questions.json") goldPath = (Join-Path $runDir "gold.json") outputDirectory = $runDir candidates = $candidateArray } $manifestPath = Join-Path $runDir "run-manifest.json" $runManifest | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $manifestPath Write-Host "" Write-Host "Benchmark run created:" Write-Host $runDir Write-Host "" Write-Host "Review each candidate answer-sheet.md before opening gold.json. Don't cheat unless you enjoy measuring pasteurized vibes."