e625283342
Every Faust render, and every trial-loop lane, runs dotnet with a fresh DOTNET_CLI_HOME under the temp directory. The SDK's first-run experience appends "<home>\.dotnet\tools" to the persisted user PATH each time, and DOTNET_SKIP_FIRST_TIME_EXPERIENCE does not stop that part. Seventy-nine dead entries had accumulated (9,634 chars), past cmd's environment limit, so anything spawned through cmd lost the tail of PATH — node included. DOTNET_ADD_GLOBAL_TOOLS_TO_PATH=0 is the switch that governs the PATH write. Set it everywhere a throwaway dotnet home is used; telemetry opt-out rides along for the renderer since these homes never persist. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
577 lines
34 KiB
PowerShell
577 lines
34 KiB
PowerShell
param(
|
|
[int]$Rounds = 5,
|
|
[string]$BatchId = "five-seed-trials",
|
|
[string]$CodexCommand = "codex",
|
|
[string]$CodexModel = "gpt-5.3-codex",
|
|
[switch]$AllowCodexSourceEdits,
|
|
[switch]$SkipLocalTrialRun
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Resolve-RepoRoot {
|
|
$dir = Get-Item -LiteralPath (Get-Location)
|
|
while ($null -ne $dir) {
|
|
if (Test-Path -LiteralPath (Join-Path $dir.FullName "AquaSynth.sln")) {
|
|
return $dir.FullName
|
|
}
|
|
|
|
$dir = $dir.Parent
|
|
}
|
|
|
|
throw "Could not find AquaSynth.sln above the current directory."
|
|
}
|
|
|
|
function New-Timestamp {
|
|
return [DateTimeOffset]::UtcNow.ToString("yyyyMMddTHHmmssfff")
|
|
}
|
|
|
|
function Invoke-LoggedProcess {
|
|
param(
|
|
[string]$FilePath,
|
|
[string[]]$ArgumentList,
|
|
[string]$LogPath,
|
|
[hashtable]$Environment = @{}
|
|
)
|
|
|
|
$logDir = Split-Path -Parent $LogPath
|
|
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
|
|
$oldValues = @{}
|
|
foreach ($key in $Environment.Keys) {
|
|
$oldValues[$key] = [Environment]::GetEnvironmentVariable($key, "Process")
|
|
[Environment]::SetEnvironmentVariable($key, [string]$Environment[$key], "Process")
|
|
}
|
|
|
|
try {
|
|
$display = "$FilePath $($ArgumentList -join ' ')"
|
|
Add-Content -LiteralPath $LogPath -Value "[$([DateTimeOffset]::UtcNow.ToString("O"))] $display"
|
|
& $FilePath @ArgumentList *>&1 | Tee-Object -FilePath $LogPath -Append
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Command failed with exit code $LASTEXITCODE`: $display"
|
|
}
|
|
}
|
|
finally {
|
|
foreach ($key in $Environment.Keys) {
|
|
[Environment]::SetEnvironmentVariable($key, $oldValues[$key], "Process")
|
|
}
|
|
}
|
|
}
|
|
|
|
function Invoke-CodexAgent {
|
|
param(
|
|
[string]$RepoRoot,
|
|
[string]$Prompt,
|
|
[string]$OutputPath,
|
|
[string]$LogPath,
|
|
[string]$CodexCommand,
|
|
[string]$CodexModel,
|
|
[bool]$AllowSourceEdits
|
|
)
|
|
|
|
$promptPath = [IO.Path]::ChangeExtension($OutputPath, ".prompt.md")
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $OutputPath) | Out-Null
|
|
Set-Content -LiteralPath $promptPath -Value $Prompt
|
|
|
|
$args = @(
|
|
"exec",
|
|
"--cd", $RepoRoot,
|
|
"--full-auto",
|
|
"--output-last-message", $OutputPath
|
|
)
|
|
if ($CodexModel.Length -gt 0) {
|
|
$args += @("--model", $CodexModel)
|
|
}
|
|
if (-not $AllowSourceEdits) {
|
|
$args += @(
|
|
"--config", 'shell_environment_policy.inherit="all"',
|
|
"--config", 'sandbox_permissions=["workspace-write"]'
|
|
)
|
|
}
|
|
$args += "-"
|
|
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
Get-Content -LiteralPath $promptPath -Raw | & $CodexCommand @args 2>&1 | Tee-Object -FilePath $LogPath -Append
|
|
$exitCode = $LASTEXITCODE
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
|
|
if ($exitCode -ne 0) {
|
|
throw "External Codex agent failed with exit code $exitCode. See $LogPath."
|
|
}
|
|
}
|
|
|
|
function Assert-CandidateBatch {
|
|
param(
|
|
[string]$PatchDirectory
|
|
)
|
|
|
|
$targetSets = @(
|
|
@("a", "i", "u", "e", "o"),
|
|
@("m", "n", "ng", "l", "r"),
|
|
@("s", "z", "f", "v", "th"),
|
|
@("p", "b", "t", "d", "k"),
|
|
@("mix-a", "mix-m", "mix-s", "mix-p", "mix-u")
|
|
)
|
|
$files = @(Get-ChildItem -LiteralPath $PatchDirectory -Filter "*.aqua" -File -Recurse)
|
|
if ($files.Count -ne 25) {
|
|
throw "Hypothesis worker wrote $($files.Count) .aqua candidates; expected exactly 25."
|
|
}
|
|
|
|
$stems = $files | ForEach-Object { [IO.Path]::GetFileNameWithoutExtension($_.Name) }
|
|
foreach ($targetSet in $targetSets) {
|
|
foreach ($target in $targetSet) {
|
|
$covered = $false
|
|
foreach ($stem in $stems) {
|
|
if ($stem.StartsWith("$target`__", [StringComparison]::OrdinalIgnoreCase)) {
|
|
$covered = $true
|
|
break
|
|
}
|
|
}
|
|
|
|
if (-not $covered) {
|
|
throw "Hypothesis worker did not write a candidate for target '$target'. Candidate files must be named <targetId>__<family>__<hypothesis-name>.aqua."
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach ($stem in $stems) {
|
|
$parts = $stem.Split([string[]]@("__"), [StringSplitOptions]::None)
|
|
if ($parts.Count -lt 3 -or [string]::IsNullOrWhiteSpace($parts[1]) -or [string]::IsNullOrWhiteSpace($parts[2])) {
|
|
throw "Candidate '$stem' does not follow <targetId>__<family>__<hypothesis-name>.aqua."
|
|
}
|
|
|
|
if ($parts[1] -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$') {
|
|
throw "Candidate '$stem' uses invalid family '$($parts[1])'. Family ids must be lowercase kebab-case."
|
|
}
|
|
}
|
|
}
|
|
|
|
function Invoke-SourceEditVerification {
|
|
param(
|
|
[string]$LogPath,
|
|
[bool]$AllowSourceEdits
|
|
)
|
|
|
|
if (-not $AllowSourceEdits) {
|
|
return
|
|
}
|
|
|
|
Invoke-LoggedProcess `
|
|
-FilePath "dotnet" `
|
|
-ArgumentList @(
|
|
"build",
|
|
"tests\AquaSynth.Dsl.Tests\AquaSynth.Dsl.Tests.csproj",
|
|
"--no-restore",
|
|
"--disable-build-servers",
|
|
"-p:UseSharedCompilation=false",
|
|
"-p:BuildInParallel=false",
|
|
"-v",
|
|
"minimal"
|
|
) `
|
|
-LogPath $LogPath
|
|
}
|
|
|
|
$repoRoot = Resolve-RepoRoot
|
|
$codex = Get-Command $CodexCommand -ErrorAction Stop
|
|
$loopId = New-Timestamp
|
|
$loopRoot = Join-Path $repoRoot "artifacts/parity/ipa-trial-loops/$loopId"
|
|
$logRoot = Join-Path $loopRoot "logs"
|
|
New-Item -ItemType Directory -Force -Path $loopRoot, $logRoot | Out-Null
|
|
$storePath = Join-Path $loopRoot "ipa-trial-results.cc"
|
|
$workerProject = Join-Path $repoRoot "tools/IpaTrialWorker/IpaTrialWorker.csproj"
|
|
|
|
$indexPath = Join-Path $loopRoot "loop-index.md"
|
|
Set-Content -LiteralPath $indexPath -Value @"
|
|
# IPA Trial Loop $loopId
|
|
|
|
- repo: $repoRoot
|
|
- codex: $($codex.Source)
|
|
- rounds: $Rounds
|
|
- batch_id: $BatchId
|
|
- allow_source_edits: $AllowCodexSourceEdits
|
|
- trial_results_store: $storePath
|
|
- worker: $workerProject
|
|
- trial_shape: five rounds, exactly 25 candidates per round, one per phoneme lane
|
|
|
|
"@
|
|
|
|
if (-not $SkipLocalTrialRun) {
|
|
$seedLog = Join-Path $logRoot "seed-trial-run.log"
|
|
Invoke-LoggedProcess `
|
|
-FilePath "dotnet" `
|
|
-ArgumentList @(
|
|
"run",
|
|
"--project",
|
|
$workerProject,
|
|
"--",
|
|
"seed",
|
|
"--artifact-root",
|
|
$loopRoot,
|
|
"--batch-id",
|
|
$BatchId,
|
|
"--store",
|
|
$storePath
|
|
) `
|
|
-LogPath $seedLog
|
|
}
|
|
else {
|
|
Add-Content -LiteralPath $indexPath -Value "- seed: skipped"
|
|
}
|
|
|
|
for ($round = 1; $round -le $Rounds; $round++) {
|
|
$roundId = "round-{0:000}" -f $round
|
|
$roundDir = Join-Path $loopRoot $roundId
|
|
$patchDir = Join-Path $roundDir "proposed-patches"
|
|
New-Item -ItemType Directory -Force -Path $roundDir | Out-Null
|
|
New-Item -ItemType Directory -Force -Path $patchDir | Out-Null
|
|
Add-Content -LiteralPath $indexPath -Value "- ${roundId}: $roundDir"
|
|
|
|
$indexBeforeLog = Join-Path $logRoot "$roundId-index-before.log"
|
|
Invoke-LoggedProcess `
|
|
-FilePath "dotnet" `
|
|
-ArgumentList @(
|
|
"run",
|
|
"--project",
|
|
$workerProject,
|
|
"--",
|
|
"index",
|
|
"--store",
|
|
$storePath,
|
|
"--output",
|
|
(Join-Path $roundDir "vector-index-before.md")
|
|
) `
|
|
-LogPath $indexBeforeLog
|
|
|
|
$preEvidence = Join-Path $roundDir "semantic-search-before.md"
|
|
$preDumpLog = Join-Path $logRoot "$roundId-search-before.log"
|
|
Invoke-LoggedProcess `
|
|
-FilePath "dotnet" `
|
|
-ArgumentList @(
|
|
"run",
|
|
"--project",
|
|
$workerProject,
|
|
"--",
|
|
"search",
|
|
"--store",
|
|
$storePath,
|
|
"--query",
|
|
"weak promising generalization vowel nasal fricative stop articulation owner source tract radiation gesture primitive dressing DSL lowering Faust machinery scene voices background noise",
|
|
"--limit",
|
|
"40",
|
|
"--output",
|
|
$preEvidence,
|
|
"--skip-index",
|
|
"true"
|
|
) `
|
|
-LogPath $preDumpLog
|
|
|
|
$hypothesisOutput = Join-Path $roundDir "hypothesis-agent.md"
|
|
$hypothesisLog = Join-Path $logRoot "$roundId-hypothesis-agent.log"
|
|
$previousRoundId = "round-{0:000}" -f ($round - 1)
|
|
$previousEvaluation = if ($round -gt 1) { Join-Path (Join-Path $loopRoot $previousRoundId) "evaluation.md" } else { "none" }
|
|
$hypothesisPrompt = @"
|
|
You are an AquaSynth IPA loss-landscape hypothesis and patch-writing worker.
|
|
|
|
Read this semantic search result from the CultCache trial-results database:
|
|
$preEvidence
|
|
|
|
The backing store is:
|
|
$storePath
|
|
|
|
Previous round evaluation:
|
|
$previousEvaluation
|
|
|
|
Use the worker as your search organ. Do not ask for crude full dumps unless retrieval fails:
|
|
`$env:DOTNET_CLI_HOME="$roundDir/dotnet-home"; `$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE="1"; `$env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH="0"; `$env:DOTNET_CLI_TELEMETRY_OPTOUT="1"; dotnet run --no-build --no-restore --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- search --store "$storePath" --query "<your evidence question>" --limit 20 --output "$roundDir/search-<topic>.md" --skip-index true
|
|
`$env:DOTNET_CLI_HOME="$roundDir/dotnet-home"; `$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE="1"; `$env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH="0"; `$env:DOTNET_CLI_TELEMETRY_OPTOUT="1"; dotnet run --no-build --no-restore --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- show --store "$storePath" --trial-id "<trial id from search>" --output "$roundDir/detail-<candidate>.md"
|
|
|
|
Retrieval failure policy:
|
|
- A failed search/show command is itself evidence about the worker environment, but it is low-quality evidence for speech hypotheses.
|
|
- Fallback artifact reads may appear in `Weakest Missing Evidence`, but they do not satisfy the search/show receipt counts.
|
|
- Do not write patches until live `search` and `show` receipts exist.
|
|
- Search-summary-only claims are low-quality until corroborated by a `show` receipt with concrete metric values or artifact paths.
|
|
|
|
Useful repo surfaces:
|
|
- docs/ipa-vocal-tract-roadmap.md
|
|
- docs/audio-doctrine.md
|
|
- src/AquaSynth.Core/IpaGestureExperiment.cs
|
|
- src/AquaSynth.Core/Model.cs
|
|
- src/AquaSynth.Core/PatchScript.cs
|
|
- src/AquaSynth.Core/ControlSplineTimeline.cs
|
|
- src/AquaSynth.Core/ProbeTimelineReport.cs
|
|
- src/AquaSynth.Core/FaustExport.cs
|
|
- src/AquaSynth.Faust/IpaTrialOrchestrator.cs
|
|
- tools/IpaTrialWorker/Program.cs
|
|
- artifacts under $loopRoot
|
|
|
|
Before writing candidates:
|
|
0. Open $preEvidence and include a `PreEvidence Digest` section in $roundDir/hypotheses.md with five metric-bearing facts from it.
|
|
1. Run exactly three targeted semantic searches and save the outputs under ${roundDir}:
|
|
- one for weakest stop/plosive closure evidence;
|
|
- one for best vowel/nasal/fricative transferable successes;
|
|
- one for DSL lowering/Faust machinery, scene voices, background-noise modeling, and dressing/FM/AM/noise versus articulation-owner failures.
|
|
2. Open exactly three detailed trial records with `show`: one weak stop/plosive, one transferable success, and one dressing-vs-articulation case when available.
|
|
3. In $roundDir/hypotheses.md, include a `Retrieval Receipts` section listing the exact search/show files you used.
|
|
4. If fewer than three search receipts and three show receipts exist on disk, stop and create the missing retrieval artifacts before writing `.aqua` files.
|
|
5. Do not run additional retrieval unless one of the six required receipts is empty or unrelated; spend the rest of the turn reasoning over the evidence.
|
|
6. For every seam you name, cite at least one contrast pair: a weak record and a stronger related record, or two variants that isolate one axis. If the receipts do not contain a contrast pair, mark the seam `not actionable`.
|
|
7. Extract primitive timeline facts when available: closure duration, burst/release window, voicing onset, constriction window, branch admittance, radiation flow, or passivity/energy. If timelines are absent, mark timeline evidence `missing` instead of implying it.
|
|
8. Read the previous round evaluation when it exists. A failed, flat, or regressed family is falsified until you state exactly what changed. Do not reuse family names or rotate old family ideas across new targets unless the report includes a `Previous Round Learning` section with the prior metric movement, accepted lesson, rejected lesson, and mutation that makes the new test different.
|
|
|
|
Task:
|
|
1. Read the current loss landscape across gesture, log-mel, RMS, articulation, and primitive timeline evidence.
|
|
2. Invent the next candidate patches across five five-phoneme target sets:
|
|
vowels: a, i, u, e, o
|
|
nasals/approximants: m, n, ng, l, r
|
|
fricatives: s, z, f, v, th
|
|
stops: p, b, t, d, k
|
|
mixed generalization: mix-a, mix-m, mix-s, mix-p, mix-u
|
|
3. Write exactly twenty-five new .aqua candidates under:
|
|
$patchDir
|
|
Name each file <targetId>__<family>__<hypothesis-name>.aqua so the scorer can infer the reference target and hypothesis family.
|
|
4. Make the candidates test transferable hypotheses, not one-off phoneme golf. Reuse the same contour idea across multiple places/manners when that is the claim being tested.
|
|
5. If source edits are allowed, refine the DSL or lowering code only when the evidence says the graph owner is wrong rather than the patch values.
|
|
6. Write a concise hypothesis report to:
|
|
$roundDir/hypotheses.md
|
|
|
|
Source-code golf contract:
|
|
- Source edits allowed: $AllowCodexSourceEdits
|
|
- When source edits are allowed, you may edit `src/`, `tests/`, `docs/`, and `tools/` to test DSL syntax, DSL lowering, Faust emission, worker scoring, or scene-voice evidence surfaces.
|
|
- Do not edit generated artifacts, existing `.cc` stores, existing trial results, or previous round reports.
|
|
- Source edits must move one named owner, not add compensators. Use the sentence `X owns Y so Z remains true`.
|
|
- Valid source hypotheses include: new patch DSL syntax, parser/model records, Faust lowering changes, ProbeTimeline fields, IpaTrialWorker evidence/reporting, or tests that prove the new syntax/lowering path.
|
|
- Invalid source hypotheses include: broad rewrites, changing metric thresholds to flatter candidates, deleting hard evidence, mutating references, or hardcoding a candidate name.
|
|
- If you edit source, write `$roundDir/source-change-plan.md` with: owner map, files edited, invariant, expected metric movement, tests/build commands to run, and rollback trigger.
|
|
- If you do not edit source, write `Source Edit Decision: none` and explain which evidence is still missing.
|
|
|
|
Scene voice model:
|
|
- Every candidate patch represents a scene, not a single naked waveform. The report must name the scene voices and say how each is synthesized.
|
|
- Required scene voice roles: `primary-articulatory`, `tract-noise`, `release/transient`, `background-room`, and `recording-condition`.
|
|
- Use ordinary AquaSynth primitives for helper voices where useful: `voice` layers, noise, FM, AM/tremolo, envelopes, filters, formants, additive/spectral voices, macros, and modulation. These helper voices are allowed in full parity.
|
|
- At least one hypothesis family must implement real helper/background scene voices in the `.aqua` scripts using ordinary `voice` declarations, such as filtered `wave=noise`, FM partials, AM/tremolo beds, or formant-colored capture layers. Do not satisfy the scene voice requirement with prose alone.
|
|
- When adding helper voices, keep gain low and name the role in the report. The primary articulatory `vocal network=...` voice must remain present unless the family explicitly tests a source-edit replacement.
|
|
- Helper/background voices must not count as articulation evidence unless gesture, primitive timeline, and clean vocal metrics move coherently.
|
|
- For each family, say which voices are active, which controls they expose, and which metric/timeline witness would prove the helper is not merely dressing.
|
|
|
|
Candidate design contract:
|
|
- Produce exactly 25 files: exactly one candidate for each target lane. No extras.
|
|
- Use 3-5 named hypothesis families across the 25 files, for example source-carrier, constriction-place, stop-release, nasal-branch, scene-noise-bed, and lowering-release-owner.
|
|
- Filename schema is mandatory: <targetId>__<family>__<hypothesis-name>.aqua.
|
|
- Family ids must be lowercase kebab-case and reused verbatim in filenames and `Hypothesis Families`.
|
|
- The family segment in filenames is the canonical hypothesis-family id used by the evaluator.
|
|
- The same family name should appear in filenames across multiple target sets when you are testing generalization.
|
|
- A family is transferable only if it appears in at least three target sets; otherwise label it exploratory in the report.
|
|
- A family must be phonetic-class aligned. Stop-release families belong on stop and `mix-p` lanes; nasal-branch families belong on nasal/approximant and `mix-m` lanes; frication families belong on fricative and `mix-s` lanes; vowel-shape/source families belong on vowel and vowel-like mixed lanes. Cross-class use is allowed only when the family explicitly tests transfer and the candidate changes a class-appropriate control subset for that target.
|
|
- Do not distribute families round-robin across target ids. A candidate matrix that assigns stop-release to vowels, nasal-branch to stops, or scene-noise to arbitrary phonemes without a class-specific control plan is failed science even when the file count is correct.
|
|
- Each family must include a target-class lane plan: which target classes it applies to, why those classes are lawful for the owner, and which class-specific controls change. If a target is outside the family class, the matrix must say `transfer-test` and name the class-specific mutation.
|
|
- Keep `.aqua` scripts parseable and self-contained. Prefer modifying values and gesture contours already visible in seed candidates before inventing new syntax.
|
|
- Do not use source edits as a way to rescue malformed candidate patches; use source edits to make a coherent syntax/lowering/Faust owner possible.
|
|
- Only edit source if two or more retrieved trials indicate the same owner-layer failure pattern, or if at least two scene-voice candidates need the same missing DSL/lowering primitive; otherwise `Source Edit Decision: none`.
|
|
- If Source edits allowed is False, do not edit any file outside $patchDir and $roundDir/hypotheses.md.
|
|
- Before finishing, list $patchDir and verify exact count, target coverage, and filename schema in `Acceptance Checklist`.
|
|
- A seam claim must use an owner sentence: `X owns Y so Z remains true`.
|
|
- A seam is actionable only if the next candidate family proposes at least five micro-sweep perturbations across that seam, spread across target lanes when possible.
|
|
- Novelty gate: a family must move a new owner, control axis, timing contour, or primitive relationship. Raising loudness, burst gain, or dressing alone is not novel.
|
|
- Scene novelty can be valid only when it changes a named voice role, routing/lowering primitive, modulation relationship, or background-noise synthesis model and predicts a non-articulation witness such as noise-floor, spectral tilt, RMS stability, or room/recording match. A scene-noise or recording-condition family that does not add an actual helper voice is not novel.
|
|
- Include a class-consistent reference matrix that maps each target id to the exact reference trial or fixture id used for comparison; mixed-set evidence cannot silently stand in for single-phoneme evidence.
|
|
|
|
Required `$roundDir/hypotheses.md` shape:
|
|
- `Previous Round Learning`: if a previous evaluation exists, summarize which family claims were falsified/flat/improved, what lesson survives, and exactly how this round mutates the next test. If no previous round exists, write `first round`.
|
|
- `PreEvidence Digest`: five facts with exact metric keys and values from $preEvidence.
|
|
- `Retrieval Receipts`: search/show files used.
|
|
- `Loss Landscape Read`: strongest and weakest evidence by family.
|
|
- `Reference Matrix`: one row per target id with reference trial/fixture id, candidate baseline if known, and whether the evidence is class-consistent or mixed/weak.
|
|
- `Hypothesis Families`: 3-5 families, each with owner layer, transfer status, evidence_refs, cited trial_ids, at least one concrete metric key/value from `show`, a contrast pair, an owner sentence of the form `X owns Y so Z remains true`, five planned micro-sweep perturbations, and predicted metric movement using `gesture:+/-/flat`, `logmel:+/-/flat`, `articulation:+/-/flat`, `rms:+/-/flat`, `timeline:+/-/flat/risk`.
|
|
- `Target-Class Lane Plan`: one row per family naming allowed target classes, forbidden target classes, lawful transfer lanes, and class-specific controls that change.
|
|
- `Scene Voice Model`: one row per family with primary voice, helper/background voices, AquaSynth primitives used, controls exposed, expected non-articulation witnesses, and dressing risk.
|
|
- `Claim Audit`: one row per family: claim -> evidence file(s) -> contrast pair -> metric key/value(s) -> primitive timeline excerpt or `missing` -> predicted movement.
|
|
- `Evidence Quality Ledger`: one row per family with specificity, comparability, falsifiability, reuse value for the next round, and the weakest missing evidence. Use `high`, `medium`, or `low` for each quality field.
|
|
- `Novelty Gate`: one row per family saying which new owner/control axis/timing contour/primitive relationship is being explored; write `dressing-only` if the family mainly changes loudness/FM/AM/noise/envelopes.
|
|
- `Candidate Matrix`: exactly 25 rows with target id, filename, family, expected metric movement, and risk.
|
|
- `Source Edit Decision`: `none` unless source edits are allowed and justified by evidence; if source was edited, link `$roundDir/source-change-plan.md`, list files changed, tests/builds run or pending, and rollback trigger.
|
|
- `Acceptance Checklist`: 25 files, one per target id, filename schema, required report sections, >=3 search receipts, >=3 show receipts, target-class lane plan present, no round-robin family assignment, and at least one family with actual auxiliary `voice` helper/background declarations in its candidate scripts.
|
|
|
|
Authority boundary:
|
|
- Source edits allowed: $AllowCodexSourceEdits
|
|
- Do not rewrite existing trial artifacts or the CultCache store directly.
|
|
- The worker process owns measurement and .cc writes; you own candidate invention and optional source refinements.
|
|
- Keep gesture_score, clean vocal/audio evidence, and full parity dressing separate.
|
|
- Patch candidates should make full use of AquaSynth general synth primitives when modeling the scene: FM, AM, modulators, envelopes, filters, formants, noise voices, additive/spectral layers, and extra animated voices. Call out when these are background/recording-condition voices rather than articulation.
|
|
- Prefer fixing primitive owners over adding audio dressing when the evidence says air is missing.
|
|
|
|
Return a short final summary naming the files you wrote.
|
|
"@
|
|
Invoke-CodexAgent `
|
|
-RepoRoot $repoRoot `
|
|
-Prompt $hypothesisPrompt `
|
|
-OutputPath $hypothesisOutput `
|
|
-LogPath $hypothesisLog `
|
|
-CodexCommand $CodexCommand `
|
|
-CodexModel $CodexModel `
|
|
-AllowSourceEdits:$AllowCodexSourceEdits
|
|
|
|
Assert-CandidateBatch -PatchDirectory $patchDir
|
|
|
|
$sourceVerifyLog = Join-Path $logRoot "$roundId-source-edit-verification.log"
|
|
Invoke-SourceEditVerification `
|
|
-LogPath $sourceVerifyLog `
|
|
-AllowSourceEdits:$AllowCodexSourceEdits
|
|
|
|
$scoreLog = Join-Path $logRoot "$roundId-score-candidates.log"
|
|
Invoke-LoggedProcess `
|
|
-FilePath "dotnet" `
|
|
-ArgumentList @(
|
|
"run",
|
|
"--project",
|
|
$workerProject,
|
|
"--",
|
|
"score",
|
|
"--patch-root",
|
|
$patchDir,
|
|
"--artifact-root",
|
|
$roundDir,
|
|
"--batch-id",
|
|
$roundId,
|
|
"--store",
|
|
$storePath,
|
|
"--hypothesizer",
|
|
"external-codex-$roundId"
|
|
) `
|
|
-LogPath $scoreLog
|
|
|
|
$indexAfterLog = Join-Path $logRoot "$roundId-index-after.log"
|
|
Invoke-LoggedProcess `
|
|
-FilePath "dotnet" `
|
|
-ArgumentList @(
|
|
"run",
|
|
"--project",
|
|
$workerProject,
|
|
"--",
|
|
"index",
|
|
"--store",
|
|
$storePath,
|
|
"--output",
|
|
(Join-Path $roundDir "vector-index-after.md")
|
|
) `
|
|
-LogPath $indexAfterLog
|
|
|
|
$postEvidence = Join-Path $roundDir "semantic-search-after.md"
|
|
$postDumpLog = Join-Path $logRoot "$roundId-search-after.log"
|
|
Invoke-LoggedProcess `
|
|
-FilePath "dotnet" `
|
|
-ArgumentList @(
|
|
"run",
|
|
"--project",
|
|
$workerProject,
|
|
"--",
|
|
"search",
|
|
"--store",
|
|
$storePath,
|
|
"--query",
|
|
"latest round weak promising transfer generalization vowel nasal fricative stop articulation owner source tract radiation gesture primitive dressing DSL lowering Faust machinery scene voices background noise",
|
|
"--limit",
|
|
"60",
|
|
"--output",
|
|
$postEvidence,
|
|
"--skip-index",
|
|
"true"
|
|
) `
|
|
-LogPath $postDumpLog
|
|
|
|
$evaluatorOutput = Join-Path $roundDir "evaluator-agent.md"
|
|
$evaluatorLog = Join-Path $logRoot "$roundId-evaluator-agent.log"
|
|
$evaluatorPrompt = @"
|
|
You are an AquaSynth IPA trial science evaluator.
|
|
|
|
Read the updated semantic retrieval from the trial-results database:
|
|
$postEvidence
|
|
|
|
Also read the hypothesis worker output:
|
|
$hypothesisOutput
|
|
|
|
Round artifacts are under:
|
|
$roundDir
|
|
|
|
Before judging:
|
|
1. Run exactly three targeted semantic searches into ${roundDir}:
|
|
- current round candidates by hypothesizer id or round id;
|
|
- weak regressions / failed articulation;
|
|
- promising transfer / generalization, including DSL lowering/Faust/scene-voice source-edit evidence when source edits were allowed.
|
|
2. Use `show` on at least one promising and one weak candidate from the current round.
|
|
3. If the hypothesis worker skipped the required 25-lane matrix or retrieval receipts in `$roundDir/hypotheses.md`, mark orchestration/prompt compliance as failed even if some audio metrics improved.
|
|
4. If source edits were allowed, inspect `$roundDir/source-change-plan.md` when present and verify the source edit is a coherent owner move, not a metric hack.
|
|
|
|
Use the worker as your search organ:
|
|
`$env:DOTNET_CLI_HOME="$roundDir/dotnet-home"; `$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE="1"; `$env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH="0"; `$env:DOTNET_CLI_TELEMETRY_OPTOUT="1"; dotnet run --no-build --no-restore --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- search --store "$storePath" --query "$roundId external-codex-$roundId current round candidates" --limit 30 --output "$roundDir/eval-search-current-round.md" --skip-index true
|
|
`$env:DOTNET_CLI_HOME="$roundDir/dotnet-home"; `$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE="1"; `$env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH="0"; `$env:DOTNET_CLI_TELEMETRY_OPTOUT="1"; dotnet run --no-build --no-restore --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- search --store "$storePath" --query "weak regression failed articulation stop plosive source tract radiation" --limit 30 --output "$roundDir/eval-search-weak.md" --skip-index true
|
|
`$env:DOTNET_CLI_HOME="$roundDir/dotnet-home"; `$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE="1"; `$env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH="0"; `$env:DOTNET_CLI_TELEMETRY_OPTOUT="1"; dotnet run --no-build --no-restore --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- search --store "$storePath" --query "promising transfer generalization vowel nasal fricative articulation" --limit 30 --output "$roundDir/eval-search-promising.md" --skip-index true
|
|
`$env:DOTNET_CLI_HOME="$roundDir/dotnet-home"; `$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE="1"; `$env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH="0"; `$env:DOTNET_CLI_TELEMETRY_OPTOUT="1"; dotnet run --no-build --no-restore --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- show --store "$storePath" --trial-id "<trial id from current round search>" --output "$roundDir/eval-detail-<candidate>.md"
|
|
|
|
Retrieval failure policy:
|
|
- Failed search/show commands do not count as evidence receipts for family verdicts.
|
|
- If live worker retrieval fails, set `Round Compliance: failed` and focus the report on the retrieval blocker instead of inventing metric conclusions.
|
|
- Search-summary-only claims are low-quality until corroborated by `show` receipts.
|
|
|
|
Verify compliance from disk as well as the hypothesis report:
|
|
- count .aqua files under $patchDir;
|
|
- validate exactly 25 files;
|
|
- validate one target each for a, i, u, e, o, m, n, ng, l, r, s, z, f, v, th, p, b, t, d, k, mix-a, mix-m, mix-s, mix-p, mix-u;
|
|
- validate filename schema <targetId>__<family>__<hypothesis-name>.aqua.
|
|
- validate family ids are lowercase kebab-case.
|
|
- derive canonical family ids only from the filename family segment; ignore prose family names when they disagree.
|
|
- validate that each actionable seam has a contrast pair, a primitive timeline excerpt or explicit `missing`, an owner sentence, and at least five planned micro-sweep perturbations.
|
|
- validate that `$roundDir/hypotheses.md` includes a class-consistent reference matrix, scene voice model, and novelty gate. `hypothesis-agent.md` is only the external agent's final chat summary and must not be treated as the full report.
|
|
- validate that `$roundDir/hypotheses.md` includes `Previous Round Learning` and `Target-Class Lane Plan`.
|
|
- validate target-class alignment from filenames and family plan: stop-release families should not be assigned to vowel lanes, nasal-branch families should not be assigned to stop lanes, frication families should not be assigned to vowel lanes, and scene-noise families should not be arbitrary catchalls unless each off-class lane is marked `transfer-test` with a class-specific mutation.
|
|
- fail `class alignment` when families are distributed round-robin across target ids or when candidate files reuse the same family script shape without target-class-specific controls.
|
|
- validate that every family identifies primary-articulatory, tract-noise, release/transient, background-room, and recording-condition roles, even if some are explicitly silent.
|
|
- validate that at least one family actually adds ordinary helper/background `voice` declarations in its `.aqua` scripts for scene noise or recording-condition modeling.
|
|
- validate that helper/background voices are evaluated separately from articulation evidence.
|
|
- validate source edit compliance if source edits were allowed: coherent owner map, edited file list, invariant, expected metric movement, verification command, and rollback trigger.
|
|
- If any required section, receipt class, per-target validation, filename validation, family extraction, reference matrix, previous-round learning, target-class lane plan, class alignment, scene voice model, contrast pair, owner sentence, or novelty gate is incomplete, set `Round Compliance: failed`.
|
|
|
|
Task:
|
|
1. Evaluate whether the worker's proposed hypotheses follow from the measured evidence.
|
|
2. Call out any hypothesis that is likely full-patch dressing impersonating articulation.
|
|
3. Recommend exactly one next implementation/refinement target, with file/module owner and why. It may be DSL syntax, Faust lowering, scene-voice modeling, or worker evidence if the receipts support it.
|
|
4. Write your evaluator report to:
|
|
$roundDir/evaluation.md
|
|
|
|
Required `$roundDir/evaluation.md` shape:
|
|
- `Retrieval Receipts`: exactly five receipts: three search outputs plus one weak show output and one promising show output. Cite no other evidence files in this section. Only cite claims backed by these file paths; write `unknown` when evidence is absent.
|
|
- `Commands Run`: exact search/show commands and output paths used to support claims.
|
|
- `Round Compliance`: passed/failed, candidate count, target coverage, naming validity, family extraction, reference matrix, previous-round learning, target-class lane plan, class alignment, contrast-pair coverage, owner-sentence coverage, micro-sweep coverage, novelty gate, auxiliary scene voice presence, and whether `hypotheses.md` had the required matrix.
|
|
- `Family Verdicts`: one row per canonical filename family with improved/flat/regressed/unknown metrics. Treat |delta| < 0.01 as flat; if deltas are not present in cited show artifacts, write `unknown` and reason=`no delta in receipts`.
|
|
- `Evidence Quality Verdicts`: one row per family with specificity, comparability, falsifiability, reuse value, primitive timeline support, and weakest missing evidence. Penalize vague receipts, search-summary-only claims, missing contrast pairs, missing reference ids, and missing timeline excerpts even when metrics improve.
|
|
- `Seam Audit`: one row per named seam with owner sentence, contrast pair, timeline excerpt, planned perturbation count, and verdict `actionable` or `not actionable`; fewer than five perturbations means `not actionable`.
|
|
- `Scene Voice Audit`: one row per family naming scene voices, helper/background synthesis primitives, non-articulation witness, and whether the helper voices are honest context or dressing.
|
|
- `Source Edit Audit`: if source edits were allowed, summarize source files changed, invariant claimed, verification result, and whether the edit should survive; otherwise `not allowed`.
|
|
- `Novelty Audit`: one row per family saying whether it moves a new owner/control axis/timing contour/primitive relationship/scene-voice model or is mainly loudness/dressing.
|
|
- `Generalization Read`: whether effects transferred across target sets or overfit one phoneme.
|
|
- `Learning Audit`: whether failed/flat previous families were mutated, retired, or blindly repeated.
|
|
- `Dressing Audit`: articulation evidence requires movement in gesture/articulation/primitive timeline metrics; FM/AM/noise/envelope evidence without those is dressing.
|
|
- `Next Target`: exactly one markdown bullet, final line of the file, and no continuation text: `<file-or-module> | <invariant> | <expected metric movement>`.
|
|
|
|
Authority boundary:
|
|
- Do not edit source files.
|
|
- Do not rewrite trial artifacts.
|
|
- Be explicit about which layer owns the failure: gesture DSL, scene voice model, primitive source, tract/radiation lowering, Faust machinery, audio scoring, or orchestration.
|
|
- Treat the CultCache .cc store as the shared trial memory that future hypothesis workers will query through the worker search/show commands.
|
|
|
|
Return a short final summary naming the report you wrote.
|
|
"@
|
|
Invoke-CodexAgent `
|
|
-RepoRoot $repoRoot `
|
|
-Prompt $evaluatorPrompt `
|
|
-OutputPath $evaluatorOutput `
|
|
-LogPath $evaluatorLog `
|
|
-CodexCommand $CodexCommand `
|
|
-CodexModel $CodexModel `
|
|
-AllowSourceEdits:$false
|
|
}
|
|
|
|
Write-Host "IPA trial loop complete: $loopRoot"
|