Automate IPA trial exploration loop
This commit is contained in:
parent
05cf7d36d6
commit
d1789d6677
12 changed files with 1289 additions and 17 deletions
|
|
@ -17,6 +17,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{07C2787E
|
|||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdSyrinxGolf", "tools\BirdSyrinxGolf\BirdSyrinxGolf.csproj", "{A1914F24-9CE5-47DA-943E-E26D3AC4DFD3}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IpaTrialWorker", "tools\IpaTrialWorker\IpaTrialWorker.csproj", "{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
|
|
@ -75,6 +77,18 @@ Global
|
|||
{A1914F24-9CE5-47DA-943E-E26D3AC4DFD3}.Release|x64.Build.0 = Release|Any CPU
|
||||
{A1914F24-9CE5-47DA-943E-E26D3AC4DFD3}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{A1914F24-9CE5-47DA-943E-E26D3AC4DFD3}.Release|x86.Build.0 = Release|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Release|x64.Build.0 = Release|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
@ -84,5 +98,6 @@ Global
|
|||
{10EDBE53-4957-44AB-90C9-82C2ABD526CD} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{D6CDBB4A-A783-4AA8-8715-E9E71A9BF9CB} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
|
||||
{A1914F24-9CE5-47DA-943E-E26D3AC4DFD3} = {07C2787E-EAC7-C090-1BA3-A61EC2A24D84}
|
||||
{9596767B-8FDC-49E5-8F70-0A1CECE35C5E} = {07C2787E-EAC7-C090-1BA3-A61EC2A24D84}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
|
|||
|
|
@ -660,6 +660,44 @@ The evidence says the source-carrier cut fixed the voiced-air failure. The next
|
|||
pressure is plosive closure/release ownership and stronger vowel/nasal tract
|
||||
color, not decorative full-patch dressing.
|
||||
|
||||
### External Codex Trial Loop
|
||||
|
||||
`tools/IpaTrialWorker` is the command-line trial organ for external agents. It
|
||||
keeps measurement and CultCache writes inside AquaSynth instead of asking Codex
|
||||
workers to hand-edit evidence:
|
||||
|
||||
- `seed` renders the baseline five trial sets into a shared
|
||||
`ipa-trial-results.cc` store.
|
||||
- `score` reads agent-authored `.aqua` candidates, renders and scores them
|
||||
against the PT fixtures, and upserts typed `IpaTrialResult` records.
|
||||
- `search` is the semantic retrieval surface for the `.cc` store. It expands
|
||||
speech-control vocabulary such as vowel/nasal/fricative/stop, ranks records
|
||||
with metric-aware evidence bias, and writes a compact markdown result set.
|
||||
- `show` drills into one trial or candidate with full metrics, artifacts,
|
||||
known contamination, hypothesis, and evaluator summary.
|
||||
- `dump` remains an audit escape hatch, not the normal agent memory path.
|
||||
|
||||
`tools/run-ipa-trial-loop.ps1` orchestrates the outer loop with external
|
||||
`codex exec` workers. Each round searches accumulated trial memory, asks a
|
||||
hypothesis worker to write a batch of candidates, scores those candidates
|
||||
through `IpaTrialWorker`, searches the updated store, then asks a science
|
||||
evaluator to judge which hypotheses held up.
|
||||
|
||||
The trial shape is five target sets with five phoneme lanes each:
|
||||
|
||||
- vowels: `a`, `i`, `u`, `e`, `o`;
|
||||
- nasals/approximants: `m`, `n`, `ng`, `l`, `r`;
|
||||
- fricatives: `s`, `z`, `f`, `v`, `th`;
|
||||
- stops: `p`, `b`, `t`, `d`, `k`;
|
||||
- mixed generalization: `mix-a`, `mix-m`, `mix-s`, `mix-p`, `mix-u`.
|
||||
|
||||
Agent-authored candidates must be named
|
||||
`<targetId>__<hypothesis-name>.aqua`. The point is not one-patch golf. A useful
|
||||
round tests whether a contour, source, tract, radiation, or DSL-lowering idea
|
||||
generalizes across the target sets. Full-patch FM/AM/noise dressing is allowed
|
||||
as parity pressure, but the evaluator must label it as dressing when primitive
|
||||
or gesture evidence does not improve.
|
||||
|
||||
## CultMesh Render/Scoring Work
|
||||
|
||||
CultMesh is the transport, admission, and worker-orchestration layer for speech
|
||||
|
|
|
|||
|
|
@ -21,6 +21,13 @@ public sealed record IpaTrialTargetSet(
|
|||
public IReadOnlyList<IpaGestureExperimentVariant> Variants { get; init; } = Variants ?? Array.Empty<IpaGestureExperimentVariant>();
|
||||
}
|
||||
|
||||
public sealed record IpaTrialScriptCandidate(
|
||||
string TargetId,
|
||||
string CandidateId,
|
||||
string ScriptPath,
|
||||
string HypothesizerId,
|
||||
string Hypothesis);
|
||||
|
||||
public sealed record IpaTrialOrchestrationOptions(
|
||||
string BatchId = "ipa-seed-trials",
|
||||
string HypothesizerId = "local-hypothesis-worker",
|
||||
|
|
@ -48,34 +55,57 @@ public static class IpaTrialOrchestrator
|
|||
[
|
||||
new(
|
||||
"trial-001-vowels",
|
||||
"Open and front vowel hypotheses against static tract references.",
|
||||
"Five vowel-space hypotheses against static tract references.",
|
||||
[
|
||||
Target("a", "a", "voiced_open_back_unrounded_vowel", "open-vowel", .20f),
|
||||
Target("i", "i", "voiced_close_front_unrounded_vowel", "front-vowel", .20f)
|
||||
Target("i", "i", "voiced_close_front_unrounded_vowel", "front-vowel", .20f),
|
||||
Target("u", "u", "voiced_close_back_rounded_vowel", "nasal-vowel", .20f),
|
||||
Target("e", "e", "voiced_close_mid_front_unrounded_vowel", "front-vowel", .20f),
|
||||
Target("o", "o", "voiced_close_mid_back_rounded_vowel", "open-vowel", .20f)
|
||||
],
|
||||
[Variant("baseline", 1, 1, "vowel-seed")]),
|
||||
new(
|
||||
"trial-002-nasal",
|
||||
"Bilabial nasal velum/contact hypothesis against the ma pressure fixture.",
|
||||
[Target("m", "m", "voiced_bilabial_nasal", "bilabial-nasal-ma", .18f)],
|
||||
"trial-002-nasals-approximants",
|
||||
"Five sonorant hypotheses that should generalize voicing, velum, lips, and lateral/rhotic shaping.",
|
||||
[
|
||||
Target("m", "m", "voiced_bilabial_nasal", "bilabial-nasal-ma", .18f),
|
||||
Target("n", "n", "voiced_alveolar_nasal", "bilabial-nasal-ma", .18f),
|
||||
Target("ng", "ŋ", "voiced_velar_nasal", "nasal-vowel", .18f),
|
||||
Target("l", "l", "voiced_alveolar_lateral_approximant", "front-vowel", .18f),
|
||||
Target("r", "r", "voiced_alveolar_rhotic_approximant", "open-vowel", .18f)
|
||||
],
|
||||
[Variant("nasal-base", 1, 1, "nasal-seed")]),
|
||||
new(
|
||||
"trial-003-sibilant",
|
||||
"Voiceless alveolar fricative hypothesis against the high-turbulence sibilant fixture.",
|
||||
[Target("s", "s", "voiceless_alveolar_fricative", "sibilant", .16f)],
|
||||
"trial-003-fricatives",
|
||||
"Five fricative hypotheses that should generalize narrow constriction, turbulence, place, and voicing.",
|
||||
[
|
||||
Target("s", "s", "voiceless_alveolar_fricative", "sibilant", .16f),
|
||||
Target("z", "z", "voiced_alveolar_fricative", "sibilant", .16f),
|
||||
Target("f", "f", "voiceless_labiodental_fricative", "sibilant", .16f),
|
||||
Target("v", "v", "voiced_labiodental_fricative", "sibilant", .16f),
|
||||
Target("th", "θ", "voiceless_dental_fricative", "sibilant", .16f)
|
||||
],
|
||||
[Variant("hiss-base", 1.15f, .9f, "fricative-seed")]),
|
||||
new(
|
||||
"trial-004-plosive",
|
||||
"Bilabial stop closure/release hypothesis against obstruction-history pressure.",
|
||||
[Target("p", "p", "voiceless_bilabial_plosive", "closure-release", .14f)],
|
||||
"trial-004-stops",
|
||||
"Five stop hypotheses that should generalize closure, reservoir pressure, release, place, and voicing.",
|
||||
[
|
||||
Target("p", "p", "voiceless_bilabial_plosive", "closure-release", .14f),
|
||||
Target("b", "b", "voiced_bilabial_plosive", "closure-release", .14f),
|
||||
Target("t", "t", "voiceless_alveolar_plosive", "closure-release", .14f),
|
||||
Target("d", "d", "voiced_alveolar_plosive", "closure-release", .14f),
|
||||
Target("k", "k", "voiceless_velar_plosive", "closure-release", .14f)
|
||||
],
|
||||
[Variant("stop-base", 1.2f, .85f, "plosive-seed")]),
|
||||
new(
|
||||
"trial-005-mixed",
|
||||
"Small mixed set used to check whether the same DSL abstractions separate vowel, nasal, and fricative pressure.",
|
||||
"trial-005-mixed-generalization",
|
||||
"Five mixed phones used to check whether one DSL/control vocabulary transfers across vowel, nasal, fricative, stop, and rounded classes.",
|
||||
[
|
||||
Target("mix-a", "a", "voiced_open_back_unrounded_vowel", "open-vowel", .18f),
|
||||
Target("mix-m", "m", "voiced_bilabial_nasal", "bilabial-nasal-ma", .16f),
|
||||
Target("mix-s", "s", "voiceless_alveolar_fricative", "sibilant", .14f)
|
||||
Target("mix-s", "s", "voiceless_alveolar_fricative", "sibilant", .14f),
|
||||
Target("mix-p", "p", "voiceless_bilabial_plosive", "closure-release", .14f),
|
||||
Target("mix-u", "u", "voiced_close_back_rounded_vowel", "nasal-vowel", .18f)
|
||||
],
|
||||
[Variant("mixed-base", 1, 1, "mixed-seed")])
|
||||
];
|
||||
|
|
@ -227,6 +257,162 @@ public static class IpaTrialOrchestrator
|
|||
results);
|
||||
}
|
||||
|
||||
public static async Task<IpaTrialOrchestrationResult> RunCandidateScriptsAsync(
|
||||
string artifactRoot,
|
||||
IReadOnlyList<IpaTrialScriptCandidate> candidates,
|
||||
IReadOnlyList<IpaTrialTargetSet>? targetSets = null,
|
||||
IpaTrialOrchestrationOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactRoot);
|
||||
ArgumentNullException.ThrowIfNull(candidates);
|
||||
options ??= new IpaTrialOrchestrationOptions(BatchId: "ipa-agent-candidate-trials");
|
||||
targetSets ??= DefaultFiveSeedTrialSets;
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one IPA script candidate is required.", nameof(candidates));
|
||||
}
|
||||
|
||||
var batchDirectory = Path.Combine(artifactRoot, options.BatchId);
|
||||
var candidateRoot = Path.Combine(batchDirectory, "agent-candidates");
|
||||
var timelineRoot = Path.Combine(batchDirectory, "agent-timelines");
|
||||
Directory.CreateDirectory(candidateRoot);
|
||||
Directory.CreateDirectory(timelineRoot);
|
||||
var storePath = Path.Combine(batchDirectory, options.TrialResultStoreFileName);
|
||||
var targetById = targetSets
|
||||
.SelectMany(set => set.Targets.Select(target => (set, target)))
|
||||
.GroupBy(pair => pair.target.Target.Id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(group => group.Key, group => group.First(), StringComparer.OrdinalIgnoreCase);
|
||||
var referenceRenderer = new PinkTromboneReferenceRenderer();
|
||||
var analyzer = new AudioAnalyzer(new AudioAnalysisConfig(SampleRate: options.SampleRate));
|
||||
var results = new List<IpaTrialResult>();
|
||||
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (!targetById.TryGetValue(candidate.TargetId, out var mapped))
|
||||
{
|
||||
throw new ArgumentException($"Candidate `{candidate.CandidateId}` names unknown IPA target `{candidate.TargetId}`.", nameof(candidates));
|
||||
}
|
||||
|
||||
var startedAt = DateTimeOffset.UtcNow;
|
||||
var startedStamp = Stopwatch.GetTimestamp();
|
||||
var safeCandidateId = SafeName(candidate.CandidateId);
|
||||
var candidateDir = Path.Combine(candidateRoot, safeCandidateId);
|
||||
Directory.CreateDirectory(candidateDir);
|
||||
var audioDir = Path.Combine(candidateDir, "audio");
|
||||
Directory.CreateDirectory(audioDir);
|
||||
var scriptCopy = Path.Combine(candidateDir, "candidate.aqua");
|
||||
File.Copy(candidate.ScriptPath, scriptCopy, overwrite: true);
|
||||
var script = await File.ReadAllTextAsync(scriptCopy, cancellationToken).ConfigureAwait(false);
|
||||
var patch = PatchScript.Parse(script);
|
||||
var timeline = ProbeTimelineReport.Build(patch, "voice", Math.Max(1, options.TimelineBlocks));
|
||||
var timelinePath = Path.Combine(timelineRoot, $"{safeCandidateId}.csv");
|
||||
await File.WriteAllTextAsync(timelinePath, ProbeTimelineReport.ToCsv(timeline), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var source = FaustEmitter.EmitScript(script, new FaustExportOptions(safeCandidateId)).Source;
|
||||
var dspPath = Path.Combine(candidateDir, "candidate.dsp");
|
||||
await File.WriteAllTextAsync(dspPath, source, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var referenceFixture = PinkTromboneParityFixtures.ById(mapped.target.ReferenceFixtureId);
|
||||
var reference = referenceRenderer.Render(referenceFixture.Controls, mapped.target.Target.DurationSeconds + .18f);
|
||||
var candidateRender = await FaustCompiler.RenderAsync(
|
||||
source,
|
||||
new FaustRenderOptions(reference.SampleRate, reference.Samples.Length / (float)reference.SampleRate),
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var referenceWav = Path.Combine(audioDir, "reference.wav");
|
||||
var candidateWav = Path.Combine(audioDir, "candidate.wav");
|
||||
WriteWav(referenceWav, reference.Samples, reference.SampleRate);
|
||||
var metrics = new List<SpeechScoreMetric>
|
||||
{
|
||||
new("gesture_score", AgentGestureScore(mapped.target.Target, patch, timeline), .25f)
|
||||
};
|
||||
string verdict;
|
||||
string evaluation;
|
||||
var artifacts = new List<SpeechRenderArtifact>
|
||||
{
|
||||
Artifact("candidate-script", scriptCopy),
|
||||
Artifact("candidate-dsp", dspPath),
|
||||
Artifact("primitive-timeline", timelinePath),
|
||||
Artifact("reference-wav", referenceWav)
|
||||
};
|
||||
|
||||
if (candidateRender is null || candidateRender.Samples.Length == 0)
|
||||
{
|
||||
verdict = "render-failed";
|
||||
evaluation = candidateRender is null
|
||||
? "Faust was not available to render this agent-authored candidate."
|
||||
: $"Faust render produced no samples. stderr: {candidateRender.Stderr}";
|
||||
metrics.Add(new SpeechScoreMetric("render_failed", 1, 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteWav(candidateWav, candidateRender.Samples, candidateRender.SampleRate);
|
||||
artifacts.Add(Artifact("candidate-wav", candidateWav));
|
||||
var comparison = analyzer.Compare(reference.Samples, candidateRender.Samples);
|
||||
metrics.AddRange(ComparisonMetrics(comparison));
|
||||
verdict = Verdict(comparison);
|
||||
evaluation = EvaluationSentence(mapped.target.Target.Ipa, mapped.target.ReferenceFixtureId, comparison, verdict);
|
||||
var comparisonPath = Path.Combine(audioDir, "comparison.txt");
|
||||
await WriteComparisonReportAsync(
|
||||
comparisonPath,
|
||||
mapped.target,
|
||||
new IpaGestureExperimentCandidate(safeCandidateId, candidate.TargetId, "agent-authored", scriptCopy, timelinePath, []),
|
||||
comparison,
|
||||
verdict,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
artifacts.Add(Artifact("comparison-report", comparisonPath));
|
||||
}
|
||||
|
||||
var latency = Stopwatch.GetElapsedTime(startedStamp).TotalMilliseconds;
|
||||
results.Add(new IpaTrialResult(
|
||||
$"{options.BatchId}:{mapped.set.Id}:{safeCandidateId}",
|
||||
options.BatchId,
|
||||
startedAt.ToString("O", CultureInfo.InvariantCulture),
|
||||
mapped.set.Id,
|
||||
mapped.set.Targets.Select(item => item.Target.Ipa).Distinct(StringComparer.Ordinal).ToArray(),
|
||||
mapped.target.ReferenceFixtureId,
|
||||
safeCandidateId,
|
||||
candidate.HypothesizerId,
|
||||
candidate.Hypothesis,
|
||||
scriptCopy,
|
||||
referenceWav,
|
||||
File.Exists(candidateWav) ? candidateWav : "",
|
||||
timelinePath,
|
||||
metrics.ToArray(),
|
||||
artifacts.ToArray(),
|
||||
options.EvaluatorId,
|
||||
evaluation,
|
||||
verdict,
|
||||
KnownLies(options, mapped.target.ReferenceFixtureId),
|
||||
[
|
||||
new SpeechTimingReceipt(
|
||||
"ipa-agent-candidate-render-score",
|
||||
startedAt.ToString("O", CultureInfo.InvariantCulture),
|
||||
latency,
|
||||
0,
|
||||
Confidence(metrics),
|
||||
"Local worker rendered an external Codex-authored patch, compared log-mel/articulation evidence, and wrote CultCache trial result data.")
|
||||
]));
|
||||
}
|
||||
|
||||
await IpaTrialResultCultCacheStore.UpsertResultsAsync(storePath, results).ConfigureAwait(false);
|
||||
var summaryPath = Path.Combine(batchDirectory, "summary.csv");
|
||||
await File.WriteAllTextAsync(summaryPath, SummaryCsv(results), cancellationToken).ConfigureAwait(false);
|
||||
var evaluatorReport = Path.Combine(batchDirectory, "evaluator-report.md");
|
||||
await File.WriteAllTextAsync(evaluatorReport, EvaluatorReport(options, results), cancellationToken).ConfigureAwait(false);
|
||||
await IpaTrialResultCultCacheStore.UpsertResultsAsync(storePath, results).ConfigureAwait(false);
|
||||
|
||||
return new IpaTrialOrchestrationResult(
|
||||
options.BatchId,
|
||||
batchDirectory,
|
||||
storePath,
|
||||
summaryPath,
|
||||
evaluatorReport,
|
||||
results);
|
||||
}
|
||||
|
||||
private static IpaTrialReferenceTarget Target(string id, string ipa, string descriptor, string fixture, float duration) =>
|
||||
new(new IpaGestureExperimentTarget(id, ipa, descriptor, duration), fixture);
|
||||
|
||||
|
|
@ -253,6 +439,102 @@ public static class IpaTrialOrchestrator
|
|||
];
|
||||
}
|
||||
|
||||
private static IReadOnlyList<SpeechScoreMetric> ComparisonMetrics(AudioComparison comparison) =>
|
||||
[
|
||||
new("log_mel_cosine", comparison.LogMelCosineSimilarity, .25f),
|
||||
new("log_mel_distance", comparison.LogMelDistance, .15f),
|
||||
new("audio_score", comparison.Score, .15f),
|
||||
new("articulation_score", comparison.Articulation.ArticulationScore, .10f),
|
||||
new("rms_ratio", comparison.RmsRatio, .05f),
|
||||
new("speech_band_ratio", comparison.Articulation.SpeechBandRatio, .05f)
|
||||
];
|
||||
|
||||
private static float AgentGestureScore(
|
||||
IpaGestureExperimentTarget target,
|
||||
SynthPatch patch,
|
||||
IReadOnlyList<ProbeTimelineSample> timeline)
|
||||
{
|
||||
var tokens = DescriptorTokens(target.Descriptor);
|
||||
var expected = ExpectedSurfaces(tokens);
|
||||
var touched = patch.ControlSplines
|
||||
.Where(spline => spline.Enabled)
|
||||
.Select(spline => spline.SurfacePath)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var coverage = expected.Length == 0
|
||||
? 1
|
||||
: expected.Count(surface => touched.Contains(surface)) / (float)expected.Length;
|
||||
var motion = patch.ControlSplines.Any(spline => spline.Enabled && spline.Points.Count >= 2) ? 1f : 0f;
|
||||
var primitive = timeline.Any(sample =>
|
||||
(sample.Signal.Equals("output", StringComparison.OrdinalIgnoreCase) ||
|
||||
sample.Signal.Equals("flow", StringComparison.OrdinalIgnoreCase) ||
|
||||
sample.Signal.Equals("released_flow", StringComparison.OrdinalIgnoreCase)) &&
|
||||
float.IsFinite(sample.Value))
|
||||
? 1f
|
||||
: 0f;
|
||||
return Math.Clamp(.5f * coverage + .25f * motion + .25f * primitive, 0, 1);
|
||||
}
|
||||
|
||||
private static string[] ExpectedSurfaces(HashSet<string> tokens)
|
||||
{
|
||||
var surfaces = new List<string>
|
||||
{
|
||||
"/vocal/sources/0/pressure",
|
||||
"/vocal/sources/0/tension",
|
||||
"/vocal/sources/0/opening"
|
||||
};
|
||||
|
||||
if (tokens.Contains("vowel"))
|
||||
{
|
||||
surfaces.AddRange(
|
||||
[
|
||||
"/vocal/areas/0/area/tongue_index",
|
||||
"/vocal/areas/0/area/tongue_diameter",
|
||||
"/vocal/areas/0/area/lip_opening",
|
||||
"/vocal/radiation/0/aperture"
|
||||
]);
|
||||
}
|
||||
else
|
||||
{
|
||||
surfaces.AddRange(
|
||||
[
|
||||
"/vocal/areas/0/area/constriction_index",
|
||||
"/vocal/areas/0/area/constriction_diameter",
|
||||
"/vocal/contacts/0/opening"
|
||||
]);
|
||||
}
|
||||
|
||||
if (tokens.Contains("nasal"))
|
||||
{
|
||||
surfaces.Add("/vocal/branches/0/opening");
|
||||
}
|
||||
|
||||
if (tokens.Contains("fricative"))
|
||||
{
|
||||
surfaces.Add("/vocal/sources/0/noise");
|
||||
surfaces.Add("/vocal/contacts/0/resistance");
|
||||
}
|
||||
|
||||
if (tokens.Contains("plosive") || tokens.Contains("stop"))
|
||||
{
|
||||
surfaces.Add("/vocal/contacts/0/stored_pressure");
|
||||
}
|
||||
|
||||
if (tokens.Contains("bilabial") || tokens.Contains("labial") || tokens.Contains("labiodental") || tokens.Contains("rounded"))
|
||||
{
|
||||
surfaces.Add("/vocal/areas/0/area/lip_opening");
|
||||
surfaces.Add("/vocal/radiation/0/aperture");
|
||||
}
|
||||
|
||||
return surfaces.Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
|
||||
private static HashSet<string> DescriptorTokens(string descriptor) =>
|
||||
descriptor
|
||||
.ToLowerInvariant()
|
||||
.Replace('-', '_')
|
||||
.Split(['_', ',', '+', ' '], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static string Verdict(AudioComparison comparison)
|
||||
{
|
||||
if (comparison.LogMelCosineSimilarity >= .70f && comparison.Articulation.ArticulationScore >= .45f)
|
||||
|
|
|
|||
|
|
@ -237,3 +237,4 @@
|
|||
{"date":"2026-05-29","area":"ipa-training-score-contract","summary":"Documented the speech training scoring contract. `gesture_score` measures descriptor-to-surface coverage, direction, spline contour/timing, primitive timeline consequences, and optional external articulation evidence. Training reports must keep `gesture_score`, `clean_vocal_score`, and `full_parity_score` separate. Full parity scripts may use FM/AM/modulators/envelopes/filters, auxiliary animated voices, and recording-condition helpers, but those helpers cannot be accepted as articulation evidence unless lower-layer gesture and clean-vocal scores also improve.","evidence":"docs/ipa-vocal-tract-roadmap.md; docs/utterance-parity-research.md; state/spine.yaml; state/scratch.md"}
|
||||
{"date":"2026-05-29","area":"ipa-gesture-experiment-harness","summary":"Added `IpaGestureExperiment.WriteRound`, the first frozen batch surface for IPA hypothesis work. A round writes deterministic candidate `.aqua` scripts, primitive `ProbeTimelineReport` CSVs, `manifest.yaml`, gesture-layer `metrics.csv`, and per-candidate `evidence.jsonl`. `IpaGestureExperiment.AnalyzeRound` adds metric summaries, score-band candidate clusters, and a science brief for sub-agent analysis. The harness scores descriptor/spline/primitive evidence only and explicitly leaves clean vocal identity, full spectrogram parity, optimizer checkpoints, and worker orchestration downstream. PatchScriptTests pass 93/93.","evidence":"src/AquaSynth.Core/IpaGestureExperiment.cs; tests/AquaSynth.Dsl.Tests/PatchScriptTests.cs; docs/ipa-vocal-tract-roadmap.md; docs/utterance-parity-research.md; state/map.yaml; state/spine.yaml; state/scratch.md"}
|
||||
{"date":"2026-05-29","area":"ipa-trial-render-orchestration","summary":"Added the first local IPA trial worker/evaluator loop. `IpaTrialOrchestrator.RunAsync` generates candidates through `IpaGestureExperiment`, renders them through Faust, compares spectrogram/articulation evidence against local Pink Trombone fixture references, writes WAV/report artifacts, and stores typed `IpaTrialResult` records in `ipa-trial-results.cc`. The first five-seed run proved the pipeline but exposed silent vowel/nasal candidates; the accepted refinement moved voiced excitation into primitive `SourcePort` lowering. The second run at `artifacts/parity/ipa-trials/20260529T214127955/five-seed-trials` improved open `a` to log-mel cosine `0.5693`, `m` to `0.5199`, and `s` to `0.5498`, while `p` stayed weak at `-0.1024`, pointing to closure/release ownership next. Focused verification passes 94/94 plus opt-in IPA trial run.","evidence":"src/AquaSynth.Core/IpaTrialResults.cs; src/AquaSynth.Faust/IpaTrialOrchestrator.cs; src/AquaSynth.Core/FaustExport.cs; src/AquaSynth.Core/IpaGestureExperiment.cs; tests/AquaSynth.Dsl.Tests/IpaTrialOrchestratorTests.cs; docs/ipa-vocal-tract-roadmap.md; state/spine.yaml; state/scratch.md"}
|
||||
{"date":"2026-05-29","area":"ipa-trial-loop","lesson":"External IPA exploration needs a semantic retrieval tool over the CultCache .cc trial store, not crude full-memory dumps. Hypothesis agents should use search/show to retrieve ranked metric and artifact context, then write batches across the five five-phoneme target sets; the worker owns rendering, scoring, and .cc upserts."}
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ modules:
|
|||
ipa_gesture_experiment: src/AquaSynth.Core/IpaGestureExperiment.cs
|
||||
ipa_trial_results: src/AquaSynth.Core/IpaTrialResults.cs
|
||||
ipa_trial_orchestrator: src/AquaSynth.Faust/IpaTrialOrchestrator.cs
|
||||
ipa_trial_worker: tools/IpaTrialWorker
|
||||
ipa_trial_external_loop: tools/run-ipa-trial-loop.ps1
|
||||
acoustic_graph_missing_pieces_audit: docs/acoustic-graph-missing-pieces-audit.md
|
||||
acoustic_graph_proprioception: docs/acoustic-graph-proprioception.md
|
||||
pink_trombone_parity_fixtures: src/AquaSynth.Core/PinkTromboneParityFixtures.cs
|
||||
|
|
|
|||
|
|
@ -92,10 +92,10 @@
|
|||
"Surge XT for modern hybrid synth modulation",
|
||||
"Cardinal/VCV-style patches for modular cable/rate semantics"
|
||||
],
|
||||
"rule": "For speech, add schema fields and learned-control lanes only when the parity harness has a loss surface that can exercise them. For reference synths, add DSL abstractions only when rebuilds prove the missing invariant. Parsing alone is inventory, not evidence of expressivity. Pink Trombone should be decomposed into reusable low-level Aqua primitives before audio golf, not copied as a bespoke clone or inflated into PT-shaped modules. After the 2026-05-29 proprioception correction, collapse graph-vocal lowering into a small set of instrumented physical primitives before more utterance golf. Gesture motion belongs in ControlSurface/ControlSpline timeline ownership, not primitive-local smoothing; cut any generated contact/port spline that invents motion not supported by primitive reference timelines. Public clients author gestures through ControlSurfaceCatalog plus ControlSplineTimeline and stream surface control dictionaries; higher-level gesture templates and phoneme_gesture are only spline emitters. IPA descriptors seed which anatomical surfaces move, while local datasets golf numeric spline defaults and patch voice code. Training reports must separate gesture_score, clean_vocal_score, and full_parity_score so full-patch recording-condition helpers cannot impersonate articulation. Frozen IPA gesture rounds own candidate scripts, primitive timelines, gesture metrics, manifests, evidence receipts, metric summaries, candidate clusters, and science briefs; `IpaTrialResult` CultCache records own rendered trial outcomes. Statistical science workers and audio parity stages consume those bundles without rewriting gesture evidence. If gesture scores rise but rendered audio is silent, fix the primitive source/tract owner before adding full-patch dressing."
|
||||
"rule": "For speech, add schema fields and learned-control lanes only when the parity harness has a loss surface that can exercise them. For reference synths, add DSL abstractions only when rebuilds prove the missing invariant. Parsing alone is inventory, not evidence of expressivity. Pink Trombone should be decomposed into reusable low-level Aqua primitives before audio golf, not copied as a bespoke clone or inflated into PT-shaped modules. After the 2026-05-29 proprioception correction, collapse graph-vocal lowering into a small set of instrumented physical primitives before more utterance golf. Gesture motion belongs in ControlSurface/ControlSpline timeline ownership, not primitive-local smoothing; cut any generated contact/port spline that invents motion not supported by primitive reference timelines. Public clients author gestures through ControlSurfaceCatalog plus ControlSplineTimeline and stream surface control dictionaries; higher-level gesture templates and phoneme_gesture are only spline emitters. IPA descriptors seed which anatomical surfaces move, while local datasets golf numeric spline defaults and patch voice code. Training reports must separate gesture_score, clean_vocal_score, and full_parity_score so full-patch recording-condition helpers cannot impersonate articulation. Frozen IPA gesture rounds own candidate scripts, primitive timelines, gesture metrics, manifests, evidence receipts, metric summaries, candidate clusters, and science briefs; `IpaTrialResult` CultCache records own rendered trial outcomes. External Codex IPA exploration must query the `.cc` store through the worker's semantic `search`/`show` organ, then write candidate batches across the five five-phoneme target sets; crude full-store dumps are audit fallback, not agent memory. Statistical science workers and audio parity stages consume those bundles without rewriting gesture evidence. If gesture scores rise but rendered audio is silent, fix the primitive source/tract owner before adding full-patch dressing."
|
||||
},
|
||||
"ipa_vocal_tract_path": {
|
||||
"status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets. CultMesh speech training now has payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application; workers do not own model updates. Real remote artifact chunk transfer and process supervision remain CultMesh infrastructure surfaces, not AquaSynth workaround territory. IpaGestureExperiment writes frozen descriptor/variant rounds with candidate scripts, primitive timelines, gesture metrics, manifests, and evidence JSONL, then analyzes them into metric summaries, score-band clusters, and a science brief. IpaTrialOrchestrator now writes local five-seed IPA render trials, WAV/report artifacts, evaluator reports, and `ipa-trial-results.cc` CultCache records. The latest seed run improved voiced vowels/nasals after primitive `SourcePort` lowering gained an oscillatory carrier; plosive closure/release remains the next weak owner.",
|
||||
"status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets. CultMesh speech training now has payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application; workers do not own model updates. Real remote artifact chunk transfer and process supervision remain CultMesh infrastructure surfaces, not AquaSynth workaround territory. IpaGestureExperiment writes frozen descriptor/variant rounds with candidate scripts, primitive timelines, gesture metrics, manifests, and evidence JSONL, then analyzes them into metric summaries, score-band clusters, and a science brief. IpaTrialOrchestrator now writes local five-seed IPA render trials, accepts externally authored candidate scripts through RunCandidateScriptsAsync, writes WAV/report artifacts, evaluator reports, and `ipa-trial-results.cc` CultCache records. `tools/IpaTrialWorker` is the public worker tool for seed/score/search/show over the store, and `tools/run-ipa-trial-loop.ps1` dispatches external Codex hypothesis/evaluator workers around five five-phoneme target sets. The latest seed run improved voiced vowels/nasals after primitive `SourcePort` lowering gained an oscillatory carrier; plosive closure/release remains the next weak owner.",
|
||||
"lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.",
|
||||
"next": "Start the next speech gradient descent pass from Weksa's emitted v0.1 seed batch at `E:\\Projects\\weksa\\examples\\speech-training\\tiny-panphon-v0.1\\batch.json`: 1024 bge-m3 semantic speech-text floats, variable-length PanPhon sequence evidence, 256 AquaSynth-owned phonetic-realization embedding floats from `aquasynth.panphon_sequence_encoder.v0.1`, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Next tighten the parity harness: load that batch into AquaSynth training records, render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop. For distributed work, use RunCultMeshTrainingStep as the boring authority path: admitted leased workers return gradients/results, then the trainer applies centrally. Do not attempt federated averaging until this path is exercised against real Faust-rendered speech jobs."
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2,6 +2,34 @@
|
|||
|
||||
Current slice:
|
||||
|
||||
- Added the external IPA trial loop as a real worker boundary instead of an
|
||||
in-chat report shuffle. `tools/IpaTrialWorker` now exposes `seed`, `score`,
|
||||
`search`, `show`, and audit-only `dump` commands over a shared
|
||||
`ipa-trial-results.cc` CultCache store. `search` is the semantic retrieval
|
||||
organ for agents: it expands speech/control vocabulary and ranks records with
|
||||
metric-aware evidence bias; `show` drills into one trial. `score` renders
|
||||
agent-authored `.aqua` candidates through
|
||||
`IpaTrialOrchestrator.RunCandidateScriptsAsync` and upserts new typed trial
|
||||
results. `tools/run-ipa-trial-loop.ps1` now runs five external Codex rounds by
|
||||
default, each asking for a batch across five five-phoneme target sets rather
|
||||
than one-off patch golf; the script rejects a round before scoring unless it
|
||||
contains at least one candidate for every target lane.
|
||||
- Active IPA exploration shape: vowels `a/i/u/e/o`,
|
||||
nasals/approximants `m/n/ng/l/r`, fricatives `s/z/f/v/th`, stops
|
||||
`p/b/t/d/k`, and mixed generalization `mix-a/mix-m/mix-s/mix-p/mix-u`.
|
||||
Agent patch files must be named `<targetId>__<hypothesis-name>.aqua` so the
|
||||
scorer can bind them to target/reference lanes. Full-patch dressing remains
|
||||
allowed, but evaluator reports must say when it is compensating for failed
|
||||
primitive or gesture evidence.
|
||||
- Verification so far: `dotnet build tools\IpaTrialWorker\IpaTrialWorker.csproj
|
||||
--disable-build-servers -p:UseSharedCompilation=false
|
||||
-p:BuildInParallel=false -v minimal` passes; `IpaTrialWorker search` returns
|
||||
ranked evidence from
|
||||
`artifacts/parity/ipa-trial-worker-smoke/20260529T2204/ipa-trial-results.cc`;
|
||||
`tools\run-ipa-trial-loop.ps1 -Rounds 0 -SkipLocalTrialRun` creates a loop
|
||||
skeleton. Need final focused tests and optionally a one-round external Codex
|
||||
smoke if time/tooling allows.
|
||||
|
||||
- Implemented the first IPA trial render/scoring orchestration. Core now has
|
||||
typed `IpaTrialResult` CultCache records plus `IpaTrialResultCultCacheStore`;
|
||||
Faust now has `IpaTrialOrchestrator.RunAsync`, which generates candidate
|
||||
|
|
|
|||
|
|
@ -145,6 +145,7 @@ live_invariants:
|
|||
- IPA training scoring contract is documented. Reports must keep `gesture_score`, `clean_vocal_score`, and `full_parity_score` separate. Gesture score measures descriptor-to-surface coverage, motion direction, spline contour/timing, primitive timeline consequences, and optional external articulation landmarks. Full parity patches may use FM/AM/modulators/envelopes/filters, auxiliary animated voices, and recording-condition helpers, but those helpers cannot count as articulation evidence unless gesture and clean-vocal scores move coherently.
|
||||
- `IpaGestureExperiment.WriteRound` is the first frozen batch harness for IPA hypothesis work. It writes `manifest.yaml`, candidate `.aqua` scripts, primitive `ProbeTimelineReport` CSVs, gesture-layer `metrics.csv`, and per-candidate `evidence.jsonl` under a caller-provided artifact root. `IpaGestureExperiment.AnalyzeRound` writes `analysis/metric-summary.csv`, `analysis/candidate-clusters.csv`, and `analysis/science-brief.md` for the science-worker handoff. This harness owns frozen descriptor/spline/primitive evidence only; clean vocal identity, full spectrogram parity, optimizer checkpoints, and worker orchestration stay downstream. Focused verification: `PatchScriptTests` pass 93/93.
|
||||
- `IpaTrialOrchestrator.RunAsync` is the first local render/scoring worker for IPA trials. It generates candidate patches through `IpaGestureExperiment`, renders them through Faust, compares against local PT fixture references with `AudioAnalyzer`, writes WAV/report artifacts, and stores typed `IpaTrialResult` records in `ipa-trial-results.cc`. The five-seed trial run at `tests/AquaSynth.Dsl.Tests/bin/Debug/net10.0/artifacts/parity/ipa-trials/20260529T214127955/five-seed-trials` showed voiced vowels/nasals became audible only after primitive `SourcePort` lowering gained a true oscillatory voiced carrier; sibilants are pressure, plosives remain weak. Verification: opt-in IPA trial test passed, and focused `PatchScriptTests|IpaTrialResultsRoundTripThroughCultCache` pass 94/94.
|
||||
- External IPA exploration now has a worker/tool boundary. `tools/IpaTrialWorker` owns `seed`, `score`, `search`, `show`, and audit-only `dump` commands over the typed `.cc` trial-result store; `score` renders external Codex-authored `.aqua` candidates through `IpaTrialOrchestrator.RunCandidateScriptsAsync`, while `search` is the semantic retrieval organ for hypothesis agents. `tools/run-ipa-trial-loop.ps1` orchestrates external `codex exec` hypothesis/evaluator workers around that store. The active trial shape is five target sets of five phoneme lanes each: vowels, nasals/approximants, fricatives, stops, and mixed generalization. Verification: `IpaTrialWorker` builds, `search` smoke reads the seed store, and the loop skeleton runs with `-Rounds 0 -SkipLocalTrialRun`.
|
||||
- TurbulenceJet now includes a release-timed burst-noise component in addition to the deterministic pressure pulse, but a stronger trial barely moved papa; the source is not the main remaining articulation bottleneck. Graph radiation now mixes 30% raw boundary flow with 70% high-passed flow after local admittance weighting, down from the over-bright 15/85 blend. Latest radiation/source artifact under `artifacts/parity/pink-trombone-utterance-logmel/20260527T174236049`: mama/papa/thrombosis cosine 0.6578/0.7692/0.5585, rmsRatio 3.0006/2.6153/2.3488, articulation 0.1437/0.1173/0.1191. Static `closure-release` under `artifacts/parity/pink-trombone-logmel/20260527T174139661` has rmsRatio 0.9900 and speechBandRatio 0.0842. This is still not speaking; next pressure is true tract resonance/passive scattering and pressure storage, not more burst gain.
|
||||
- Same-path two-port area discontinuities now emit `graph_area_energy_in_*` and `graph_area_energy_out_*` probes alongside the three-port branch `graph_connection_energy_*` probes. Use these to inspect passivity and formant-body failures at the scatter sites rather than judging only final output loudness.
|
||||
- `FaustExportOptions.DebugProbeUi` wraps graph probe signals in Faust `vbargraph` UI controls under `/debug/...` without changing audio output arity. Native streaming Faust patches now separate writable controls from read-only probe zones and can read/snapshot probe values after `ProcessBlock`; use this for acoustic graph passivity/radiation inspection, leave it off for normal parity renders.
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ public sealed class IpaTrialOrchestratorTests
|
|||
Assert.True(File.Exists(result.TrialResultStorePath));
|
||||
Assert.True(File.Exists(result.SummaryPath));
|
||||
Assert.True(File.Exists(result.EvaluatorReportPath));
|
||||
Assert.Equal(8, result.TrialResults.Count);
|
||||
Assert.Equal(25, result.TrialResults.Count);
|
||||
Assert.Equal(5, result.TrialResults.Select(item => item.TargetSetId).Distinct(StringComparer.Ordinal).Count());
|
||||
Assert.All(result.TrialResults, trial => Assert.NotEmpty(trial.Metrics));
|
||||
Assert.Contains(result.TrialResults, trial => trial.Metrics.Any(metric => metric.Name == "log_mel_cosine"));
|
||||
|
|
@ -84,6 +84,45 @@ public sealed class IpaTrialOrchestratorTests
|
|||
Assert.Equal(result.TrialResults.Count, loaded.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IpaTrialOrchestratorScoresAgentAuthoredCandidateScriptsWhenEnabled()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("AQUASYNTH_RUN_IPA_TRIALS") != "1")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (FaustCompiler.FindFaust() is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var directory = TempDirectory();
|
||||
try
|
||||
{
|
||||
var script = IpaGestureExperiment.BuildCandidateScript(
|
||||
new IpaGestureExperimentTarget("a", "a", "voiced_open_back_unrounded_vowel", .12f),
|
||||
new IpaGestureExperimentVariant("agent-smoke"));
|
||||
var scriptPath = Path.Combine(directory, "a__agent-smoke.aqua");
|
||||
await File.WriteAllTextAsync(scriptPath, script);
|
||||
|
||||
var result = await IpaTrialOrchestrator.RunCandidateScriptsAsync(
|
||||
Path.Combine(directory, "artifacts"),
|
||||
[new IpaTrialScriptCandidate("a", "a__agent-smoke", scriptPath, "test-agent", "score externally authored candidate")],
|
||||
options: new IpaTrialOrchestrationOptions(BatchId: "agent-smoke"));
|
||||
|
||||
var single = Assert.Single(result.TrialResults);
|
||||
Assert.Equal("a__agent_smoke", single.CandidateId);
|
||||
Assert.Contains(single.Metrics, metric => metric.Name == "log_mel_cosine");
|
||||
Assert.True(File.Exists(single.CandidatePatchUri));
|
||||
Assert.True(File.Exists(result.TrialResultStorePath));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static string TempDirectory()
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), $"aquasynth-ipa-trial-{Guid.NewGuid():N}");
|
||||
|
|
|
|||
15
tools/IpaTrialWorker/IpaTrialWorker.csproj
Normal file
15
tools/IpaTrialWorker/IpaTrialWorker.csproj
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\AquaSynth.Core\AquaSynth.Core.csproj" />
|
||||
<ProjectReference Include="..\..\src\AquaSynth.Faust\AquaSynth.Faust.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
492
tools/IpaTrialWorker/Program.cs
Normal file
492
tools/IpaTrialWorker/Program.cs
Normal file
|
|
@ -0,0 +1,492 @@
|
|||
using System.Globalization;
|
||||
using System.Text;
|
||||
|
||||
using AquaSynth.Dsl;
|
||||
using AquaSynth.Faust;
|
||||
|
||||
var command = args.FirstOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(command) || command is "-h" or "--help")
|
||||
{
|
||||
PrintHelp();
|
||||
return 0;
|
||||
}
|
||||
|
||||
var options = ParseOptions(args.Skip(1).ToArray());
|
||||
try
|
||||
{
|
||||
switch (command)
|
||||
{
|
||||
case "seed":
|
||||
await SeedAsync(options);
|
||||
return 0;
|
||||
case "score":
|
||||
await ScoreAsync(options);
|
||||
return 0;
|
||||
case "dump":
|
||||
await DumpAsync(options);
|
||||
return 0;
|
||||
case "search":
|
||||
await SearchAsync(options);
|
||||
return 0;
|
||||
case "show":
|
||||
await ShowAsync(options);
|
||||
return 0;
|
||||
default:
|
||||
Console.Error.WriteLine($"Unknown command `{command}`.");
|
||||
PrintHelp();
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine(ex);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static async Task SeedAsync(Dictionary<string, string> options)
|
||||
{
|
||||
var artifactRoot = Required(options, "artifact-root");
|
||||
var batchId = Value(options, "batch-id", "five-seed-trials");
|
||||
var store = Value(options, "store", Path.Combine(artifactRoot, "ipa-trial-results.cc"));
|
||||
var result = await IpaTrialOrchestrator.RunAsync(
|
||||
artifactRoot,
|
||||
options: new IpaTrialOrchestrationOptions(BatchId: batchId));
|
||||
await IpaTrialResultCultCacheStore.UpsertResultsAsync(store, result.TrialResults);
|
||||
Console.WriteLine(result.ArtifactDirectory);
|
||||
Console.WriteLine(store);
|
||||
}
|
||||
|
||||
static async Task ScoreAsync(Dictionary<string, string> options)
|
||||
{
|
||||
var patchRoot = Required(options, "patch-root");
|
||||
var artifactRoot = Required(options, "artifact-root");
|
||||
var batchId = Value(options, "batch-id", $"agent-candidates-{DateTimeOffset.UtcNow:yyyyMMddTHHmmssfff}");
|
||||
var store = Value(options, "store", Path.Combine(artifactRoot, "ipa-trial-results.cc"));
|
||||
var hypothesizer = Value(options, "hypothesizer", "external-codex-hypothesis-worker");
|
||||
var candidates = CandidateScripts(patchRoot, hypothesizer);
|
||||
var result = await IpaTrialOrchestrator.RunCandidateScriptsAsync(
|
||||
artifactRoot,
|
||||
candidates,
|
||||
options: new IpaTrialOrchestrationOptions(BatchId: batchId, HypothesizerId: hypothesizer));
|
||||
await IpaTrialResultCultCacheStore.UpsertResultsAsync(store, result.TrialResults);
|
||||
Console.WriteLine(result.ArtifactDirectory);
|
||||
Console.WriteLine(store);
|
||||
}
|
||||
|
||||
static async Task DumpAsync(Dictionary<string, string> options)
|
||||
{
|
||||
var store = Required(options, "store");
|
||||
var output = Required(options, "output");
|
||||
var results = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
||||
await File.WriteAllTextAsync(output, StoreReport(store, results), Encoding.UTF8);
|
||||
Console.WriteLine(output);
|
||||
}
|
||||
|
||||
static async Task SearchAsync(Dictionary<string, string> options)
|
||||
{
|
||||
var store = Required(options, "store");
|
||||
var query = Required(options, "query");
|
||||
var output = Required(options, "output");
|
||||
var limit = int.TryParse(Value(options, "limit", "12"), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)
|
||||
? Math.Clamp(parsed, 1, 100)
|
||||
: 12;
|
||||
var results = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
||||
await File.WriteAllTextAsync(output, SearchReport(store, query, Rank(results, query, limit)), Encoding.UTF8);
|
||||
Console.WriteLine(output);
|
||||
}
|
||||
|
||||
static async Task ShowAsync(Dictionary<string, string> options)
|
||||
{
|
||||
var store = Required(options, "store");
|
||||
var trialId = Required(options, "trial-id");
|
||||
var output = Required(options, "output");
|
||||
var results = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
||||
var result = results.FirstOrDefault(item =>
|
||||
item.TrialId.Equals(trialId, StringComparison.OrdinalIgnoreCase) ||
|
||||
item.CandidateId.Equals(trialId, StringComparison.OrdinalIgnoreCase))
|
||||
?? throw new InvalidDataException($"No trial or candidate found for `{trialId}`.");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
||||
await File.WriteAllTextAsync(output, DetailReport(store, result), Encoding.UTF8);
|
||||
Console.WriteLine(output);
|
||||
}
|
||||
|
||||
static IReadOnlyList<IpaTrialScriptCandidate> CandidateScripts(string patchRoot, string hypothesizer)
|
||||
{
|
||||
if (!Directory.Exists(patchRoot))
|
||||
{
|
||||
throw new DirectoryNotFoundException(patchRoot);
|
||||
}
|
||||
|
||||
var targetIds = IpaTrialOrchestrator.DefaultFiveSeedTrialSets
|
||||
.SelectMany(set => set.Targets)
|
||||
.Select(target => target.Target.Id)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderByDescending(id => id.Length)
|
||||
.ToArray();
|
||||
return Directory.EnumerateFiles(patchRoot, "*.aqua", SearchOption.AllDirectories)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(path =>
|
||||
{
|
||||
var stem = Path.GetFileNameWithoutExtension(path);
|
||||
var target = TargetIdFromStem(stem, targetIds);
|
||||
return new IpaTrialScriptCandidate(
|
||||
target,
|
||||
stem,
|
||||
path,
|
||||
hypothesizer,
|
||||
$"External Codex-authored IPA patch candidate `{stem}` for target `{target}`.");
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
static string TargetIdFromStem(string stem, IReadOnlyList<string> targetIds)
|
||||
{
|
||||
var explicitSplit = stem.Split(["__"], StringSplitOptions.None);
|
||||
if (explicitSplit.Length > 1 && targetIds.Contains(explicitSplit[0], StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
return explicitSplit[0];
|
||||
}
|
||||
|
||||
var matched = targetIds.FirstOrDefault(id =>
|
||||
stem.Equals(id, StringComparison.OrdinalIgnoreCase) ||
|
||||
stem.StartsWith($"{id}-", StringComparison.OrdinalIgnoreCase) ||
|
||||
stem.StartsWith($"{id}_", StringComparison.OrdinalIgnoreCase));
|
||||
if (!string.IsNullOrWhiteSpace(matched))
|
||||
{
|
||||
return matched;
|
||||
}
|
||||
|
||||
throw new InvalidDataException(
|
||||
$"Could not infer IPA target id from `{stem}`. Name files as `<targetId>__candidate-name.aqua`; known target ids: {string.Join(", ", targetIds)}.");
|
||||
}
|
||||
|
||||
static IReadOnlyList<(IpaTrialResult Result, float Score, string[] MatchedTerms)> Rank(
|
||||
IReadOnlyList<IpaTrialResult> results,
|
||||
string query,
|
||||
int limit)
|
||||
{
|
||||
var terms = ExpandTerms(Tokenize(query)).ToArray();
|
||||
return results
|
||||
.Select(result =>
|
||||
{
|
||||
var document = Tokenize(DocumentText(result)).ToArray();
|
||||
var termCounts = document.GroupBy(term => term, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(group => group.Key, group => group.Count(), StringComparer.OrdinalIgnoreCase);
|
||||
var matched = terms.Where(term => termCounts.ContainsKey(term)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
var lexical = matched.Sum(term => 1f + MathF.Log(1 + termCounts[term]));
|
||||
var metricBias = MetricBias(result, terms);
|
||||
var score = lexical + metricBias;
|
||||
return (result, score, matched);
|
||||
})
|
||||
.Where(item => item.score > 0)
|
||||
.OrderByDescending(item => item.score)
|
||||
.ThenByDescending(item => ScoreSort(item.result))
|
||||
.ThenBy(item => item.result.TrialId, StringComparer.Ordinal)
|
||||
.Take(limit)
|
||||
.Select(item => (item.result, item.score, item.matched))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
static IEnumerable<string> ExpandTerms(IEnumerable<string> terms)
|
||||
{
|
||||
var set = new HashSet<string>(terms, StringComparer.OrdinalIgnoreCase);
|
||||
var aliases = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["vowel"] = ["a", "i", "u", "e", "o", "formant", "tongue", "lip", "radiation"],
|
||||
["nasal"] = ["m", "n", "ng", "ŋ", "velum", "branch", "velopharynx", "admittance"],
|
||||
["sonorant"] = ["nasal", "approximant", "l", "r", "voiced"],
|
||||
["fricative"] = ["s", "z", "f", "v", "th", "θ", "turbulence", "noise", "constriction"],
|
||||
["stop"] = ["p", "b", "t", "d", "k", "plosive", "closure", "release", "reservoir"],
|
||||
["plosive"] = ["stop", "closure", "release", "reservoir", "stored", "pressure"],
|
||||
["weak"] = ["render-failed", "weak", "low", "failed", "silent"],
|
||||
["promising"] = ["pressure", "promising", "high", "improved"],
|
||||
["articulation"] = ["gesture", "primitive", "timeline", "opening", "flow"],
|
||||
["dressing"] = ["fm", "am", "modulator", "envelope", "noise", "patch"],
|
||||
["owner"] = ["source", "lowering", "tract", "radiation", "gesture", "orchestration"]
|
||||
};
|
||||
|
||||
var queue = set.ToArray();
|
||||
foreach (var term in queue)
|
||||
{
|
||||
if (!aliases.TryGetValue(term, out var expanded))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var alias in expanded)
|
||||
{
|
||||
set.Add(alias);
|
||||
}
|
||||
}
|
||||
|
||||
return set;
|
||||
}
|
||||
|
||||
static float MetricBias(IpaTrialResult result, IReadOnlyCollection<string> terms)
|
||||
{
|
||||
var bias = 0f;
|
||||
var cosine = result.Metrics.FirstOrDefault(metric => metric.Name == "log_mel_cosine")?.Value ?? 0;
|
||||
var articulation = result.Metrics.FirstOrDefault(metric => metric.Name == "articulation_score")?.Value ?? 0;
|
||||
var gesture = result.Metrics.FirstOrDefault(metric => metric.Name == "gesture_score")?.Value ?? 0;
|
||||
if (terms.Contains("weak", StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
bias += Math.Clamp(1 - cosine, 0, 1);
|
||||
bias += Math.Clamp(1 - articulation, 0, 1);
|
||||
}
|
||||
|
||||
if (terms.Contains("promising", StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
bias += Math.Clamp(cosine, 0, 1);
|
||||
bias += Math.Clamp(articulation, 0, 1);
|
||||
}
|
||||
|
||||
if (terms.Contains("gesture", StringComparer.OrdinalIgnoreCase) ||
|
||||
terms.Contains("articulation", StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
bias += Math.Clamp(gesture, 0, 1);
|
||||
}
|
||||
|
||||
return bias;
|
||||
}
|
||||
|
||||
static string DocumentText(IpaTrialResult result)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.Append(result.TrialId).Append(' ');
|
||||
builder.Append(result.BatchId).Append(' ');
|
||||
builder.Append(result.TargetSetId).Append(' ');
|
||||
builder.AppendJoin(' ', result.Phonemes).Append(' ');
|
||||
builder.Append(result.ReferenceId).Append(' ');
|
||||
builder.Append(result.CandidateId).Append(' ');
|
||||
builder.Append(result.HypothesizerId).Append(' ');
|
||||
builder.Append(result.Hypothesis).Append(' ');
|
||||
builder.Append(result.EvaluatorId).Append(' ');
|
||||
builder.Append(result.EvaluationSummary).Append(' ');
|
||||
builder.Append(result.Verdict).Append(' ');
|
||||
builder.AppendJoin(' ', result.KnownLies).Append(' ');
|
||||
foreach (var metric in result.Metrics)
|
||||
{
|
||||
builder.Append(metric.Name).Append(' ');
|
||||
}
|
||||
|
||||
foreach (var artifact in result.Artifacts)
|
||||
{
|
||||
builder.Append(artifact.Kind).Append(' ').Append(artifact.Uri).Append(' ');
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
static string[] Tokenize(string text) =>
|
||||
text.ToLowerInvariant()
|
||||
.Split([' ', '\t', '\r', '\n', ',', '.', ':', ';', '/', '\\', '`', '"', '\'', '[', ']', '(', ')', '{', '}', '<', '>', '|', '_', '-'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Where(term => term.Length > 1 || "aeioubdklmnprstvzθŋ".Contains(term, StringComparison.Ordinal))
|
||||
.ToArray();
|
||||
|
||||
static string SearchReport(
|
||||
string store,
|
||||
string query,
|
||||
IReadOnlyList<(IpaTrialResult Result, float Score, string[] MatchedTerms)> ranked)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine("# IPA Trial Semantic Search");
|
||||
builder.AppendLine();
|
||||
builder.AppendLine($"store: `{store}`");
|
||||
builder.AppendLine($"query: `{query}`");
|
||||
builder.AppendLine($"matches: {ranked.Count}");
|
||||
builder.AppendLine();
|
||||
foreach (var (result, score, matched) in ranked)
|
||||
{
|
||||
builder.Append("- ");
|
||||
builder.Append(result.TrialId);
|
||||
builder.Append(" / ");
|
||||
builder.Append(result.CandidateId);
|
||||
builder.Append(" / ");
|
||||
builder.Append(result.Verdict);
|
||||
builder.Append(" / score=");
|
||||
builder.Append(score.ToString("0.###", CultureInfo.InvariantCulture));
|
||||
builder.Append(" / matched=");
|
||||
builder.AppendLine(string.Join('|', matched.Take(12)));
|
||||
builder.Append(" metrics: gesture=");
|
||||
builder.Append(Metric(result, "gesture_score"));
|
||||
builder.Append(", logMelCosine=");
|
||||
builder.Append(Metric(result, "log_mel_cosine"));
|
||||
builder.Append(", articulation=");
|
||||
builder.Append(Metric(result, "articulation_score"));
|
||||
builder.Append(", rmsRatio=");
|
||||
builder.AppendLine(Metric(result, "rms_ratio"));
|
||||
builder.Append(" hypothesis: ");
|
||||
builder.AppendLine(result.Hypothesis);
|
||||
builder.Append(" evaluation: ");
|
||||
builder.AppendLine(result.EvaluationSummary);
|
||||
builder.Append(" patch: ");
|
||||
builder.AppendLine(result.CandidatePatchUri);
|
||||
builder.Append(" show: dotnet run --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- show --store \"");
|
||||
builder.Append(store);
|
||||
builder.Append("\" --trial-id \"");
|
||||
builder.Append(result.TrialId);
|
||||
builder.AppendLine("\" --output <detail.md>");
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
static string DetailReport(string store, IpaTrialResult result)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine($"# IPA Trial Detail: {result.TrialId}");
|
||||
builder.AppendLine();
|
||||
builder.AppendLine($"store: `{store}`");
|
||||
builder.AppendLine($"batch: `{result.BatchId}`");
|
||||
builder.AppendLine($"target_set: `{result.TargetSetId}`");
|
||||
builder.AppendLine($"phonemes: `{string.Join(", ", result.Phonemes)}`");
|
||||
builder.AppendLine($"reference: `{result.ReferenceId}`");
|
||||
builder.AppendLine($"candidate: `{result.CandidateId}`");
|
||||
builder.AppendLine($"verdict: `{result.Verdict}`");
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("## Hypothesis");
|
||||
builder.AppendLine(result.Hypothesis);
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("## Evaluation");
|
||||
builder.AppendLine(result.EvaluationSummary);
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("## Metrics");
|
||||
foreach (var metric in result.Metrics.OrderBy(metric => metric.Name, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
builder.Append("- ");
|
||||
builder.Append(metric.Name);
|
||||
builder.Append(": ");
|
||||
builder.Append(metric.Value.ToString("0.######", CultureInfo.InvariantCulture));
|
||||
builder.Append(" weight=");
|
||||
builder.AppendLine(metric.Weight.ToString("0.######", CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("## Artifacts");
|
||||
foreach (var artifact in result.Artifacts)
|
||||
{
|
||||
builder.Append("- ");
|
||||
builder.Append(artifact.Kind);
|
||||
builder.Append(": ");
|
||||
builder.Append(artifact.Uri);
|
||||
if (!string.IsNullOrWhiteSpace(artifact.ContentHash))
|
||||
{
|
||||
builder.Append(" ");
|
||||
builder.Append(artifact.ContentHash);
|
||||
}
|
||||
|
||||
builder.AppendLine();
|
||||
}
|
||||
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("## Known Lies");
|
||||
foreach (var lie in result.KnownLies)
|
||||
{
|
||||
builder.Append("- ");
|
||||
builder.AppendLine(lie);
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
static string StoreReport(string store, IReadOnlyList<IpaTrialResult> results)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine($"# IPA Trial Results Store");
|
||||
builder.AppendLine();
|
||||
builder.AppendLine($"store: `{store}`");
|
||||
builder.AppendLine($"records: {results.Count}");
|
||||
builder.AppendLine();
|
||||
foreach (var group in results.GroupBy(result => result.TargetSetId).OrderBy(group => group.Key, StringComparer.Ordinal))
|
||||
{
|
||||
builder.AppendLine($"## {group.Key}");
|
||||
foreach (var result in group.OrderByDescending(ScoreSort).ThenBy(result => result.TrialId, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append("- ");
|
||||
builder.Append(result.TrialId);
|
||||
builder.Append(" / ");
|
||||
builder.Append(result.CandidateId);
|
||||
builder.Append(" / ");
|
||||
builder.Append(result.Verdict);
|
||||
builder.Append(": ");
|
||||
builder.Append("gesture=");
|
||||
builder.Append(Metric(result, "gesture_score"));
|
||||
builder.Append(", logMelCosine=");
|
||||
builder.Append(Metric(result, "log_mel_cosine"));
|
||||
builder.Append(", articulation=");
|
||||
builder.Append(Metric(result, "articulation_score"));
|
||||
builder.Append(", rmsRatio=");
|
||||
builder.Append(Metric(result, "rms_ratio"));
|
||||
builder.AppendLine();
|
||||
builder.Append(" patch: ");
|
||||
builder.AppendLine(result.CandidatePatchUri);
|
||||
builder.Append(" timeline: ");
|
||||
builder.AppendLine(result.PrimitiveTimelineUri);
|
||||
builder.Append(" evaluation: ");
|
||||
builder.AppendLine(result.EvaluationSummary);
|
||||
}
|
||||
|
||||
builder.AppendLine();
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
static float ScoreSort(IpaTrialResult result) =>
|
||||
result.Metrics.FirstOrDefault(metric => metric.Name == "log_mel_cosine")?.Value ?? float.NegativeInfinity;
|
||||
|
||||
static string Metric(IpaTrialResult result, string name) =>
|
||||
(result.Metrics.FirstOrDefault(metric => metric.Name == name)?.Value ?? 0)
|
||||
.ToString("0.######", CultureInfo.InvariantCulture);
|
||||
|
||||
static Dictionary<string, string> ParseOptions(IReadOnlyList<string> args)
|
||||
{
|
||||
var options = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
for (var i = 0; i < args.Count; i++)
|
||||
{
|
||||
var arg = args[i];
|
||||
if (!arg.StartsWith("--", StringComparison.Ordinal))
|
||||
{
|
||||
throw new ArgumentException($"Expected option, got `{arg}`.");
|
||||
}
|
||||
|
||||
var key = arg[2..];
|
||||
if (i + 1 >= args.Count || args[i + 1].StartsWith("--", StringComparison.Ordinal))
|
||||
{
|
||||
options[key] = "true";
|
||||
}
|
||||
else
|
||||
{
|
||||
options[key] = args[++i];
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
static string Required(Dictionary<string, string> options, string key) =>
|
||||
options.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)
|
||||
? value
|
||||
: throw new ArgumentException($"Missing required --{key} option.");
|
||||
|
||||
static string Value(Dictionary<string, string> options, string key, string fallback) =>
|
||||
options.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value) ? value : fallback;
|
||||
|
||||
static void PrintHelp()
|
||||
{
|
||||
Console.WriteLine("""
|
||||
IPA trial worker
|
||||
|
||||
Commands:
|
||||
seed --artifact-root <dir> [--batch-id five-seed-trials] [--store <trial-results.cc>]
|
||||
score --patch-root <dir> --artifact-root <dir> [--batch-id round-001] [--store <trial-results.cc>] [--hypothesizer id]
|
||||
dump --store <trial-results.cc> --output <report.md>
|
||||
search --store <trial-results.cc> --query <text> --output <report.md> [--limit 12]
|
||||
show --store <trial-results.cc> --trial-id <id-or-candidate> --output <detail.md>
|
||||
|
||||
Agent-authored patch files must be named <targetId>__candidate-name.aqua.
|
||||
Known seed target ids: a, i, u, e, o, m, n, ng, l, r, s, z, f, v, th, p, b, t, d, k, mix-a, mix-m, mix-s, mix-p, mix-u.
|
||||
""");
|
||||
}
|
||||
359
tools/run-ipa-trial-loop.ps1
Normal file
359
tools/run-ipa-trial-loop.ps1
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
param(
|
||||
[int]$Rounds = 5,
|
||||
[string]$BatchId = "five-seed-trials",
|
||||
[string]$CodexCommand = "codex",
|
||||
[string]$CodexModel = "gpt-5.3-codex",
|
||||
[switch]$AllowCodexSourceEdits,
|
||||
[switch]$SkipLocalTrialRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Resolve-RepoRoot {
|
||||
$dir = Get-Item -LiteralPath (Get-Location)
|
||||
while ($null -ne $dir) {
|
||||
if (Test-Path -LiteralPath (Join-Path $dir.FullName "AquaSynth.sln")) {
|
||||
return $dir.FullName
|
||||
}
|
||||
|
||||
$dir = $dir.Parent
|
||||
}
|
||||
|
||||
throw "Could not find AquaSynth.sln above the current directory."
|
||||
}
|
||||
|
||||
function New-Timestamp {
|
||||
return [DateTimeOffset]::UtcNow.ToString("yyyyMMddTHHmmssfff")
|
||||
}
|
||||
|
||||
function Invoke-LoggedProcess {
|
||||
param(
|
||||
[string]$FilePath,
|
||||
[string[]]$ArgumentList,
|
||||
[string]$LogPath,
|
||||
[hashtable]$Environment = @{}
|
||||
)
|
||||
|
||||
$logDir = Split-Path -Parent $LogPath
|
||||
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
|
||||
$oldValues = @{}
|
||||
foreach ($key in $Environment.Keys) {
|
||||
$oldValues[$key] = [Environment]::GetEnvironmentVariable($key, "Process")
|
||||
[Environment]::SetEnvironmentVariable($key, [string]$Environment[$key], "Process")
|
||||
}
|
||||
|
||||
try {
|
||||
$display = "$FilePath $($ArgumentList -join ' ')"
|
||||
Add-Content -LiteralPath $LogPath -Value "[$([DateTimeOffset]::UtcNow.ToString("O"))] $display"
|
||||
& $FilePath @ArgumentList *>&1 | Tee-Object -FilePath $LogPath -Append
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Command failed with exit code $LASTEXITCODE`: $display"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
foreach ($key in $Environment.Keys) {
|
||||
[Environment]::SetEnvironmentVariable($key, $oldValues[$key], "Process")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-CodexAgent {
|
||||
param(
|
||||
[string]$RepoRoot,
|
||||
[string]$Prompt,
|
||||
[string]$OutputPath,
|
||||
[string]$LogPath,
|
||||
[string]$CodexCommand,
|
||||
[string]$CodexModel,
|
||||
[bool]$AllowSourceEdits
|
||||
)
|
||||
|
||||
$promptPath = [IO.Path]::ChangeExtension($OutputPath, ".prompt.md")
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $OutputPath) | Out-Null
|
||||
Set-Content -LiteralPath $promptPath -Value $Prompt
|
||||
|
||||
$args = @(
|
||||
"exec",
|
||||
"--cd", $RepoRoot,
|
||||
"--full-auto",
|
||||
"--output-last-message", $OutputPath
|
||||
)
|
||||
if ($CodexModel.Length -gt 0) {
|
||||
$args += @("--model", $CodexModel)
|
||||
}
|
||||
if (-not $AllowSourceEdits) {
|
||||
$args += @(
|
||||
"--config", 'shell_environment_policy.inherit="all"',
|
||||
"--config", 'sandbox_permissions=["workspace-write"]'
|
||||
)
|
||||
}
|
||||
$args += "-"
|
||||
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
Get-Content -LiteralPath $promptPath -Raw | & $CodexCommand @args 2>&1 | Tee-Object -FilePath $LogPath -Append
|
||||
$exitCode = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
|
||||
if ($exitCode -ne 0) {
|
||||
throw "External Codex agent failed with exit code $exitCode. See $LogPath."
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-CandidateBatch {
|
||||
param(
|
||||
[string]$PatchDirectory
|
||||
)
|
||||
|
||||
$targetSets = @(
|
||||
@("a", "i", "u", "e", "o"),
|
||||
@("m", "n", "ng", "l", "r"),
|
||||
@("s", "z", "f", "v", "th"),
|
||||
@("p", "b", "t", "d", "k"),
|
||||
@("mix-a", "mix-m", "mix-s", "mix-p", "mix-u")
|
||||
)
|
||||
$files = @(Get-ChildItem -LiteralPath $PatchDirectory -Filter "*.aqua" -File -Recurse)
|
||||
if ($files.Count -lt 25) {
|
||||
throw "Hypothesis worker wrote $($files.Count) .aqua candidates; expected at least 25."
|
||||
}
|
||||
|
||||
$stems = $files | ForEach-Object { [IO.Path]::GetFileNameWithoutExtension($_.Name) }
|
||||
foreach ($targetSet in $targetSets) {
|
||||
foreach ($target in $targetSet) {
|
||||
$covered = $false
|
||||
foreach ($stem in $stems) {
|
||||
if ($stem.StartsWith("$target`__", [StringComparison]::OrdinalIgnoreCase)) {
|
||||
$covered = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $covered) {
|
||||
throw "Hypothesis worker did not write a candidate for target '$target'. Candidate files must be named <targetId>__<hypothesis-name>.aqua."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$repoRoot = Resolve-RepoRoot
|
||||
$codex = Get-Command $CodexCommand -ErrorAction Stop
|
||||
$loopId = New-Timestamp
|
||||
$loopRoot = Join-Path $repoRoot "artifacts/parity/ipa-trial-loops/$loopId"
|
||||
$logRoot = Join-Path $loopRoot "logs"
|
||||
New-Item -ItemType Directory -Force -Path $loopRoot, $logRoot | Out-Null
|
||||
$storePath = Join-Path $loopRoot "ipa-trial-results.cc"
|
||||
$workerProject = Join-Path $repoRoot "tools/IpaTrialWorker/IpaTrialWorker.csproj"
|
||||
|
||||
$indexPath = Join-Path $loopRoot "loop-index.md"
|
||||
Set-Content -LiteralPath $indexPath -Value @"
|
||||
# IPA Trial Loop $loopId
|
||||
|
||||
- repo: $repoRoot
|
||||
- codex: $($codex.Source)
|
||||
- rounds: $Rounds
|
||||
- batch_id: $BatchId
|
||||
- allow_source_edits: $AllowCodexSourceEdits
|
||||
- trial_results_store: $storePath
|
||||
- worker: $workerProject
|
||||
- trial_shape: five rounds, each asking for five five-phoneme target-set candidates
|
||||
|
||||
"@
|
||||
|
||||
if (-not $SkipLocalTrialRun) {
|
||||
$seedLog = Join-Path $logRoot "seed-trial-run.log"
|
||||
Invoke-LoggedProcess `
|
||||
-FilePath "dotnet" `
|
||||
-ArgumentList @(
|
||||
"run",
|
||||
"--project",
|
||||
$workerProject,
|
||||
"--",
|
||||
"seed",
|
||||
"--artifact-root",
|
||||
$loopRoot,
|
||||
"--batch-id",
|
||||
$BatchId,
|
||||
"--store",
|
||||
$storePath
|
||||
) `
|
||||
-LogPath $seedLog
|
||||
}
|
||||
else {
|
||||
Add-Content -LiteralPath $indexPath -Value "- seed: skipped"
|
||||
}
|
||||
|
||||
for ($round = 1; $round -le $Rounds; $round++) {
|
||||
$roundId = "round-{0:000}" -f $round
|
||||
$roundDir = Join-Path $loopRoot $roundId
|
||||
$patchDir = Join-Path $roundDir "proposed-patches"
|
||||
New-Item -ItemType Directory -Force -Path $roundDir | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path $patchDir | Out-Null
|
||||
Add-Content -LiteralPath $indexPath -Value "- ${roundId}: $roundDir"
|
||||
|
||||
$preEvidence = Join-Path $roundDir "semantic-search-before.md"
|
||||
$preDumpLog = Join-Path $logRoot "$roundId-search-before.log"
|
||||
Invoke-LoggedProcess `
|
||||
-FilePath "dotnet" `
|
||||
-ArgumentList @(
|
||||
"run",
|
||||
"--project",
|
||||
$workerProject,
|
||||
"--",
|
||||
"search",
|
||||
"--store",
|
||||
$storePath,
|
||||
"--query",
|
||||
"weak promising generalization vowel nasal fricative stop articulation owner source tract radiation gesture primitive dressing",
|
||||
"--limit",
|
||||
"40",
|
||||
"--output",
|
||||
$preEvidence
|
||||
) `
|
||||
-LogPath $preDumpLog
|
||||
|
||||
$hypothesisOutput = Join-Path $roundDir "hypothesis-agent.md"
|
||||
$hypothesisLog = Join-Path $logRoot "$roundId-hypothesis-agent.log"
|
||||
$hypothesisPrompt = @"
|
||||
You are an AquaSynth IPA loss-landscape hypothesis and patch-writing worker.
|
||||
|
||||
Read this semantic search result from the CultCache trial-results database:
|
||||
$preEvidence
|
||||
|
||||
The backing store is:
|
||||
$storePath
|
||||
|
||||
Use the worker as your search organ. Do not ask for crude full dumps unless retrieval fails:
|
||||
dotnet run --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- search --store "$storePath" --query "<your evidence question>" --limit 20 --output "$roundDir/search-<topic>.md"
|
||||
dotnet run --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- show --store "$storePath" --trial-id "<trial id from search>" --output "$roundDir/detail-<candidate>.md"
|
||||
|
||||
Useful repo surfaces:
|
||||
- docs/ipa-vocal-tract-roadmap.md
|
||||
- src/AquaSynth.Core/IpaGestureExperiment.cs
|
||||
- src/AquaSynth.Core/FaustExport.cs
|
||||
- src/AquaSynth.Faust/IpaTrialOrchestrator.cs
|
||||
- artifacts under $loopRoot
|
||||
|
||||
Task:
|
||||
1. Read the current loss landscape across gesture, log-mel, RMS, articulation, and primitive timeline evidence.
|
||||
2. Invent the next candidate patches across five five-phoneme target sets. Cover at least one target from each set, and prefer all 25 when the evidence points to reusable changes:
|
||||
vowels: a, i, u, e, o
|
||||
nasals/approximants: m, n, ng, l, r
|
||||
fricatives: s, z, f, v, th
|
||||
stops: p, b, t, d, k
|
||||
mixed generalization: mix-a, mix-m, mix-s, mix-p, mix-u
|
||||
3. Write at least twenty-five new .aqua candidates under:
|
||||
$patchDir
|
||||
Name each file <targetId>__<hypothesis-name>.aqua so the scorer can infer the reference target.
|
||||
4. Make the candidates test transferable hypotheses, not one-off phoneme golf. Reuse the same contour idea across multiple places/manners when that is the claim being tested.
|
||||
5. If source edits are allowed, refine the DSL or lowering code only when the evidence says the graph owner is wrong rather than the patch values.
|
||||
6. Write a concise hypothesis report to:
|
||||
$roundDir/hypotheses.md
|
||||
|
||||
Authority boundary:
|
||||
- Source edits allowed: $AllowCodexSourceEdits
|
||||
- Do not rewrite existing trial artifacts or the CultCache store directly.
|
||||
- The worker process owns measurement and .cc writes; you own candidate invention and optional source refinements.
|
||||
- Keep gesture_score, clean vocal/audio evidence, and full parity dressing separate.
|
||||
- Patch candidates may use FM, AM, modulators, envelopes, and extra animated voices, but call out when dressing is compensating for failed articulation.
|
||||
- Prefer fixing primitive owners over adding audio dressing when the evidence says air is missing.
|
||||
|
||||
Return a short final summary naming the files you wrote.
|
||||
"@
|
||||
Invoke-CodexAgent `
|
||||
-RepoRoot $repoRoot `
|
||||
-Prompt $hypothesisPrompt `
|
||||
-OutputPath $hypothesisOutput `
|
||||
-LogPath $hypothesisLog `
|
||||
-CodexCommand $CodexCommand `
|
||||
-CodexModel $CodexModel `
|
||||
-AllowSourceEdits:$AllowCodexSourceEdits
|
||||
|
||||
Assert-CandidateBatch -PatchDirectory $patchDir
|
||||
|
||||
$scoreLog = Join-Path $logRoot "$roundId-score-candidates.log"
|
||||
Invoke-LoggedProcess `
|
||||
-FilePath "dotnet" `
|
||||
-ArgumentList @(
|
||||
"run",
|
||||
"--project",
|
||||
$workerProject,
|
||||
"--",
|
||||
"score",
|
||||
"--patch-root",
|
||||
$patchDir,
|
||||
"--artifact-root",
|
||||
$roundDir,
|
||||
"--batch-id",
|
||||
$roundId,
|
||||
"--store",
|
||||
$storePath,
|
||||
"--hypothesizer",
|
||||
"external-codex-$roundId"
|
||||
) `
|
||||
-LogPath $scoreLog
|
||||
|
||||
$postEvidence = Join-Path $roundDir "semantic-search-after.md"
|
||||
$postDumpLog = Join-Path $logRoot "$roundId-search-after.log"
|
||||
Invoke-LoggedProcess `
|
||||
-FilePath "dotnet" `
|
||||
-ArgumentList @(
|
||||
"run",
|
||||
"--project",
|
||||
$workerProject,
|
||||
"--",
|
||||
"search",
|
||||
"--store",
|
||||
$storePath,
|
||||
"--query",
|
||||
"latest round weak promising transfer generalization vowel nasal fricative stop articulation owner source tract radiation gesture primitive dressing",
|
||||
"--limit",
|
||||
"60",
|
||||
"--output",
|
||||
$postEvidence
|
||||
) `
|
||||
-LogPath $postDumpLog
|
||||
|
||||
$evaluatorOutput = Join-Path $roundDir "evaluator-agent.md"
|
||||
$evaluatorLog = Join-Path $logRoot "$roundId-evaluator-agent.log"
|
||||
$evaluatorPrompt = @"
|
||||
You are an AquaSynth IPA trial science evaluator.
|
||||
|
||||
Read the updated semantic retrieval from the trial-results database:
|
||||
$postEvidence
|
||||
|
||||
Also read the hypothesis worker output:
|
||||
$hypothesisOutput
|
||||
|
||||
Round artifacts are under:
|
||||
$roundDir
|
||||
|
||||
Task:
|
||||
1. Evaluate whether the worker's proposed hypotheses follow from the measured evidence.
|
||||
2. Call out any hypothesis that is likely full-patch dressing impersonating articulation.
|
||||
3. Recommend exactly one next implementation/refinement target, with file/module owner and why.
|
||||
4. Write your evaluator report to:
|
||||
$roundDir/evaluation.md
|
||||
|
||||
Authority boundary:
|
||||
- Do not edit source files.
|
||||
- Do not rewrite trial artifacts.
|
||||
- Be explicit about which layer owns the failure: gesture DSL, primitive source, tract/radiation lowering, audio scoring, or orchestration.
|
||||
- Treat the CultCache .cc store as the shared trial memory that future hypothesis workers will query through the worker search/show commands.
|
||||
|
||||
Return a short final summary naming the report you wrote.
|
||||
"@
|
||||
Invoke-CodexAgent `
|
||||
-RepoRoot $repoRoot `
|
||||
-Prompt $evaluatorPrompt `
|
||||
-OutputPath $evaluatorOutput `
|
||||
-LogPath $evaluatorLog `
|
||||
-CodexCommand $CodexCommand `
|
||||
-CodexModel $CodexModel `
|
||||
-AllowSourceEdits:$false
|
||||
}
|
||||
|
||||
Write-Host "IPA trial loop complete: $loopRoot"
|
||||
Loading…
Reference in a new issue