Add IPA trial render orchestration
This commit is contained in:
Meta Vens
parent
62178531f9
commit
05cf7d36d6
11 changed files with 683 additions and 11 deletions
|
|
@ -629,6 +629,37 @@ The intended scale-out loop is brutally simple:
|
|||
`gesture_score` first, then `clean_vocal_score`, then
|
||||
`full_parity_score`.
|
||||
|
||||
### IPA Trial Render Workers
|
||||
|
||||
`IpaTrialOrchestrator.RunAsync` is the first local render/scoring worker for
|
||||
that loop. It takes trial target sets, asks `IpaGestureExperiment` to generate
|
||||
candidate patches, renders each candidate through Faust, compares it against a
|
||||
local PT fixture reference with `AudioAnalyzer`, writes WAV/report artifacts,
|
||||
and stores typed `IpaTrialResult` records in `ipa-trial-results.cc`.
|
||||
|
||||
The `.cc` store is per-record CultCache MessagePack data, not a hand-written
|
||||
JSON ledger. `IpaTrialResult` records include hypothesis text, candidate patch
|
||||
URI, reference/candidate artifacts, primitive timeline URI, metrics, evaluator
|
||||
summary, verdict, known contamination, and timing receipts. This is the trial
|
||||
result database that later hypothesis workers consume.
|
||||
|
||||
The first five seed trials were run locally on 2026-05-29 under
|
||||
`artifacts/parity/ipa-trials/20260529T214127955/five-seed-trials`. The first
|
||||
attempt proved the pipeline but showed vowels/nasals were nearly silent. The
|
||||
accepted refinement moved voiced source excitation into primitive `SourcePort`
|
||||
lowering instead of relying on radiation to high-pass DC flow. After that cut:
|
||||
|
||||
- open `a`: log-mel cosine `0.5693`, RMS ratio `0.3033`;
|
||||
- front `i`: log-mel cosine `0.3536`, RMS ratio `0.5721`;
|
||||
- bilabial nasal `m`: log-mel cosine `0.5199`, RMS ratio `0.2688`;
|
||||
- alveolar fricative `s`: log-mel cosine `0.5498`, articulation `0.4111`;
|
||||
- bilabial plosive `p`: still weak, log-mel cosine `-0.1024` despite RMS ratio
|
||||
`0.6481`.
|
||||
|
||||
The evidence says the source-carrier cut fixed the voiced-air failure. The next
|
||||
pressure is plosive closure/release ownership and stronger vowel/nasal tract
|
||||
color, not decorative full-patch dressing.
|
||||
|
||||
## CultMesh Render/Scoring Work
|
||||
|
||||
CultMesh is the transport, admission, and worker-orchestration layer for speech
|
||||
|
|
|
|||
|
|
@ -699,10 +699,13 @@ public static class FaustEmitter
|
|||
var impedance = parameters.Expression(OwnerField(owner, "impedance"), port.Impedance);
|
||||
var flowScale = parameters.Expression(OwnerField(owner, "flow_scale"), port.FlowScale);
|
||||
var load = $"{voiceName}_primitive_source_{safe}_load_pressure";
|
||||
var voiced = $"{voiceName}_primitive_source_{safe}_voiced";
|
||||
var flow = $"{voiceName}_primitive_source_{safe}_flow";
|
||||
source.AppendLine($" {load} = {ProbeSignal(options, $"/debug/{voiceName}/source/{port.Name}/load_pressure", $"({impedance}) * ({pressure})", 0, 2)};");
|
||||
source.AppendLine($" {flow} = {ProbeSignal(options, $"/debug/{voiceName}/source/{port.Name}/flow", $"({flowScale}) * ma.tanh((({pressure}) * max(0.0, {opening}) * (0.5 + {tension}) + 0.08 * no.noise * {parameters.Expression(OwnerField(owner, "noise"), port.Noise)}) / max(0.05, {impedance}))", -2, 2)};");
|
||||
source.AppendLine($" {voiced} = os.osc({frequency}) * clip01({opening}) * clip01(0.25 + {tension});");
|
||||
source.AppendLine($" {flow} = {ProbeSignal(options, $"/debug/{voiceName}/source/{port.Name}/flow", $"({flowScale}) * (({voiced}) * ma.tanh((({pressure}) * (0.5 + {tension})) / max(0.05, {impedance})) + 0.08 * no.noise * {parameters.Expression(OwnerField(owner, "noise"), port.Noise)})", -2, 2)};");
|
||||
keepAlive.Add(load);
|
||||
keepAlive.Add(voiced);
|
||||
keepAlive.Add(flow);
|
||||
return flow;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -201,19 +201,19 @@ public static class IpaGestureExperiment
|
|||
var frequency = DescriptorTokens(target.Descriptor).Contains("vowel") ? 150 : 135;
|
||||
|
||||
return $$"""
|
||||
patch gain=.22
|
||||
patch gain=.45
|
||||
morphology name=oral length_cm=17 diameters=.55,.75,1.05,1.45,1.25,.8 tongue_index=3 tongue_diameter=1.4 constriction_index=4 constriction_diameter=1 lip_opening=1.2
|
||||
morphology name=nasal length_cm=12 diameters=.04,.28,.52,.72
|
||||
waveguide_path name=oral_path morphology=oral strategy=thiran order=1 max_delay=4096 loss=.998
|
||||
waveguide_path name=nasal_path morphology=nasal strategy=thiran order=1 max_delay=4096 loss=.997
|
||||
source_port name=folds path=oral_path kind=glottal position=0 pressure=.66 tension=.54 opening=.42 noise=.04 impedance=.32 flow_scale=.02
|
||||
source_port name=folds path=oral_path kind=glottal position=0 pressure=.66 tension=.54 opening=.42 noise=.04 impedance=.32 flow_scale=.12
|
||||
branch_port name=velopharynx from=oral_path from_position=.45 to=nasal_path opening=.01 coupling=1
|
||||
constriction_contact name=contact path=oral_path position=.92 opening=.5 resistance=.45 stored_pressure=.12
|
||||
radiation_load name=mouth path=oral_path kind=lip position=1 aperture=.8 reflection=-.82 impedance=.28
|
||||
probe_timeline name=flow network=voice blocks=12 block_size=64
|
||||
vocal_network name=voice paths=oral_path,nasal_path sources=folds contacts=contact branches=velopharynx radiation=mouth probe=flow
|
||||
phoneme_gesture name={{SafeDslName(target.Id)}} ipa={{target.Ipa}} descriptor={{descriptor}} start=0 dur={{Format(duration)}} intensity={{Format(intensity)}}
|
||||
vocal network=voice freq={{Format(frequency)}} gain=.2 sustain={{Format(sustain)}}
|
||||
vocal network=voice freq={{Format(frequency)}} gain=.7 sustain={{Format(sustain)}}
|
||||
""";
|
||||
}
|
||||
|
||||
|
|
|
|||
73
src/AquaSynth.Core/IpaTrialResults.cs
Normal file
73
src/AquaSynth.Core/IpaTrialResults.cs
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
using GameCult.Caching;
|
||||
using GameCult.Caching.MessagePack;
|
||||
using MessagePack;
|
||||
|
||||
namespace AquaSynth.Dsl;
|
||||
|
||||
[CultDocument("aquasynth.ipa_trial_result", "aquasynth.ipa_trial_result.v1")]
|
||||
[MessagePackObject]
|
||||
public sealed record IpaTrialResult(
|
||||
[property: Key(0)]
|
||||
[property: CultName]
|
||||
string TrialId,
|
||||
[property: Key(1)] string BatchId,
|
||||
[property: Key(2)] string CreatedAtUtc,
|
||||
[property: Key(3)] string TargetSetId,
|
||||
[property: Key(4)] string[] Phonemes,
|
||||
[property: Key(5)] string ReferenceId,
|
||||
[property: Key(6)] string CandidateId,
|
||||
[property: Key(7)] string HypothesizerId,
|
||||
[property: Key(8)] string Hypothesis,
|
||||
[property: Key(9)] string CandidatePatchUri,
|
||||
[property: Key(10)] string ReferenceArtifactUri,
|
||||
[property: Key(11)] string CandidateArtifactUri,
|
||||
[property: Key(12)] string PrimitiveTimelineUri,
|
||||
[property: Key(13)] SpeechScoreMetric[] Metrics,
|
||||
[property: Key(14)] SpeechRenderArtifact[] Artifacts,
|
||||
[property: Key(15)] string EvaluatorId,
|
||||
[property: Key(16)] string EvaluationSummary,
|
||||
[property: Key(17)] string Verdict,
|
||||
[property: Key(18)] string[] KnownLies,
|
||||
[property: Key(19)] SpeechTimingReceipt[] TimingReceipts);
|
||||
|
||||
public static class IpaTrialResultCultCacheStore
|
||||
{
|
||||
public static Task UpsertResultsAsync(string filePath, IEnumerable<IpaTrialResult> results) =>
|
||||
UpsertAsync(filePath, results, result => $"ipa-trial-result:{result.TrialId}");
|
||||
|
||||
public static async Task<IReadOnlyList<IpaTrialResult>> ReadResultsAsync(string filePath)
|
||||
{
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false);
|
||||
return cache.GetAll<IpaTrialResult>()
|
||||
.OrderBy(result => result.TrialId, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static async Task UpsertAsync<T>(
|
||||
string filePath,
|
||||
IEnumerable<T> documents,
|
||||
Func<T, string> key)
|
||||
where T : class
|
||||
{
|
||||
var directory = Path.GetDirectoryName(filePath);
|
||||
if (!string.IsNullOrWhiteSpace(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
using var cache = await CultCacheMessagePack.OpenAsync(
|
||||
filePath,
|
||||
new CultCacheOpenOptions { PullOnOpen = File.Exists(filePath) }).ConfigureAwait(false);
|
||||
foreach (var document in documents)
|
||||
{
|
||||
await cache.UpsertAsync(document, new CultRecordHandle<T>(new CultRecordKey(key(document)))).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await cache.FlushAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
432
src/AquaSynth.Faust/IpaTrialOrchestrator.cs
Normal file
432
src/AquaSynth.Faust/IpaTrialOrchestrator.cs
Normal file
|
|
@ -0,0 +1,432 @@
|
|||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
using AquaSynth.Dsl;
|
||||
|
||||
namespace AquaSynth.Faust;
|
||||
|
||||
public sealed record IpaTrialReferenceTarget(
|
||||
IpaGestureExperimentTarget Target,
|
||||
string ReferenceFixtureId);
|
||||
|
||||
public sealed record IpaTrialTargetSet(
|
||||
string Id,
|
||||
string Description,
|
||||
IReadOnlyList<IpaTrialReferenceTarget> Targets,
|
||||
IReadOnlyList<IpaGestureExperimentVariant> Variants)
|
||||
{
|
||||
public IReadOnlyList<IpaTrialReferenceTarget> Targets { get; init; } = Targets ?? Array.Empty<IpaTrialReferenceTarget>();
|
||||
public IReadOnlyList<IpaGestureExperimentVariant> Variants { get; init; } = Variants ?? Array.Empty<IpaGestureExperimentVariant>();
|
||||
}
|
||||
|
||||
public sealed record IpaTrialOrchestrationOptions(
|
||||
string BatchId = "ipa-seed-trials",
|
||||
string HypothesizerId = "local-hypothesis-worker",
|
||||
string EvaluatorId = "local-science-evaluator",
|
||||
int TimelineBlocks = 12,
|
||||
int SampleRate = 44100,
|
||||
string RendererToolchain = "local-faust-render",
|
||||
string ReferenceRenderer = "pink-trombone-local-fixture",
|
||||
string TrialResultStoreFileName = "ipa-trial-results.cc");
|
||||
|
||||
public sealed record IpaTrialOrchestrationResult(
|
||||
string BatchId,
|
||||
string ArtifactDirectory,
|
||||
string TrialResultStorePath,
|
||||
string SummaryPath,
|
||||
string EvaluatorReportPath,
|
||||
IReadOnlyList<IpaTrialResult> TrialResults)
|
||||
{
|
||||
public IReadOnlyList<IpaTrialResult> TrialResults { get; init; } = TrialResults ?? Array.Empty<IpaTrialResult>();
|
||||
}
|
||||
|
||||
public static class IpaTrialOrchestrator
|
||||
{
|
||||
public static IReadOnlyList<IpaTrialTargetSet> DefaultFiveSeedTrialSets { get; } =
|
||||
[
|
||||
new(
|
||||
"trial-001-vowels",
|
||||
"Open and front vowel hypotheses against static tract references.",
|
||||
[
|
||||
Target("a", "a", "voiced_open_back_unrounded_vowel", "open-vowel", .20f),
|
||||
Target("i", "i", "voiced_close_front_unrounded_vowel", "front-vowel", .20f)
|
||||
],
|
||||
[Variant("baseline", 1, 1, "vowel-seed")]),
|
||||
new(
|
||||
"trial-002-nasal",
|
||||
"Bilabial nasal velum/contact hypothesis against the ma pressure fixture.",
|
||||
[Target("m", "m", "voiced_bilabial_nasal", "bilabial-nasal-ma", .18f)],
|
||||
[Variant("nasal-base", 1, 1, "nasal-seed")]),
|
||||
new(
|
||||
"trial-003-sibilant",
|
||||
"Voiceless alveolar fricative hypothesis against the high-turbulence sibilant fixture.",
|
||||
[Target("s", "s", "voiceless_alveolar_fricative", "sibilant", .16f)],
|
||||
[Variant("hiss-base", 1.15f, .9f, "fricative-seed")]),
|
||||
new(
|
||||
"trial-004-plosive",
|
||||
"Bilabial stop closure/release hypothesis against obstruction-history pressure.",
|
||||
[Target("p", "p", "voiceless_bilabial_plosive", "closure-release", .14f)],
|
||||
[Variant("stop-base", 1.2f, .85f, "plosive-seed")]),
|
||||
new(
|
||||
"trial-005-mixed",
|
||||
"Small mixed set used to check whether the same DSL abstractions separate vowel, nasal, and fricative pressure.",
|
||||
[
|
||||
Target("mix-a", "a", "voiced_open_back_unrounded_vowel", "open-vowel", .18f),
|
||||
Target("mix-m", "m", "voiced_bilabial_nasal", "bilabial-nasal-ma", .16f),
|
||||
Target("mix-s", "s", "voiceless_alveolar_fricative", "sibilant", .14f)
|
||||
],
|
||||
[Variant("mixed-base", 1, 1, "mixed-seed")])
|
||||
];
|
||||
|
||||
public static async Task<IpaTrialOrchestrationResult> RunAsync(
|
||||
string artifactRoot,
|
||||
IReadOnlyList<IpaTrialTargetSet>? trialSets = null,
|
||||
IpaTrialOrchestrationOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactRoot);
|
||||
options ??= new IpaTrialOrchestrationOptions();
|
||||
trialSets ??= DefaultFiveSeedTrialSets;
|
||||
if (trialSets.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one IPA trial target set is required.", nameof(trialSets));
|
||||
}
|
||||
|
||||
var batchDirectory = Path.Combine(artifactRoot, options.BatchId);
|
||||
Directory.CreateDirectory(batchDirectory);
|
||||
var storePath = Path.Combine(batchDirectory, options.TrialResultStoreFileName);
|
||||
var referenceRenderer = new PinkTromboneReferenceRenderer();
|
||||
var analyzer = new AudioAnalyzer(new AudioAnalysisConfig(SampleRate: options.SampleRate));
|
||||
var results = new List<IpaTrialResult>();
|
||||
|
||||
foreach (var targetSet in trialSets)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var round = IpaGestureExperiment.WriteRound(
|
||||
batchDirectory,
|
||||
targetSet.Id,
|
||||
targetSet.Targets.Select(target => target.Target),
|
||||
targetSet.Variants,
|
||||
options.TimelineBlocks);
|
||||
var analysis = IpaGestureExperiment.AnalyzeRound(round);
|
||||
var targetById = targetSet.Targets.ToDictionary(target => target.Target.Id, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var candidate in round.Candidates)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var startedAt = DateTimeOffset.UtcNow;
|
||||
var startedStamp = Stopwatch.GetTimestamp();
|
||||
if (!targetById.TryGetValue(candidate.TargetId, out var target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var candidateDir = Path.Combine(round.RoundDirectory, "audio", candidate.Id);
|
||||
Directory.CreateDirectory(candidateDir);
|
||||
var script = await File.ReadAllTextAsync(candidate.ScriptPath, cancellationToken).ConfigureAwait(false);
|
||||
var source = FaustEmitter.EmitScript(script, new FaustExportOptions(SafeName(candidate.Id))).Source;
|
||||
var dspPath = Path.Combine(candidateDir, "candidate.dsp");
|
||||
await File.WriteAllTextAsync(dspPath, source, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var referenceFixture = PinkTromboneParityFixtures.ById(target.ReferenceFixtureId);
|
||||
var reference = referenceRenderer.Render(referenceFixture.Controls, target.Target.DurationSeconds + .18f);
|
||||
var candidateRender = await FaustCompiler.RenderAsync(
|
||||
source,
|
||||
new FaustRenderOptions(reference.SampleRate, reference.Samples.Length / (float)reference.SampleRate),
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var referenceWav = Path.Combine(candidateDir, "reference.wav");
|
||||
var candidateWav = Path.Combine(candidateDir, "candidate.wav");
|
||||
WriteWav(referenceWav, reference.Samples, reference.SampleRate);
|
||||
var metrics = new List<SpeechScoreMetric>();
|
||||
string verdict;
|
||||
string evaluation;
|
||||
var artifacts = new List<SpeechRenderArtifact>
|
||||
{
|
||||
Artifact("candidate-script", candidate.ScriptPath),
|
||||
Artifact("candidate-dsp", dspPath),
|
||||
Artifact("primitive-timeline", candidate.TimelinePath),
|
||||
Artifact("reference-wav", referenceWav),
|
||||
Artifact("round-science-brief", analysis.ScienceBriefPath)
|
||||
};
|
||||
|
||||
if (candidateRender is null || candidateRender.Samples.Length == 0)
|
||||
{
|
||||
verdict = "render-failed";
|
||||
evaluation = candidateRender is null
|
||||
? "Faust was not available to render this candidate."
|
||||
: $"Faust render produced no samples. stderr: {candidateRender.Stderr}";
|
||||
metrics.Add(new SpeechScoreMetric("render_failed", 1, 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteWav(candidateWav, candidateRender.Samples, candidateRender.SampleRate);
|
||||
artifacts.Add(Artifact("candidate-wav", candidateWav));
|
||||
var comparison = analyzer.Compare(reference.Samples, candidateRender.Samples);
|
||||
metrics.AddRange(Metrics(candidate, round, comparison));
|
||||
verdict = Verdict(comparison);
|
||||
evaluation = EvaluationSentence(target.Target.Ipa, target.ReferenceFixtureId, comparison, verdict);
|
||||
await WriteComparisonReportAsync(
|
||||
Path.Combine(candidateDir, "comparison.txt"),
|
||||
target,
|
||||
candidate,
|
||||
comparison,
|
||||
verdict,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
artifacts.Add(Artifact("comparison-report", Path.Combine(candidateDir, "comparison.txt")));
|
||||
}
|
||||
|
||||
var latency = Stopwatch.GetElapsedTime(startedStamp).TotalMilliseconds;
|
||||
results.Add(new IpaTrialResult(
|
||||
$"{targetSet.Id}:{candidate.Id}",
|
||||
options.BatchId,
|
||||
startedAt.ToString("O", CultureInfo.InvariantCulture),
|
||||
targetSet.Id,
|
||||
targetSet.Targets.Select(item => item.Target.Ipa).Distinct(StringComparer.Ordinal).ToArray(),
|
||||
target.ReferenceFixtureId,
|
||||
candidate.Id,
|
||||
options.HypothesizerId,
|
||||
Hypothesis(targetSet, target, candidate),
|
||||
candidate.ScriptPath,
|
||||
referenceWav,
|
||||
File.Exists(candidateWav) ? candidateWav : "",
|
||||
candidate.TimelinePath,
|
||||
metrics.ToArray(),
|
||||
artifacts.ToArray(),
|
||||
options.EvaluatorId,
|
||||
evaluation,
|
||||
verdict,
|
||||
KnownLies(options, target.ReferenceFixtureId),
|
||||
[
|
||||
new SpeechTimingReceipt(
|
||||
"ipa-trial-render-score",
|
||||
startedAt.ToString("O", CultureInfo.InvariantCulture),
|
||||
latency,
|
||||
0,
|
||||
Confidence(metrics),
|
||||
"Local worker generated the candidate patch, rendered Faust audio, compared log-mel/articulation evidence, and wrote CultCache trial result data.")
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
await IpaTrialResultCultCacheStore.UpsertResultsAsync(storePath, results).ConfigureAwait(false);
|
||||
var summaryPath = Path.Combine(batchDirectory, "summary.csv");
|
||||
await File.WriteAllTextAsync(summaryPath, SummaryCsv(results), cancellationToken).ConfigureAwait(false);
|
||||
var evaluatorReport = Path.Combine(batchDirectory, "evaluator-report.md");
|
||||
await File.WriteAllTextAsync(evaluatorReport, EvaluatorReport(options, results), cancellationToken).ConfigureAwait(false);
|
||||
await IpaTrialResultCultCacheStore.UpsertResultsAsync(storePath, results).ConfigureAwait(false);
|
||||
|
||||
return new IpaTrialOrchestrationResult(
|
||||
options.BatchId,
|
||||
batchDirectory,
|
||||
storePath,
|
||||
summaryPath,
|
||||
evaluatorReport,
|
||||
results);
|
||||
}
|
||||
|
||||
private static IpaTrialReferenceTarget Target(string id, string ipa, string descriptor, string fixture, float duration) =>
|
||||
new(new IpaGestureExperimentTarget(id, ipa, descriptor, duration), fixture);
|
||||
|
||||
private static IpaGestureExperimentVariant Variant(string id, float intensity, float duration, string tag) =>
|
||||
new(id, intensity, duration, [tag]);
|
||||
|
||||
private static IReadOnlyList<SpeechScoreMetric> Metrics(
|
||||
IpaGestureExperimentCandidate candidate,
|
||||
IpaGestureExperimentResult round,
|
||||
AudioComparison comparison)
|
||||
{
|
||||
var gesture = round.Metrics.FirstOrDefault(metric =>
|
||||
metric.CandidateId.Equals(candidate.Id, StringComparison.OrdinalIgnoreCase) &&
|
||||
metric.Metric.Equals("gesture_score", StringComparison.OrdinalIgnoreCase))?.Value ?? 0;
|
||||
return
|
||||
[
|
||||
new("gesture_score", gesture, .25f),
|
||||
new("log_mel_cosine", comparison.LogMelCosineSimilarity, .25f),
|
||||
new("log_mel_distance", comparison.LogMelDistance, .15f),
|
||||
new("audio_score", comparison.Score, .15f),
|
||||
new("articulation_score", comparison.Articulation.ArticulationScore, .10f),
|
||||
new("rms_ratio", comparison.RmsRatio, .05f),
|
||||
new("speech_band_ratio", comparison.Articulation.SpeechBandRatio, .05f)
|
||||
];
|
||||
}
|
||||
|
||||
private static string Verdict(AudioComparison comparison)
|
||||
{
|
||||
if (comparison.LogMelCosineSimilarity >= .70f && comparison.Articulation.ArticulationScore >= .45f)
|
||||
{
|
||||
return "promising";
|
||||
}
|
||||
|
||||
if (comparison.LogMelCosineSimilarity >= .35f)
|
||||
{
|
||||
return "pressure";
|
||||
}
|
||||
|
||||
return "weak";
|
||||
}
|
||||
|
||||
private static string EvaluationSentence(string ipa, string referenceId, AudioComparison comparison, string verdict) =>
|
||||
$"IPA `{ipa}` against `{referenceId}` is `{verdict}`: logMelCosine={comparison.LogMelCosineSimilarity:0.0000}, logMelDistance={comparison.LogMelDistance:0.0000}, articulation={comparison.Articulation.ArticulationScore:0.0000}, rmsRatio={comparison.RmsRatio:0.0000}.";
|
||||
|
||||
private static string Hypothesis(IpaTrialTargetSet set, IpaTrialReferenceTarget target, IpaGestureExperimentCandidate candidate) =>
|
||||
$"{set.Description} Candidate `{candidate.Id}` tests whether descriptor `{target.Target.Descriptor}` drives the public anatomical control surfaces enough to approach `{target.ReferenceFixtureId}` spectrogram pressure.";
|
||||
|
||||
private static string[] KnownLies(IpaTrialOrchestrationOptions options, string referenceId) =>
|
||||
[
|
||||
$"Ground truth is `{referenceId}` from {options.ReferenceRenderer}, not an open IPA recording dataset.",
|
||||
"Log-mel evidence is normalized shape pressure and must be read with RMS/articulation witnesses.",
|
||||
"The first seed trials render one candidate per hypothesis set; they explore the orchestration surface before optimizer scale-out."
|
||||
];
|
||||
|
||||
private static float Confidence(IReadOnlyList<SpeechScoreMetric> metrics)
|
||||
{
|
||||
var cosine = metrics.FirstOrDefault(metric => metric.Name == "log_mel_cosine")?.Value ?? 0;
|
||||
var gesture = metrics.FirstOrDefault(metric => metric.Name == "gesture_score")?.Value ?? 0;
|
||||
return Math.Clamp((cosine + gesture) * .5f, 0, 1);
|
||||
}
|
||||
|
||||
private static SpeechRenderArtifact Artifact(string kind, string path) =>
|
||||
new(kind, path, File.Exists(path) ? Sha256(path) : "");
|
||||
|
||||
private static string Sha256(string path)
|
||||
{
|
||||
using var stream = File.OpenRead(path);
|
||||
var hash = SHA256.HashData(stream);
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static async Task WriteComparisonReportAsync(
|
||||
string path,
|
||||
IpaTrialReferenceTarget target,
|
||||
IpaGestureExperimentCandidate candidate,
|
||||
AudioComparison comparison,
|
||||
string verdict,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var text =
|
||||
$"candidate={candidate.Id}{Environment.NewLine}" +
|
||||
$"ipa={target.Target.Ipa}{Environment.NewLine}" +
|
||||
$"descriptor={target.Target.Descriptor}{Environment.NewLine}" +
|
||||
$"reference={target.ReferenceFixtureId}{Environment.NewLine}" +
|
||||
$"verdict={verdict}{Environment.NewLine}" +
|
||||
$"logMelCosine={comparison.LogMelCosineSimilarity:0.######}{Environment.NewLine}" +
|
||||
$"logMelDistance={comparison.LogMelDistance:0.######}{Environment.NewLine}" +
|
||||
$"score={comparison.Score:0.######}{Environment.NewLine}" +
|
||||
$"rmsRatio={comparison.RmsRatio:0.######}{Environment.NewLine}" +
|
||||
$"centroidRatio={comparison.CentroidRatio:0.######}{Environment.NewLine}" +
|
||||
$"articulation={comparison.Articulation.ArticulationScore:0.######}{Environment.NewLine}" +
|
||||
$"speechBandRatio={comparison.Articulation.SpeechBandRatio:0.######}{Environment.NewLine}";
|
||||
await File.WriteAllTextAsync(path, text, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string SummaryCsv(IReadOnlyList<IpaTrialResult> results)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine("trial_id,target_set_id,candidate_id,reference_id,verdict,gesture_score,log_mel_cosine,log_mel_distance,audio_score,articulation_score,rms_ratio");
|
||||
foreach (var result in results)
|
||||
{
|
||||
builder.Append(Escape(result.TrialId));
|
||||
builder.Append(',');
|
||||
builder.Append(Escape(result.TargetSetId));
|
||||
builder.Append(',');
|
||||
builder.Append(Escape(result.CandidateId));
|
||||
builder.Append(',');
|
||||
builder.Append(Escape(result.ReferenceId));
|
||||
builder.Append(',');
|
||||
builder.Append(Escape(result.Verdict));
|
||||
builder.Append(',');
|
||||
builder.Append(Metric(result, "gesture_score"));
|
||||
builder.Append(',');
|
||||
builder.Append(Metric(result, "log_mel_cosine"));
|
||||
builder.Append(',');
|
||||
builder.Append(Metric(result, "log_mel_distance"));
|
||||
builder.Append(',');
|
||||
builder.Append(Metric(result, "audio_score"));
|
||||
builder.Append(',');
|
||||
builder.Append(Metric(result, "articulation_score"));
|
||||
builder.Append(',');
|
||||
builder.AppendLine(Metric(result, "rms_ratio"));
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static string EvaluatorReport(IpaTrialOrchestrationOptions options, IReadOnlyList<IpaTrialResult> results)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine($"# IPA Trial Evaluator Report: {options.BatchId}");
|
||||
builder.AppendLine();
|
||||
builder.AppendLine("Authority: this evaluator reads generated candidate patches, primitive timelines, rendered audio comparisons, and CultCache trial records. It does not mutate optimizer checkpoints.");
|
||||
builder.AppendLine();
|
||||
foreach (var group in results.GroupBy(result => result.TargetSetId).OrderBy(group => group.Key, StringComparer.Ordinal))
|
||||
{
|
||||
builder.AppendLine($"## {group.Key}");
|
||||
foreach (var result in group.OrderBy(result => result.TrialId, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append("- ");
|
||||
builder.Append(result.CandidateId);
|
||||
builder.Append(": ");
|
||||
builder.Append(result.EvaluationSummary);
|
||||
builder.Append(" Known contamination: ");
|
||||
builder.AppendLine(string.Join(" / ", result.KnownLies));
|
||||
}
|
||||
builder.AppendLine();
|
||||
}
|
||||
|
||||
builder.AppendLine("## Next Hypothesis Pressure");
|
||||
foreach (var result in results
|
||||
.OrderBy(result => float.Parse(Metric(result, "log_mel_cosine"), CultureInfo.InvariantCulture))
|
||||
.ThenBy(result => result.TrialId, StringComparer.Ordinal)
|
||||
.Take(5))
|
||||
{
|
||||
builder.Append("- ");
|
||||
builder.Append(result.TrialId);
|
||||
builder.Append(": inspect ");
|
||||
builder.Append(result.PrimitiveTimelineUri);
|
||||
builder.Append(" and ");
|
||||
builder.Append(result.CandidatePatchUri);
|
||||
builder.AppendLine(" before adding more synthesis dressing.");
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static string Metric(IpaTrialResult result, string name) =>
|
||||
(result.Metrics.FirstOrDefault(metric => metric.Name == name)?.Value ?? 0)
|
||||
.ToString("0.######", CultureInfo.InvariantCulture);
|
||||
|
||||
private static string SafeName(string value) =>
|
||||
new(value.Select(character => char.IsLetterOrDigit(character) ? character : '_').ToArray());
|
||||
|
||||
private static string Escape(string value) =>
|
||||
value.Contains(',') || value.Contains('"') || value.Contains('\n') || value.Contains('\r')
|
||||
? $"\"{value.Replace("\"", "\"\"", StringComparison.Ordinal)}\""
|
||||
: value;
|
||||
|
||||
private static void WriteWav(string path, IReadOnlyList<float> samples, int sampleRate)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
using var stream = File.Create(path);
|
||||
using var writer = new BinaryWriter(stream, Encoding.ASCII);
|
||||
var dataLength = samples.Count * sizeof(short);
|
||||
writer.Write(Encoding.ASCII.GetBytes("RIFF"));
|
||||
writer.Write(36 + dataLength);
|
||||
writer.Write(Encoding.ASCII.GetBytes("WAVEfmt "));
|
||||
writer.Write(16);
|
||||
writer.Write((short)1);
|
||||
writer.Write((short)1);
|
||||
writer.Write(sampleRate);
|
||||
writer.Write(sampleRate * sizeof(short));
|
||||
writer.Write((short)sizeof(short));
|
||||
writer.Write((short)16);
|
||||
writer.Write(Encoding.ASCII.GetBytes("data"));
|
||||
writer.Write(dataLength);
|
||||
foreach (var sample in samples)
|
||||
{
|
||||
writer.Write((short)Math.Clamp(sample * short.MaxValue, short.MinValue, short.MaxValue));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -236,3 +236,4 @@
|
|||
{"date":"2026-05-29","area":"ipa-phoneme-gesture-dsl","summary":"Added deterministic IPA/anatomy gesture sugar. `phoneme_gesture` records a symbol/descriptor/time/intensity and expands known IPA symbols or explicit place/manner/voicing descriptors into ordinary `ControlSpline`s over public source, area, branch, contact, and radiation surfaces. This gives local IPA-chart/audio datasets a seeded anatomical control path while preserving FM/AM/modulators/envelopes/filters as expressive patch tools around the anatomical stream. PatchScriptTests pass 92/92.","evidence":"src/AquaSynth.Core/Model.cs; src/AquaSynth.Core/PatchScript.cs; tests/AquaSynth.Dsl.Tests/PatchScriptTests.cs; docs/acoustic-graph-proprioception.md"}
|
||||
{"date":"2026-05-29","area":"ipa-training-score-contract","summary":"Documented the speech training scoring contract. `gesture_score` measures descriptor-to-surface coverage, direction, spline contour/timing, primitive timeline consequences, and optional external articulation evidence. Training reports must keep `gesture_score`, `clean_vocal_score`, and `full_parity_score` separate. Full parity scripts may use FM/AM/modulators/envelopes/filters, auxiliary animated voices, and recording-condition helpers, but those helpers cannot be accepted as articulation evidence unless lower-layer gesture and clean-vocal scores also improve.","evidence":"docs/ipa-vocal-tract-roadmap.md; docs/utterance-parity-research.md; state/spine.yaml; state/scratch.md"}
|
||||
{"date":"2026-05-29","area":"ipa-gesture-experiment-harness","summary":"Added `IpaGestureExperiment.WriteRound`, the first frozen batch surface for IPA hypothesis work. A round writes deterministic candidate `.aqua` scripts, primitive `ProbeTimelineReport` CSVs, `manifest.yaml`, gesture-layer `metrics.csv`, and per-candidate `evidence.jsonl`. `IpaGestureExperiment.AnalyzeRound` adds metric summaries, score-band candidate clusters, and a science brief for sub-agent analysis. The harness scores descriptor/spline/primitive evidence only and explicitly leaves clean vocal identity, full spectrogram parity, optimizer checkpoints, and worker orchestration downstream. PatchScriptTests pass 93/93.","evidence":"src/AquaSynth.Core/IpaGestureExperiment.cs; tests/AquaSynth.Dsl.Tests/PatchScriptTests.cs; docs/ipa-vocal-tract-roadmap.md; docs/utterance-parity-research.md; state/map.yaml; state/spine.yaml; state/scratch.md"}
|
||||
{"date":"2026-05-29","area":"ipa-trial-render-orchestration","summary":"Added the first local IPA trial worker/evaluator loop. `IpaTrialOrchestrator.RunAsync` generates candidates through `IpaGestureExperiment`, renders them through Faust, compares spectrogram/articulation evidence against local Pink Trombone fixture references, writes WAV/report artifacts, and stores typed `IpaTrialResult` records in `ipa-trial-results.cc`. The first five-seed run proved the pipeline but exposed silent vowel/nasal candidates; the accepted refinement moved voiced excitation into primitive `SourcePort` lowering. The second run at `artifacts/parity/ipa-trials/20260529T214127955/five-seed-trials` improved open `a` to log-mel cosine `0.5693`, `m` to `0.5199`, and `s` to `0.5498`, while `p` stayed weak at `-0.1024`, pointing to closure/release ownership next. Focused verification passes 94/94 plus opt-in IPA trial run.","evidence":"src/AquaSynth.Core/IpaTrialResults.cs; src/AquaSynth.Faust/IpaTrialOrchestrator.cs; src/AquaSynth.Core/FaustExport.cs; src/AquaSynth.Core/IpaGestureExperiment.cs; tests/AquaSynth.Dsl.Tests/IpaTrialOrchestratorTests.cs; docs/ipa-vocal-tract-roadmap.md; state/spine.yaml; state/scratch.md"}
|
||||
|
|
|
|||
|
|
@ -59,6 +59,8 @@ modules:
|
|||
primitive_control_spline_timeline: src/AquaSynth.Core/ControlSplineTimeline.cs
|
||||
primitive_vocal_reference_report: src/AquaSynth.Core/PrimitiveReferenceReport.cs
|
||||
ipa_gesture_experiment: src/AquaSynth.Core/IpaGestureExperiment.cs
|
||||
ipa_trial_results: src/AquaSynth.Core/IpaTrialResults.cs
|
||||
ipa_trial_orchestrator: src/AquaSynth.Faust/IpaTrialOrchestrator.cs
|
||||
acoustic_graph_missing_pieces_audit: docs/acoustic-graph-missing-pieces-audit.md
|
||||
acoustic_graph_proprioception: docs/acoustic-graph-proprioception.md
|
||||
pink_trombone_parity_fixtures: src/AquaSynth.Core/PinkTromboneParityFixtures.cs
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@
|
|||
"Faust-generated C# candidate rendering",
|
||||
"speech timing/confidence receipts for render requests, results, and checkpoints",
|
||||
"CultMesh speech render/scoring orchestration with payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application while the authority trainer owns optimizer steps and checkpoint commits",
|
||||
"frozen IPA gesture experiment rounds with candidate scripts, primitive timelines, gesture-layer metrics, manifests, evidence JSONL, metric summaries, candidate clusters, and science briefs",
|
||||
"frozen IPA gesture experiment rounds and local render trials with candidate scripts, primitive timelines, gesture-layer metrics, manifests, evidence JSONL, metric summaries, candidate clusters, science briefs, WAV artifacts, evaluator reports, and CultCache .cc trial-result records",
|
||||
"script scoring",
|
||||
"preset catalogs",
|
||||
"Faust emission and compiler wrapper"
|
||||
|
|
@ -92,10 +92,10 @@
|
|||
"Surge XT for modern hybrid synth modulation",
|
||||
"Cardinal/VCV-style patches for modular cable/rate semantics"
|
||||
],
|
||||
"rule": "For speech, add schema fields and learned-control lanes only when the parity harness has a loss surface that can exercise them. For reference synths, add DSL abstractions only when rebuilds prove the missing invariant. Parsing alone is inventory, not evidence of expressivity. Pink Trombone should be decomposed into reusable low-level Aqua primitives before audio golf, not copied as a bespoke clone or inflated into PT-shaped modules. After the 2026-05-29 proprioception correction, collapse graph-vocal lowering into a small set of instrumented physical primitives before more utterance golf. Gesture motion belongs in ControlSurface/ControlSpline timeline ownership, not primitive-local smoothing; cut any generated contact/port spline that invents motion not supported by primitive reference timelines. Public clients author gestures through ControlSurfaceCatalog plus ControlSplineTimeline and stream surface control dictionaries; higher-level gesture templates and phoneme_gesture are only spline emitters. IPA descriptors seed which anatomical surfaces move, while local datasets golf numeric spline defaults and patch voice code. Training reports must separate gesture_score, clean_vocal_score, and full_parity_score so full-patch recording-condition helpers cannot impersonate articulation. Frozen IPA gesture rounds own candidate scripts, primitive timelines, gesture metrics, manifests, evidence receipts, metric summaries, candidate clusters, and science briefs; statistical science workers and audio parity stages consume those bundles without rewriting gesture evidence."
|
||||
"rule": "For speech, add schema fields and learned-control lanes only when the parity harness has a loss surface that can exercise them. For reference synths, add DSL abstractions only when rebuilds prove the missing invariant. Parsing alone is inventory, not evidence of expressivity. Pink Trombone should be decomposed into reusable low-level Aqua primitives before audio golf, not copied as a bespoke clone or inflated into PT-shaped modules. After the 2026-05-29 proprioception correction, collapse graph-vocal lowering into a small set of instrumented physical primitives before more utterance golf. Gesture motion belongs in ControlSurface/ControlSpline timeline ownership, not primitive-local smoothing; cut any generated contact/port spline that invents motion not supported by primitive reference timelines. Public clients author gestures through ControlSurfaceCatalog plus ControlSplineTimeline and stream surface control dictionaries; higher-level gesture templates and phoneme_gesture are only spline emitters. IPA descriptors seed which anatomical surfaces move, while local datasets golf numeric spline defaults and patch voice code. Training reports must separate gesture_score, clean_vocal_score, and full_parity_score so full-patch recording-condition helpers cannot impersonate articulation. Frozen IPA gesture rounds own candidate scripts, primitive timelines, gesture metrics, manifests, evidence receipts, metric summaries, candidate clusters, and science briefs; `IpaTrialResult` CultCache records own rendered trial outcomes. Statistical science workers and audio parity stages consume those bundles without rewriting gesture evidence. If gesture scores rise but rendered audio is silent, fix the primitive source/tract owner before adding full-patch dressing."
|
||||
},
|
||||
"ipa_vocal_tract_path": {
|
||||
"status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets. CultMesh speech training now has payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application; workers do not own model updates. Real remote artifact chunk transfer and process supervision remain CultMesh infrastructure surfaces, not AquaSynth workaround territory. IpaGestureExperiment now writes frozen descriptor/variant rounds with candidate scripts, primitive timelines, gesture metrics, manifests, and evidence JSONL, then analyzes them into metric summaries, score-band clusters, and a science brief so statistical sub-agents can inspect immutable evidence while foreground generation continues.",
|
||||
"status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets. CultMesh speech training now has payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application; workers do not own model updates. Real remote artifact chunk transfer and process supervision remain CultMesh infrastructure surfaces, not AquaSynth workaround territory. IpaGestureExperiment writes frozen descriptor/variant rounds with candidate scripts, primitive timelines, gesture metrics, manifests, and evidence JSONL, then analyzes them into metric summaries, score-band clusters, and a science brief. IpaTrialOrchestrator now writes local five-seed IPA render trials, WAV/report artifacts, evaluator reports, and `ipa-trial-results.cc` CultCache records. The latest seed run improved voiced vowels/nasals after primitive `SourcePort` lowering gained an oscillatory carrier; plosive closure/release remains the next weak owner.",
|
||||
"lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.",
|
||||
"next": "Start the next speech gradient descent pass from Weksa's emitted v0.1 seed batch at `E:\\Projects\\weksa\\examples\\speech-training\\tiny-panphon-v0.1\\batch.json`: 1024 bge-m3 semantic speech-text floats, variable-length PanPhon sequence evidence, 256 AquaSynth-owned phonetic-realization embedding floats from `aquasynth.panphon_sequence_encoder.v0.1`, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Next tighten the parity harness: load that batch into AquaSynth training records, render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop. For distributed work, use RunCultMeshTrainingStep as the boring authority path: admitted leased workers return gradients/results, then the trainer applies centrally. Do not attempt federated averaging until this path is exercised against real Faust-rendered speech jobs."
|
||||
},
|
||||
|
|
@ -122,8 +122,8 @@
|
|||
},
|
||||
"last_verified": {
|
||||
"command": "dotnet test tests\\AquaSynth.Dsl.Tests\\AquaSynth.Dsl.Tests.csproj --no-restore --disable-build-servers -p:UseSharedCompilation=false -p:BuildInParallel=false --filter \"FullyQualifiedName~PatchScriptTests\" -v minimal",
|
||||
"result": "IpaGestureExperiment frozen-round harness and science-brief analysis added and documented. PatchScriptTests pass 93/93. Earlier full unfiltered runs hit the five-minute wall after leaving orphan dotnet workers; focused suites are the current reliable verification path.",
|
||||
"commit": "pending ipa gesture experiment harness"
|
||||
"result": "IPA trial render orchestration added. Opt-in five-seed trial run passed with Faust installed and wrote artifacts/parity/ipa-trials/20260529T214127955/five-seed-trials. Focused PatchScriptTests plus IpaTrialResultsRoundTripThroughCultCache pass 94/94. Earlier full unfiltered runs hit the five-minute wall after leaving orphan dotnet workers; focused suites are the current reliable verification path.",
|
||||
"commit": "pending ipa trial orchestration"
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,28 @@
|
|||
|
||||
Current slice:
|
||||
|
||||
- Implemented the first IPA trial render/scoring orchestration. Core now has
|
||||
typed `IpaTrialResult` CultCache records plus `IpaTrialResultCultCacheStore`;
|
||||
Faust now has `IpaTrialOrchestrator.RunAsync`, which generates candidate
|
||||
patches through `IpaGestureExperiment`, renders them with Faust, compares
|
||||
spectrogram/articulation evidence against local PT fixture references, writes
|
||||
WAV/report artifacts, and stores the general trial-result `.cc` database.
|
||||
The opt-in five-seed run wrote
|
||||
`artifacts/parity/ipa-trials/20260529T214127955/five-seed-trials`.
|
||||
- First run showed the pipeline worked but vowels/nasals were effectively
|
||||
silent despite decent gesture scores. The refinement was architectural:
|
||||
primitive `SourcePort` Faust lowering now emits an oscillatory voiced carrier
|
||||
tied to the network frequency, instead of feeding mostly DC pressure into a
|
||||
high-passed radiation load. Seed patch defaults now use stronger clean
|
||||
primitive source flow. Second run moved the intended layer: open `a`
|
||||
log-mel cosine `0.5693`, `m` `0.5199`, `s` `0.5498` with articulation
|
||||
`0.4111`; plosive `p` remains weak at `-0.1024`, pointing to closure/release
|
||||
ownership rather than source loudness.
|
||||
- Verification: opt-in
|
||||
`IpaTrialOrchestratorWritesFiveSeedTrialsWhenEnabled` passed with Faust
|
||||
installed; focused
|
||||
`PatchScriptTests|IpaTrialResultsRoundTripThroughCultCache` pass 94/94.
|
||||
|
||||
- Added the first scalable IPA gesture experiment harness.
|
||||
`IpaGestureExperiment.WriteRound` writes a frozen round bundle with
|
||||
`manifest.yaml`, candidate `.aqua` scripts, primitive timeline CSVs from
|
||||
|
|
@ -1504,4 +1526,3 @@ Next likely slice:
|
|||
current frequency/gate controls.
|
||||
- Keep tests focused on structure first, then add rendered audio comparison once
|
||||
the render path is explicit.
|
||||
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@ live_invariants:
|
|||
- First IPA/anatomy sugar is live. `phoneme_gesture` accepts known IPA symbols or explicit descriptors such as `descriptor=voiceless_labial-velar_fricative`, maps place/manner/voicing onto primary anatomical surfaces, and emits ordinary `ControlSpline`s. It is deterministic seed material for local reference-data golf, not exact articulatory truth. Patches may still use normal FM/AM/modulators/envelopes/filters around the anatomical control stream. Verification: `PatchScriptTests` pass 92/92.
|
||||
- IPA training scoring contract is documented. Reports must keep `gesture_score`, `clean_vocal_score`, and `full_parity_score` separate. Gesture score measures descriptor-to-surface coverage, motion direction, spline contour/timing, primitive timeline consequences, and optional external articulation landmarks. Full parity patches may use FM/AM/modulators/envelopes/filters, auxiliary animated voices, and recording-condition helpers, but those helpers cannot count as articulation evidence unless gesture and clean-vocal scores move coherently.
|
||||
- `IpaGestureExperiment.WriteRound` is the first frozen batch harness for IPA hypothesis work. It writes `manifest.yaml`, candidate `.aqua` scripts, primitive `ProbeTimelineReport` CSVs, gesture-layer `metrics.csv`, and per-candidate `evidence.jsonl` under a caller-provided artifact root. `IpaGestureExperiment.AnalyzeRound` writes `analysis/metric-summary.csv`, `analysis/candidate-clusters.csv`, and `analysis/science-brief.md` for the science-worker handoff. This harness owns frozen descriptor/spline/primitive evidence only; clean vocal identity, full spectrogram parity, optimizer checkpoints, and worker orchestration stay downstream. Focused verification: `PatchScriptTests` pass 93/93.
|
||||
- `IpaTrialOrchestrator.RunAsync` is the first local render/scoring worker for IPA trials. It generates candidate patches through `IpaGestureExperiment`, renders them through Faust, compares against local PT fixture references with `AudioAnalyzer`, writes WAV/report artifacts, and stores typed `IpaTrialResult` records in `ipa-trial-results.cc`. The five-seed trial run at `tests/AquaSynth.Dsl.Tests/bin/Debug/net10.0/artifacts/parity/ipa-trials/20260529T214127955/five-seed-trials` showed voiced vowels/nasals became audible only after primitive `SourcePort` lowering gained a true oscillatory voiced carrier; sibilants are pressure, plosives remain weak. Verification: opt-in IPA trial test passed, and focused `PatchScriptTests|IpaTrialResultsRoundTripThroughCultCache` pass 94/94.
|
||||
- TurbulenceJet now includes a release-timed burst-noise component in addition to the deterministic pressure pulse, but a stronger trial barely moved papa; the source is not the main remaining articulation bottleneck. Graph radiation now mixes 30% raw boundary flow with 70% high-passed flow after local admittance weighting, down from the over-bright 15/85 blend. Latest radiation/source artifact under `artifacts/parity/pink-trombone-utterance-logmel/20260527T174236049`: mama/papa/thrombosis cosine 0.6578/0.7692/0.5585, rmsRatio 3.0006/2.6153/2.3488, articulation 0.1437/0.1173/0.1191. Static `closure-release` under `artifacts/parity/pink-trombone-logmel/20260527T174139661` has rmsRatio 0.9900 and speechBandRatio 0.0842. This is still not speaking; next pressure is true tract resonance/passive scattering and pressure storage, not more burst gain.
|
||||
- Same-path two-port area discontinuities now emit `graph_area_energy_in_*` and `graph_area_energy_out_*` probes alongside the three-port branch `graph_connection_energy_*` probes. Use these to inspect passivity and formant-body failures at the scatter sites rather than judging only final output loudness.
|
||||
- `FaustExportOptions.DebugProbeUi` wraps graph probe signals in Faust `vbargraph` UI controls under `/debug/...` without changing audio output arity. Native streaming Faust patches now separate writable controls from read-only probe zones and can read/snapshot probe values after `ProcessBlock`; use this for acoustic graph passivity/radiation inspection, leave it off for normal parity renders.
|
||||
|
|
@ -956,4 +957,3 @@ cut_line:
|
|||
- "Cut any design that requires recompiling a patch to move an exposed musical parameter."
|
||||
- "Split a lower-level graph DSL if Faust/Cardinal-style references make the voice DSL lie."
|
||||
- "Cut any new graph-vocal feature that lands before primitive-collapse and flow-probe instrumentation make the tract body's sample/flow behavior observable."
|
||||
|
||||
|
|
|
|||
109
tests/AquaSynth.Dsl.Tests/IpaTrialOrchestratorTests.cs
Normal file
109
tests/AquaSynth.Dsl.Tests/IpaTrialOrchestratorTests.cs
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
using AquaSynth.Dsl;
|
||||
using AquaSynth.Faust;
|
||||
|
||||
namespace AquaSynth.Dsl.Tests;
|
||||
|
||||
public sealed class IpaTrialOrchestratorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task IpaTrialResultsRoundTripThroughCultCache()
|
||||
{
|
||||
var directory = TempDirectory();
|
||||
try
|
||||
{
|
||||
var store = Path.Combine(directory, "ipa-trial-results.cc");
|
||||
var result = new IpaTrialResult(
|
||||
"trial-a",
|
||||
"batch-a",
|
||||
"2026-05-29T12:00:00.0000000Z",
|
||||
"vowels",
|
||||
["a"],
|
||||
"open-vowel",
|
||||
"candidate-a",
|
||||
"hypothesis-worker",
|
||||
"Try an open vowel descriptor against an open vowel fixture.",
|
||||
"candidate.aqua",
|
||||
"reference.wav",
|
||||
"candidate.wav",
|
||||
"timeline.csv",
|
||||
[new SpeechScoreMetric("log_mel_cosine", .42f, 1)],
|
||||
[new SpeechRenderArtifact("comparison-report", "report.txt", "sha256:test")],
|
||||
"science-evaluator",
|
||||
"Weak but measurable spectrogram pressure.",
|
||||
"pressure",
|
||||
["fixture reference is not an IPA sample"],
|
||||
[new SpeechTimingReceipt("ipa-trial-render-score", "2026-05-29T12:00:00.0000000Z", 3, 20, .5f, "test")]);
|
||||
|
||||
await IpaTrialResultCultCacheStore.UpsertResultsAsync(store, [result]);
|
||||
var loaded = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
||||
|
||||
var single = Assert.Single(loaded);
|
||||
Assert.Equal("trial-a", single.TrialId);
|
||||
Assert.Equal("candidate-a", single.CandidateId);
|
||||
Assert.Contains(single.Metrics, metric => metric.Name == "log_mel_cosine");
|
||||
Assert.Contains(single.KnownLies, lie => lie.Contains("not an IPA sample", StringComparison.Ordinal));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IpaTrialOrchestratorWritesFiveSeedTrialsWhenEnabled()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("AQUASYNTH_RUN_IPA_TRIALS") != "1")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (FaustCompiler.FindFaust() is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var artifactRoot = Path.Combine(
|
||||
RepositoryRoot(),
|
||||
"artifacts",
|
||||
"parity",
|
||||
"ipa-trials",
|
||||
DateTimeOffset.UtcNow.ToString("yyyyMMddTHHmmssfff"));
|
||||
var result = await IpaTrialOrchestrator.RunAsync(
|
||||
artifactRoot,
|
||||
options: new IpaTrialOrchestrationOptions(BatchId: "five-seed-trials"));
|
||||
|
||||
Assert.True(File.Exists(result.TrialResultStorePath));
|
||||
Assert.True(File.Exists(result.SummaryPath));
|
||||
Assert.True(File.Exists(result.EvaluatorReportPath));
|
||||
Assert.Equal(8, result.TrialResults.Count);
|
||||
Assert.Equal(5, result.TrialResults.Select(item => item.TargetSetId).Distinct(StringComparer.Ordinal).Count());
|
||||
Assert.All(result.TrialResults, trial => Assert.NotEmpty(trial.Metrics));
|
||||
Assert.Contains(result.TrialResults, trial => trial.Metrics.Any(metric => metric.Name == "log_mel_cosine"));
|
||||
|
||||
var loaded = await IpaTrialResultCultCacheStore.ReadResultsAsync(result.TrialResultStorePath);
|
||||
Assert.Equal(result.TrialResults.Count, loaded.Count);
|
||||
}
|
||||
|
||||
private static string TempDirectory()
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), $"aquasynth-ipa-trial-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
private static string RepositoryRoot()
|
||||
{
|
||||
var directory = new DirectoryInfo(Environment.CurrentDirectory);
|
||||
while (directory is not null)
|
||||
{
|
||||
if (File.Exists(Path.Combine(directory.FullName, "AquaSynth.sln")))
|
||||
{
|
||||
return directory.FullName;
|
||||
}
|
||||
|
||||
directory = directory.Parent;
|
||||
}
|
||||
|
||||
return Environment.CurrentDirectory;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue