GameCult/AquaSynth
Watch
9
0
Fork
You've already forked AquaSynth
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add log-mel evidence to eSpeak workout

Browse source
This commit is contained in:
Meta Vens 2026-05-19 16:15:44 +01:00
commit fdb9d76cfd
3 changed files with 66 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

7
docs/ipa-vocal-tract-roadmap.md
View file

@ -455,9 +455,10 @@ artifacts/parity/espeak-ng-ipa-workout/<timestamp>-tiny/
```
The report records the source text, eSpeak IPA transcription, sample counts, and
basic audio features. This is the first external speech judge for the vocal
tract lane. It should pressure AquaSynth's future IPA/gesture/render path, not
replace it.
basic audio features. It also records normalized log-mel spectrogram summaries
per fixture and a compact log-mel distance matrix across the workout. This is
the first external speech judge for the vocal tract lane. It should pressure
AquaSynth's future IPA/gesture/render path, not replace it.
## First Work Packets

1
state/evidence.jsonl
View file

@ -108,3 +108,4 @@
{"date":"2026-05-19","topic":"vocal-tract-boundary-contracts","lesson":"Defend the Weksa/AquaSynth tract boundary with three real bones: PhoneticIntent for language-owned phones/features/timing, ArticulatoryPlan for AquaSynth-owned gesture realization, and ArticulatoryConstraintReport for host-visible anatomy vetoes. Morphology capabilities must reject impossible phones such as bilabials for a beaked speaker instead of silently substituting liar-slurry.","evidence":"Added VocalTractModel.cs contracts and VocalTractModelTests.cs coverage for inspectable Weksa intent, bilabial veto on a beaked morphology, and acceptance on a human baseline; updated audio doctrine and IPA roadmap boundary section."}
{"date":"2026-05-19","evidence":"Added VocalTractPlanner.Plan, VocalTractPlanResult, VocalTractHostReaction, rejection coverage for /pa/ against a beaked morphology, and accepted /a/ coverage for the human baseline gesture path.","lesson":"The first vocal-tract proof artifact is VocalTractPlanResult: host-visible branching around one intent and one morphology. Rejected morphology constraints carry diagnostic codes and no plan; accepted vowel intents emit inspectable glottal-source and tract-area gestures before any renderer exists.","topic":"vocal-tract-plan-result-proof"}
{"date":"2026-05-19","evidence":"Added EspeakNgReferenceTests with optional renderer discovery, tiny fixtures for a/pa/ta/ka/sa/ma, WAV artifact/report generation under artifacts/parity/espeak-ng-ipa-workout, and docs/state entries naming eSpeak as intelligibility/timing pressure only.","lesson":"The first vocal-tract parity workout uses optional eSpeak NG as a development speech reference, not an anatomy oracle. The lane renders tiny speech fixtures and captures eSpeak IPA transcription/audio features when ESPEAK_NG or espeak-ng is available, while normal builds skip cleanly when it is absent.","topic":"espeak-ng-ipa-workout"}
{"date":"2026-05-19","evidence":"Updated EspeakNgReferenceTests to record log_mel_frames, log_mel_bands, log_mel_mean, and log_mel_distance_matrix in report.md; updated the IPA vocal tract roadmap accordingly.","lesson":"The eSpeak NG speech workout should expose log-mel evidence immediately, not just basic audio features. Each fixture report now carries normalized log-mel band means plus a cross-fixture log-mel distance matrix so future AquaSynth IPA renders can plug into the same spectral comparison shape.","topic":"espeak-ng-logmel-workout"}

63
tests/AquaSynth.Dsl.Tests/EspeakNgReferenceTests.cs
View file

@ -32,8 +32,10 @@ public sealed class EspeakNgReferenceTests
{
"eSpeak NG IPA workout",
$"renderer: {renderer.CommandPath}",
"role: optional development reference for intelligibility, phoneme timing, and broad IPA/text coverage; not anatomy truth"
"role: optional development reference for intelligibility, phoneme timing, and broad IPA/text coverage; not anatomy truth",
"log_mel: 32 normalized bands; values are averages across frames from AudioAnalyzer"
};
var analyses = new List<FixtureAnalysis>();
foreach (var fixture in fixtures)
{
@ -45,7 +47,10 @@ public sealed class EspeakNgReferenceTests
var samples = ReadMonoPcm16Wav(wavPath);
Assert.True(samples.Length > 1024, $"{fixture.Id} rendered too few samples.");
var features = AudioAnalyzer.AnalyzeAudio(samples).Features;
var analysis = AudioAnalyzer.AnalyzeAudio(samples);
var features = analysis.Features;
var logMelMean = MeanBands(analysis.LogMelSpectrogram);
analyses.Add(new FixtureAnalysis(fixture, analysis, logMelMean));
Assert.True(features.Peak > 0.001f, $"{fixture.Id} rendered near silence.");
report.Add($"fixture: {fixture.Id}");
@ -56,6 +61,18 @@ public sealed class EspeakNgReferenceTests
report.Add($" peak: {features.Peak:0.######}");
report.Add($" rms: {features.Rms:0.######}");
report.Add($" centroid_hz: {features.SpectralCentroidHz:0.######}");
report.Add($" log_mel_frames: {analysis.LogMelSpectrogram.Frames}");
report.Add($" log_mel_bands: {analysis.LogMelSpectrogram.Bands}");
report.Add($" log_mel_mean: {FormatBands(logMelMean)}");
}
report.Add("log_mel_distance_matrix:");
report.Add($" fixture,{string.Join(',', analyses.Select(item => item.Fixture.Id))}");
foreach (var row in analyses)
{
var distances = analyses.Select(column =>
LogMelDistance(row.Analysis.LogMelSpectrogram, column.Analysis.LogMelSpectrogram).ToString("0.######"));
report.Add($" {row.Fixture.Id},{string.Join(',', distances)}");
}
await File.WriteAllLinesAsync(Path.Combine(artifactDir, "report.md"), report);
@ -152,6 +169,48 @@ public sealed class EspeakNgReferenceTests
private sealed record EspeakNgFixture(string Id, string Text, string Voice);
private sealed record FixtureAnalysis(EspeakNgFixture Fixture, AudioAnalysis Analysis, float[] LogMelMean);
private static float[] MeanBands(Spectrogram spectrogram)
{
var means = new float[spectrogram.Bands];
for (var frame = 0; frame < spectrogram.Frames; frame++)
{
for (var band = 0; band < spectrogram.Bands; band++)
{
means[band] += spectrogram.At(frame, band);
}
}
for (var band = 0; band < means.Length; band++)
{
means[band] /= Math.Max(1, spectrogram.Frames);
}
return means;
}
private static float LogMelDistance(Spectrogram left, Spectrogram right)
{
var count = Math.Min(left.Values.Length, right.Values.Length);
if (count == 0)
{
return 0;
}
var sum = 0f;
for (var index = 0; index < count; index++)
{
var difference = left.Values[index] - right.Values[index];
sum += difference * difference;
}
return MathF.Sqrt(sum / count);
}
private static string FormatBands(IReadOnlyList<float> bands) =>
string.Join(',', bands.Select(value => value.ToString("0.###")));
private sealed class EspeakNgReferenceRenderer(string commandPath)
{
public const string MissingToolMessage =