From fdb9d76cfdced79808a328d37a641eac98362b5b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 19 May 2026 16:15:44 +0100 Subject: [PATCH] Add log-mel evidence to eSpeak workout --- docs/ipa-vocal-tract-roadmap.md | 7 ++- state/evidence.jsonl | 1 + .../EspeakNgReferenceTests.cs | 63 ++++++++++++++++++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/docs/ipa-vocal-tract-roadmap.md b/docs/ipa-vocal-tract-roadmap.md index 4c29d64..8f5d8bf 100644 --- a/docs/ipa-vocal-tract-roadmap.md +++ b/docs/ipa-vocal-tract-roadmap.md @@ -455,9 +455,10 @@ artifacts/parity/espeak-ng-ipa-workout/-tiny/ ``` The report records the source text, eSpeak IPA transcription, sample counts, and -basic audio features. This is the first external speech judge for the vocal -tract lane. It should pressure AquaSynth's future IPA/gesture/render path, not -replace it. +basic audio features. It also records normalized log-mel spectrogram summaries +per fixture and a compact log-mel distance matrix across the workout. This is +the first external speech judge for the vocal tract lane. It should pressure +AquaSynth's future IPA/gesture/render path, not replace it. ## First Work Packets diff --git a/state/evidence.jsonl b/state/evidence.jsonl index a186fa2..fcb65a1 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -108,3 +108,4 @@ {"date":"2026-05-19","topic":"vocal-tract-boundary-contracts","lesson":"Defend the Weksa/AquaSynth tract boundary with three real bones: PhoneticIntent for language-owned phones/features/timing, ArticulatoryPlan for AquaSynth-owned gesture realization, and ArticulatoryConstraintReport for host-visible anatomy vetoes. Morphology capabilities must reject impossible phones such as bilabials for a beaked speaker instead of silently substituting liar-slurry.","evidence":"Added VocalTractModel.cs contracts and VocalTractModelTests.cs coverage for inspectable Weksa intent, bilabial veto on a beaked morphology, and acceptance on a human baseline; updated audio doctrine and IPA roadmap boundary section."} {"date":"2026-05-19","evidence":"Added VocalTractPlanner.Plan, VocalTractPlanResult, VocalTractHostReaction, rejection coverage for /pa/ against a beaked morphology, and accepted /a/ coverage for the human baseline gesture path.","lesson":"The first vocal-tract proof artifact is VocalTractPlanResult: host-visible branching around one intent and one morphology. Rejected morphology constraints carry diagnostic codes and no plan; accepted vowel intents emit inspectable glottal-source and tract-area gestures before any renderer exists.","topic":"vocal-tract-plan-result-proof"} {"date":"2026-05-19","evidence":"Added EspeakNgReferenceTests with optional renderer discovery, tiny fixtures for a/pa/ta/ka/sa/ma, WAV artifact/report generation under artifacts/parity/espeak-ng-ipa-workout, and docs/state entries naming eSpeak as intelligibility/timing pressure only.","lesson":"The first vocal-tract parity workout uses optional eSpeak NG as a development speech reference, not an anatomy oracle. The lane renders tiny speech fixtures and captures eSpeak IPA transcription/audio features when ESPEAK_NG or espeak-ng is available, while normal builds skip cleanly when it is absent.","topic":"espeak-ng-ipa-workout"} +{"date":"2026-05-19","evidence":"Updated EspeakNgReferenceTests to record log_mel_frames, log_mel_bands, log_mel_mean, and log_mel_distance_matrix in report.md; updated the IPA vocal tract roadmap accordingly.","lesson":"The eSpeak NG speech workout should expose log-mel evidence immediately, not just basic audio features. Each fixture report now carries normalized log-mel band means plus a cross-fixture log-mel distance matrix so future AquaSynth IPA renders can plug into the same spectral comparison shape.","topic":"espeak-ng-logmel-workout"} diff --git a/tests/AquaSynth.Dsl.Tests/EspeakNgReferenceTests.cs b/tests/AquaSynth.Dsl.Tests/EspeakNgReferenceTests.cs index ffce1af..d57cd22 100644 --- a/tests/AquaSynth.Dsl.Tests/EspeakNgReferenceTests.cs +++ b/tests/AquaSynth.Dsl.Tests/EspeakNgReferenceTests.cs @@ -32,8 +32,10 @@ public sealed class EspeakNgReferenceTests { "eSpeak NG IPA workout", $"renderer: {renderer.CommandPath}", - "role: optional development reference for intelligibility, phoneme timing, and broad IPA/text coverage; not anatomy truth" + "role: optional development reference for intelligibility, phoneme timing, and broad IPA/text coverage; not anatomy truth", + "log_mel: 32 normalized bands; values are averages across frames from AudioAnalyzer" }; + var analyses = new List(); foreach (var fixture in fixtures) { @@ -45,7 +47,10 @@ public sealed class EspeakNgReferenceTests var samples = ReadMonoPcm16Wav(wavPath); Assert.True(samples.Length > 1024, $"{fixture.Id} rendered too few samples."); - var features = AudioAnalyzer.AnalyzeAudio(samples).Features; + var analysis = AudioAnalyzer.AnalyzeAudio(samples); + var features = analysis.Features; + var logMelMean = MeanBands(analysis.LogMelSpectrogram); + analyses.Add(new FixtureAnalysis(fixture, analysis, logMelMean)); Assert.True(features.Peak > 0.001f, $"{fixture.Id} rendered near silence."); report.Add($"fixture: {fixture.Id}"); @@ -56,6 +61,18 @@ public sealed class EspeakNgReferenceTests report.Add($" peak: {features.Peak:0.######}"); report.Add($" rms: {features.Rms:0.######}"); report.Add($" centroid_hz: {features.SpectralCentroidHz:0.######}"); + report.Add($" log_mel_frames: {analysis.LogMelSpectrogram.Frames}"); + report.Add($" log_mel_bands: {analysis.LogMelSpectrogram.Bands}"); + report.Add($" log_mel_mean: {FormatBands(logMelMean)}"); + } + + report.Add("log_mel_distance_matrix:"); + report.Add($" fixture,{string.Join(',', analyses.Select(item => item.Fixture.Id))}"); + foreach (var row in analyses) + { + var distances = analyses.Select(column => + LogMelDistance(row.Analysis.LogMelSpectrogram, column.Analysis.LogMelSpectrogram).ToString("0.######")); + report.Add($" {row.Fixture.Id},{string.Join(',', distances)}"); } await File.WriteAllLinesAsync(Path.Combine(artifactDir, "report.md"), report); @@ -152,6 +169,48 @@ public sealed class EspeakNgReferenceTests private sealed record EspeakNgFixture(string Id, string Text, string Voice); + private sealed record FixtureAnalysis(EspeakNgFixture Fixture, AudioAnalysis Analysis, float[] LogMelMean); + + private static float[] MeanBands(Spectrogram spectrogram) + { + var means = new float[spectrogram.Bands]; + for (var frame = 0; frame < spectrogram.Frames; frame++) + { + for (var band = 0; band < spectrogram.Bands; band++) + { + means[band] += spectrogram.At(frame, band); + } + } + + for (var band = 0; band < means.Length; band++) + { + means[band] /= Math.Max(1, spectrogram.Frames); + } + + return means; + } + + private static float LogMelDistance(Spectrogram left, Spectrogram right) + { + var count = Math.Min(left.Values.Length, right.Values.Length); + if (count == 0) + { + return 0; + } + + var sum = 0f; + for (var index = 0; index < count; index++) + { + var difference = left.Values[index] - right.Values[index]; + sum += difference * difference; + } + + return MathF.Sqrt(sum / count); + } + + private static string FormatBands(IReadOnlyList bands) => + string.Join(',', bands.Select(value => value.ToString("0.###"))); + private sealed class EspeakNgReferenceRenderer(string commandPath) { public const string MissingToolMessage =