diff --git a/docs/ipa-vocal-tract-roadmap.md b/docs/ipa-vocal-tract-roadmap.md index a21c492..026593d 100644 --- a/docs/ipa-vocal-tract-roadmap.md +++ b/docs/ipa-vocal-tract-roadmap.md @@ -190,12 +190,16 @@ the gradient descent. `UtteranceEmbeddingNeuralEncoder` trains a tiny model to compress those ingredients into one utterance embedding. `VocalTractNeuralMapper` then takes phonetic features plus that embedding and predicts tract controls. Both controllers use the same native C# packed-network path: row-major buffers, -unsafe hot loops, batched gradients, and Adam or SGD updates. The synth-driver -output includes core tract parameters plus expressive synthesis lanes: AM depth, -FM depth, LFO rate/depth, filter cutoff, filter resonance, and a full -mel-frequency spectral envelope vector for PAD-style wavetable/formant -coloration. Those lanes are not license to bypass tract anatomy; they are -renderable expression controls the later synth can choose to honor. +unsafe hot loops, batched gradients, and Adam or SGD updates. For chained speech +training, `SpeechBackpropagationPipeline` runs the synth-driver loss backward +through the utterance embedding, slices out the embedding gradient from the +synth-driver input gradient, and updates the utterance encoder from that upstream +loss. The synth-driver output includes core tract parameters plus expressive +synthesis lanes: AM depth, FM depth, LFO rate/depth, filter cutoff, filter +resonance, and a full mel-frequency spectral envelope vector for PAD-style +wavetable/formant coloration. Those lanes are not license to bypass tract +anatomy; they are renderable expression controls the later synth can choose to +honor. ### 4. Morphology Model @@ -476,10 +480,11 @@ AquaSynth's future IPA/gesture/render path, not replace it. The optional test `EspeakNgGroundTruthTrainsUtteranceEmbeddingAndSynthDriverWhenInstalled` turns the same idea into a gradient-descent fixture. When eSpeak is present it -renders the tiny syllable set, extracts normalized log-mel means, trains -`UtteranceEmbeddingNeuralEncoder` toward compressed eSpeak evidence, then trains -`VocalTractNeuralMapper` toward tract/mod/filter/mel controls seeded by that -evidence. It writes WAVs plus a training report under: +renders the tiny syllable set, extracts normalized log-mel means, and runs +`SpeechBackpropagationPipeline`: the eSpeak-seeded synth-driver target loss +updates `VocalTractNeuralMapper` and backpropagates through the utterance +embedding into `UtteranceEmbeddingNeuralEncoder`. It writes WAVs plus a training +report under: ```text artifacts/parity/espeak-ng-gradient-descent/-tiny/ diff --git a/src/AquaSynth.Core/PackedNeuralNetwork.cs b/src/AquaSynth.Core/PackedNeuralNetwork.cs index bf42e1d..9c2e5fb 100644 --- a/src/AquaSynth.Core/PackedNeuralNetwork.cs +++ b/src/AquaSynth.Core/PackedNeuralNetwork.cs @@ -27,6 +27,11 @@ public sealed record PackedNeuralTrainingResult( PackedNeuralNetwork Network, IReadOnlyList Steps); +public sealed record PackedNeuralBackpropagation( + float Loss, + float[] Output, + float[] InputGradient); + public sealed class PackedNeuralNetwork { private readonly Layer[] layers; @@ -148,6 +153,42 @@ public sealed class PackedNeuralNetwork return new PackedNeuralTrainingResult(this, steps); } + public PackedNeuralBackpropagation TrainSingle( + IReadOnlyList input, + IReadOnlyList target, + float learningRate) + { + if (learningRate <= 0) throw new ArgumentOutOfRangeException(nameof(learningRate), "learning rate must be positive"); + if (target.Count != OutputSize) throw new ArgumentException($"target must have {OutputSize} values", nameof(target)); + + var outputGradient = new float[OutputSize]; + var scratch = TrainingScratch.Create(layers, InputSize, OutputSize); + CopyChecked(input, scratch.Activations[0], "input"); + Copy(target, scratch.Target); + Forward(scratch); + var loss = MeanSquaredError(scratch.Output, scratch.Target); + for (var index = 0; index < outputGradient.Length; index++) + { + outputGradient[index] = 2f * (scratch.Output[index] - scratch.Target[index]) / Math.Max(1, OutputSize); + } + + return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, loss); + } + + public PackedNeuralBackpropagation TrainSingleFromOutputGradient( + IReadOnlyList input, + IReadOnlyList outputGradient, + float learningRate) + { + if (learningRate <= 0) throw new ArgumentOutOfRangeException(nameof(learningRate), "learning rate must be positive"); + if (outputGradient.Count != OutputSize) throw new ArgumentException($"output gradient must have {OutputSize} values", nameof(outputGradient)); + + var scratch = TrainingScratch.Create(layers, InputSize, OutputSize); + CopyChecked(input, scratch.Activations[0], "input"); + Forward(scratch); + return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, 0); + } + private static void Validate(PackedNeuralTrainingOptions options) { if (options.Epochs <= 0) throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive"); @@ -163,7 +204,31 @@ public sealed class PackedNeuralNetwork } } + private PackedNeuralBackpropagation TrainSingleFromOutputGradient( + TrainingScratch scratch, + IReadOnlyList outputGradient, + float learningRate, + float loss) + { + var gradients = LayerBuffers.Create(layers); + AccumulateGradientsFromOutputGradient(scratch, outputGradient, gradients); + var inputGradient = InputGradient(scratch); + ApplySgd(gradients, 1f, learningRate); + return new PackedNeuralBackpropagation(loss, scratch.Output.ToArray(), inputGradient); + } + private void AccumulateGradients(TrainingScratch scratch, LayerBuffers gradients) + { + var outputGradient = new float[OutputSize]; + for (var index = 0; index < outputGradient.Length; index++) + { + outputGradient[index] = 2f * (scratch.Output[index] - scratch.Target[index]); + } + + AccumulateGradientsFromOutputGradient(scratch, outputGradient, gradients); + } + + private void AccumulateGradientsFromOutputGradient(TrainingScratch scratch, IReadOnlyList outputGradient, LayerBuffers gradients) { var lastLayerIndex = layers.Length - 1; var outputDelta = scratch.Deltas[lastLayerIndex]; @@ -171,7 +236,7 @@ public sealed class PackedNeuralNetwork for (var index = 0; index < outputDelta.Length; index++) { var actual = output[index]; - outputDelta[index] = 2f * (actual - scratch.Target[index]) * layers[lastLayerIndex].Derivative(actual); + outputDelta[index] = outputGradient[index] * layers[lastLayerIndex].Derivative(actual); } for (var layerIndex = lastLayerIndex - 1; layerIndex >= 0; layerIndex--) @@ -185,6 +250,13 @@ public sealed class PackedNeuralNetwork } } + private float[] InputGradient(TrainingScratch scratch) + { + var inputGradient = new float[InputSize]; + layers[0].BackpropagateInputGradient(scratch.Deltas[0], inputGradient); + return inputGradient; + } + private void ApplySgd(LayerBuffers gradients, float scale, float learningRate) { for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++) @@ -407,6 +479,28 @@ public sealed class PackedNeuralNetwork } } + public void BackpropagateInputGradient(float[] nextDelta, float[] inputGradient) + { + unsafe + { + fixed (float* nextDeltaBase = nextDelta) + fixed (float* inputGradientBase = inputGradient) + fixed (float* weightBase = Weights) + { + for (var input = 0; input < InputSize; input++) + { + var sum = 0f; + for (var output = 0; output < OutputSize; output++) + { + sum += weightBase[output * InputSize + input] * nextDeltaBase[output]; + } + + inputGradientBase[input] = sum; + } + } + } + } + public float Derivative(float activated) => activation switch { diff --git a/src/AquaSynth.Core/SpeechBackpropagationPipeline.cs b/src/AquaSynth.Core/SpeechBackpropagationPipeline.cs new file mode 100644 index 0000000..0c4d5b4 --- /dev/null +++ b/src/AquaSynth.Core/SpeechBackpropagationPipeline.cs @@ -0,0 +1,111 @@ +namespace AquaSynth.Dsl; + +public sealed record SpeechBackpropagationTrainingExample( + UtteranceEmbeddingInput UtteranceInput, + PhoneticEvent Event, + VocalTractControlTarget Target); + +public sealed record SpeechBackpropagationTrainingOptions( + int Epochs = 200, + float UtteranceLearningRate = 0.01f, + float SynthDriverLearningRate = 0.01f, + int Seed = 1337, + bool Shuffle = true); + +public sealed record SpeechBackpropagationTrainingStep(int Epoch, float Loss); + +public sealed record SpeechBackpropagationTrainingResult( + SpeechBackpropagationPipeline Pipeline, + IReadOnlyList Steps); + +public sealed class SpeechBackpropagationPipeline( + UtteranceEmbeddingNeuralEncoder utteranceEncoder, + VocalTractNeuralMapper synthDriver) +{ + public UtteranceEmbeddingNeuralEncoder UtteranceEncoder { get; } = utteranceEncoder; + + public VocalTractNeuralMapper SynthDriver { get; } = synthDriver; + + public SpeechBackpropagationTrainingResult Train( + IReadOnlyList examples, + SpeechBackpropagationTrainingOptions? options = null) + { + if (examples.Count == 0) + { + throw new ArgumentException("at least one training example is required", nameof(examples)); + } + + options ??= new SpeechBackpropagationTrainingOptions(); + if (options.Epochs <= 0) throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive"); + if (options.UtteranceLearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(options), "utterance learning rate must be positive"); + if (options.SynthDriverLearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(options), "synth-driver learning rate must be positive"); + + var order = Enumerable.Range(0, examples.Count).ToArray(); + var random = new Random(options.Seed); + var steps = new List(); + + for (var epoch = 0; epoch < options.Epochs; epoch++) + { + if (options.Shuffle) + { + Shuffle(order, random); + } + + var loss = 0f; + foreach (var index in order) + { + loss += TrainSingle(examples[index], options); + } + + steps.Add(new SpeechBackpropagationTrainingStep(epoch + 1, loss / examples.Count)); + } + + return new SpeechBackpropagationTrainingResult(this, steps); + } + + public VocalTractControlTarget Predict(SpeechBackpropagationTrainingExample example) + { + var embedding = UtteranceEncoder.Encode(example.UtteranceInput).ToSemanticEmbedding(); + return SynthDriver.Predict(example.Event, embedding); + } + + private float TrainSingle(SpeechBackpropagationTrainingExample example, SpeechBackpropagationTrainingOptions options) + { + var utteranceFeatures = example.UtteranceInput.ToFeatureVector(); + var embedding = UtteranceEncoder.Encode(utteranceFeatures); + var synthResult = SynthDriver.TrainSingle( + example.Event, + embedding.ToSemanticEmbedding(), + example.Target, + options.SynthDriverLearningRate); + var embeddingGradient = SemanticEmbeddingGradient(synthResult.InputGradient); + UtteranceEncoder.TrainSingleFromOutputGradient(utteranceFeatures, embeddingGradient, options.UtteranceLearningRate); + return synthResult.Loss; + } + + private float[] SemanticEmbeddingGradient(IReadOnlyList synthInputGradient) + { + if (synthInputGradient.Count != SynthDriver.InputSize) + { + throw new InvalidOperationException("synth input gradient length does not match synth driver input size"); + } + + var gradient = new float[SynthDriver.SemanticEmbeddingSize]; + var offset = SynthDriver.InputSize - SynthDriver.SemanticEmbeddingSize; + for (var index = 0; index < gradient.Length; index++) + { + gradient[index] = synthInputGradient[offset + index]; + } + + return gradient; + } + + private static void Shuffle(int[] values, Random random) + { + for (var index = values.Length - 1; index > 0; index--) + { + var swapIndex = random.Next(index + 1); + (values[index], values[swapIndex]) = (values[swapIndex], values[index]); + } + } +} diff --git a/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs b/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs index e099c86..dbf40f7 100644 --- a/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs +++ b/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs @@ -80,6 +80,15 @@ public sealed class UtteranceEmbeddingNeuralEncoder return new UtteranceEmbedding(network.Predict(features.Values)); } + public PackedNeuralBackpropagation TrainSingleFromOutputGradient( + UtteranceEmbeddingFeatureVector features, + IReadOnlyList outputGradient, + float learningRate) + { + ValidateSize(features.Values, InputSize, nameof(features)); + return network.TrainSingleFromOutputGradient(features.Values, outputGradient, learningRate); + } + public UtteranceEmbeddingTrainingResult Train( IReadOnlyList examples, PackedNeuralTrainingOptions? options = null) diff --git a/src/AquaSynth.Core/VocalTractLearning.cs b/src/AquaSynth.Core/VocalTractLearning.cs index c84c7cb..3717752 100644 --- a/src/AquaSynth.Core/VocalTractLearning.cs +++ b/src/AquaSynth.Core/VocalTractLearning.cs @@ -99,9 +99,22 @@ public sealed class VocalTractNeuralMapper public VocalTractControlTarget Predict(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding) { - return TargetFromVector(network.Predict(EncodeInput(phoneticEvent, semanticEmbedding))); + return TargetFromVector(PredictVector(phoneticEvent, semanticEmbedding)); } + public float[] PredictVector(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding) => + network.Predict(EncodeInput(phoneticEvent, semanticEmbedding)); + + public PackedNeuralBackpropagation TrainSingle( + PhoneticEvent phoneticEvent, + VocalTractSemanticEmbedding semanticEmbedding, + VocalTractControlTarget target, + float learningRate) => + network.TrainSingle( + EncodeInput(phoneticEvent, semanticEmbedding), + target.ToVector(MelBandCount, NeutralMelBand), + learningRate); + public VocalTractTrainingResult Train( IReadOnlyList examples, VocalTractTrainingOptions? options = null) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 8b19a34..5718e76 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -113,3 +113,4 @@ {"date":"2026-05-19","topic":"vocal-tract-optimizer-hot-path","lesson":"The first learned vocal-tract optimizer should stay native and inspectable until the render-loss boundary proves it needs a tensor framework. The trainer now uses packed row-major weight buffers, owned scratch activation/delta/target buffers, unsafe pointer loops, reusable gradient/moment buffers, batched gradients, deterministic shuffling, and Adam by default instead of allocation-heavy per-example jagged-array SGD.","evidence":"Rebuilt VocalTractLearning.cs hot path around flat float arrays and unsafe loops; enabled unsafe blocks in AquaSynth.Core; VocalTractLearningTests pass against the packed-buffer Adam trainer."} {"date":"2026-05-19","topic":"utterance-embedding-learning-boundary","lesson":"AquaSynth should own gradient descent for Weksa speech handoff. Weksa supplies speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors; AquaSynth trains the tiny utterance-embedding encoder and passes that embedding into the synth-driver model.","evidence":"Extracted PackedNeuralNetwork as the shared packed-buffer Adam/SGD primitive, rewired VocalTractNeuralMapper through it, and added UtteranceEmbeddingInput/UtteranceEmbeddingNeuralEncoder plus tests for packing, training, and feeding the vocal-tract semantic embedding."} {"date":"2026-05-19","topic":"espeak-gradient-descent-fixture","lesson":"The first speech-learning fixture should train on generated eSpeak evidence before pretending AquaSynth has waveform-loss vocal-tract backprop. eSpeak log-mel means can supervise both utterance embedding targets and synth-driver mel/control lanes, giving the packed optimizer real generated speech pressure while the renderer is still catching up.","evidence":"Added EspeakNgGradientDescentTests. When ESPEAK_NG or espeak/espeak-ng is available, it renders a/pa/ta/ka/sa/ma, extracts log-mel means, trains UtteranceEmbeddingNeuralEncoder, trains VocalTractNeuralMapper against eSpeak-seeded control targets, asserts loss reductions, and writes artifacts/parity/espeak-ng-gradient-descent/-tiny/training-report.md."} +{"date":"2026-05-19","topic":"speech-pipeline-chained-backprop","lesson":"The speech-learning path now has real chained backprop between the utterance encoder and synth driver. The packed network can return gradients with respect to its input, so downstream synth-driver loss can update the upstream utterance encoder instead of training the two models as separate supervised islands.","evidence":"Added PackedNeuralBackpropagation, TrainSingle/TrainSingleFromOutputGradient, SpeechBackpropagationPipeline, and SpeechBackpropagationPipelineTests; updated the eSpeak gradient fixture to train the utterance encoder from synth-driver loss rather than from a separate compressed-mel target."} diff --git a/state/map.yaml b/state/map.yaml index 790827a..86e298a 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -34,6 +34,7 @@ modules: vocal_tract_model: src/AquaSynth.Core/VocalTractModel.cs packed_neural_network: src/AquaSynth.Core/PackedNeuralNetwork.cs utterance_embedding_learning: src/AquaSynth.Core/UtteranceEmbeddingLearning.cs + speech_backpropagation_pipeline: src/AquaSynth.Core/SpeechBackpropagationPipeline.cs vocal_tract_learning: src/AquaSynth.Core/VocalTractLearning.cs faust_emitter: src/AquaSynth.Core/FaustExport.cs faust_compiler: src/AquaSynth.Faust/FaustCompiler.cs @@ -47,6 +48,7 @@ modules: package_boundary_tests: tests/AquaSynth.Dsl.Tests/PackageBoundaryTests.cs vocal_tract_model_tests: tests/AquaSynth.Dsl.Tests/VocalTractModelTests.cs utterance_embedding_learning_tests: tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs + speech_backpropagation_pipeline_tests: tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs vocal_tract_learning_tests: tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs sfxr: src/AquaSynth.Core/SfxrParams.cs analysis: src/AquaSynth.Core/AudioAnalysis.cs @@ -73,6 +75,7 @@ boundaries: - packed-buffer native backprop/Adam as the shared AquaSynth gradient-descent primitive - learned utterance embedding encoder from speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors - learned phonetic-plus-utterance-embedding synth driver with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls + - chained utterance-embedding-to-synth-driver backpropagation that returns synth input gradients and updates the utterance encoder from downstream speech loss - optional eSpeak NG generated ground-truth fixture for supervised utterance-embedding and synth-driver gradient descent dev_only: - public-domain DX7 SysEx fixtures diff --git a/state/memory.json b/state/memory.json index bfc7ead..6af70f3 100644 --- a/state/memory.json +++ b/state/memory.json @@ -86,7 +86,7 @@ "rule": "Add DSL abstractions only when reference rebuilds prove the missing invariant. Do not consume references that cannot be backed by parity tests; parsing alone is inventory, not evidence of expressivity." }, "ipa_vocal_tract_path": { - "status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. AquaSynth now owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. The first generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, trains the utterance encoder, then trains the synth driver toward mel/control targets seeded by that evidence.", + "status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. AquaSynth now owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline now chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets.", "lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.", "next": "Implement Stage 0/1 fixtures: IPA token spans, vowel feature bundles, human baseline vowel tract-area targets, and a tiny audible vowel render proof before expanding consonants." }, diff --git a/tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs b/tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs index 133984e..af0b057 100644 --- a/tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs +++ b/tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs @@ -30,7 +30,7 @@ public sealed class EspeakNgGradientDescentTests var samples = ReadMonoPcm16Wav(wavPath); var analysis = AudioAnalyzer.AnalyzeAudio(samples); var logMelMean = MeanBands(analysis.LogMelSpectrogram); - groundTruth.Add(new GroundTruthFixture(fixture, logMelMean, Downsample(logMelMean, 16))); + groundTruth.Add(new GroundTruthFixture(fixture, logMelMean)); } var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( @@ -38,42 +38,34 @@ public sealed class EspeakNgGradientDescentTests embeddingSize: 16, hiddenLayerSizes: [64, 48, 32], seed: 101); - var utteranceExamples = groundTruth - .Select(item => new UtteranceEmbeddingTrainingExample( - item.Fixture.UtteranceInput.ToFeatureVector(), - new UtteranceEmbedding(item.TargetEmbedding))) - .ToArray(); - - var utteranceBefore = AverageUtteranceLoss(utteranceEncoder, utteranceExamples); - var utteranceResult = utteranceEncoder.Train(utteranceExamples, new PackedNeuralTrainingOptions(Epochs: 420, LearningRate: 0.03f, Seed: 101, BatchSize: 3)); - var utteranceAfter = AverageUtteranceLoss(utteranceEncoder, utteranceExamples); - var vocalMapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, melBandCount: 32, hiddenLayerSizes: [80, 56, 40], seed: 103); - var vocalExamples = groundTruth - .Select(item => new VocalTractTrainingExample( + var pipeline = new SpeechBackpropagationPipeline(utteranceEncoder, vocalMapper); + var examples = groundTruth + .Select(item => new SpeechBackpropagationTrainingExample( + item.Fixture.UtteranceInput, item.Fixture.Event, - utteranceEncoder.Encode(item.Fixture.UtteranceInput).ToSemanticEmbedding(), TargetFromGroundTruth(item.Fixture.Event, item.LogMelMean))) .ToArray(); - var vocalBefore = AverageVocalLoss(vocalMapper, vocalExamples); - var vocalResult = vocalMapper.Train(vocalExamples, new VocalTractTrainingOptions(Epochs: 520, LearningRate: 0.035f, Seed: 103, BatchSize: 3)); - var vocalAfter = AverageVocalLoss(vocalMapper, vocalExamples); + var pipelineBefore = AveragePipelineLoss(pipeline, examples); + var pipelineResult = pipeline.Train(examples, new SpeechBackpropagationTrainingOptions( + Epochs: 640, + UtteranceLearningRate: 0.04f, + SynthDriverLearningRate: 0.04f, + Seed: 107)); + var pipelineAfter = AveragePipelineLoss(pipeline, examples); - Assert.True(utteranceAfter < utteranceBefore * 0.45f, $"utterance embedding loss did not drop enough; before={utteranceBefore}, after={utteranceAfter}"); - Assert.True(vocalAfter < vocalBefore * 0.45f, $"vocal driver loss did not drop enough; before={vocalBefore}, after={vocalAfter}"); + Assert.True(pipelineAfter < pipelineBefore * 0.55f, $"end-to-end speech pipeline loss did not drop enough; before={pipelineBefore}, after={pipelineAfter}"); + Assert.True(pipelineResult.Steps[^1].Loss < pipelineResult.Steps[0].Loss, "end-to-end pipeline loss should decrease across epochs"); var report = new List { "eSpeak NG gradient descent fixture", $"renderer: {renderer.CommandPath}", - "role: generated supervised ground truth for AquaSynth utterance embedding and synth-driver training", - $"utterance_loss_before: {utteranceBefore:0.######}", - $"utterance_loss_after: {utteranceAfter:0.######}", - $"utterance_epochs: {utteranceResult.Steps.Count}", - $"vocal_loss_before: {vocalBefore:0.######}", - $"vocal_loss_after: {vocalAfter:0.######}", - $"vocal_epochs: {vocalResult.Steps.Count}", + "role: generated supervised ground truth for end-to-end AquaSynth utterance embedding -> synth-driver backpropagation", + $"pipeline_loss_before: {pipelineBefore:0.######}", + $"pipeline_loss_after: {pipelineAfter:0.######}", + $"pipeline_epochs: {pipelineResult.Steps.Count}", "fixtures:" }; foreach (var item in groundTruth) @@ -127,25 +119,13 @@ public sealed class EspeakNgGradientDescentTests MelSpectralEnvelope: logMelMean); } - private static float AverageUtteranceLoss(UtteranceEmbeddingNeuralEncoder encoder, IReadOnlyList examples) + private static float AveragePipelineLoss(SpeechBackpropagationPipeline pipeline, IReadOnlyList examples) { var loss = 0f; foreach (var example in examples) { - var prediction = encoder.Encode(example.Features); - loss += MeanSquaredError(prediction.Values, example.Target.Values); - } - - return loss / examples.Count; - } - - private static float AverageVocalLoss(VocalTractNeuralMapper mapper, IReadOnlyList examples) - { - var loss = 0f; - foreach (var example in examples) - { - var prediction = mapper.Predict(example.Event, example.SemanticEmbedding); - loss += MeanSquaredError(prediction.ToVector(mapper.MelBandCount), example.Target.ToVector(mapper.MelBandCount)); + var prediction = pipeline.Predict(example); + loss += MeanSquaredError(prediction.ToVector(pipeline.SynthDriver.MelBandCount), example.Target.ToVector(pipeline.SynthDriver.MelBandCount)); } return loss / examples.Count; @@ -164,18 +144,6 @@ public sealed class EspeakNgGradientDescentTests return loss / Math.Max(1, count); } - private static float[] Downsample(IReadOnlyList values, int count) - { - var output = new float[count]; - for (var index = 0; index < count; index++) - { - var source = (int)MathF.Round(index * (values.Count - 1) / Math.Max(1f, count - 1f)); - output[index] = values[source]; - } - - return output; - } - private static float[] MeanBands(Spectrogram spectrogram) { var means = new float[spectrogram.Bands]; @@ -283,8 +251,7 @@ public sealed class EspeakNgGradientDescentTests private sealed record GroundTruthFixture( TrainingFixture Fixture, - float[] LogMelMean, - float[] TargetEmbedding); + float[] LogMelMean); private sealed class EspeakNgReferenceRenderer(string commandPath) { diff --git a/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs b/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs new file mode 100644 index 0000000..3647d2c --- /dev/null +++ b/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs @@ -0,0 +1,105 @@ +using AquaSynth.Dsl; + +namespace AquaSynth.Dsl.Tests; + +public sealed class SpeechBackpropagationPipelineTests +{ + [Fact] + public void SpeechPipelineBackpropagatesSynthDriverLossIntoUtteranceEncoder() + { + var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( + inputSize: 10, + embeddingSize: 16, + hiddenLayerSizes: [48, 32, 24], + seed: 211); + var synthDriver = VocalTractNeuralMapper.Create( + semanticEmbeddingSize: 16, + melBandCount: 12, + hiddenLayerSizes: [72, 48, 32], + seed: 223); + var pipeline = new SpeechBackpropagationPipeline(utteranceEncoder, synthDriver); + var examples = TrainingSet(); + + var before = AverageLoss(pipeline, examples); + var result = pipeline.Train(examples, new SpeechBackpropagationTrainingOptions( + Epochs: 520, + UtteranceLearningRate: 0.055f, + SynthDriverLearningRate: 0.055f, + Seed: 229)); + var after = AverageLoss(pipeline, examples); + + Assert.Equal(520, result.Steps.Count); + Assert.True(after < before * 0.55f, $"expected end-to-end loss to drop; before={before}, after={after}"); + Assert.True(result.Steps[^1].Loss < result.Steps[0].Loss, "pipeline loss should decrease across epochs"); + } + + private static IReadOnlyList TrainingSet() => + [ + Example( + "a", + new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), + new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]), + Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f, 0.34f, 0.28f, 0.22f, 0.18f, 0.16f, 0.14f])), + Example( + "sa", + new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless), + new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]), + Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f, 0.76f, 0.88f, 0.94f, 0.90f, 0.82f, 0.72f])), + Example( + "ma", + new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true), + new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]), + Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f, 0.40f, 0.36f, 0.32f, 0.30f, 0.28f, 0.26f])) + ]; + + private static SpeechBackpropagationTrainingExample Example( + string ipa, + PhoneticFeatures features, + UtteranceEmbeddingInput input, + VocalTractControlTarget target) => + new(input, new PhoneticEvent($"phone-{ipa}", ipa, features, DurationSeconds: 0.2), target); + + private static VocalTractControlTarget Target(IReadOnlyList mel) => + new( + TongueBody: mel[3], + TongueTip: mel[8], + LipAperture: mel[0], + LipRounding: 0.08f, + Velum: mel[1], + GlottalTenseness: mel[4], + Turbulence: mel[9], + Pressure: mel[6], + AmDepth: mel[2], + FmDepth: mel[7], + LfoRate: 0.22f, + LfoDepth: mel[5], + FilterCutoff: mel[10], + FilterResonance: mel[11], + MelSpectralEnvelope: mel); + + private static float AverageLoss(SpeechBackpropagationPipeline pipeline, IReadOnlyList examples) + { + var loss = 0f; + foreach (var example in examples) + { + var prediction = pipeline.Predict(example); + loss += MeanSquaredError( + prediction.ToVector(pipeline.SynthDriver.MelBandCount), + example.Target.ToVector(pipeline.SynthDriver.MelBandCount)); + } + + return loss / examples.Count; + } + + private static float MeanSquaredError(IReadOnlyList actual, IReadOnlyList target) + { + var loss = 0f; + for (var index = 0; index < actual.Count; index++) + { + var difference = actual[index] - target[index]; + loss += difference * difference; + } + + return loss / actual.Count; + } +}