Chain speech model backpropagation

docs/ipa-vocal-tract-roadmap.md

@@ -190,12 +190,16 @@ the gradient descent. `UtteranceEmbeddingNeuralEncoder` trains a tiny model to
 compress those ingredients into one utterance embedding. `VocalTractNeuralMapper`
 then takes phonetic features plus that embedding and predicts tract controls.
 Both controllers use the same native C# packed-network path: row-major buffers,
-unsafe hot loops, batched gradients, and Adam or SGD updates. The synth-driver
-output includes core tract parameters plus expressive synthesis lanes: AM depth,
-FM depth, LFO rate/depth, filter cutoff, filter resonance, and a full
-mel-frequency spectral envelope vector for PAD-style wavetable/formant
-coloration. Those lanes are not license to bypass tract anatomy; they are
-renderable expression controls the later synth can choose to honor.
+unsafe hot loops, batched gradients, and Adam or SGD updates. For chained speech
+training, `SpeechBackpropagationPipeline` runs the synth-driver loss backward
+through the utterance embedding, slices out the embedding gradient from the
+synth-driver input gradient, and updates the utterance encoder from that upstream
+loss. The synth-driver output includes core tract parameters plus expressive
+synthesis lanes: AM depth, FM depth, LFO rate/depth, filter cutoff, filter
+resonance, and a full mel-frequency spectral envelope vector for PAD-style
+wavetable/formant coloration. Those lanes are not license to bypass tract
+anatomy; they are renderable expression controls the later synth can choose to
+honor.
 
 ### 4. Morphology Model
 
@@ -476,10 +480,11 @@ AquaSynth's future IPA/gesture/render path, not replace it.
 
 The optional test `EspeakNgGroundTruthTrainsUtteranceEmbeddingAndSynthDriverWhenInstalled`
 turns the same idea into a gradient-descent fixture. When eSpeak is present it
-renders the tiny syllable set, extracts normalized log-mel means, trains
-`UtteranceEmbeddingNeuralEncoder` toward compressed eSpeak evidence, then trains
-`VocalTractNeuralMapper` toward tract/mod/filter/mel controls seeded by that
-evidence. It writes WAVs plus a training report under:
+renders the tiny syllable set, extracts normalized log-mel means, and runs
+`SpeechBackpropagationPipeline`: the eSpeak-seeded synth-driver target loss
+updates `VocalTractNeuralMapper` and backpropagates through the utterance
+embedding into `UtteranceEmbeddingNeuralEncoder`. It writes WAVs plus a training
+report under:
 
 ```text
 artifacts/parity/espeak-ng-gradient-descent/<timestamp>-tiny/

src/AquaSynth.Core/PackedNeuralNetwork.cs

@@ -27,6 +27,11 @@ public sealed record PackedNeuralTrainingResult(
     PackedNeuralNetwork Network,
     IReadOnlyList<PackedNeuralTrainingStep> Steps);
 
+public sealed record PackedNeuralBackpropagation(
+    float Loss,
+    float[] Output,
+    float[] InputGradient);
+
 public sealed class PackedNeuralNetwork
 {
     private readonly Layer[] layers;
@@ -148,6 +153,42 @@ public sealed class PackedNeuralNetwork
         return new PackedNeuralTrainingResult(this, steps);
     }
 
+    public PackedNeuralBackpropagation TrainSingle(
+        IReadOnlyList<float> input,
+        IReadOnlyList<float> target,
+        float learningRate)
+    {
+        if (learningRate <= 0) throw new ArgumentOutOfRangeException(nameof(learningRate), "learning rate must be positive");
+        if (target.Count != OutputSize) throw new ArgumentException($"target must have {OutputSize} values", nameof(target));
+
+        var outputGradient = new float[OutputSize];
+        var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
+        CopyChecked(input, scratch.Activations[0], "input");
+        Copy(target, scratch.Target);
+        Forward(scratch);
+        var loss = MeanSquaredError(scratch.Output, scratch.Target);
+        for (var index = 0; index < outputGradient.Length; index++)
+        {
+            outputGradient[index] = 2f * (scratch.Output[index] - scratch.Target[index]) / Math.Max(1, OutputSize);
+        }
+
+        return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, loss);
+    }
+
+    public PackedNeuralBackpropagation TrainSingleFromOutputGradient(
+        IReadOnlyList<float> input,
+        IReadOnlyList<float> outputGradient,
+        float learningRate)
+    {
+        if (learningRate <= 0) throw new ArgumentOutOfRangeException(nameof(learningRate), "learning rate must be positive");
+        if (outputGradient.Count != OutputSize) throw new ArgumentException($"output gradient must have {OutputSize} values", nameof(outputGradient));
+
+        var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
+        CopyChecked(input, scratch.Activations[0], "input");
+        Forward(scratch);
+        return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, 0);
+    }
+
     private static void Validate(PackedNeuralTrainingOptions options)
     {
         if (options.Epochs <= 0) throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive");
@@ -163,7 +204,31 @@ public sealed class PackedNeuralNetwork
         }
     }
 
+    private PackedNeuralBackpropagation TrainSingleFromOutputGradient(
+        TrainingScratch scratch,
+        IReadOnlyList<float> outputGradient,
+        float learningRate,
+        float loss)
+    {
+        var gradients = LayerBuffers.Create(layers);
+        AccumulateGradientsFromOutputGradient(scratch, outputGradient, gradients);
+        var inputGradient = InputGradient(scratch);
+        ApplySgd(gradients, 1f, learningRate);
+        return new PackedNeuralBackpropagation(loss, scratch.Output.ToArray(), inputGradient);
+    }
+
     private void AccumulateGradients(TrainingScratch scratch, LayerBuffers gradients)
+    {
+        var outputGradient = new float[OutputSize];
+        for (var index = 0; index < outputGradient.Length; index++)
+        {
+            outputGradient[index] = 2f * (scratch.Output[index] - scratch.Target[index]);
+        }
+
+        AccumulateGradientsFromOutputGradient(scratch, outputGradient, gradients);
+    }
+
+    private void AccumulateGradientsFromOutputGradient(TrainingScratch scratch, IReadOnlyList<float> outputGradient, LayerBuffers gradients)
     {
         var lastLayerIndex = layers.Length - 1;
         var outputDelta = scratch.Deltas[lastLayerIndex];
@@ -171,7 +236,7 @@ public sealed class PackedNeuralNetwork
         for (var index = 0; index < outputDelta.Length; index++)
         {
             var actual = output[index];
-            outputDelta[index] = 2f * (actual - scratch.Target[index]) * layers[lastLayerIndex].Derivative(actual);
+            outputDelta[index] = outputGradient[index] * layers[lastLayerIndex].Derivative(actual);
         }
 
         for (var layerIndex = lastLayerIndex - 1; layerIndex >= 0; layerIndex--)
@@ -185,6 +250,13 @@ public sealed class PackedNeuralNetwork
         }
     }
 
+    private float[] InputGradient(TrainingScratch scratch)
+    {
+        var inputGradient = new float[InputSize];
+        layers[0].BackpropagateInputGradient(scratch.Deltas[0], inputGradient);
+        return inputGradient;
+    }
+
     private void ApplySgd(LayerBuffers gradients, float scale, float learningRate)
     {
         for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++)
@@ -407,6 +479,28 @@ public sealed class PackedNeuralNetwork
             }
         }
 
+        public void BackpropagateInputGradient(float[] nextDelta, float[] inputGradient)
+        {
+            unsafe
+            {
+                fixed (float* nextDeltaBase = nextDelta)
+                fixed (float* inputGradientBase = inputGradient)
+                fixed (float* weightBase = Weights)
+                {
+                    for (var input = 0; input < InputSize; input++)
+                    {
+                        var sum = 0f;
+                        for (var output = 0; output < OutputSize; output++)
+                        {
+                            sum += weightBase[output * InputSize + input] * nextDeltaBase[output];
+                        }
+
+                        inputGradientBase[input] = sum;
+                    }
+                }
+            }
+        }
+
         public float Derivative(float activated) =>
             activation switch
             {

src/AquaSynth.Core/SpeechBackpropagationPipeline.cs

@@ -0,0 +1,111 @@
+namespace AquaSynth.Dsl;
+
+public sealed record SpeechBackpropagationTrainingExample(
+    UtteranceEmbeddingInput UtteranceInput,
+    PhoneticEvent Event,
+    VocalTractControlTarget Target);
+
+public sealed record SpeechBackpropagationTrainingOptions(
+    int Epochs = 200,
+    float UtteranceLearningRate = 0.01f,
+    float SynthDriverLearningRate = 0.01f,
+    int Seed = 1337,
+    bool Shuffle = true);
+
+public sealed record SpeechBackpropagationTrainingStep(int Epoch, float Loss);
+
+public sealed record SpeechBackpropagationTrainingResult(
+    SpeechBackpropagationPipeline Pipeline,
+    IReadOnlyList<SpeechBackpropagationTrainingStep> Steps);
+
+public sealed class SpeechBackpropagationPipeline(
+    UtteranceEmbeddingNeuralEncoder utteranceEncoder,
+    VocalTractNeuralMapper synthDriver)
+{
+    public UtteranceEmbeddingNeuralEncoder UtteranceEncoder { get; } = utteranceEncoder;
+
+    public VocalTractNeuralMapper SynthDriver { get; } = synthDriver;
+
+    public SpeechBackpropagationTrainingResult Train(
+        IReadOnlyList<SpeechBackpropagationTrainingExample> examples,
+        SpeechBackpropagationTrainingOptions? options = null)
+    {
+        if (examples.Count == 0)
+        {
+            throw new ArgumentException("at least one training example is required", nameof(examples));
+        }
+
+        options ??= new SpeechBackpropagationTrainingOptions();
+        if (options.Epochs <= 0) throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive");
+        if (options.UtteranceLearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(options), "utterance learning rate must be positive");
+        if (options.SynthDriverLearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(options), "synth-driver learning rate must be positive");
+
+        var order = Enumerable.Range(0, examples.Count).ToArray();
+        var random = new Random(options.Seed);
+        var steps = new List<SpeechBackpropagationTrainingStep>();
+
+        for (var epoch = 0; epoch < options.Epochs; epoch++)
+        {
+            if (options.Shuffle)
+            {
+                Shuffle(order, random);
+            }
+
+            var loss = 0f;
+            foreach (var index in order)
+            {
+                loss += TrainSingle(examples[index], options);
+            }
+
+            steps.Add(new SpeechBackpropagationTrainingStep(epoch + 1, loss / examples.Count));
+        }
+
+        return new SpeechBackpropagationTrainingResult(this, steps);
+    }
+
+    public VocalTractControlTarget Predict(SpeechBackpropagationTrainingExample example)
+    {
+        var embedding = UtteranceEncoder.Encode(example.UtteranceInput).ToSemanticEmbedding();
+        return SynthDriver.Predict(example.Event, embedding);
+    }
+
+    private float TrainSingle(SpeechBackpropagationTrainingExample example, SpeechBackpropagationTrainingOptions options)
+    {
+        var utteranceFeatures = example.UtteranceInput.ToFeatureVector();
+        var embedding = UtteranceEncoder.Encode(utteranceFeatures);
+        var synthResult = SynthDriver.TrainSingle(
+            example.Event,
+            embedding.ToSemanticEmbedding(),
+            example.Target,
+            options.SynthDriverLearningRate);
+        var embeddingGradient = SemanticEmbeddingGradient(synthResult.InputGradient);
+        UtteranceEncoder.TrainSingleFromOutputGradient(utteranceFeatures, embeddingGradient, options.UtteranceLearningRate);
+        return synthResult.Loss;
+    }
+
+    private float[] SemanticEmbeddingGradient(IReadOnlyList<float> synthInputGradient)
+    {
+        if (synthInputGradient.Count != SynthDriver.InputSize)
+        {
+            throw new InvalidOperationException("synth input gradient length does not match synth driver input size");
+        }
+
+        var gradient = new float[SynthDriver.SemanticEmbeddingSize];
+        var offset = SynthDriver.InputSize - SynthDriver.SemanticEmbeddingSize;
+        for (var index = 0; index < gradient.Length; index++)
+        {
+            gradient[index] = synthInputGradient[offset + index];
+        }
+
+        return gradient;
+    }
+
+    private static void Shuffle(int[] values, Random random)
+    {
+        for (var index = values.Length - 1; index > 0; index--)
+        {
+            var swapIndex = random.Next(index + 1);
+            (values[index], values[swapIndex]) = (values[swapIndex], values[index]);
+        }
+    }
+}

src/AquaSynth.Core/UtteranceEmbeddingLearning.cs

@@ -80,6 +80,15 @@ public sealed class UtteranceEmbeddingNeuralEncoder
         return new UtteranceEmbedding(network.Predict(features.Values));
     }
 
+    public PackedNeuralBackpropagation TrainSingleFromOutputGradient(
+        UtteranceEmbeddingFeatureVector features,
+        IReadOnlyList<float> outputGradient,
+        float learningRate)
+    {
+        ValidateSize(features.Values, InputSize, nameof(features));
+        return network.TrainSingleFromOutputGradient(features.Values, outputGradient, learningRate);
+    }
+
     public UtteranceEmbeddingTrainingResult Train(
         IReadOnlyList<UtteranceEmbeddingTrainingExample> examples,
         PackedNeuralTrainingOptions? options = null)

src/AquaSynth.Core/VocalTractLearning.cs

@@ -99,9 +99,22 @@ public sealed class VocalTractNeuralMapper
 
     public VocalTractControlTarget Predict(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding)
     {
-        return TargetFromVector(network.Predict(EncodeInput(phoneticEvent, semanticEmbedding)));
+        return TargetFromVector(PredictVector(phoneticEvent, semanticEmbedding));
     }
 
+    public float[] PredictVector(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding) =>
+        network.Predict(EncodeInput(phoneticEvent, semanticEmbedding));
+
+    public PackedNeuralBackpropagation TrainSingle(
+        PhoneticEvent phoneticEvent,
+        VocalTractSemanticEmbedding semanticEmbedding,
+        VocalTractControlTarget target,
+        float learningRate) =>
+        network.TrainSingle(
+            EncodeInput(phoneticEvent, semanticEmbedding),
+            target.ToVector(MelBandCount, NeutralMelBand),
+            learningRate);
+
     public VocalTractTrainingResult Train(
         IReadOnlyList<VocalTractTrainingExample> examples,
         VocalTractTrainingOptions? options = null)

state/evidence.jsonl

@@ -113,3 +113,4 @@
 {"date":"2026-05-19","topic":"vocal-tract-optimizer-hot-path","lesson":"The first learned vocal-tract optimizer should stay native and inspectable until the render-loss boundary proves it needs a tensor framework. The trainer now uses packed row-major weight buffers, owned scratch activation/delta/target buffers, unsafe pointer loops, reusable gradient/moment buffers, batched gradients, deterministic shuffling, and Adam by default instead of allocation-heavy per-example jagged-array SGD.","evidence":"Rebuilt VocalTractLearning.cs hot path around flat float arrays and unsafe loops; enabled unsafe blocks in AquaSynth.Core; VocalTractLearningTests pass against the packed-buffer Adam trainer."}
 {"date":"2026-05-19","topic":"utterance-embedding-learning-boundary","lesson":"AquaSynth should own gradient descent for Weksa speech handoff. Weksa supplies speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors; AquaSynth trains the tiny utterance-embedding encoder and passes that embedding into the synth-driver model.","evidence":"Extracted PackedNeuralNetwork as the shared packed-buffer Adam/SGD primitive, rewired VocalTractNeuralMapper through it, and added UtteranceEmbeddingInput/UtteranceEmbeddingNeuralEncoder plus tests for packing, training, and feeding the vocal-tract semantic embedding."}
 {"date":"2026-05-19","topic":"espeak-gradient-descent-fixture","lesson":"The first speech-learning fixture should train on generated eSpeak evidence before pretending AquaSynth has waveform-loss vocal-tract backprop. eSpeak log-mel means can supervise both utterance embedding targets and synth-driver mel/control lanes, giving the packed optimizer real generated speech pressure while the renderer is still catching up.","evidence":"Added EspeakNgGradientDescentTests. When ESPEAK_NG or espeak/espeak-ng is available, it renders a/pa/ta/ka/sa/ma, extracts log-mel means, trains UtteranceEmbeddingNeuralEncoder, trains VocalTractNeuralMapper against eSpeak-seeded control targets, asserts loss reductions, and writes artifacts/parity/espeak-ng-gradient-descent/<timestamp>-tiny/training-report.md."}
+{"date":"2026-05-19","topic":"speech-pipeline-chained-backprop","lesson":"The speech-learning path now has real chained backprop between the utterance encoder and synth driver. The packed network can return gradients with respect to its input, so downstream synth-driver loss can update the upstream utterance encoder instead of training the two models as separate supervised islands.","evidence":"Added PackedNeuralBackpropagation, TrainSingle/TrainSingleFromOutputGradient, SpeechBackpropagationPipeline, and SpeechBackpropagationPipelineTests; updated the eSpeak gradient fixture to train the utterance encoder from synth-driver loss rather than from a separate compressed-mel target."}

state/map.yaml

@@ -34,6 +34,7 @@ modules:
   vocal_tract_model: src/AquaSynth.Core/VocalTractModel.cs
   packed_neural_network: src/AquaSynth.Core/PackedNeuralNetwork.cs
   utterance_embedding_learning: src/AquaSynth.Core/UtteranceEmbeddingLearning.cs
+  speech_backpropagation_pipeline: src/AquaSynth.Core/SpeechBackpropagationPipeline.cs
   vocal_tract_learning: src/AquaSynth.Core/VocalTractLearning.cs
   faust_emitter: src/AquaSynth.Core/FaustExport.cs
   faust_compiler: src/AquaSynth.Faust/FaustCompiler.cs
@@ -47,6 +48,7 @@ modules:
   package_boundary_tests: tests/AquaSynth.Dsl.Tests/PackageBoundaryTests.cs
   vocal_tract_model_tests: tests/AquaSynth.Dsl.Tests/VocalTractModelTests.cs
   utterance_embedding_learning_tests: tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs
+  speech_backpropagation_pipeline_tests: tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs
   vocal_tract_learning_tests: tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs
   sfxr: src/AquaSynth.Core/SfxrParams.cs
   analysis: src/AquaSynth.Core/AudioAnalysis.cs
@@ -73,6 +75,7 @@ boundaries:
     - packed-buffer native backprop/Adam as the shared AquaSynth gradient-descent primitive
     - learned utterance embedding encoder from speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors
     - learned phonetic-plus-utterance-embedding synth driver with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls
+    - chained utterance-embedding-to-synth-driver backpropagation that returns synth input gradients and updates the utterance encoder from downstream speech loss
     - optional eSpeak NG generated ground-truth fixture for supervised utterance-embedding and synth-driver gradient descent
   dev_only:
     - public-domain DX7 SysEx fixtures

state/memory.json

@@ -86,7 +86,7 @@
     "rule": "Add DSL abstractions only when reference rebuilds prove the missing invariant. Do not consume references that cannot be backed by parity tests; parsing alone is inventory, not evidence of expressivity."
   },
   "ipa_vocal_tract_path": {
-    "status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. AquaSynth now owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. The first generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, trains the utterance encoder, then trains the synth driver toward mel/control targets seeded by that evidence.",
+    "status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. AquaSynth now owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline now chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets.",
     "lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.",
     "next": "Implement Stage 0/1 fixtures: IPA token spans, vowel feature bundles, human baseline vowel tract-area targets, and a tiny audible vowel render proof before expanding consonants."
   },

tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs

@@ -30,7 +30,7 @@ public sealed class EspeakNgGradientDescentTests
             var samples = ReadMonoPcm16Wav(wavPath);
             var analysis = AudioAnalyzer.AnalyzeAudio(samples);
             var logMelMean = MeanBands(analysis.LogMelSpectrogram);
-            groundTruth.Add(new GroundTruthFixture(fixture, logMelMean, Downsample(logMelMean, 16)));
+            groundTruth.Add(new GroundTruthFixture(fixture, logMelMean));
         }
 
         var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
@@ -38,42 +38,34 @@ public sealed class EspeakNgGradientDescentTests
             embeddingSize: 16,
             hiddenLayerSizes: [64, 48, 32],
             seed: 101);
-        var utteranceExamples = groundTruth
-            .Select(item => new UtteranceEmbeddingTrainingExample(
-                item.Fixture.UtteranceInput.ToFeatureVector(),
-                new UtteranceEmbedding(item.TargetEmbedding)))
-            .ToArray();
-
-        var utteranceBefore = AverageUtteranceLoss(utteranceEncoder, utteranceExamples);
-        var utteranceResult = utteranceEncoder.Train(utteranceExamples, new PackedNeuralTrainingOptions(Epochs: 420, LearningRate: 0.03f, Seed: 101, BatchSize: 3));
-        var utteranceAfter = AverageUtteranceLoss(utteranceEncoder, utteranceExamples);
-
         var vocalMapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, melBandCount: 32, hiddenLayerSizes: [80, 56, 40], seed: 103);
-        var vocalExamples = groundTruth
-            .Select(item => new VocalTractTrainingExample(
+        var pipeline = new SpeechBackpropagationPipeline(utteranceEncoder, vocalMapper);
+        var examples = groundTruth
+            .Select(item => new SpeechBackpropagationTrainingExample(
+                item.Fixture.UtteranceInput,
                 item.Fixture.Event,
-                utteranceEncoder.Encode(item.Fixture.UtteranceInput).ToSemanticEmbedding(),
                 TargetFromGroundTruth(item.Fixture.Event, item.LogMelMean)))
             .ToArray();
 
-        var vocalBefore = AverageVocalLoss(vocalMapper, vocalExamples);
-        var vocalResult = vocalMapper.Train(vocalExamples, new VocalTractTrainingOptions(Epochs: 520, LearningRate: 0.035f, Seed: 103, BatchSize: 3));
-        var vocalAfter = AverageVocalLoss(vocalMapper, vocalExamples);
+        var pipelineBefore = AveragePipelineLoss(pipeline, examples);
+        var pipelineResult = pipeline.Train(examples, new SpeechBackpropagationTrainingOptions(
+            Epochs: 640,
+            UtteranceLearningRate: 0.04f,
+            SynthDriverLearningRate: 0.04f,
+            Seed: 107));
+        var pipelineAfter = AveragePipelineLoss(pipeline, examples);
 
-        Assert.True(utteranceAfter < utteranceBefore * 0.45f, $"utterance embedding loss did not drop enough; before={utteranceBefore}, after={utteranceAfter}");
-        Assert.True(vocalAfter < vocalBefore * 0.45f, $"vocal driver loss did not drop enough; before={vocalBefore}, after={vocalAfter}");
+        Assert.True(pipelineAfter < pipelineBefore * 0.55f, $"end-to-end speech pipeline loss did not drop enough; before={pipelineBefore}, after={pipelineAfter}");
+        Assert.True(pipelineResult.Steps[^1].Loss < pipelineResult.Steps[0].Loss, "end-to-end pipeline loss should decrease across epochs");
 
         var report = new List<string>
         {
             "eSpeak NG gradient descent fixture",
             $"renderer: {renderer.CommandPath}",
-            "role: generated supervised ground truth for AquaSynth utterance embedding and synth-driver training",
-            $"utterance_loss_before: {utteranceBefore:0.######}",
-            $"utterance_loss_after: {utteranceAfter:0.######}",
-            $"utterance_epochs: {utteranceResult.Steps.Count}",
-            $"vocal_loss_before: {vocalBefore:0.######}",
-            $"vocal_loss_after: {vocalAfter:0.######}",
-            $"vocal_epochs: {vocalResult.Steps.Count}",
+            "role: generated supervised ground truth for end-to-end AquaSynth utterance embedding -> synth-driver backpropagation",
+            $"pipeline_loss_before: {pipelineBefore:0.######}",
+            $"pipeline_loss_after: {pipelineAfter:0.######}",
+            $"pipeline_epochs: {pipelineResult.Steps.Count}",
             "fixtures:"
         };
         foreach (var item in groundTruth)
@@ -127,25 +119,13 @@ public sealed class EspeakNgGradientDescentTests
             MelSpectralEnvelope: logMelMean);
     }
 
-    private static float AverageUtteranceLoss(UtteranceEmbeddingNeuralEncoder encoder, IReadOnlyList<UtteranceEmbeddingTrainingExample> examples)
+    private static float AveragePipelineLoss(SpeechBackpropagationPipeline pipeline, IReadOnlyList<SpeechBackpropagationTrainingExample> examples)
     {
         var loss = 0f;
         foreach (var example in examples)
         {
-            var prediction = encoder.Encode(example.Features);
-            loss += MeanSquaredError(prediction.Values, example.Target.Values);
-        }
-
-        return loss / examples.Count;
-    }
-
-    private static float AverageVocalLoss(VocalTractNeuralMapper mapper, IReadOnlyList<VocalTractTrainingExample> examples)
-    {
-        var loss = 0f;
-        foreach (var example in examples)
-        {
-            var prediction = mapper.Predict(example.Event, example.SemanticEmbedding);
-            loss += MeanSquaredError(prediction.ToVector(mapper.MelBandCount), example.Target.ToVector(mapper.MelBandCount));
+            var prediction = pipeline.Predict(example);
+            loss += MeanSquaredError(prediction.ToVector(pipeline.SynthDriver.MelBandCount), example.Target.ToVector(pipeline.SynthDriver.MelBandCount));
         }
 
         return loss / examples.Count;
@@ -164,18 +144,6 @@ public sealed class EspeakNgGradientDescentTests
         return loss / Math.Max(1, count);
     }
 
-    private static float[] Downsample(IReadOnlyList<float> values, int count)
-    {
-        var output = new float[count];
-        for (var index = 0; index < count; index++)
-        {
-            var source = (int)MathF.Round(index * (values.Count - 1) / Math.Max(1f, count - 1f));
-            output[index] = values[source];
-        }
-
-        return output;
-    }
-
     private static float[] MeanBands(Spectrogram spectrogram)
     {
         var means = new float[spectrogram.Bands];
@@ -283,8 +251,7 @@ public sealed class EspeakNgGradientDescentTests
 
     private sealed record GroundTruthFixture(
         TrainingFixture Fixture,
-        float[] LogMelMean,
-        float[] TargetEmbedding);
+        float[] LogMelMean);
 
     private sealed class EspeakNgReferenceRenderer(string commandPath)
     {

tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs

@@ -0,0 +1,105 @@
+using AquaSynth.Dsl;
+
+namespace AquaSynth.Dsl.Tests;
+
+public sealed class SpeechBackpropagationPipelineTests
+{
+    [Fact]
+    public void SpeechPipelineBackpropagatesSynthDriverLossIntoUtteranceEncoder()
+    {
+        var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
+            inputSize: 10,
+            embeddingSize: 16,
+            hiddenLayerSizes: [48, 32, 24],
+            seed: 211);
+        var synthDriver = VocalTractNeuralMapper.Create(
+            semanticEmbeddingSize: 16,
+            melBandCount: 12,
+            hiddenLayerSizes: [72, 48, 32],
+            seed: 223);
+        var pipeline = new SpeechBackpropagationPipeline(utteranceEncoder, synthDriver);
+        var examples = TrainingSet();
+
+        var before = AverageLoss(pipeline, examples);
+        var result = pipeline.Train(examples, new SpeechBackpropagationTrainingOptions(
+            Epochs: 520,
+            UtteranceLearningRate: 0.055f,
+            SynthDriverLearningRate: 0.055f,
+            Seed: 229));
+        var after = AverageLoss(pipeline, examples);
+
+        Assert.Equal(520, result.Steps.Count);
+        Assert.True(after < before * 0.55f, $"expected end-to-end loss to drop; before={before}, after={after}");
+        Assert.True(result.Steps[^1].Loss < result.Steps[0].Loss, "pipeline loss should decrease across epochs");
+    }
+
+    private static IReadOnlyList<SpeechBackpropagationTrainingExample> TrainingSet() =>
+    [
+        Example(
+            "a",
+            new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
+            new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]),
+            Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f, 0.34f, 0.28f, 0.22f, 0.18f, 0.16f, 0.14f])),
+        Example(
+            "sa",
+            new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless),
+            new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]),
+            Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f, 0.76f, 0.88f, 0.94f, 0.90f, 0.82f, 0.72f])),
+        Example(
+            "ma",
+            new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true),
+            new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]),
+            Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f, 0.40f, 0.36f, 0.32f, 0.30f, 0.28f, 0.26f]))
+    ];
+
+    private static SpeechBackpropagationTrainingExample Example(
+        string ipa,
+        PhoneticFeatures features,
+        UtteranceEmbeddingInput input,
+        VocalTractControlTarget target) =>
+        new(input, new PhoneticEvent($"phone-{ipa}", ipa, features, DurationSeconds: 0.2), target);
+
+    private static VocalTractControlTarget Target(IReadOnlyList<float> mel) =>
+        new(
+            TongueBody: mel[3],
+            TongueTip: mel[8],
+            LipAperture: mel[0],
+            LipRounding: 0.08f,
+            Velum: mel[1],
+            GlottalTenseness: mel[4],
+            Turbulence: mel[9],
+            Pressure: mel[6],
+            AmDepth: mel[2],
+            FmDepth: mel[7],
+            LfoRate: 0.22f,
+            LfoDepth: mel[5],
+            FilterCutoff: mel[10],
+            FilterResonance: mel[11],
+            MelSpectralEnvelope: mel);
+
+    private static float AverageLoss(SpeechBackpropagationPipeline pipeline, IReadOnlyList<SpeechBackpropagationTrainingExample> examples)
+    {
+        var loss = 0f;
+        foreach (var example in examples)
+        {
+            var prediction = pipeline.Predict(example);
+            loss += MeanSquaredError(
+                prediction.ToVector(pipeline.SynthDriver.MelBandCount),
+                example.Target.ToVector(pipeline.SynthDriver.MelBandCount));
+        }
+
+        return loss / examples.Count;
+    }
+
+    private static float MeanSquaredError(IReadOnlyList<float> actual, IReadOnlyList<float> target)
+    {
+        var loss = 0f;
+        for (var index = 0; index < actual.Count; index++)
+        {
+            var difference = actual[index] - target[index];
+            loss += difference * difference;
+        }
+
+        return loss / actual.Count;
+    }
+}