Add Faust rendered speech loss surface

src/AquaSynth.Core/PackedNeuralNetwork.cs

@@ -178,7 +178,8 @@ public sealed class PackedNeuralNetwork
     public PackedNeuralBackpropagation TrainSingleFromOutputGradient(
         IReadOnlyList<float> input,
         IReadOnlyList<float> outputGradient,
-        float learningRate)
+        float learningRate,
+        float loss = 0)
     {
         if (learningRate <= 0) throw new ArgumentOutOfRangeException(nameof(learningRate), "learning rate must be positive");
         if (outputGradient.Count != OutputSize) throw new ArgumentException($"output gradient must have {OutputSize} values", nameof(outputGradient));
@@ -186,7 +187,7 @@ public sealed class PackedNeuralNetwork
         var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
         CopyChecked(input, scratch.Activations[0], "input");
         Forward(scratch);
-        return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, 0);
+        return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, loss);
     }
 
     private static void Validate(PackedNeuralTrainingOptions options)

src/AquaSynth.Core/SpeechBackpropagationPipeline.cs

@@ -69,6 +69,34 @@ public sealed class SpeechBackpropagationPipeline(
         return SynthDriver.Predict(example.Event, embedding);
     }
 
+    public PackedNeuralBackpropagation TrainSingleFromSynthOutputGradient(
+        UtteranceEmbeddingInput utteranceInput,
+        PhoneticEvent phoneticEvent,
+        IReadOnlyList<float> synthOutputGradient,
+        float utteranceLearningRate,
+        float synthDriverLearningRate,
+        float loss = 0)
+    {
+        if (utteranceLearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(utteranceLearningRate), "utterance learning rate must be positive");
+        if (synthDriverLearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(synthDriverLearningRate), "synth-driver learning rate must be positive");
+        if (synthOutputGradient.Count != SynthDriver.OutputSize)
+        {
+            throw new ArgumentException($"synth output gradient must have {SynthDriver.OutputSize} values", nameof(synthOutputGradient));
+        }
+
+        var utteranceFeatures = utteranceInput.ToFeatureVector();
+        var embedding = UtteranceEncoder.Encode(utteranceFeatures);
+        var synthResult = SynthDriver.TrainSingleFromOutputGradient(
+            phoneticEvent,
+            embedding.ToSemanticEmbedding(),
+            synthOutputGradient,
+            synthDriverLearningRate,
+            loss);
+        var embeddingGradient = SemanticEmbeddingGradient(synthResult.InputGradient);
+        UtteranceEncoder.TrainSingleFromOutputGradient(utteranceFeatures, embeddingGradient, utteranceLearningRate);
+        return synthResult;
+    }
+
     private float TrainSingle(SpeechBackpropagationTrainingExample example, SpeechBackpropagationTrainingOptions options)
     {
         var utteranceFeatures = example.UtteranceInput.ToFeatureVector();

src/AquaSynth.Core/VocalTractLearning.cs

@@ -115,6 +115,18 @@ public sealed class VocalTractNeuralMapper
             target.ToVector(MelBandCount, NeutralMelBand),
             learningRate);
 
+    public PackedNeuralBackpropagation TrainSingleFromOutputGradient(
+        PhoneticEvent phoneticEvent,
+        VocalTractSemanticEmbedding semanticEmbedding,
+        IReadOnlyList<float> outputGradient,
+        float learningRate,
+        float loss = 0) =>
+        network.TrainSingleFromOutputGradient(
+            EncodeInput(phoneticEvent, semanticEmbedding),
+            outputGradient,
+            learningRate,
+            loss);
+
     public VocalTractTrainingResult Train(
         IReadOnlyList<VocalTractTrainingExample> examples,
         VocalTractTrainingOptions? options = null)

src/AquaSynth.Faust/FaustRenderedSpeechLossSurface.cs

@@ -0,0 +1,169 @@
+using System.Globalization;
+using System.Text;
+
+using AquaSynth.Dsl;
+
+namespace AquaSynth.Faust;
+
+public sealed record FaustRenderedSpeechLossOptions(
+    int SampleRate = 22050,
+    float DurationSeconds = 0.22f,
+    int MelBandCount = 12,
+    float MinFrequencyHz = 90,
+    float MaxFrequencyHz = 7000,
+    float FiniteDifferenceStep = 0.035f,
+    IReadOnlyList<int>? GradientOutputIndices = null);
+
+public sealed record FaustRenderedSpeechLoss(
+    float Loss,
+    float[] OutputGradient,
+    FaustRender Reference,
+    FaustRender Candidate,
+    AudioComparison Comparison);
+
+public sealed class FaustRenderedSpeechLossSurface(FaustRenderedSpeechLossOptions? options = null, string? faustPath = null)
+{
+    private readonly FaustRenderedSpeechLossOptions options = options ?? new FaustRenderedSpeechLossOptions();
+
+    public async Task<FaustRender?> RenderAsync(
+        VocalTractControlTarget target,
+        CancellationToken cancellationToken = default) =>
+        await FaustCompiler.RenderAsync(
+            SourceFor(target),
+            new FaustRenderOptions(options.SampleRate, options.DurationSeconds),
+            faustPath,
+            cancellationToken);
+
+    public async Task<FaustRenderedSpeechLoss?> EvaluateAsync(
+        VocalTractControlTarget reference,
+        VocalTractControlTarget candidate,
+        CancellationToken cancellationToken = default)
+    {
+        var referenceRender = await RenderAsync(reference, cancellationToken);
+        if (referenceRender is null || referenceRender.Samples.Length == 0)
+        {
+            return null;
+        }
+
+        var candidateRender = await RenderAsync(candidate, cancellationToken);
+        if (candidateRender is null || candidateRender.Samples.Length == 0)
+        {
+            return null;
+        }
+
+        var analyzer = Analyzer();
+        var comparison = analyzer.Compare(referenceRender.Samples, candidateRender.Samples);
+        var gradient = await EstimateGradientAsync(referenceRender.Samples, candidate, comparison.LogMelDistance, cancellationToken);
+        return new FaustRenderedSpeechLoss(comparison.LogMelDistance, gradient, referenceRender, candidateRender, comparison);
+    }
+
+    public string SourceFor(VocalTractControlTarget target)
+    {
+        var mel = target.MelSpectralEnvelope ?? [];
+        var source = new StringBuilder();
+        source.AppendLine("patch gain=0.85");
+        source.AppendLine(VoiceLine("saw", F(120 + target.GlottalTenseness * 110), F(0.08f + target.Pressure * 0.22f), target, target.Turbulence * 0.22f));
+        if (target.Turbulence > 0.02f)
+        {
+            source.AppendLine(VoiceLine("noise", F(900 + target.Turbulence * 2600), F(target.Turbulence * 0.16f), target, 1));
+        }
+
+        for (var band = 0; band < options.MelBandCount; band++)
+        {
+            var value = band < mel.Count ? Math.Clamp(mel[band], 0, 1) : 0.5f;
+            var hz = MelBandCenterHz(band);
+            var gain = value * value * 0.08f;
+            source.AppendLine(VoiceLine("sine", F(hz), F(gain), target, 0));
+        }
+
+        return FaustEmitter.EmitScript(source.ToString(), new FaustExportOptions("aquasynth_speech_loss")).Source;
+    }
+
+    private async Task<float[]> EstimateGradientAsync(
+        IReadOnlyList<float> referenceSamples,
+        VocalTractControlTarget candidate,
+        float baselineLoss,
+        CancellationToken cancellationToken)
+    {
+        var gradient = new float[14 + options.MelBandCount];
+        foreach (var index in GradientIndices())
+        {
+            var perturbed = Perturb(candidate, index, options.FiniteDifferenceStep);
+            var render = await RenderAsync(perturbed, cancellationToken);
+            if (render is null || render.Samples.Length == 0)
+            {
+                continue;
+            }
+
+            var loss = Analyzer().Compare(referenceSamples.ToArray(), render.Samples).LogMelDistance;
+            gradient[index] = (loss - baselineLoss) / options.FiniteDifferenceStep;
+        }
+
+        return gradient;
+    }
+
+    private IReadOnlyList<int> GradientIndices() =>
+        options.GradientOutputIndices ?? Enumerable.Range(14, options.MelBandCount).ToArray();
+
+    private AudioAnalyzer Analyzer() =>
+        new(new AudioAnalysisConfig(
+            SampleRate: options.SampleRate,
+            FftSize: 256,
+            HopSize: 96,
+            MelBandCount: options.MelBandCount,
+            MinFrequencyHz: options.MinFrequencyHz,
+            MaxFrequencyHz: options.MaxFrequencyHz));
+
+    private float MelBandCenterHz(int band)
+    {
+        var minMel = HzToMel(options.MinFrequencyHz);
+        var maxMel = HzToMel(options.MaxFrequencyHz);
+        var t = (band + 0.5f) / Math.Max(1, options.MelBandCount);
+        return MelToHz(minMel + (maxMel - minMel) * t);
+    }
+
+    private static VocalTractControlTarget Perturb(VocalTractControlTarget target, int outputIndex, float delta)
+    {
+        var values = target.ToVector(Math.Max(0, outputIndex - 13));
+        if (outputIndex >= values.Length)
+        {
+            Array.Resize(ref values, outputIndex + 1);
+        }
+
+        values[outputIndex] = Math.Clamp(values[outputIndex] + delta, 0, 1);
+        return TargetFromVector(values);
+    }
+
+    private static VocalTractControlTarget TargetFromVector(IReadOnlyList<float> values)
+    {
+        var mel = new float[Math.Max(0, values.Count - 14)];
+        for (var index = 0; index < mel.Length; index++)
+        {
+            mel[index] = values[14 + index];
+        }
+
+        return new VocalTractControlTarget(
+            values.ElementAtOrDefault(0),
+            values.ElementAtOrDefault(1),
+            values.ElementAtOrDefault(2),
+            values.ElementAtOrDefault(3),
+            values.ElementAtOrDefault(4),
+            values.ElementAtOrDefault(5),
+            values.ElementAtOrDefault(6),
+            values.ElementAtOrDefault(7),
+            values.ElementAtOrDefault(8),
+            values.ElementAtOrDefault(9),
+            values.ElementAtOrDefault(10),
+            values.ElementAtOrDefault(11),
+            values.ElementAtOrDefault(12),
+            values.ElementAtOrDefault(13),
+            mel);
+    }
+
+    private static string VoiceLine(string wave, string freq, string gain, VocalTractControlTarget target, float noise) =>
+        string.Create(CultureInfo.InvariantCulture, $"voice wave={wave} freq={freq} gain={gain} attack=0.004 sustain=0.16 decay=0.055 lpf={Math.Clamp(target.FilterCutoff, 0.04f, 1):0.######} lpf_q={Math.Clamp(0.2f + target.FilterResonance * 8, 0.2f, 12):0.######} noise={Math.Clamp(noise, 0, 1):0.######} fm=2.0 fm_index={Math.Clamp(target.FmDepth, 0, 1):0.######} tremolo={Math.Clamp(target.AmDepth, 0, 1):0.######} tremolo_hz={Math.Clamp(2 + target.LfoRate * 11, 0.1f, 20):0.######}");
+
+    private static string F(float value) => value.ToString("0.######", CultureInfo.InvariantCulture);
+    private static float HzToMel(float hz) => 2595 * MathF.Log10(1 + hz / 700);
+    private static float MelToHz(float mel) => 700 * (MathF.Pow(10, mel / 2595) - 1);
+}

state/evidence.jsonl

@@ -114,3 +114,4 @@
 {"date":"2026-05-19","topic":"utterance-embedding-learning-boundary","lesson":"AquaSynth should own gradient descent for Weksa speech handoff. Weksa supplies speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors; AquaSynth trains the tiny utterance-embedding encoder and passes that embedding into the synth-driver model.","evidence":"Extracted PackedNeuralNetwork as the shared packed-buffer Adam/SGD primitive, rewired VocalTractNeuralMapper through it, and added UtteranceEmbeddingInput/UtteranceEmbeddingNeuralEncoder plus tests for packing, training, and feeding the vocal-tract semantic embedding."}
 {"date":"2026-05-19","topic":"espeak-gradient-descent-fixture","lesson":"The first speech-learning fixture should train on generated eSpeak evidence before pretending AquaSynth has waveform-loss vocal-tract backprop. eSpeak log-mel means can supervise both utterance embedding targets and synth-driver mel/control lanes, giving the packed optimizer real generated speech pressure while the renderer is still catching up.","evidence":"Added EspeakNgGradientDescentTests. When ESPEAK_NG or espeak/espeak-ng is available, it renders a/pa/ta/ka/sa/ma, extracts log-mel means, trains UtteranceEmbeddingNeuralEncoder, trains VocalTractNeuralMapper against eSpeak-seeded control targets, asserts loss reductions, and writes artifacts/parity/espeak-ng-gradient-descent/<timestamp>-tiny/training-report.md."}
 {"date":"2026-05-19","topic":"speech-pipeline-chained-backprop","lesson":"The speech-learning path now has real chained backprop between the utterance encoder and synth driver. The packed network can return gradients with respect to its input, so downstream synth-driver loss can update the upstream utterance encoder instead of training the two models as separate supervised islands.","evidence":"Added PackedNeuralBackpropagation, TrainSingle/TrainSingleFromOutputGradient, SpeechBackpropagationPipeline, and SpeechBackpropagationPipelineTests; updated the eSpeak gradient fixture to train the utterance encoder from synth-driver loss rather than from a separate compressed-mel target."}
+{"date":"2026-05-19","topic":"faust-rendered-speech-loss","lesson":"For speech learning, Core should accept external synth-output gradients while Faust owns rendered loss evaluation. Finite-difference gradients over controller outputs are honest black-box renderer pressure, but too slow for default tests until render compilation is cached or proxied.","evidence":"Added SpeechBackpropagationPipeline.TrainSingleFromSynthOutputGradient and FaustRenderedSpeechLossSurface. The opt-in AQUASYNTH_RUN_FAUST_SPEECH_LOSS=1 proof renders reference and candidate speech targets through Faust, compares log-mel evidence, and returns a finite-difference output gradient for backpropagation through the learned chain."}

state/map.yaml

@@ -39,6 +39,7 @@ modules:
   faust_emitter: src/AquaSynth.Core/FaustExport.cs
   faust_compiler: src/AquaSynth.Faust/FaustCompiler.cs
   native_faust_runtime: src/AquaSynth.Faust/NativeFaustRuntime.cs
+  faust_rendered_speech_loss: src/AquaSynth.Faust/FaustRenderedSpeechLossSurface.cs
   dx7_reference_render_tests: tests/AquaSynth.Dsl.Tests/Dx7ReferenceParityTests.cs
   espeak_ng_reference_tests: tests/AquaSynth.Dsl.Tests/EspeakNgReferenceTests.cs
   espeak_ng_gradient_descent_tests: tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs
@@ -76,7 +77,9 @@ boundaries:
     - learned utterance embedding encoder from speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors
     - learned phonetic-plus-utterance-embedding synth driver with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls
     - chained utterance-embedding-to-synth-driver backpropagation that returns synth input gradients and updates the utterance encoder from downstream speech loss
+    - external-gradient speech backpropagation entry point for rendered loss surfaces
     - optional eSpeak NG generated ground-truth fixture for supervised utterance-embedding and synth-driver gradient descent
+    - Faust-rendered speech loss surface in AquaSynth.Faust, using rendered audio analysis and finite-difference controller-output gradients
   dev_only:
     - public-domain DX7 SysEx fixtures
     - pinned GPL ZynAddSubFX source for parity tests

state/scratch.md

@@ -1,5 +1,40 @@
 # Scratch
 
+Current slice:
+
+- Added the first Faust-rendered speech loss surface. `AquaSynth.Faust`
+  now owns `FaustRenderedSpeechLossSurface`, which turns a
+  `VocalTractControlTarget` into an inspectable AquaSynth script, renders it
+  through Faust, compares rendered log-mel evidence, and estimates
+  `d(loss)/d(controller output)` with finite differences over selected output
+  indices.
+- Added the missing Core backprop seam without making Core depend on Faust:
+  `SpeechBackpropagationPipeline.TrainSingleFromSynthOutputGradient` accepts an
+  external synth-output gradient, updates the synth driver, slices the returned
+  semantic embedding gradient, and updates the utterance encoder. This is the
+  handoff a Faust-rendered loss surface needs.
+- Added `VocalTractNeuralMapper.TrainSingleFromOutputGradient` and allowed
+  `PackedNeuralNetwork.TrainSingleFromOutputGradient` to preserve the external
+  loss value in its result.
+- The normal focused tests verify external-gradient backprop without invoking
+  Faust. The heavier rendered proof is opt-in via
+  `AQUASYNTH_RUN_FAUST_SPEECH_LOSS=1` because the current renderer recompiles
+  per finite-difference evaluation.
+- Verification:
+  `dotnet test tests\AquaSynth.Dsl.Tests\AquaSynth.Dsl.Tests.csproj --filter "SpeechBackpropagationPipelineTests|FaustRenderedSpeechLossSurfaceTests" -v minimal`:
+  3 passed.
+- Rendered proof verification:
+  `AQUASYNTH_RUN_FAUST_SPEECH_LOSS=1 dotnet test tests\AquaSynth.Dsl.Tests\AquaSynth.Dsl.Tests.csproj --filter "FaustRenderedSpeechLossSurfaceTests" -v minimal`:
+  1 passed in about 1m22s.
+- Build verification:
+  `dotnet build AquaSynth.sln -v minimal`: succeeded.
+- Current honesty: this is a real Faust-rendered loss surface and a real
+  backprop seam through both learned models. It is still black-box finite
+  difference over controller outputs, not analytic differentiation through
+  Faust DSP. The next coherent performance cut is render/compile caching or a
+  differentiable proxy that is explicitly trained against Faust, not pretending
+  recompiling Faust dozens of times is the final supercar.
+
 Current standing order: grow AquaSynth through external reference
 targets, not through speculative field sprawl. `state/spine.yaml` and
 `docs/reference-synth-roadmap.md` are the handoff surfaces for agents working on

tests/AquaSynth.Dsl.Tests/FaustRenderedSpeechLossSurfaceTests.cs

@@ -0,0 +1,50 @@
+using AquaSynth.Dsl;
+using AquaSynth.Faust;
+
+namespace AquaSynth.Dsl.Tests;
+
+public sealed class FaustRenderedSpeechLossSurfaceTests
+{
+    [Fact]
+    public async Task FaustRenderedSpeechLossSurfaceReturnsFiniteDifferenceGradientWhenToolchainIsAvailable()
+    {
+        if (FaustCompiler.FindFaust() is null ||
+            Environment.GetEnvironmentVariable("AQUASYNTH_RUN_FAUST_SPEECH_LOSS") != "1")
+        {
+            return;
+        }
+
+        var surface = new FaustRenderedSpeechLossSurface(new FaustRenderedSpeechLossOptions(
+            SampleRate: 16000,
+            DurationSeconds: 0.12f,
+            MelBandCount: 4,
+            GradientOutputIndices: [14]));
+        var reference = Target([0.90f, 0.78f, 0.28f, 0.12f], turbulence: 0.08f, cutoff: 0.46f);
+        var candidate = Target([0.18f, 0.25f, 0.76f, 0.88f], turbulence: 0.40f, cutoff: 0.80f);
+
+        var loss = await surface.EvaluateAsync(reference, candidate);
+
+        Assert.NotNull(loss);
+        Assert.True(loss.Loss > 0.01f, $"rendered log-mel loss should see the mismatch; loss={loss.Loss}");
+        Assert.Contains(loss.OutputGradient, value => Math.Abs(value) > 0.0001f);
+        Assert.Contains("process =", surface.SourceFor(candidate));
+    }
+
+    private static VocalTractControlTarget Target(IReadOnlyList<float> mel, float turbulence, float cutoff) =>
+        new(
+            TongueBody: 0.55f,
+            TongueTip: 0.45f,
+            LipAperture: 0.70f,
+            LipRounding: 0.08f,
+            Velum: 0.05f,
+            GlottalTenseness: 0.48f,
+            Turbulence: turbulence,
+            Pressure: 0.62f,
+            AmDepth: 0.10f,
+            FmDepth: 0.12f,
+            LfoRate: 0.25f,
+            LfoDepth: 0.08f,
+            FilterCutoff: cutoff,
+            FilterResonance: 0.20f,
+            MelSpectralEnvelope: mel);
+}

tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs

@@ -33,6 +33,45 @@ public sealed class SpeechBackpropagationPipelineTests
         Assert.True(result.Steps[^1].Loss < result.Steps[0].Loss, "pipeline loss should decrease across epochs");
     }
 
+    [Fact]
+    public void SpeechPipelineAcceptsExternalRenderedLossGradient()
+    {
+        var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
+            inputSize: 10,
+            embeddingSize: 8,
+            hiddenLayerSizes: [24, 16],
+            seed: 301);
+        var synthDriver = VocalTractNeuralMapper.Create(
+            semanticEmbeddingSize: 8,
+            melBandCount: 4,
+            hiddenLayerSizes: [28, 20],
+            seed: 307);
+        var pipeline = new SpeechBackpropagationPipeline(utteranceEncoder, synthDriver);
+        var example = Example(
+            "a",
+            new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
+            new UtteranceEmbeddingInput([0.9f, 0.1f, 0.2f, 0.7f], [0.8f, 0.3f, 0.2f], [0.6f, 0.2f, 0.4f]),
+            SmallTarget([0.9f, 0.7f, 0.3f, 0.1f]));
+        var before = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount);
+        var gradient = new float[pipeline.SynthDriver.OutputSize];
+        gradient[14] = before[14] - 0.95f;
+        gradient[15] = before[15] - 0.75f;
+        gradient[16] = before[16] - 0.25f;
+        gradient[17] = before[17] - 0.05f;
+
+        var result = pipeline.TrainSingleFromSynthOutputGradient(
+            example.UtteranceInput,
+            example.Event,
+            gradient,
+            utteranceLearningRate: 0.08f,
+            synthDriverLearningRate: 0.08f,
+            loss: 1.25f);
+        var after = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount);
+
+        Assert.Equal(1.25f, result.Loss);
+        Assert.True(Math.Abs(after[14] - 0.95f) < Math.Abs(before[14] - 0.95f), "external output gradient should move the synth-driver output toward the rendered-loss target");
+    }
+
     private static IReadOnlyList<SpeechBackpropagationTrainingExample> TrainingSet() =>
     [
         Example(
@@ -77,6 +116,24 @@ public sealed class SpeechBackpropagationPipelineTests
             FilterResonance: mel[11],
             MelSpectralEnvelope: mel);
 
+    private static VocalTractControlTarget SmallTarget(IReadOnlyList<float> mel) =>
+        new(
+            TongueBody: 0.55f,
+            TongueTip: 0.45f,
+            LipAperture: 0.70f,
+            LipRounding: 0.08f,
+            Velum: 0.05f,
+            GlottalTenseness: 0.48f,
+            Turbulence: 0.20f,
+            Pressure: 0.62f,
+            AmDepth: 0.10f,
+            FmDepth: 0.12f,
+            LfoRate: 0.22f,
+            LfoDepth: 0.08f,
+            FilterCutoff: 0.54f,
+            FilterResonance: 0.20f,
+            MelSpectralEnvelope: mel);
+
     private static float AverageLoss(SpeechBackpropagationPipeline pipeline, IReadOnlyList<SpeechBackpropagationTrainingExample> examples)
     {
         var loss = 0f;