diff --git a/src/AquaSynth.Core/PackedNeuralNetwork.cs b/src/AquaSynth.Core/PackedNeuralNetwork.cs index 9c2e5fb..0113d37 100644 --- a/src/AquaSynth.Core/PackedNeuralNetwork.cs +++ b/src/AquaSynth.Core/PackedNeuralNetwork.cs @@ -178,7 +178,8 @@ public sealed class PackedNeuralNetwork public PackedNeuralBackpropagation TrainSingleFromOutputGradient( IReadOnlyList input, IReadOnlyList outputGradient, - float learningRate) + float learningRate, + float loss = 0) { if (learningRate <= 0) throw new ArgumentOutOfRangeException(nameof(learningRate), "learning rate must be positive"); if (outputGradient.Count != OutputSize) throw new ArgumentException($"output gradient must have {OutputSize} values", nameof(outputGradient)); @@ -186,7 +187,7 @@ public sealed class PackedNeuralNetwork var scratch = TrainingScratch.Create(layers, InputSize, OutputSize); CopyChecked(input, scratch.Activations[0], "input"); Forward(scratch); - return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, 0); + return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, loss); } private static void Validate(PackedNeuralTrainingOptions options) diff --git a/src/AquaSynth.Core/SpeechBackpropagationPipeline.cs b/src/AquaSynth.Core/SpeechBackpropagationPipeline.cs index 0c4d5b4..cf4db0f 100644 --- a/src/AquaSynth.Core/SpeechBackpropagationPipeline.cs +++ b/src/AquaSynth.Core/SpeechBackpropagationPipeline.cs @@ -69,6 +69,34 @@ public sealed class SpeechBackpropagationPipeline( return SynthDriver.Predict(example.Event, embedding); } + public PackedNeuralBackpropagation TrainSingleFromSynthOutputGradient( + UtteranceEmbeddingInput utteranceInput, + PhoneticEvent phoneticEvent, + IReadOnlyList synthOutputGradient, + float utteranceLearningRate, + float synthDriverLearningRate, + float loss = 0) + { + if (utteranceLearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(utteranceLearningRate), "utterance learning rate must be positive"); + if (synthDriverLearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(synthDriverLearningRate), "synth-driver learning rate must be positive"); + if (synthOutputGradient.Count != SynthDriver.OutputSize) + { + throw new ArgumentException($"synth output gradient must have {SynthDriver.OutputSize} values", nameof(synthOutputGradient)); + } + + var utteranceFeatures = utteranceInput.ToFeatureVector(); + var embedding = UtteranceEncoder.Encode(utteranceFeatures); + var synthResult = SynthDriver.TrainSingleFromOutputGradient( + phoneticEvent, + embedding.ToSemanticEmbedding(), + synthOutputGradient, + synthDriverLearningRate, + loss); + var embeddingGradient = SemanticEmbeddingGradient(synthResult.InputGradient); + UtteranceEncoder.TrainSingleFromOutputGradient(utteranceFeatures, embeddingGradient, utteranceLearningRate); + return synthResult; + } + private float TrainSingle(SpeechBackpropagationTrainingExample example, SpeechBackpropagationTrainingOptions options) { var utteranceFeatures = example.UtteranceInput.ToFeatureVector(); diff --git a/src/AquaSynth.Core/VocalTractLearning.cs b/src/AquaSynth.Core/VocalTractLearning.cs index 3717752..f899a88 100644 --- a/src/AquaSynth.Core/VocalTractLearning.cs +++ b/src/AquaSynth.Core/VocalTractLearning.cs @@ -115,6 +115,18 @@ public sealed class VocalTractNeuralMapper target.ToVector(MelBandCount, NeutralMelBand), learningRate); + public PackedNeuralBackpropagation TrainSingleFromOutputGradient( + PhoneticEvent phoneticEvent, + VocalTractSemanticEmbedding semanticEmbedding, + IReadOnlyList outputGradient, + float learningRate, + float loss = 0) => + network.TrainSingleFromOutputGradient( + EncodeInput(phoneticEvent, semanticEmbedding), + outputGradient, + learningRate, + loss); + public VocalTractTrainingResult Train( IReadOnlyList examples, VocalTractTrainingOptions? options = null) diff --git a/src/AquaSynth.Faust/FaustRenderedSpeechLossSurface.cs b/src/AquaSynth.Faust/FaustRenderedSpeechLossSurface.cs new file mode 100644 index 0000000..886bea8 --- /dev/null +++ b/src/AquaSynth.Faust/FaustRenderedSpeechLossSurface.cs @@ -0,0 +1,169 @@ +using System.Globalization; +using System.Text; + +using AquaSynth.Dsl; + +namespace AquaSynth.Faust; + +public sealed record FaustRenderedSpeechLossOptions( + int SampleRate = 22050, + float DurationSeconds = 0.22f, + int MelBandCount = 12, + float MinFrequencyHz = 90, + float MaxFrequencyHz = 7000, + float FiniteDifferenceStep = 0.035f, + IReadOnlyList? GradientOutputIndices = null); + +public sealed record FaustRenderedSpeechLoss( + float Loss, + float[] OutputGradient, + FaustRender Reference, + FaustRender Candidate, + AudioComparison Comparison); + +public sealed class FaustRenderedSpeechLossSurface(FaustRenderedSpeechLossOptions? options = null, string? faustPath = null) +{ + private readonly FaustRenderedSpeechLossOptions options = options ?? new FaustRenderedSpeechLossOptions(); + + public async Task RenderAsync( + VocalTractControlTarget target, + CancellationToken cancellationToken = default) => + await FaustCompiler.RenderAsync( + SourceFor(target), + new FaustRenderOptions(options.SampleRate, options.DurationSeconds), + faustPath, + cancellationToken); + + public async Task EvaluateAsync( + VocalTractControlTarget reference, + VocalTractControlTarget candidate, + CancellationToken cancellationToken = default) + { + var referenceRender = await RenderAsync(reference, cancellationToken); + if (referenceRender is null || referenceRender.Samples.Length == 0) + { + return null; + } + + var candidateRender = await RenderAsync(candidate, cancellationToken); + if (candidateRender is null || candidateRender.Samples.Length == 0) + { + return null; + } + + var analyzer = Analyzer(); + var comparison = analyzer.Compare(referenceRender.Samples, candidateRender.Samples); + var gradient = await EstimateGradientAsync(referenceRender.Samples, candidate, comparison.LogMelDistance, cancellationToken); + return new FaustRenderedSpeechLoss(comparison.LogMelDistance, gradient, referenceRender, candidateRender, comparison); + } + + public string SourceFor(VocalTractControlTarget target) + { + var mel = target.MelSpectralEnvelope ?? []; + var source = new StringBuilder(); + source.AppendLine("patch gain=0.85"); + source.AppendLine(VoiceLine("saw", F(120 + target.GlottalTenseness * 110), F(0.08f + target.Pressure * 0.22f), target, target.Turbulence * 0.22f)); + if (target.Turbulence > 0.02f) + { + source.AppendLine(VoiceLine("noise", F(900 + target.Turbulence * 2600), F(target.Turbulence * 0.16f), target, 1)); + } + + for (var band = 0; band < options.MelBandCount; band++) + { + var value = band < mel.Count ? Math.Clamp(mel[band], 0, 1) : 0.5f; + var hz = MelBandCenterHz(band); + var gain = value * value * 0.08f; + source.AppendLine(VoiceLine("sine", F(hz), F(gain), target, 0)); + } + + return FaustEmitter.EmitScript(source.ToString(), new FaustExportOptions("aquasynth_speech_loss")).Source; + } + + private async Task EstimateGradientAsync( + IReadOnlyList referenceSamples, + VocalTractControlTarget candidate, + float baselineLoss, + CancellationToken cancellationToken) + { + var gradient = new float[14 + options.MelBandCount]; + foreach (var index in GradientIndices()) + { + var perturbed = Perturb(candidate, index, options.FiniteDifferenceStep); + var render = await RenderAsync(perturbed, cancellationToken); + if (render is null || render.Samples.Length == 0) + { + continue; + } + + var loss = Analyzer().Compare(referenceSamples.ToArray(), render.Samples).LogMelDistance; + gradient[index] = (loss - baselineLoss) / options.FiniteDifferenceStep; + } + + return gradient; + } + + private IReadOnlyList GradientIndices() => + options.GradientOutputIndices ?? Enumerable.Range(14, options.MelBandCount).ToArray(); + + private AudioAnalyzer Analyzer() => + new(new AudioAnalysisConfig( + SampleRate: options.SampleRate, + FftSize: 256, + HopSize: 96, + MelBandCount: options.MelBandCount, + MinFrequencyHz: options.MinFrequencyHz, + MaxFrequencyHz: options.MaxFrequencyHz)); + + private float MelBandCenterHz(int band) + { + var minMel = HzToMel(options.MinFrequencyHz); + var maxMel = HzToMel(options.MaxFrequencyHz); + var t = (band + 0.5f) / Math.Max(1, options.MelBandCount); + return MelToHz(minMel + (maxMel - minMel) * t); + } + + private static VocalTractControlTarget Perturb(VocalTractControlTarget target, int outputIndex, float delta) + { + var values = target.ToVector(Math.Max(0, outputIndex - 13)); + if (outputIndex >= values.Length) + { + Array.Resize(ref values, outputIndex + 1); + } + + values[outputIndex] = Math.Clamp(values[outputIndex] + delta, 0, 1); + return TargetFromVector(values); + } + + private static VocalTractControlTarget TargetFromVector(IReadOnlyList values) + { + var mel = new float[Math.Max(0, values.Count - 14)]; + for (var index = 0; index < mel.Length; index++) + { + mel[index] = values[14 + index]; + } + + return new VocalTractControlTarget( + values.ElementAtOrDefault(0), + values.ElementAtOrDefault(1), + values.ElementAtOrDefault(2), + values.ElementAtOrDefault(3), + values.ElementAtOrDefault(4), + values.ElementAtOrDefault(5), + values.ElementAtOrDefault(6), + values.ElementAtOrDefault(7), + values.ElementAtOrDefault(8), + values.ElementAtOrDefault(9), + values.ElementAtOrDefault(10), + values.ElementAtOrDefault(11), + values.ElementAtOrDefault(12), + values.ElementAtOrDefault(13), + mel); + } + + private static string VoiceLine(string wave, string freq, string gain, VocalTractControlTarget target, float noise) => + string.Create(CultureInfo.InvariantCulture, $"voice wave={wave} freq={freq} gain={gain} attack=0.004 sustain=0.16 decay=0.055 lpf={Math.Clamp(target.FilterCutoff, 0.04f, 1):0.######} lpf_q={Math.Clamp(0.2f + target.FilterResonance * 8, 0.2f, 12):0.######} noise={Math.Clamp(noise, 0, 1):0.######} fm=2.0 fm_index={Math.Clamp(target.FmDepth, 0, 1):0.######} tremolo={Math.Clamp(target.AmDepth, 0, 1):0.######} tremolo_hz={Math.Clamp(2 + target.LfoRate * 11, 0.1f, 20):0.######}"); + + private static string F(float value) => value.ToString("0.######", CultureInfo.InvariantCulture); + private static float HzToMel(float hz) => 2595 * MathF.Log10(1 + hz / 700); + private static float MelToHz(float mel) => 700 * (MathF.Pow(10, mel / 2595) - 1); +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 5718e76..3e1df6a 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -114,3 +114,4 @@ {"date":"2026-05-19","topic":"utterance-embedding-learning-boundary","lesson":"AquaSynth should own gradient descent for Weksa speech handoff. Weksa supplies speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors; AquaSynth trains the tiny utterance-embedding encoder and passes that embedding into the synth-driver model.","evidence":"Extracted PackedNeuralNetwork as the shared packed-buffer Adam/SGD primitive, rewired VocalTractNeuralMapper through it, and added UtteranceEmbeddingInput/UtteranceEmbeddingNeuralEncoder plus tests for packing, training, and feeding the vocal-tract semantic embedding."} {"date":"2026-05-19","topic":"espeak-gradient-descent-fixture","lesson":"The first speech-learning fixture should train on generated eSpeak evidence before pretending AquaSynth has waveform-loss vocal-tract backprop. eSpeak log-mel means can supervise both utterance embedding targets and synth-driver mel/control lanes, giving the packed optimizer real generated speech pressure while the renderer is still catching up.","evidence":"Added EspeakNgGradientDescentTests. When ESPEAK_NG or espeak/espeak-ng is available, it renders a/pa/ta/ka/sa/ma, extracts log-mel means, trains UtteranceEmbeddingNeuralEncoder, trains VocalTractNeuralMapper against eSpeak-seeded control targets, asserts loss reductions, and writes artifacts/parity/espeak-ng-gradient-descent/-tiny/training-report.md."} {"date":"2026-05-19","topic":"speech-pipeline-chained-backprop","lesson":"The speech-learning path now has real chained backprop between the utterance encoder and synth driver. The packed network can return gradients with respect to its input, so downstream synth-driver loss can update the upstream utterance encoder instead of training the two models as separate supervised islands.","evidence":"Added PackedNeuralBackpropagation, TrainSingle/TrainSingleFromOutputGradient, SpeechBackpropagationPipeline, and SpeechBackpropagationPipelineTests; updated the eSpeak gradient fixture to train the utterance encoder from synth-driver loss rather than from a separate compressed-mel target."} +{"date":"2026-05-19","topic":"faust-rendered-speech-loss","lesson":"For speech learning, Core should accept external synth-output gradients while Faust owns rendered loss evaluation. Finite-difference gradients over controller outputs are honest black-box renderer pressure, but too slow for default tests until render compilation is cached or proxied.","evidence":"Added SpeechBackpropagationPipeline.TrainSingleFromSynthOutputGradient and FaustRenderedSpeechLossSurface. The opt-in AQUASYNTH_RUN_FAUST_SPEECH_LOSS=1 proof renders reference and candidate speech targets through Faust, compares log-mel evidence, and returns a finite-difference output gradient for backpropagation through the learned chain."} diff --git a/state/map.yaml b/state/map.yaml index 86e298a..bc0461b 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -39,6 +39,7 @@ modules: faust_emitter: src/AquaSynth.Core/FaustExport.cs faust_compiler: src/AquaSynth.Faust/FaustCompiler.cs native_faust_runtime: src/AquaSynth.Faust/NativeFaustRuntime.cs + faust_rendered_speech_loss: src/AquaSynth.Faust/FaustRenderedSpeechLossSurface.cs dx7_reference_render_tests: tests/AquaSynth.Dsl.Tests/Dx7ReferenceParityTests.cs espeak_ng_reference_tests: tests/AquaSynth.Dsl.Tests/EspeakNgReferenceTests.cs espeak_ng_gradient_descent_tests: tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs @@ -76,7 +77,9 @@ boundaries: - learned utterance embedding encoder from speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors - learned phonetic-plus-utterance-embedding synth driver with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls - chained utterance-embedding-to-synth-driver backpropagation that returns synth input gradients and updates the utterance encoder from downstream speech loss + - external-gradient speech backpropagation entry point for rendered loss surfaces - optional eSpeak NG generated ground-truth fixture for supervised utterance-embedding and synth-driver gradient descent + - Faust-rendered speech loss surface in AquaSynth.Faust, using rendered audio analysis and finite-difference controller-output gradients dev_only: - public-domain DX7 SysEx fixtures - pinned GPL ZynAddSubFX source for parity tests diff --git a/state/scratch.md b/state/scratch.md index d589301..eb11596 100644 --- a/state/scratch.md +++ b/state/scratch.md @@ -1,5 +1,40 @@ # Scratch +Current slice: + +- Added the first Faust-rendered speech loss surface. `AquaSynth.Faust` + now owns `FaustRenderedSpeechLossSurface`, which turns a + `VocalTractControlTarget` into an inspectable AquaSynth script, renders it + through Faust, compares rendered log-mel evidence, and estimates + `d(loss)/d(controller output)` with finite differences over selected output + indices. +- Added the missing Core backprop seam without making Core depend on Faust: + `SpeechBackpropagationPipeline.TrainSingleFromSynthOutputGradient` accepts an + external synth-output gradient, updates the synth driver, slices the returned + semantic embedding gradient, and updates the utterance encoder. This is the + handoff a Faust-rendered loss surface needs. +- Added `VocalTractNeuralMapper.TrainSingleFromOutputGradient` and allowed + `PackedNeuralNetwork.TrainSingleFromOutputGradient` to preserve the external + loss value in its result. +- The normal focused tests verify external-gradient backprop without invoking + Faust. The heavier rendered proof is opt-in via + `AQUASYNTH_RUN_FAUST_SPEECH_LOSS=1` because the current renderer recompiles + per finite-difference evaluation. +- Verification: + `dotnet test tests\AquaSynth.Dsl.Tests\AquaSynth.Dsl.Tests.csproj --filter "SpeechBackpropagationPipelineTests|FaustRenderedSpeechLossSurfaceTests" -v minimal`: + 3 passed. +- Rendered proof verification: + `AQUASYNTH_RUN_FAUST_SPEECH_LOSS=1 dotnet test tests\AquaSynth.Dsl.Tests\AquaSynth.Dsl.Tests.csproj --filter "FaustRenderedSpeechLossSurfaceTests" -v minimal`: + 1 passed in about 1m22s. +- Build verification: + `dotnet build AquaSynth.sln -v minimal`: succeeded. +- Current honesty: this is a real Faust-rendered loss surface and a real + backprop seam through both learned models. It is still black-box finite + difference over controller outputs, not analytic differentiation through + Faust DSP. The next coherent performance cut is render/compile caching or a + differentiable proxy that is explicitly trained against Faust, not pretending + recompiling Faust dozens of times is the final supercar. + Current standing order: grow AquaSynth through external reference targets, not through speculative field sprawl. `state/spine.yaml` and `docs/reference-synth-roadmap.md` are the handoff surfaces for agents working on diff --git a/tests/AquaSynth.Dsl.Tests/FaustRenderedSpeechLossSurfaceTests.cs b/tests/AquaSynth.Dsl.Tests/FaustRenderedSpeechLossSurfaceTests.cs new file mode 100644 index 0000000..dae8beb --- /dev/null +++ b/tests/AquaSynth.Dsl.Tests/FaustRenderedSpeechLossSurfaceTests.cs @@ -0,0 +1,50 @@ +using AquaSynth.Dsl; +using AquaSynth.Faust; + +namespace AquaSynth.Dsl.Tests; + +public sealed class FaustRenderedSpeechLossSurfaceTests +{ + [Fact] + public async Task FaustRenderedSpeechLossSurfaceReturnsFiniteDifferenceGradientWhenToolchainIsAvailable() + { + if (FaustCompiler.FindFaust() is null || + Environment.GetEnvironmentVariable("AQUASYNTH_RUN_FAUST_SPEECH_LOSS") != "1") + { + return; + } + + var surface = new FaustRenderedSpeechLossSurface(new FaustRenderedSpeechLossOptions( + SampleRate: 16000, + DurationSeconds: 0.12f, + MelBandCount: 4, + GradientOutputIndices: [14])); + var reference = Target([0.90f, 0.78f, 0.28f, 0.12f], turbulence: 0.08f, cutoff: 0.46f); + var candidate = Target([0.18f, 0.25f, 0.76f, 0.88f], turbulence: 0.40f, cutoff: 0.80f); + + var loss = await surface.EvaluateAsync(reference, candidate); + + Assert.NotNull(loss); + Assert.True(loss.Loss > 0.01f, $"rendered log-mel loss should see the mismatch; loss={loss.Loss}"); + Assert.Contains(loss.OutputGradient, value => Math.Abs(value) > 0.0001f); + Assert.Contains("process =", surface.SourceFor(candidate)); + } + + private static VocalTractControlTarget Target(IReadOnlyList mel, float turbulence, float cutoff) => + new( + TongueBody: 0.55f, + TongueTip: 0.45f, + LipAperture: 0.70f, + LipRounding: 0.08f, + Velum: 0.05f, + GlottalTenseness: 0.48f, + Turbulence: turbulence, + Pressure: 0.62f, + AmDepth: 0.10f, + FmDepth: 0.12f, + LfoRate: 0.25f, + LfoDepth: 0.08f, + FilterCutoff: cutoff, + FilterResonance: 0.20f, + MelSpectralEnvelope: mel); +} diff --git a/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs b/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs index 3647d2c..3f46bc0 100644 --- a/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs +++ b/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs @@ -33,6 +33,45 @@ public sealed class SpeechBackpropagationPipelineTests Assert.True(result.Steps[^1].Loss < result.Steps[0].Loss, "pipeline loss should decrease across epochs"); } + [Fact] + public void SpeechPipelineAcceptsExternalRenderedLossGradient() + { + var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( + inputSize: 10, + embeddingSize: 8, + hiddenLayerSizes: [24, 16], + seed: 301); + var synthDriver = VocalTractNeuralMapper.Create( + semanticEmbeddingSize: 8, + melBandCount: 4, + hiddenLayerSizes: [28, 20], + seed: 307); + var pipeline = new SpeechBackpropagationPipeline(utteranceEncoder, synthDriver); + var example = Example( + "a", + new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), + new UtteranceEmbeddingInput([0.9f, 0.1f, 0.2f, 0.7f], [0.8f, 0.3f, 0.2f], [0.6f, 0.2f, 0.4f]), + SmallTarget([0.9f, 0.7f, 0.3f, 0.1f])); + var before = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount); + var gradient = new float[pipeline.SynthDriver.OutputSize]; + gradient[14] = before[14] - 0.95f; + gradient[15] = before[15] - 0.75f; + gradient[16] = before[16] - 0.25f; + gradient[17] = before[17] - 0.05f; + + var result = pipeline.TrainSingleFromSynthOutputGradient( + example.UtteranceInput, + example.Event, + gradient, + utteranceLearningRate: 0.08f, + synthDriverLearningRate: 0.08f, + loss: 1.25f); + var after = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount); + + Assert.Equal(1.25f, result.Loss); + Assert.True(Math.Abs(after[14] - 0.95f) < Math.Abs(before[14] - 0.95f), "external output gradient should move the synth-driver output toward the rendered-loss target"); + } + private static IReadOnlyList TrainingSet() => [ Example( @@ -77,6 +116,24 @@ public sealed class SpeechBackpropagationPipelineTests FilterResonance: mel[11], MelSpectralEnvelope: mel); + private static VocalTractControlTarget SmallTarget(IReadOnlyList mel) => + new( + TongueBody: 0.55f, + TongueTip: 0.45f, + LipAperture: 0.70f, + LipRounding: 0.08f, + Velum: 0.05f, + GlottalTenseness: 0.48f, + Turbulence: 0.20f, + Pressure: 0.62f, + AmDepth: 0.10f, + FmDepth: 0.12f, + LfoRate: 0.22f, + LfoDepth: 0.08f, + FilterCutoff: 0.54f, + FilterResonance: 0.20f, + MelSpectralEnvelope: mel); + private static float AverageLoss(SpeechBackpropagationPipeline pipeline, IReadOnlyList examples) { var loss = 0f;