From e20b3a51cdb8495957f3fac8acc845defe9c62ec Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 19 May 2026 16:29:37 +0100 Subject: [PATCH] Add learned vocal tract controller --- docs/ipa-vocal-tract-roadmap.md | 20 +- src/AquaSynth.Core/VocalTractLearning.cs | 379 ++++++++++++++++++ state/evidence.jsonl | 1 + state/map.yaml | 3 + state/memory.json | 2 +- .../VocalTractLearningTests.cs | 230 +++++++++++ 6 files changed, 631 insertions(+), 4 deletions(-) create mode 100644 src/AquaSynth.Core/VocalTractLearning.cs create mode 100644 tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs diff --git a/docs/ipa-vocal-tract-roadmap.md b/docs/ipa-vocal-tract-roadmap.md index 8f5d8bf..cbb3b13 100644 --- a/docs/ipa-vocal-tract-roadmap.md +++ b/docs/ipa-vocal-tract-roadmap.md @@ -183,6 +183,17 @@ features become overlapping constriction tasks and source events. It should not linearly interpolate phone presets unless a test proves that shortcut works for the current stage. +The first learned controller lives below the handoff and above rendering: +`VocalTractNeuralMapper` takes phonetic features plus a semantic embedding vector +and predicts tract controls. The default shape is deliberately small but real: +three hidden dense layers with tanh activations, sigmoid output controls, and +plain SGD/backprop training. Its output includes core tract parameters plus +expressive synthesis lanes: AM depth, FM depth, LFO rate/depth, filter cutoff, +filter resonance, and a full mel-frequency spectral envelope vector for +PAD-style wavetable/formant coloration. Those lanes are not license to bypass +tract anatomy; they are renderable expression controls the later synth can +choose to honor. + ### 4. Morphology Model Owns anatomy. @@ -467,10 +478,13 @@ AquaSynth's future IPA/gesture/render path, not replace it. 3. Add a human baseline vowel planner that emits tract-area targets, not audio. 4. Add a tiny tract renderer prototype in AquaSynth.Faust or a test-only lab lane, then decide the package boundary once the prototype proves itself. -5. Render five vowels and three CVC syllables through fixtures. -6. Add the first Weksa phonology profile with explicit unsupported-feature +5. Train the learned controller against eSpeak/log-mel, hand-authored tract + targets, and PAD-style spectral-table targets once the renderer can produce + comparable buffers. +6. Render five vowels and three CVC syllables through fixtures. +7. Add the first Weksa phonology profile with explicit unsupported-feature diagnostics. -7. Only then add Stage 2 consonant synthesis to the shipping path. +8. Only then add Stage 2 consonant synthesis to the shipping path. ## Non-Goals diff --git a/src/AquaSynth.Core/VocalTractLearning.cs b/src/AquaSynth.Core/VocalTractLearning.cs new file mode 100644 index 0000000..512e21b --- /dev/null +++ b/src/AquaSynth.Core/VocalTractLearning.cs @@ -0,0 +1,379 @@ +namespace AquaSynth.Dsl; + +public sealed record VocalTractSemanticEmbedding(IReadOnlyList Values); + +public sealed record VocalTractControlTarget( + float TongueBody, + float TongueTip, + float LipAperture, + float LipRounding, + float Velum, + float GlottalTenseness, + float Turbulence, + float Pressure, + float AmDepth = 0, + float FmDepth = 0, + float LfoRate = 0, + float LfoDepth = 0, + float FilterCutoff = 0.5f, + float FilterResonance = 0, + IReadOnlyList? MelSpectralEnvelope = null); + +public sealed record VocalTractTrainingExample( + PhoneticEvent Event, + VocalTractSemanticEmbedding SemanticEmbedding, + VocalTractControlTarget Target); + +public sealed record VocalTractTrainingOptions( + int Epochs = 200, + float LearningRate = 0.03f, + int Seed = 1337); + +public sealed record VocalTractTrainingStep(int Epoch, float Loss); + +public sealed record VocalTractTrainingResult( + VocalTractNeuralMapper Mapper, + IReadOnlyList Steps); + +public sealed class VocalTractNeuralMapper +{ + private const int PhoneticFeatureCount = 48; + private const int BaseOutputCount = 14; + private const float NeutralMelBand = 0.5f; + private readonly Layer[] layers; + + private VocalTractNeuralMapper(int semanticEmbeddingSize, int melBandCount, Layer[] layers) + { + SemanticEmbeddingSize = semanticEmbeddingSize; + MelBandCount = melBandCount; + this.layers = layers; + } + + public int SemanticEmbeddingSize { get; } + + public int MelBandCount { get; } + + public int InputSize => PhoneticFeatureCount + 4 + SemanticEmbeddingSize; + + public int OutputSize => BaseOutputCount + MelBandCount; + + public IReadOnlyList HiddenLayerSizes => layers.Take(layers.Length - 1).Select(layer => layer.OutputSize).ToArray(); + + public static VocalTractNeuralMapper Create( + int semanticEmbeddingSize, + int melBandCount = 32, + IReadOnlyList? hiddenLayerSizes = null, + int seed = 1337) + { + if (semanticEmbeddingSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(semanticEmbeddingSize), "semantic embedding size must be positive"); + } + + if (melBandCount <= 0) + { + throw new ArgumentOutOfRangeException(nameof(melBandCount), "mel band count must be positive"); + } + + hiddenLayerSizes ??= [64, 64, 32]; + var random = new Random(seed); + var sizes = new List { PhoneticFeatureCount + 4 + semanticEmbeddingSize }; + sizes.AddRange(hiddenLayerSizes); + sizes.Add(BaseOutputCount + melBandCount); + + var layers = new Layer[sizes.Count - 1]; + for (var index = 0; index < layers.Length; index++) + { + var activation = index == layers.Length - 1 + ? Activation.Sigmoid + : Activation.Tanh; + layers[index] = Layer.Create(sizes[index], sizes[index + 1], activation, random); + } + + return new VocalTractNeuralMapper(semanticEmbeddingSize, melBandCount, layers); + } + + public VocalTractControlTarget Predict(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding) + { + var output = Forward(BuildInput(phoneticEvent, semanticEmbedding)); + return TargetFromVector(output[^1]); + } + + public VocalTractTrainingResult Train( + IReadOnlyList examples, + VocalTractTrainingOptions? options = null) + { + if (examples.Count == 0) + { + throw new ArgumentException("at least one training example is required", nameof(examples)); + } + + options ??= new VocalTractTrainingOptions(); + var steps = new List(); + for (var epoch = 0; epoch < options.Epochs; epoch++) + { + var loss = 0f; + foreach (var example in examples) + { + var input = BuildInput(example.Event, example.SemanticEmbedding); + var activations = Forward(input); + var target = VectorFromTarget(example.Target); + loss += MeanSquaredError(activations[^1], target); + Backward(activations, target, options.LearningRate); + } + + steps.Add(new VocalTractTrainingStep(epoch + 1, loss / examples.Count)); + } + + return new VocalTractTrainingResult(this, steps); + } + + public float[] EncodeInput(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding) => + BuildInput(phoneticEvent, semanticEmbedding); + + private float[][] Forward(float[] input) + { + var activations = new float[layers.Length + 1][]; + activations[0] = input; + for (var index = 0; index < layers.Length; index++) + { + activations[index + 1] = layers[index].Forward(activations[index]); + } + + return activations; + } + + private void Backward(float[][] activations, float[] target, float learningRate) + { + var deltas = new float[layers.Length][]; + var lastLayerIndex = layers.Length - 1; + deltas[lastLayerIndex] = new float[layers[lastLayerIndex].OutputSize]; + for (var output = 0; output < deltas[lastLayerIndex].Length; output++) + { + var actual = activations[^1][output]; + deltas[lastLayerIndex][output] = 2f * (actual - target[output]) * layers[lastLayerIndex].Derivative(actual); + } + + for (var layerIndex = lastLayerIndex - 1; layerIndex >= 0; layerIndex--) + { + var layer = layers[layerIndex]; + var next = layers[layerIndex + 1]; + deltas[layerIndex] = new float[layer.OutputSize]; + for (var output = 0; output < layer.OutputSize; output++) + { + var sum = 0f; + for (var nextOutput = 0; nextOutput < next.OutputSize; nextOutput++) + { + sum += next.Weights[nextOutput][output] * deltas[layerIndex + 1][nextOutput]; + } + + deltas[layerIndex][output] = sum * layer.Derivative(activations[layerIndex + 1][output]); + } + } + + for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++) + { + layers[layerIndex].ApplyGradient(activations[layerIndex], deltas[layerIndex], learningRate); + } + } + + private float[] BuildInput(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding) + { + if (semanticEmbedding.Values.Count != SemanticEmbeddingSize) + { + throw new ArgumentException($"semantic embedding must have {SemanticEmbeddingSize} values", nameof(semanticEmbedding)); + } + + var input = new float[InputSize]; + var offset = 0; + offset = OneHot(input, offset, (int)phoneticEvent.Features.Manner, Enum.GetValues().Length); + offset = OneHot(input, offset, (int)phoneticEvent.Features.Place, Enum.GetValues().Length); + offset = OneHot(input, offset, (int)phoneticEvent.Features.Phonation, Enum.GetValues().Length); + offset = OneHot(input, offset, (int)phoneticEvent.Features.Airstream, Enum.GetValues().Length); + offset = OneHot(input, offset, (int)phoneticEvent.Features.Height, Enum.GetValues().Length); + offset = OneHot(input, offset, (int)phoneticEvent.Features.Backness, Enum.GetValues().Length); + input[offset++] = phoneticEvent.Features.Rounded ? 1 : 0; + input[offset++] = phoneticEvent.Features.Long ? 1 : 0; + input[offset++] = phoneticEvent.Features.Nasalized ? 1 : 0; + input[offset++] = phoneticEvent.Features.Lateral ? 1 : 0; + input[offset++] = Clamp01(phoneticEvent.Prosody.Stress); + input[offset++] = Clamp01((phoneticEvent.Prosody.PitchTarget + 1) * 0.5f); + input[offset++] = Clamp01(phoneticEvent.Prosody.Intensity); + input[offset++] = Clamp01((float)phoneticEvent.DurationSeconds); + foreach (var value in semanticEmbedding.Values) + { + input[offset++] = Math.Clamp(value, -1, 1); + } + + return input; + } + + private static int OneHot(float[] input, int offset, int value, int count) + { + input[offset + Math.Clamp(value, 0, count - 1)] = 1; + return offset + count; + } + + private float[] VectorFromTarget(VocalTractControlTarget target) => + target.ToVector(MelBandCount, NeutralMelBand); + + private static VocalTractControlTarget TargetFromVector(IReadOnlyList values) + { + var mel = new float[Math.Max(0, values.Count - BaseOutputCount)]; + for (var index = 0; index < mel.Length; index++) + { + mel[index] = values[BaseOutputCount + index]; + } + + return new( + values[0], + values[1], + values[2], + values[3], + values[4], + values[5], + values[6], + values[7], + values[8], + values[9], + values[10], + values[11], + values[12], + values[13], + mel); + } + + private static float MeanSquaredError(IReadOnlyList actual, IReadOnlyList target) + { + var sum = 0f; + for (var index = 0; index < actual.Count; index++) + { + var difference = actual[index] - target[index]; + sum += difference * difference; + } + + return sum / actual.Count; + } + + private static float Clamp01(float value) => Math.Clamp(value, 0, 1); + + private sealed class Layer + { + private readonly Activation activation; + + private Layer(float[][] weights, float[] biases, Activation activation) + { + Weights = weights; + Biases = biases; + this.activation = activation; + } + + public float[][] Weights { get; } + + public float[] Biases { get; } + + public int OutputSize => Biases.Length; + + public static Layer Create(int inputSize, int outputSize, Activation activation, Random random) + { + var scale = MathF.Sqrt(2f / Math.Max(1, inputSize)); + var weights = new float[outputSize][]; + for (var output = 0; output < outputSize; output++) + { + weights[output] = new float[inputSize]; + for (var input = 0; input < inputSize; input++) + { + weights[output][input] = ((float)random.NextDouble() * 2f - 1f) * scale; + } + } + + return new Layer(weights, new float[outputSize], activation); + } + + public float[] Forward(IReadOnlyList input) + { + var output = new float[OutputSize]; + for (var row = 0; row < OutputSize; row++) + { + var sum = Biases[row]; + for (var column = 0; column < input.Count; column++) + { + sum += Weights[row][column] * input[column]; + } + + output[row] = Activate(sum); + } + + return output; + } + + public float Derivative(float activated) => + activation switch + { + Activation.Sigmoid => activated * (1f - activated), + Activation.Tanh => 1f - activated * activated, + _ => 1 + }; + + public void ApplyGradient(IReadOnlyList input, IReadOnlyList delta, float learningRate) + { + for (var output = 0; output < OutputSize; output++) + { + for (var column = 0; column < input.Count; column++) + { + Weights[output][column] -= learningRate * delta[output] * input[column]; + } + + Biases[output] -= learningRate * delta[output]; + } + } + + private float Activate(float value) => + activation switch + { + Activation.Sigmoid => 1f / (1f + MathF.Exp(-value)), + Activation.Tanh => MathF.Tanh(value), + _ => value + }; + } + + private enum Activation + { + Tanh, + Sigmoid + } +} + +public static class VocalTractControlTargetExtensions +{ + public static float[] ToVector(this VocalTractControlTarget target, int melBandCount = 0, float neutralMelBand = 0.5f) + { + var output = new float[14 + melBandCount]; + output[0] = Clamp01(target.TongueBody); + output[1] = Clamp01(target.TongueTip); + output[2] = Clamp01(target.LipAperture); + output[3] = Clamp01(target.LipRounding); + output[4] = Clamp01(target.Velum); + output[5] = Clamp01(target.GlottalTenseness); + output[6] = Clamp01(target.Turbulence); + output[7] = Clamp01(target.Pressure); + output[8] = Clamp01(target.AmDepth); + output[9] = Clamp01(target.FmDepth); + output[10] = Clamp01(target.LfoRate); + output[11] = Clamp01(target.LfoDepth); + output[12] = Clamp01(target.FilterCutoff); + output[13] = Clamp01(target.FilterResonance); + + for (var index = 0; index < melBandCount; index++) + { + var band = target.MelSpectralEnvelope is not null && index < target.MelSpectralEnvelope.Count + ? target.MelSpectralEnvelope[index] + : neutralMelBand; + output[14 + index] = Clamp01(band); + } + + return output; + } + + private static float Clamp01(float value) => Math.Clamp(value, 0, 1); +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl index fcb65a1..3153b27 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -109,3 +109,4 @@ {"date":"2026-05-19","evidence":"Added VocalTractPlanner.Plan, VocalTractPlanResult, VocalTractHostReaction, rejection coverage for /pa/ against a beaked morphology, and accepted /a/ coverage for the human baseline gesture path.","lesson":"The first vocal-tract proof artifact is VocalTractPlanResult: host-visible branching around one intent and one morphology. Rejected morphology constraints carry diagnostic codes and no plan; accepted vowel intents emit inspectable glottal-source and tract-area gestures before any renderer exists.","topic":"vocal-tract-plan-result-proof"} {"date":"2026-05-19","evidence":"Added EspeakNgReferenceTests with optional renderer discovery, tiny fixtures for a/pa/ta/ka/sa/ma, WAV artifact/report generation under artifacts/parity/espeak-ng-ipa-workout, and docs/state entries naming eSpeak as intelligibility/timing pressure only.","lesson":"The first vocal-tract parity workout uses optional eSpeak NG as a development speech reference, not an anatomy oracle. The lane renders tiny speech fixtures and captures eSpeak IPA transcription/audio features when ESPEAK_NG or espeak-ng is available, while normal builds skip cleanly when it is absent.","topic":"espeak-ng-ipa-workout"} {"date":"2026-05-19","evidence":"Updated EspeakNgReferenceTests to record log_mel_frames, log_mel_bands, log_mel_mean, and log_mel_distance_matrix in report.md; updated the IPA vocal tract roadmap accordingly.","lesson":"The eSpeak NG speech workout should expose log-mel evidence immediately, not just basic audio features. Each fixture report now carries normalized log-mel band means plus a cross-fixture log-mel distance matrix so future AquaSynth IPA renders can plug into the same spectral comparison shape.","topic":"espeak-ng-logmel-workout"} +{"date":"2026-05-19","topic":"vocal-tract-learned-controller","lesson":"AquaSynth now has a small learned controller scaffold for the vocal-tract lane: phonetic features plus a semantic embedding vector feed a multilayer tanh/sigmoid MLP trained with SGD/backprop to predict tract controls plus AM/FM, LFO, filter, and mel spectral-envelope expression lanes. This is a trainable controller surface, not proof of eSpeak-level speech yet.","evidence":"Added VocalTractLearning.cs and VocalTractLearningTests.cs with feature encoding, semantic embeddings, 3-layer default network, loss-reduction tests, same-phone semantic steering, expressive modulator/filter control tests, and learned dark/bright mel-envelope steering for PAD-style spectral-table coloration."} diff --git a/state/map.yaml b/state/map.yaml index 4327cc1..4924db9 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -32,6 +32,7 @@ modules: model: src/AquaSynth.Core/Model.cs parser: src/AquaSynth.Core/PatchScript.cs vocal_tract_model: src/AquaSynth.Core/VocalTractModel.cs + vocal_tract_learning: src/AquaSynth.Core/VocalTractLearning.cs faust_emitter: src/AquaSynth.Core/FaustExport.cs faust_compiler: src/AquaSynth.Faust/FaustCompiler.cs native_faust_runtime: src/AquaSynth.Faust/NativeFaustRuntime.cs @@ -42,6 +43,7 @@ modules: ipa_vocal_tract_roadmap: docs/ipa-vocal-tract-roadmap.md package_boundary_tests: tests/AquaSynth.Dsl.Tests/PackageBoundaryTests.cs vocal_tract_model_tests: tests/AquaSynth.Dsl.Tests/VocalTractModelTests.cs + vocal_tract_learning_tests: tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs sfxr: src/AquaSynth.Core/SfxrParams.cs analysis: src/AquaSynth.Core/AudioAnalysis.cs scoring: src/AquaSynth.Core/ScriptMetrics.cs @@ -64,6 +66,7 @@ boundaries: - live patch compilation, native Faust factory lifetime, and offline sample-buffer render proof in AquaSynth.Faust - IPA-to-phonetic-feature planning, articulatory gesture planning, and human/alien vocal-tract roadmap design - phonetic intent, articulatory plan, morphology capability, and constraint-report contracts for vocal tract synthesis + - learned phonetic-plus-semantic embedding controller with gradient descent/backprop for tract, AM/FM, LFO, filter, and mel spectral-envelope controls dev_only: - public-domain DX7 SysEx fixtures - pinned GPL ZynAddSubFX source for parity tests diff --git a/state/memory.json b/state/memory.json index 66067bb..df8bcf7 100644 --- a/state/memory.json +++ b/state/memory.json @@ -86,7 +86,7 @@ "rule": "Add DSL abstractions only when reference rebuilds prove the missing invariant. Do not consume references that cannot be backed by parity tests; parsing alone is inventory, not evidence of expressivity." }, "ipa_vocal_tract_path": { - "status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth.", + "status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. VocalTractNeuralMapper is the first learned controller: phonetic features plus semantic embedding vector feed a small multilayer tanh/sigmoid network trained with SGD/backprop to predict tract, AM/FM, LFO, filter, and mel spectral-envelope controls.", "lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.", "next": "Implement Stage 0/1 fixtures: IPA token spans, vowel feature bundles, human baseline vowel tract-area targets, and a tiny audible vowel render proof before expanding consonants." }, diff --git a/tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs b/tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs new file mode 100644 index 0000000..a79a3fe --- /dev/null +++ b/tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs @@ -0,0 +1,230 @@ +using AquaSynth.Dsl; + +namespace AquaSynth.Dsl.Tests; + +public sealed class VocalTractLearningTests +{ + [Fact] + public void NeuralMapperEncodesPhoneticInputAndSemanticEmbedding() + { + var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 12, hiddenLayerSizes: [32, 24, 16], seed: 7); + var input = mapper.EncodeInput( + new PhoneticEvent( + "phone-a", + "a", + new PhoneticFeatures( + PhoneticManner.Vowel, + Height: VowelHeight.Open, + Backness: VowelBackness.Front), + DurationSeconds: 0.2, + Prosody: new PhoneticProsody(Stress: 0.5f, PitchTarget: 0.25f, Intensity: 0.75f)), + new VocalTractSemanticEmbedding(Enumerable.Range(0, 12).Select(index => index / 12f).ToArray())); + + Assert.Equal(64, input.Length); + Assert.Equal(32, mapper.MelBandCount); + Assert.Equal(46, mapper.OutputSize); + Assert.Equal([32, 24, 16], mapper.HiddenLayerSizes); + Assert.All(input, value => Assert.InRange(value, -1f, 1f)); + } + + [Fact] + public void NeuralMapperTrainingReducesLossForTinyVowelSet() + { + var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, hiddenLayerSizes: [48, 32, 24], seed: 11); + var examples = TinyTrainingSet(); + + var before = AverageLoss(mapper, examples); + var result = mapper.Train(examples, new VocalTractTrainingOptions(Epochs: 260, LearningRate: 0.05f, Seed: 11)); + var after = AverageLoss(mapper, examples); + + Assert.Equal(260, result.Steps.Count); + Assert.True(after < before * 0.35f, $"expected loss to drop hard; before={before}, after={after}"); + Assert.True(result.Steps[^1].Loss < result.Steps[0].Loss, "training loss should decrease across epochs"); + } + + [Fact] + public void SemanticEmbeddingCanSteerSamePhoneToDifferentTractTargets() + { + var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, hiddenLayerSizes: [64, 48, 32], seed: 19); + var softContext = Embedding(16, 0.15f, 0.2f); + var hardContext = Embedding(16, 0.85f, -0.15f); + var eventA = new PhoneticEvent( + "phone-a", + "a", + new PhoneticFeatures( + PhoneticManner.Vowel, + Height: VowelHeight.Open, + Backness: VowelBackness.Front), + DurationSeconds: 0.2); + + var examples = new[] + { + new VocalTractTrainingExample(eventA, softContext, new VocalTractControlTarget(0.78f, 0.35f, 0.90f, 0.05f, 0.0f, 0.42f, 0.0f, 0.35f, AmDepth: 0.12f, FmDepth: 0.08f, LfoRate: 0.18f, LfoDepth: 0.10f, FilterCutoff: 0.62f, FilterResonance: 0.08f)), + new VocalTractTrainingExample(eventA, hardContext, new VocalTractControlTarget(0.54f, 0.70f, 0.42f, 0.10f, 0.0f, 0.82f, 0.0f, 0.72f, AmDepth: 0.44f, FmDepth: 0.36f, LfoRate: 0.52f, LfoDepth: 0.41f, FilterCutoff: 0.38f, FilterResonance: 0.55f)) + }; + + mapper.Train(examples, new VocalTractTrainingOptions(Epochs: 400, LearningRate: 0.04f, Seed: 19)); + var soft = mapper.Predict(eventA, softContext); + var hard = mapper.Predict(eventA, hardContext); + + Assert.True(soft.LipAperture > hard.LipAperture + 0.20f, $"soft aperture={soft.LipAperture}, hard aperture={hard.LipAperture}"); + Assert.True(hard.GlottalTenseness > soft.GlottalTenseness + 0.20f, $"hard tenseness={hard.GlottalTenseness}, soft tenseness={soft.GlottalTenseness}"); + Assert.True(hard.Pressure > soft.Pressure + 0.15f, $"hard pressure={hard.Pressure}, soft pressure={soft.Pressure}"); + Assert.True(hard.FmDepth > soft.FmDepth + 0.15f, $"hard FM={hard.FmDepth}, soft FM={soft.FmDepth}"); + Assert.True(hard.FilterResonance > soft.FilterResonance + 0.20f, $"hard resonance={hard.FilterResonance}, soft resonance={soft.FilterResonance}"); + } + + [Fact] + public void NeuralMapperCanLearnExpressiveModulatorAndFilterControls() + { + var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, hiddenLayerSizes: [64, 48, 32], seed: 31); + var fricative = new PhoneticEvent( + "phone-s", + "s", + new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless), + DurationSeconds: 0.12, + Prosody: new PhoneticProsody(Intensity: 0.9f)); + var still = Embedding(16, -0.45f, 0.05f); + var animated = Embedding(16, 0.55f, 0.30f); + var examples = new[] + { + new VocalTractTrainingExample(fricative, still, new VocalTractControlTarget(0.42f, 0.82f, 0.18f, 0.0f, 0.0f, 0.04f, 0.92f, 0.64f, AmDepth: 0.04f, FmDepth: 0.02f, LfoRate: 0.08f, LfoDepth: 0.03f, FilterCutoff: 0.72f, FilterResonance: 0.25f)), + new VocalTractTrainingExample(fricative, animated, new VocalTractControlTarget(0.35f, 0.90f, 0.12f, 0.0f, 0.0f, 0.08f, 0.98f, 0.84f, AmDepth: 0.62f, FmDepth: 0.48f, LfoRate: 0.70f, LfoDepth: 0.58f, FilterCutoff: 0.42f, FilterResonance: 0.76f)) + }; + + mapper.Train(examples, new VocalTractTrainingOptions(Epochs: 420, LearningRate: 0.04f, Seed: 31)); + var stillPrediction = mapper.Predict(fricative, still); + var animatedPrediction = mapper.Predict(fricative, animated); + + Assert.True(animatedPrediction.AmDepth > stillPrediction.AmDepth + 0.25f); + Assert.True(animatedPrediction.FmDepth > stillPrediction.FmDepth + 0.20f); + Assert.True(animatedPrediction.LfoDepth > stillPrediction.LfoDepth + 0.20f); + Assert.True(animatedPrediction.FilterResonance > stillPrediction.FilterResonance + 0.25f); + Assert.True(stillPrediction.FilterCutoff > animatedPrediction.FilterCutoff + 0.15f); + } + + [Fact] + public void NeuralMapperCanLearnMelSpectralEnvelopeControls() + { + var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, melBandCount: 12, hiddenLayerSizes: [72, 48, 32], seed: 43); + var vowel = new PhoneticEvent( + "phone-a", + "a", + new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), + DurationSeconds: 0.2, + Prosody: new PhoneticProsody(Intensity: 0.8f)); + var darkContext = Embedding(16, -0.60f, 0.10f); + var brightContext = Embedding(16, 0.60f, 0.12f); + var darkEnvelope = MelEnvelope(12, low: 0.86f, mid: 0.52f, high: 0.18f); + var brightEnvelope = MelEnvelope(12, low: 0.22f, mid: 0.58f, high: 0.90f); + var examples = new[] + { + new VocalTractTrainingExample(vowel, darkContext, new VocalTractControlTarget(0.68f, 0.34f, 0.72f, 0.10f, 0.0f, 0.48f, 0.0f, 0.42f, MelSpectralEnvelope: darkEnvelope)), + new VocalTractTrainingExample(vowel, brightContext, new VocalTractControlTarget(0.58f, 0.52f, 0.56f, 0.02f, 0.0f, 0.54f, 0.0f, 0.46f, MelSpectralEnvelope: brightEnvelope)) + }; + + mapper.Train(examples, new VocalTractTrainingOptions(Epochs: 520, LearningRate: 0.04f, Seed: 43)); + var dark = mapper.Predict(vowel, darkContext); + var bright = mapper.Predict(vowel, brightContext); + + Assert.NotNull(dark.MelSpectralEnvelope); + Assert.NotNull(bright.MelSpectralEnvelope); + Assert.Equal(12, dark.MelSpectralEnvelope.Count); + Assert.True(dark.MelSpectralEnvelope[1] > bright.MelSpectralEnvelope[1] + 0.25f); + Assert.True(bright.MelSpectralEnvelope[^2] > dark.MelSpectralEnvelope[^2] + 0.25f); + } + + private static IReadOnlyList TinyTrainingSet() => + [ + new( + new PhoneticEvent( + "a", + "a", + new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), + DurationSeconds: 0.20, + Prosody: new PhoneticProsody(Intensity: 0.8f)), + Embedding(16, 0.20f, 0.10f), + new VocalTractControlTarget(0.78f, 0.30f, 0.88f, 0.04f, 0.0f, 0.52f, 0.0f, 0.42f, AmDepth: 0.10f, FmDepth: 0.04f, LfoRate: 0.12f, LfoDepth: 0.06f, FilterCutoff: 0.62f, FilterResonance: 0.10f, MelSpectralEnvelope: MelEnvelope(32, 0.52f, 0.60f, 0.42f))), + new( + new PhoneticEvent( + "i", + "i", + new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Close, Backness: VowelBackness.Front), + DurationSeconds: 0.18, + Prosody: new PhoneticProsody(Intensity: 0.7f)), + Embedding(16, -0.15f, 0.35f), + new VocalTractControlTarget(0.25f, 0.82f, 0.24f, 0.02f, 0.0f, 0.50f, 0.0f, 0.38f, AmDepth: 0.08f, FmDepth: 0.05f, LfoRate: 0.16f, LfoDepth: 0.05f, FilterCutoff: 0.80f, FilterResonance: 0.06f, MelSpectralEnvelope: MelEnvelope(32, 0.32f, 0.78f, 0.66f))), + new( + new PhoneticEvent( + "u", + "u", + new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Close, Backness: VowelBackness.Back, Rounded: true), + DurationSeconds: 0.22, + Prosody: new PhoneticProsody(Intensity: 0.72f)), + Embedding(16, 0.40f, -0.25f), + new VocalTractControlTarget(0.18f, 0.26f, 0.22f, 0.88f, 0.0f, 0.48f, 0.0f, 0.36f, AmDepth: 0.18f, FmDepth: 0.12f, LfoRate: 0.22f, LfoDepth: 0.14f, FilterCutoff: 0.48f, FilterResonance: 0.18f, MelSpectralEnvelope: MelEnvelope(32, 0.72f, 0.46f, 0.22f))), + new( + new PhoneticEvent( + "s", + "s", + new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless), + DurationSeconds: 0.12, + Prosody: new PhoneticProsody(Intensity: 0.85f)), + Embedding(16, -0.30f, -0.20f), + new VocalTractControlTarget(0.42f, 0.86f, 0.18f, 0.0f, 0.0f, 0.05f, 0.92f, 0.70f, AmDepth: 0.45f, FmDepth: 0.35f, LfoRate: 0.55f, LfoDepth: 0.40f, FilterCutoff: 0.58f, FilterResonance: 0.62f, MelSpectralEnvelope: MelEnvelope(32, 0.18f, 0.56f, 0.92f))) + ]; + + private static float AverageLoss(VocalTractNeuralMapper mapper, IReadOnlyList examples) + { + var loss = 0f; + foreach (var example in examples) + { + var prediction = mapper.Predict(example.Event, example.SemanticEmbedding); + loss += Loss(mapper, prediction, example.Target); + } + + return loss / examples.Count; + } + + private static float Loss(VocalTractNeuralMapper mapper, VocalTractControlTarget prediction, VocalTractControlTarget target) + { + var predicted = prediction.ToVector(mapper.MelBandCount); + var expected = target.ToVector(mapper.MelBandCount); + var loss = 0f; + for (var index = 0; index < predicted.Length; index++) + { + var difference = predicted[index] - expected[index]; + loss += difference * difference; + } + + return loss / predicted.Length; + } + + private static VocalTractSemanticEmbedding Embedding(int size, float offset, float slope) + { + var values = new float[size]; + for (var index = 0; index < values.Length; index++) + { + values[index] = MathF.Tanh(offset + slope * index / Math.Max(1, size - 1)); + } + + return new VocalTractSemanticEmbedding(values); + } + + private static float[] MelEnvelope(int size, float low, float mid, float high) + { + var values = new float[size]; + for (var index = 0; index < values.Length; index++) + { + var position = index / Math.Max(1f, size - 1f); + values[index] = position < 0.5f + ? Lerp(low, mid, position * 2f) + : Lerp(mid, high, (position - 0.5f) * 2f); + } + + return values; + } + + private static float Lerp(float left, float right, float amount) => + left + (right - left) * amount; +}