From 462439c0bd01030636bfdeebc56d22f949a2b909 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 19 May 2026 17:47:08 +0100 Subject: [PATCH] Centralize utterance embedding learning --- docs/ipa-vocal-tract-roadmap.md | 24 +- src/AquaSynth.Core/PackedNeuralNetwork.cs | 544 ++++++++++++++++++ .../UtteranceEmbeddingLearning.cs | 112 ++++ src/AquaSynth.Core/VocalTractLearning.cs | 489 +--------------- state/evidence.jsonl | 1 + state/map.yaml | 7 +- state/memory.json | 2 +- .../UtteranceEmbeddingLearningTests.cs | 106 ++++ 8 files changed, 811 insertions(+), 474 deletions(-) create mode 100644 src/AquaSynth.Core/PackedNeuralNetwork.cs create mode 100644 src/AquaSynth.Core/UtteranceEmbeddingLearning.cs create mode 100644 tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs diff --git a/docs/ipa-vocal-tract-roadmap.md b/docs/ipa-vocal-tract-roadmap.md index cff2bfc..f557c77 100644 --- a/docs/ipa-vocal-tract-roadmap.md +++ b/docs/ipa-vocal-tract-roadmap.md @@ -183,17 +183,19 @@ features become overlapping constriction tasks and source events. It should not linearly interpolate phone presets unless a test proves that shortcut works for the current stage. -The first learned controller lives below the handoff and above rendering: -`VocalTractNeuralMapper` takes phonetic features plus a semantic embedding vector -and predicts tract controls. The default shape is deliberately small but real: -three hidden dense layers with tanh activations, sigmoid output controls, and a -native C# training path using packed row-major buffers, unsafe hot loops, batched -gradients, and Adam or SGD updates. Its output includes core tract parameters plus -expressive synthesis lanes: AM depth, FM depth, LFO rate/depth, filter cutoff, -filter resonance, and a full mel-frequency spectral envelope vector for -PAD-style wavetable/formant coloration. Those lanes are not license to bypass -tract anatomy; they are renderable expression controls the later synth can -choose to honor. +The first learned controllers live below the handoff and above rendering. Weksa +or another language layer may provide speech text embeddings, prosody/emphasis +hints, and Ghostlight/Epiphany-shaped character state vectors, but AquaSynth owns +the gradient descent. `UtteranceEmbeddingNeuralEncoder` trains a tiny model to +compress those ingredients into one utterance embedding. `VocalTractNeuralMapper` +then takes phonetic features plus that embedding and predicts tract controls. +Both controllers use the same native C# packed-network path: row-major buffers, +unsafe hot loops, batched gradients, and Adam or SGD updates. The synth-driver +output includes core tract parameters plus expressive synthesis lanes: AM depth, +FM depth, LFO rate/depth, filter cutoff, filter resonance, and a full +mel-frequency spectral envelope vector for PAD-style wavetable/formant +coloration. Those lanes are not license to bypass tract anatomy; they are +renderable expression controls the later synth can choose to honor. ### 4. Morphology Model diff --git a/src/AquaSynth.Core/PackedNeuralNetwork.cs b/src/AquaSynth.Core/PackedNeuralNetwork.cs new file mode 100644 index 0000000..bf42e1d --- /dev/null +++ b/src/AquaSynth.Core/PackedNeuralNetwork.cs @@ -0,0 +1,544 @@ +namespace AquaSynth.Dsl; + +public sealed record PackedNeuralTrainingExample( + IReadOnlyList Input, + IReadOnlyList Target); + +public sealed record PackedNeuralTrainingOptions( + int Epochs = 200, + float LearningRate = 0.01f, + int Seed = 1337, + PackedNeuralOptimizer Optimizer = PackedNeuralOptimizer.Adam, + int BatchSize = 32, + float Beta1 = 0.9f, + float Beta2 = 0.999f, + float Epsilon = 0.00000001f, + bool Shuffle = true); + +public enum PackedNeuralOptimizer +{ + Sgd, + Adam +} + +public sealed record PackedNeuralTrainingStep(int Epoch, float Loss); + +public sealed record PackedNeuralTrainingResult( + PackedNeuralNetwork Network, + IReadOnlyList Steps); + +public sealed class PackedNeuralNetwork +{ + private readonly Layer[] layers; + + private PackedNeuralNetwork(int inputSize, int outputSize, Layer[] layers) + { + InputSize = inputSize; + OutputSize = outputSize; + this.layers = layers; + HiddenLayerSizes = layers.Take(layers.Length - 1).Select(layer => layer.OutputSize).ToArray(); + } + + public int InputSize { get; } + + public int OutputSize { get; } + + public IReadOnlyList HiddenLayerSizes { get; } + + public static PackedNeuralNetwork Create( + int inputSize, + int outputSize, + IReadOnlyList? hiddenLayerSizes = null, + int seed = 1337) + { + if (inputSize <= 0) throw new ArgumentOutOfRangeException(nameof(inputSize), "input size must be positive"); + if (outputSize <= 0) throw new ArgumentOutOfRangeException(nameof(outputSize), "output size must be positive"); + + hiddenLayerSizes ??= [64, 64, 32]; + var random = new Random(seed); + var sizes = new List { inputSize }; + sizes.AddRange(hiddenLayerSizes); + sizes.Add(outputSize); + + var layers = new Layer[sizes.Count - 1]; + for (var index = 0; index < layers.Length; index++) + { + var activation = index == layers.Length - 1 + ? Activation.Sigmoid + : Activation.Tanh; + layers[index] = Layer.Create(sizes[index], sizes[index + 1], activation, random); + } + + return new PackedNeuralNetwork(inputSize, outputSize, layers); + } + + public float[] Predict(IReadOnlyList input) + { + if (input.Count != InputSize) + { + throw new ArgumentException($"input must have {InputSize} values", nameof(input)); + } + + var scratch = TrainingScratch.Create(layers, InputSize, OutputSize); + Copy(input, scratch.Activations[0]); + Forward(scratch); + return scratch.Output.ToArray(); + } + + public PackedNeuralTrainingResult Train( + IReadOnlyList examples, + PackedNeuralTrainingOptions? options = null) + { + if (examples.Count == 0) + { + throw new ArgumentException("at least one training example is required", nameof(examples)); + } + + options ??= new PackedNeuralTrainingOptions(); + Validate(options); + + var steps = new List(); + var order = Enumerable.Range(0, examples.Count).ToArray(); + var random = new Random(options.Seed); + var scratch = TrainingScratch.Create(layers, InputSize, OutputSize); + var gradients = LayerBuffers.Create(layers); + var adamState = options.Optimizer == PackedNeuralOptimizer.Adam + ? OptimizerState.Create(layers) + : null; + var updateStep = 0; + + for (var epoch = 0; epoch < options.Epochs; epoch++) + { + if (options.Shuffle) + { + Shuffle(order, random); + } + + var loss = 0f; + for (var batchStart = 0; batchStart < order.Length; batchStart += options.BatchSize) + { + var batchEnd = Math.Min(batchStart + options.BatchSize, order.Length); + gradients.Clear(); + + for (var orderIndex = batchStart; orderIndex < batchEnd; orderIndex++) + { + var example = examples[order[orderIndex]]; + CopyChecked(example.Input, scratch.Activations[0], "input"); + CopyChecked(example.Target, scratch.Target, "target"); + Forward(scratch); + loss += MeanSquaredError(scratch.Output, scratch.Target); + AccumulateGradients(scratch, gradients); + } + + updateStep++; + var scale = 1f / (batchEnd - batchStart); + if (options.Optimizer == PackedNeuralOptimizer.Adam) + { + adamState!.ApplyAdam(layers, gradients, scale, options.LearningRate, options.Beta1, options.Beta2, options.Epsilon, updateStep); + } + else + { + ApplySgd(gradients, scale, options.LearningRate); + } + } + + steps.Add(new PackedNeuralTrainingStep(epoch + 1, loss / examples.Count)); + } + + return new PackedNeuralTrainingResult(this, steps); + } + + private static void Validate(PackedNeuralTrainingOptions options) + { + if (options.Epochs <= 0) throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive"); + if (options.LearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(options), "learning rate must be positive"); + if (options.BatchSize <= 0) throw new ArgumentOutOfRangeException(nameof(options), "batch size must be positive"); + } + + private void Forward(TrainingScratch scratch) + { + for (var index = 0; index < layers.Length; index++) + { + layers[index].Forward(scratch.Activations[index], scratch.Activations[index + 1]); + } + } + + private void AccumulateGradients(TrainingScratch scratch, LayerBuffers gradients) + { + var lastLayerIndex = layers.Length - 1; + var outputDelta = scratch.Deltas[lastLayerIndex]; + var output = scratch.Activations[^1]; + for (var index = 0; index < outputDelta.Length; index++) + { + var actual = output[index]; + outputDelta[index] = 2f * (actual - scratch.Target[index]) * layers[lastLayerIndex].Derivative(actual); + } + + for (var layerIndex = lastLayerIndex - 1; layerIndex >= 0; layerIndex--) + { + layers[layerIndex].BackpropagateDelta(layers[layerIndex + 1], scratch.Deltas[layerIndex + 1], scratch.Activations[layerIndex + 1], scratch.Deltas[layerIndex]); + } + + for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++) + { + gradients.Accumulate(layerIndex, scratch.Activations[layerIndex], scratch.Deltas[layerIndex]); + } + } + + private void ApplySgd(LayerBuffers gradients, float scale, float learningRate) + { + for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++) + { + layers[layerIndex].ApplySgd(gradients.WeightBuffers[layerIndex], gradients.BiasBuffers[layerIndex], scale, learningRate); + } + } + + private static void CopyChecked(IReadOnlyList source, float[] target, string name) + { + if (source.Count != target.Length) + { + throw new ArgumentException($"{name} must have {target.Length} values"); + } + + Copy(source, target); + } + + private static void Copy(IReadOnlyList source, float[] target) + { + for (var index = 0; index < target.Length; index++) + { + target[index] = source[index]; + } + } + + private static void Shuffle(int[] values, Random random) + { + for (var index = values.Length - 1; index > 0; index--) + { + var swapIndex = random.Next(index + 1); + (values[index], values[swapIndex]) = (values[swapIndex], values[index]); + } + } + + private static float MeanSquaredError(IReadOnlyList actual, IReadOnlyList target) + { + var sum = 0f; + for (var index = 0; index < actual.Count; index++) + { + var difference = actual[index] - target[index]; + sum += difference * difference; + } + + return sum / actual.Count; + } + + private sealed class TrainingScratch + { + private TrainingScratch(float[][] activations, float[][] deltas, float[] target) + { + Activations = activations; + Deltas = deltas; + Target = target; + } + + public float[][] Activations { get; } + + public float[][] Deltas { get; } + + public float[] Target { get; } + + public float[] Output => Activations[^1]; + + public static TrainingScratch Create(IReadOnlyList layers, int inputSize, int outputSize) + { + var activations = new float[layers.Count + 1][]; + var deltas = new float[layers.Count][]; + activations[0] = new float[inputSize]; + for (var index = 0; index < layers.Count; index++) + { + activations[index + 1] = new float[layers[index].OutputSize]; + deltas[index] = new float[layers[index].OutputSize]; + } + + return new TrainingScratch(activations, deltas, new float[outputSize]); + } + } + + private sealed class LayerBuffers + { + private LayerBuffers(float[][] weightBuffers, float[][] biasBuffers) + { + WeightBuffers = weightBuffers; + BiasBuffers = biasBuffers; + } + + public float[][] WeightBuffers { get; } + + public float[][] BiasBuffers { get; } + + public static LayerBuffers Create(IReadOnlyList layers) + { + var weights = new float[layers.Count][]; + var biases = new float[layers.Count][]; + for (var index = 0; index < layers.Count; index++) + { + weights[index] = new float[layers[index].Weights.Length]; + biases[index] = new float[layers[index].Biases.Length]; + } + + return new LayerBuffers(weights, biases); + } + + public void Clear() + { + for (var index = 0; index < WeightBuffers.Length; index++) + { + Array.Clear(WeightBuffers[index]); + Array.Clear(BiasBuffers[index]); + } + } + + public void Accumulate(int layerIndex, float[] input, float[] delta) + { + var weightGradients = WeightBuffers[layerIndex]; + var biasGradients = BiasBuffers[layerIndex]; + var inputSize = input.Length; + + unsafe + { + fixed (float* inputBase = input) + fixed (float* deltaBase = delta) + fixed (float* weightBase = weightGradients) + fixed (float* biasBase = biasGradients) + { + for (var output = 0; output < delta.Length; output++) + { + var deltaValue = deltaBase[output]; + var weightRow = weightBase + output * inputSize; + for (var column = 0; column < inputSize; column++) + { + weightRow[column] += deltaValue * inputBase[column]; + } + + biasBase[output] += deltaValue; + } + } + } + } + } + + private sealed class Layer + { + private readonly Activation activation; + + private Layer(int inputSize, int outputSize, float[] weights, float[] biases, Activation activation) + { + InputSize = inputSize; + OutputSize = outputSize; + Weights = weights; + Biases = biases; + this.activation = activation; + } + + public int InputSize { get; } + + public int OutputSize { get; } + + public float[] Weights { get; } + + public float[] Biases { get; } + + public static Layer Create(int inputSize, int outputSize, Activation activation, Random random) + { + var scale = MathF.Sqrt(2f / Math.Max(1, inputSize)); + var weights = new float[inputSize * outputSize]; + for (var index = 0; index < weights.Length; index++) + { + weights[index] = ((float)random.NextDouble() * 2f - 1f) * scale; + } + + return new Layer(inputSize, outputSize, weights, new float[outputSize], activation); + } + + public void Forward(float[] input, float[] output) + { + unsafe + { + fixed (float* inputBase = input) + fixed (float* outputBase = output) + fixed (float* weightBase = Weights) + fixed (float* biasBase = Biases) + { + for (var row = 0; row < OutputSize; row++) + { + var sum = biasBase[row]; + var weightRow = weightBase + row * InputSize; + for (var column = 0; column < InputSize; column++) + { + sum += weightRow[column] * inputBase[column]; + } + + outputBase[row] = Activate(sum); + } + } + } + } + + public void BackpropagateDelta(Layer nextLayer, float[] nextDelta, float[] activationValues, float[] delta) + { + unsafe + { + fixed (float* nextDeltaBase = nextDelta) + fixed (float* activationBase = activationValues) + fixed (float* deltaBase = delta) + fixed (float* nextWeightBase = nextLayer.Weights) + { + for (var output = 0; output < OutputSize; output++) + { + var sum = 0f; + for (var nextOutput = 0; nextOutput < nextLayer.OutputSize; nextOutput++) + { + sum += nextWeightBase[nextOutput * nextLayer.InputSize + output] * nextDeltaBase[nextOutput]; + } + + deltaBase[output] = sum * Derivative(activationBase[output]); + } + } + } + } + + public float Derivative(float activated) => + activation switch + { + Activation.Sigmoid => activated * (1f - activated), + Activation.Tanh => 1f - activated * activated, + _ => 1 + }; + + public void ApplySgd(float[] weightGradients, float[] biasGradients, float scale, float learningRate) + { + unsafe + { + fixed (float* weightBase = Weights) + fixed (float* biasBase = Biases) + fixed (float* weightGradientBase = weightGradients) + fixed (float* biasGradientBase = biasGradients) + { + for (var index = 0; index < Weights.Length; index++) + { + weightBase[index] -= learningRate * weightGradientBase[index] * scale; + } + + for (var index = 0; index < Biases.Length; index++) + { + biasBase[index] -= learningRate * biasGradientBase[index] * scale; + } + } + } + } + + private float Activate(float value) => + activation switch + { + Activation.Sigmoid => 1f / (1f + MathF.Exp(-value)), + Activation.Tanh => MathF.Tanh(value), + _ => value + }; + } + + private enum Activation + { + Tanh, + Sigmoid + } + + private sealed class OptimizerState + { + private readonly LayerBuffers firstMoment; + private readonly LayerBuffers secondMoment; + + private OptimizerState(LayerBuffers firstMoment, LayerBuffers secondMoment) + { + this.firstMoment = firstMoment; + this.secondMoment = secondMoment; + } + + public static OptimizerState Create(IReadOnlyList layers) => + new(LayerBuffers.Create(layers), LayerBuffers.Create(layers)); + + public void ApplyAdam( + IReadOnlyList layers, + LayerBuffers gradients, + float scale, + float learningRate, + float beta1, + float beta2, + float epsilon, + int step) + { + var beta1Correction = 1f - MathF.Pow(beta1, step); + var beta2Correction = 1f - MathF.Pow(beta2, step); + + for (var layerIndex = 0; layerIndex < layers.Count; layerIndex++) + { + ApplyAdam( + layers[layerIndex].Weights, + gradients.WeightBuffers[layerIndex], + firstMoment.WeightBuffers[layerIndex], + secondMoment.WeightBuffers[layerIndex], + scale, + learningRate, + beta1, + beta2, + epsilon, + beta1Correction, + beta2Correction); + ApplyAdam( + layers[layerIndex].Biases, + gradients.BiasBuffers[layerIndex], + firstMoment.BiasBuffers[layerIndex], + secondMoment.BiasBuffers[layerIndex], + scale, + learningRate, + beta1, + beta2, + epsilon, + beta1Correction, + beta2Correction); + } + } + + private static void ApplyAdam( + float[] values, + float[] gradients, + float[] firstMoment, + float[] secondMoment, + float scale, + float learningRate, + float beta1, + float beta2, + float epsilon, + float beta1Correction, + float beta2Correction) + { + unsafe + { + fixed (float* valueBase = values) + fixed (float* gradientBase = gradients) + fixed (float* firstBase = firstMoment) + fixed (float* secondBase = secondMoment) + { + for (var index = 0; index < values.Length; index++) + { + var gradient = gradientBase[index] * scale; + firstBase[index] = beta1 * firstBase[index] + (1f - beta1) * gradient; + secondBase[index] = beta2 * secondBase[index] + (1f - beta2) * gradient * gradient; + var mHat = firstBase[index] / beta1Correction; + var vHat = secondBase[index] / beta2Correction; + valueBase[index] -= learningRate * mHat / (MathF.Sqrt(vHat) + epsilon); + } + } + } + } + } +} diff --git a/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs b/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs new file mode 100644 index 0000000..e099c86 --- /dev/null +++ b/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs @@ -0,0 +1,112 @@ +namespace AquaSynth.Dsl; + +public sealed record UtteranceEmbeddingInput( + IReadOnlyList SpeechTextEmbedding, + IReadOnlyList ProsodyAndEmphasisHints, + IReadOnlyList CharacterStateVector) +{ + public UtteranceEmbeddingFeatureVector ToFeatureVector() + { + var values = new float[SpeechTextEmbedding.Count + ProsodyAndEmphasisHints.Count + CharacterStateVector.Count]; + var offset = Copy(SpeechTextEmbedding, values, 0); + offset = Copy(ProsodyAndEmphasisHints, values, offset); + Copy(CharacterStateVector, values, offset); + return new UtteranceEmbeddingFeatureVector(values); + } + + private static int Copy(IReadOnlyList source, float[] target, int offset) + { + for (var index = 0; index < source.Count; index++) + { + target[offset + index] = source[index]; + } + + return offset + source.Count; + } +} + +public sealed record UtteranceEmbeddingFeatureVector(IReadOnlyList Values); + +public sealed record UtteranceEmbedding(IReadOnlyList Values) +{ + public VocalTractSemanticEmbedding ToSemanticEmbedding() => + new(Values); +} + +public sealed record UtteranceEmbeddingTrainingExample( + UtteranceEmbeddingFeatureVector Features, + UtteranceEmbedding Target); + +public sealed record UtteranceEmbeddingTrainingResult( + UtteranceEmbeddingNeuralEncoder Encoder, + IReadOnlyList Steps); + +public sealed class UtteranceEmbeddingNeuralEncoder +{ + private readonly PackedNeuralNetwork network; + + private UtteranceEmbeddingNeuralEncoder(int inputSize, int embeddingSize, PackedNeuralNetwork network) + { + InputSize = inputSize; + EmbeddingSize = embeddingSize; + this.network = network; + } + + public int InputSize { get; } + + public int EmbeddingSize { get; } + + public IReadOnlyList HiddenLayerSizes => network.HiddenLayerSizes; + + public static UtteranceEmbeddingNeuralEncoder Create( + int inputSize, + int embeddingSize, + IReadOnlyList? hiddenLayerSizes = null, + int seed = 1337) + { + if (inputSize <= 0) throw new ArgumentOutOfRangeException(nameof(inputSize), "input size must be positive"); + if (embeddingSize <= 0) throw new ArgumentOutOfRangeException(nameof(embeddingSize), "embedding size must be positive"); + + var network = PackedNeuralNetwork.Create(inputSize, embeddingSize, hiddenLayerSizes ?? [64, 64, 32], seed); + return new UtteranceEmbeddingNeuralEncoder(inputSize, embeddingSize, network); + } + + public UtteranceEmbedding Encode(UtteranceEmbeddingInput input) => + Encode(input.ToFeatureVector()); + + public UtteranceEmbedding Encode(UtteranceEmbeddingFeatureVector features) + { + ValidateSize(features.Values, InputSize, nameof(features)); + return new UtteranceEmbedding(network.Predict(features.Values)); + } + + public UtteranceEmbeddingTrainingResult Train( + IReadOnlyList examples, + PackedNeuralTrainingOptions? options = null) + { + if (examples.Count == 0) + { + throw new ArgumentException("at least one training example is required", nameof(examples)); + } + + var packed = new PackedNeuralTrainingExample[examples.Count]; + for (var index = 0; index < examples.Count; index++) + { + var example = examples[index]; + ValidateSize(example.Features.Values, InputSize, nameof(example.Features)); + ValidateSize(example.Target.Values, EmbeddingSize, nameof(example.Target)); + packed[index] = new PackedNeuralTrainingExample(example.Features.Values, example.Target.Values); + } + + var result = network.Train(packed, options); + return new UtteranceEmbeddingTrainingResult(this, result.Steps); + } + + private static void ValidateSize(IReadOnlyList values, int expected, string name) + { + if (values.Count != expected) + { + throw new ArgumentException($"{name} must have {expected} values", name); + } + } +} diff --git a/src/AquaSynth.Core/VocalTractLearning.cs b/src/AquaSynth.Core/VocalTractLearning.cs index e8d4cfc..c84c7cb 100644 --- a/src/AquaSynth.Core/VocalTractLearning.cs +++ b/src/AquaSynth.Core/VocalTractLearning.cs @@ -52,15 +52,13 @@ public sealed class VocalTractNeuralMapper private const int PhoneticFeatureCount = 48; private const int BaseOutputCount = 14; private const float NeutralMelBand = 0.5f; - private readonly Layer[] layers; - private readonly int[] hiddenLayerSizes; + private readonly PackedNeuralNetwork network; - private VocalTractNeuralMapper(int semanticEmbeddingSize, int melBandCount, Layer[] layers) + private VocalTractNeuralMapper(int semanticEmbeddingSize, int melBandCount, PackedNeuralNetwork network) { SemanticEmbeddingSize = semanticEmbeddingSize; MelBandCount = melBandCount; - this.layers = layers; - hiddenLayerSizes = layers.Take(layers.Length - 1).Select(layer => layer.OutputSize).ToArray(); + this.network = network; } public int SemanticEmbeddingSize { get; } @@ -71,7 +69,7 @@ public sealed class VocalTractNeuralMapper public int OutputSize => BaseOutputCount + MelBandCount; - public IReadOnlyList HiddenLayerSizes => hiddenLayerSizes; + public IReadOnlyList HiddenLayerSizes => network.HiddenLayerSizes; public static VocalTractNeuralMapper Create( int semanticEmbeddingSize, @@ -90,29 +88,18 @@ public sealed class VocalTractNeuralMapper } hiddenLayerSizes ??= [64, 64, 32]; - var random = new Random(seed); - var sizes = new List { PhoneticFeatureCount + 4 + semanticEmbeddingSize }; - sizes.AddRange(hiddenLayerSizes); - sizes.Add(BaseOutputCount + melBandCount); + var network = PackedNeuralNetwork.Create( + PhoneticFeatureCount + 4 + semanticEmbeddingSize, + BaseOutputCount + melBandCount, + hiddenLayerSizes, + seed); - var layers = new Layer[sizes.Count - 1]; - for (var index = 0; index < layers.Length; index++) - { - var activation = index == layers.Length - 1 - ? Activation.Sigmoid - : Activation.Tanh; - layers[index] = Layer.Create(sizes[index], sizes[index + 1], activation, random); - } - - return new VocalTractNeuralMapper(semanticEmbeddingSize, melBandCount, layers); + return new VocalTractNeuralMapper(semanticEmbeddingSize, melBandCount, network); } public VocalTractControlTarget Predict(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding) { - var scratch = TrainingScratch.Create(layers, InputSize, OutputSize); - EncodeInput(phoneticEvent, semanticEmbedding, scratch.Activations[0]); - Forward(scratch); - return TargetFromVector(scratch.Output); + return TargetFromVector(network.Predict(EncodeInput(phoneticEvent, semanticEmbedding))); } public VocalTractTrainingResult Train( @@ -125,56 +112,26 @@ public sealed class VocalTractNeuralMapper } options ??= new VocalTractTrainingOptions(); - Validate(options); - - var steps = new List(); - var order = Enumerable.Range(0, examples.Count).ToArray(); - var random = new Random(options.Seed); - var scratch = TrainingScratch.Create(layers, InputSize, OutputSize); - var gradients = LayerBuffers.Create(layers); - var adamState = options.Optimizer == VocalTractOptimizer.Adam - ? OptimizerState.Create(layers) - : null; - var updateStep = 0; - - for (var epoch = 0; epoch < options.Epochs; epoch++) + var packedExamples = new PackedNeuralTrainingExample[examples.Count]; + for (var index = 0; index < examples.Count; index++) { - if (options.Shuffle) - { - Shuffle(order, random); - } - - var loss = 0f; - for (var batchStart = 0; batchStart < order.Length; batchStart += options.BatchSize) - { - var batchEnd = Math.Min(batchStart + options.BatchSize, order.Length); - gradients.Clear(); - - for (var orderIndex = batchStart; orderIndex < batchEnd; orderIndex++) - { - var example = examples[order[orderIndex]]; - EncodeInput(example.Event, example.SemanticEmbedding, scratch.Activations[0]); - FillTarget(example.Target, scratch.Target); - Forward(scratch); - loss += MeanSquaredError(scratch.Output, scratch.Target); - AccumulateGradients(scratch, gradients); - } - - updateStep++; - var scale = 1f / (batchEnd - batchStart); - if (options.Optimizer == VocalTractOptimizer.Adam) - { - adamState!.ApplyAdam(layers, gradients, scale, options.LearningRate, options.Beta1, options.Beta2, options.Epsilon, updateStep); - } - else - { - ApplySgd(gradients, scale, options.LearningRate); - } - } - - steps.Add(new VocalTractTrainingStep(epoch + 1, loss / examples.Count)); + var example = examples[index]; + packedExamples[index] = new PackedNeuralTrainingExample( + EncodeInput(example.Event, example.SemanticEmbedding), + example.Target.ToVector(MelBandCount, NeutralMelBand)); } + var result = network.Train(packedExamples, new PackedNeuralTrainingOptions( + options.Epochs, + options.LearningRate, + options.Seed, + options.Optimizer == VocalTractOptimizer.Adam ? PackedNeuralOptimizer.Adam : PackedNeuralOptimizer.Sgd, + options.BatchSize, + options.Beta1, + options.Beta2, + options.Epsilon, + options.Shuffle)); + var steps = result.Steps.Select(step => new VocalTractTrainingStep(step.Epoch, step.Loss)).ToArray(); return new VocalTractTrainingResult(this, steps); } @@ -185,62 +142,6 @@ public sealed class VocalTractNeuralMapper return input; } - private static void Validate(VocalTractTrainingOptions options) - { - if (options.Epochs <= 0) - { - throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive"); - } - - if (options.LearningRate <= 0) - { - throw new ArgumentOutOfRangeException(nameof(options), "learning rate must be positive"); - } - - if (options.BatchSize <= 0) - { - throw new ArgumentOutOfRangeException(nameof(options), "batch size must be positive"); - } - } - - private void Forward(TrainingScratch scratch) - { - for (var index = 0; index < layers.Length; index++) - { - layers[index].Forward(scratch.Activations[index], scratch.Activations[index + 1]); - } - } - - private void AccumulateGradients(TrainingScratch scratch, LayerBuffers gradients) - { - var lastLayerIndex = layers.Length - 1; - var outputDelta = scratch.Deltas[lastLayerIndex]; - var output = scratch.Activations[^1]; - for (var index = 0; index < outputDelta.Length; index++) - { - var actual = output[index]; - outputDelta[index] = 2f * (actual - scratch.Target[index]) * layers[lastLayerIndex].Derivative(actual); - } - - for (var layerIndex = lastLayerIndex - 1; layerIndex >= 0; layerIndex--) - { - layers[layerIndex].BackpropagateDelta(layers[layerIndex + 1], scratch.Deltas[layerIndex + 1], scratch.Activations[layerIndex + 1], scratch.Deltas[layerIndex]); - } - - for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++) - { - gradients.Accumulate(layerIndex, scratch.Activations[layerIndex], scratch.Deltas[layerIndex]); - } - } - - private void ApplySgd(LayerBuffers gradients, float scale, float learningRate) - { - for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++) - { - layers[layerIndex].ApplySgd(gradients.WeightBuffers[layerIndex], gradients.BiasBuffers[layerIndex], scale, learningRate); - } - } - private void EncodeInput(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding, float[] input) { if (semanticEmbedding.Values.Count != SemanticEmbeddingSize) @@ -270,24 +171,12 @@ public sealed class VocalTractNeuralMapper } } - private void FillTarget(VocalTractControlTarget target, float[] output) => - target.ToVector(output, MelBandCount, NeutralMelBand); - private static int OneHot(float[] input, int offset, int value, int count) { input[offset + Math.Clamp(value, 0, count - 1)] = 1; return offset + count; } - private static void Shuffle(int[] values, Random random) - { - for (var index = values.Length - 1; index > 0; index--) - { - var swapIndex = random.Next(index + 1); - (values[index], values[swapIndex]) = (values[swapIndex], values[index]); - } - } - private static VocalTractControlTarget TargetFromVector(IReadOnlyList values) { var mel = new float[Math.Max(0, values.Count - BaseOutputCount)]; @@ -314,329 +203,7 @@ public sealed class VocalTractNeuralMapper mel); } - private static float MeanSquaredError(IReadOnlyList actual, IReadOnlyList target) - { - var sum = 0f; - for (var index = 0; index < actual.Count; index++) - { - var difference = actual[index] - target[index]; - sum += difference * difference; - } - - return sum / actual.Count; - } - private static float Clamp01(float value) => Math.Clamp(value, 0, 1); - - private sealed class TrainingScratch - { - private TrainingScratch(float[][] activations, float[][] deltas, float[] target) - { - Activations = activations; - Deltas = deltas; - Target = target; - } - - public float[][] Activations { get; } - - public float[][] Deltas { get; } - - public float[] Target { get; } - - public float[] Output => Activations[^1]; - - public static TrainingScratch Create(IReadOnlyList layers, int inputSize, int outputSize) - { - var activations = new float[layers.Count + 1][]; - var deltas = new float[layers.Count][]; - activations[0] = new float[inputSize]; - for (var index = 0; index < layers.Count; index++) - { - activations[index + 1] = new float[layers[index].OutputSize]; - deltas[index] = new float[layers[index].OutputSize]; - } - - return new TrainingScratch(activations, deltas, new float[outputSize]); - } - } - - private sealed class LayerBuffers - { - private LayerBuffers(float[][] weightBuffers, float[][] biasBuffers) - { - WeightBuffers = weightBuffers; - BiasBuffers = biasBuffers; - } - - public float[][] WeightBuffers { get; } - - public float[][] BiasBuffers { get; } - - public static LayerBuffers Create(IReadOnlyList layers) - { - var weights = new float[layers.Count][]; - var biases = new float[layers.Count][]; - for (var index = 0; index < layers.Count; index++) - { - weights[index] = new float[layers[index].Weights.Length]; - biases[index] = new float[layers[index].Biases.Length]; - } - - return new LayerBuffers(weights, biases); - } - - public void Clear() - { - for (var index = 0; index < WeightBuffers.Length; index++) - { - Array.Clear(WeightBuffers[index]); - Array.Clear(BiasBuffers[index]); - } - } - - public void Accumulate(int layerIndex, float[] input, float[] delta) - { - var weightGradients = WeightBuffers[layerIndex]; - var biasGradients = BiasBuffers[layerIndex]; - var inputSize = input.Length; - - unsafe - { - fixed (float* inputBase = input) - fixed (float* deltaBase = delta) - fixed (float* weightBase = weightGradients) - fixed (float* biasBase = biasGradients) - { - for (var output = 0; output < delta.Length; output++) - { - var deltaValue = deltaBase[output]; - var weightRow = weightBase + output * inputSize; - for (var column = 0; column < inputSize; column++) - { - weightRow[column] += deltaValue * inputBase[column]; - } - - biasBase[output] += deltaValue; - } - } - } - } - } - - private sealed class Layer - { - private readonly Activation activation; - - private Layer(int inputSize, int outputSize, float[] weights, float[] biases, Activation activation) - { - InputSize = inputSize; - OutputSize = outputSize; - Weights = weights; - Biases = biases; - this.activation = activation; - } - - public int InputSize { get; } - - public int OutputSize { get; } - - public float[] Weights { get; } - - public float[] Biases { get; } - - public static Layer Create(int inputSize, int outputSize, Activation activation, Random random) - { - var scale = MathF.Sqrt(2f / Math.Max(1, inputSize)); - var weights = new float[inputSize * outputSize]; - for (var index = 0; index < weights.Length; index++) - { - weights[index] = ((float)random.NextDouble() * 2f - 1f) * scale; - } - - return new Layer(inputSize, outputSize, weights, new float[outputSize], activation); - } - - public void Forward(float[] input, float[] output) - { - unsafe - { - fixed (float* inputBase = input) - fixed (float* outputBase = output) - fixed (float* weightBase = Weights) - fixed (float* biasBase = Biases) - { - for (var row = 0; row < OutputSize; row++) - { - var sum = biasBase[row]; - var weightRow = weightBase + row * InputSize; - for (var column = 0; column < InputSize; column++) - { - sum += weightRow[column] * inputBase[column]; - } - - outputBase[row] = Activate(sum); - } - } - } - } - - public void BackpropagateDelta(Layer nextLayer, float[] nextDelta, float[] activationValues, float[] delta) - { - unsafe - { - fixed (float* nextDeltaBase = nextDelta) - fixed (float* activationBase = activationValues) - fixed (float* deltaBase = delta) - fixed (float* nextWeightBase = nextLayer.Weights) - { - for (var output = 0; output < OutputSize; output++) - { - var sum = 0f; - for (var nextOutput = 0; nextOutput < nextLayer.OutputSize; nextOutput++) - { - sum += nextWeightBase[nextOutput * nextLayer.InputSize + output] * nextDeltaBase[nextOutput]; - } - - deltaBase[output] = sum * Derivative(activationBase[output]); - } - } - } - } - - public float Derivative(float activated) => - activation switch - { - Activation.Sigmoid => activated * (1f - activated), - Activation.Tanh => 1f - activated * activated, - _ => 1 - }; - - public void ApplySgd(float[] weightGradients, float[] biasGradients, float scale, float learningRate) - { - unsafe - { - fixed (float* weightBase = Weights) - fixed (float* biasBase = Biases) - fixed (float* weightGradientBase = weightGradients) - fixed (float* biasGradientBase = biasGradients) - { - for (var index = 0; index < Weights.Length; index++) - { - weightBase[index] -= learningRate * weightGradientBase[index] * scale; - } - - for (var index = 0; index < Biases.Length; index++) - { - biasBase[index] -= learningRate * biasGradientBase[index] * scale; - } - } - } - } - - private float Activate(float value) => - activation switch - { - Activation.Sigmoid => 1f / (1f + MathF.Exp(-value)), - Activation.Tanh => MathF.Tanh(value), - _ => value - }; - } - - private enum Activation - { - Tanh, - Sigmoid - } - - private sealed class OptimizerState - { - private readonly LayerBuffers firstMoment; - private readonly LayerBuffers secondMoment; - - private OptimizerState(LayerBuffers firstMoment, LayerBuffers secondMoment) - { - this.firstMoment = firstMoment; - this.secondMoment = secondMoment; - } - - public static OptimizerState Create(IReadOnlyList layers) => - new(LayerBuffers.Create(layers), LayerBuffers.Create(layers)); - - public void ApplyAdam( - IReadOnlyList layers, - LayerBuffers gradients, - float scale, - float learningRate, - float beta1, - float beta2, - float epsilon, - int step) - { - var beta1Correction = 1f - MathF.Pow(beta1, step); - var beta2Correction = 1f - MathF.Pow(beta2, step); - - for (var layerIndex = 0; layerIndex < layers.Count; layerIndex++) - { - ApplyAdam( - layers[layerIndex].Weights, - gradients.WeightBuffers[layerIndex], - firstMoment.WeightBuffers[layerIndex], - secondMoment.WeightBuffers[layerIndex], - scale, - learningRate, - beta1, - beta2, - epsilon, - beta1Correction, - beta2Correction); - ApplyAdam( - layers[layerIndex].Biases, - gradients.BiasBuffers[layerIndex], - firstMoment.BiasBuffers[layerIndex], - secondMoment.BiasBuffers[layerIndex], - scale, - learningRate, - beta1, - beta2, - epsilon, - beta1Correction, - beta2Correction); - } - } - - private static void ApplyAdam( - float[] values, - float[] gradients, - float[] firstMoment, - float[] secondMoment, - float scale, - float learningRate, - float beta1, - float beta2, - float epsilon, - float beta1Correction, - float beta2Correction) - { - unsafe - { - fixed (float* valueBase = values) - fixed (float* gradientBase = gradients) - fixed (float* firstBase = firstMoment) - fixed (float* secondBase = secondMoment) - { - for (var index = 0; index < values.Length; index++) - { - var gradient = gradientBase[index] * scale; - firstBase[index] = beta1 * firstBase[index] + (1f - beta1) * gradient; - secondBase[index] = beta2 * secondBase[index] + (1f - beta2) * gradient * gradient; - var mHat = firstBase[index] / beta1Correction; - var vHat = secondBase[index] / beta2Correction; - valueBase[index] -= learningRate * mHat / (MathF.Sqrt(vHat) + epsilon); - } - } - } - } - } } public static class VocalTractControlTargetExtensions diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 4dd0374..956a35d 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -111,3 +111,4 @@ {"date":"2026-05-19","evidence":"Updated EspeakNgReferenceTests to record log_mel_frames, log_mel_bands, log_mel_mean, and log_mel_distance_matrix in report.md; updated the IPA vocal tract roadmap accordingly.","lesson":"The eSpeak NG speech workout should expose log-mel evidence immediately, not just basic audio features. Each fixture report now carries normalized log-mel band means plus a cross-fixture log-mel distance matrix so future AquaSynth IPA renders can plug into the same spectral comparison shape.","topic":"espeak-ng-logmel-workout"} {"date":"2026-05-19","topic":"vocal-tract-learned-controller","lesson":"AquaSynth now has a small learned controller scaffold for the vocal-tract lane: phonetic features plus a semantic embedding vector feed a multilayer tanh/sigmoid MLP trained with SGD/backprop to predict tract controls plus AM/FM, LFO, filter, and mel spectral-envelope expression lanes. This is a trainable controller surface, not proof of eSpeak-level speech yet.","evidence":"Added VocalTractLearning.cs and VocalTractLearningTests.cs with feature encoding, semantic embeddings, 3-layer default network, loss-reduction tests, same-phone semantic steering, expressive modulator/filter control tests, and learned dark/bright mel-envelope steering for PAD-style spectral-table coloration."} {"date":"2026-05-19","topic":"vocal-tract-optimizer-hot-path","lesson":"The first learned vocal-tract optimizer should stay native and inspectable until the render-loss boundary proves it needs a tensor framework. The trainer now uses packed row-major weight buffers, owned scratch activation/delta/target buffers, unsafe pointer loops, reusable gradient/moment buffers, batched gradients, deterministic shuffling, and Adam by default instead of allocation-heavy per-example jagged-array SGD.","evidence":"Rebuilt VocalTractLearning.cs hot path around flat float arrays and unsafe loops; enabled unsafe blocks in AquaSynth.Core; VocalTractLearningTests pass against the packed-buffer Adam trainer."} +{"date":"2026-05-19","topic":"utterance-embedding-learning-boundary","lesson":"AquaSynth should own gradient descent for Weksa speech handoff. Weksa supplies speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors; AquaSynth trains the tiny utterance-embedding encoder and passes that embedding into the synth-driver model.","evidence":"Extracted PackedNeuralNetwork as the shared packed-buffer Adam/SGD primitive, rewired VocalTractNeuralMapper through it, and added UtteranceEmbeddingInput/UtteranceEmbeddingNeuralEncoder plus tests for packing, training, and feeding the vocal-tract semantic embedding."} diff --git a/state/map.yaml b/state/map.yaml index 5c83a0e..f2c5012 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -32,6 +32,8 @@ modules: model: src/AquaSynth.Core/Model.cs parser: src/AquaSynth.Core/PatchScript.cs vocal_tract_model: src/AquaSynth.Core/VocalTractModel.cs + packed_neural_network: src/AquaSynth.Core/PackedNeuralNetwork.cs + utterance_embedding_learning: src/AquaSynth.Core/UtteranceEmbeddingLearning.cs vocal_tract_learning: src/AquaSynth.Core/VocalTractLearning.cs faust_emitter: src/AquaSynth.Core/FaustExport.cs faust_compiler: src/AquaSynth.Faust/FaustCompiler.cs @@ -43,6 +45,7 @@ modules: ipa_vocal_tract_roadmap: docs/ipa-vocal-tract-roadmap.md package_boundary_tests: tests/AquaSynth.Dsl.Tests/PackageBoundaryTests.cs vocal_tract_model_tests: tests/AquaSynth.Dsl.Tests/VocalTractModelTests.cs + utterance_embedding_learning_tests: tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs vocal_tract_learning_tests: tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs sfxr: src/AquaSynth.Core/SfxrParams.cs analysis: src/AquaSynth.Core/AudioAnalysis.cs @@ -66,7 +69,9 @@ boundaries: - live patch compilation, native Faust factory lifetime, and offline sample-buffer render proof in AquaSynth.Faust - IPA-to-phonetic-feature planning, articulatory gesture planning, and human/alien vocal-tract roadmap design - phonetic intent, articulatory plan, morphology capability, and constraint-report contracts for vocal tract synthesis - - learned phonetic-plus-semantic embedding controller with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls + - packed-buffer native backprop/Adam as the shared AquaSynth gradient-descent primitive + - learned utterance embedding encoder from speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors + - learned phonetic-plus-utterance-embedding synth driver with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls dev_only: - public-domain DX7 SysEx fixtures - pinned GPL ZynAddSubFX source for parity tests diff --git a/state/memory.json b/state/memory.json index df2c6cf..1e02898 100644 --- a/state/memory.json +++ b/state/memory.json @@ -86,7 +86,7 @@ "rule": "Add DSL abstractions only when reference rebuilds prove the missing invariant. Do not consume references that cannot be backed by parity tests; parsing alone is inventory, not evidence of expressivity." }, "ipa_vocal_tract_path": { - "status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. VocalTractNeuralMapper is the first learned controller: phonetic features plus semantic embedding vector feed a small multilayer tanh/sigmoid network trained with packed-buffer native C# backprop/Adam to predict tract, AM/FM, LFO, filter, and mel spectral-envelope controls.", + "status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. AquaSynth now owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls.", "lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.", "next": "Implement Stage 0/1 fixtures: IPA token spans, vowel feature bundles, human baseline vowel tract-area targets, and a tiny audible vowel render proof before expanding consonants." }, diff --git a/tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs b/tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs new file mode 100644 index 0000000..e09b7f9 --- /dev/null +++ b/tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs @@ -0,0 +1,106 @@ +using AquaSynth.Dsl; + +namespace AquaSynth.Dsl.Tests; + +public sealed class UtteranceEmbeddingLearningTests +{ + [Fact] + public void UtteranceEmbeddingInputPacksTextProsodyAndCharacterState() + { + var input = new UtteranceEmbeddingInput( + SpeechTextEmbedding: [0.1f, 0.2f, 0.3f], + ProsodyAndEmphasisHints: [0.7f, 0.8f], + CharacterStateVector: [0.4f, 0.5f, 0.6f]); + + var features = input.ToFeatureVector(); + + Assert.Equal([0.1f, 0.2f, 0.3f, 0.7f, 0.8f, 0.4f, 0.5f, 0.6f], features.Values); + } + + [Fact] + public void UtteranceEmbeddingEncoderUsesPackedAquaSynthTraining() + { + var encoder = UtteranceEmbeddingNeuralEncoder.Create(inputSize: 10, embeddingSize: 16, hiddenLayerSizes: [48, 32, 24], seed: 61); + var examples = TrainingSet(); + + var before = AverageLoss(encoder, examples); + var result = encoder.Train(examples, new PackedNeuralTrainingOptions(Epochs: 320, LearningRate: 0.035f, Seed: 61, BatchSize: 2)); + var after = AverageLoss(encoder, examples); + + Assert.Equal([48, 32, 24], encoder.HiddenLayerSizes); + Assert.Equal(320, result.Steps.Count); + Assert.True(after < before * 0.35f, $"expected utterance-embedding loss to drop; before={before}, after={after}"); + } + + [Fact] + public void UtteranceEmbeddingCanFeedVocalTractSemanticEmbedding() + { + var vector = new UtteranceEmbedding(Enumerable.Range(0, 16).Select(index => index / 16f).ToArray()); + + var embedding = vector.ToSemanticEmbedding(); + + Assert.Equal(vector.Values, embedding.Values); + } + + [Fact] + public void UtteranceEmbeddingEncoderSeparatesContrastingDeliveryTargets() + { + var encoder = UtteranceEmbeddingNeuralEncoder.Create(inputSize: 10, embeddingSize: 16, hiddenLayerSizes: [64, 48, 32], seed: 67); + var clippedThreat = new UtteranceEmbeddingFeatureVector([0.95f, 0.85f, 0.15f, 0.80f, 0.70f, 0.10f, 0.20f, 0.90f, 0.75f, 0.30f]); + var warmReassurance = new UtteranceEmbeddingFeatureVector([0.20f, 0.25f, 0.88f, 0.15f, 0.30f, 0.80f, 0.75f, 0.18f, 0.25f, 0.70f]); + var threatTarget = Vector(16, 0.86f, 0.72f, 0.18f); + var warmTarget = Vector(16, 0.22f, 0.48f, 0.88f); + var examples = new[] + { + new UtteranceEmbeddingTrainingExample(clippedThreat, threatTarget), + new UtteranceEmbeddingTrainingExample(warmReassurance, warmTarget) + }; + + encoder.Train(examples, new PackedNeuralTrainingOptions(Epochs: 420, LearningRate: 0.035f, Seed: 67, BatchSize: 2)); + var threat = encoder.Encode(clippedThreat); + var warm = encoder.Encode(warmReassurance); + + Assert.True(threat.Values[0] > warm.Values[0] + 0.25f); + Assert.True(warm.Values[^1] > threat.Values[^1] + 0.25f); + } + + private static IReadOnlyList TrainingSet() => + [ + new(new UtteranceEmbeddingFeatureVector([0.95f, 0.82f, 0.10f, 0.75f, 0.70f, 0.12f, 0.18f, 0.88f, 0.74f, 0.28f]), Vector(16, 0.86f, 0.70f, 0.20f)), + new(new UtteranceEmbeddingFeatureVector([0.20f, 0.22f, 0.88f, 0.18f, 0.28f, 0.82f, 0.72f, 0.18f, 0.26f, 0.74f]), Vector(16, 0.20f, 0.50f, 0.88f)), + new(new UtteranceEmbeddingFeatureVector([0.55f, 0.42f, 0.48f, 0.35f, 0.92f, 0.34f, 0.42f, 0.52f, 0.62f, 0.44f]), Vector(16, 0.48f, 0.82f, 0.50f)) + ]; + + private static UtteranceEmbedding Vector(int size, float left, float center, float right) + { + var values = new float[size]; + for (var index = 0; index < size; index++) + { + var position = index / Math.Max(1f, size - 1f); + values[index] = position < 0.5f + ? Lerp(left, center, position * 2f) + : Lerp(center, right, (position - 0.5f) * 2f); + } + + return new UtteranceEmbedding(values); + } + + private static float AverageLoss(UtteranceEmbeddingNeuralEncoder encoder, IReadOnlyList examples) + { + var loss = 0f; + foreach (var example in examples) + { + var prediction = encoder.Encode(example.Features); + for (var index = 0; index < prediction.Values.Count; index++) + { + var difference = prediction.Values[index] - example.Target.Values[index]; + loss += difference * difference; + } + } + + return loss / examples.Count / encoder.EmbeddingSize; + } + + private static float Lerp(float left, float right, float amount) => + left + (right - left) * amount; +}