GameCult/AquaSynth
Watch
9
0
Fork
You've already forked AquaSynth
 0
Code Issues Pull requests 4 Projects Releases Packages Wiki Activity Actions

Centralize utterance embedding learning

Browse source
This commit is contained in:
Meta Vens 2026-05-19 17:47:08 +01:00
commit 462439c0bd
8 changed files with 814 additions and 477 deletions
Download patch file Download diff file Expand all files Collapse all files

24
docs/ipa-vocal-tract-roadmap.md
View file

@ -183,17 +183,19 @@ features become overlapping constriction tasks and source events. It should not
linearly interpolate phone presets unless a test proves that shortcut works for
the current stage.
The first learned controller lives below the handoff and above rendering:
`VocalTractNeuralMapper` takes phonetic features plus a semantic embedding vector
and predicts tract controls. The default shape is deliberately small but real:
three hidden dense layers with tanh activations, sigmoid output controls, and a
native C# training path using packed row-major buffers, unsafe hot loops, batched
gradients, and Adam or SGD updates. Its output includes core tract parameters plus
expressive synthesis lanes: AM depth, FM depth, LFO rate/depth, filter cutoff,
filter resonance, and a full mel-frequency spectral envelope vector for
PAD-style wavetable/formant coloration. Those lanes are not license to bypass
tract anatomy; they are renderable expression controls the later synth can
choose to honor.
The first learned controllers live below the handoff and above rendering. Weksa
or another language layer may provide speech text embeddings, prosody/emphasis
hints, and Ghostlight/Epiphany-shaped character state vectors, but AquaSynth owns
the gradient descent. `UtteranceEmbeddingNeuralEncoder` trains a tiny model to
compress those ingredients into one utterance embedding. `VocalTractNeuralMapper`
then takes phonetic features plus that embedding and predicts tract controls.
Both controllers use the same native C# packed-network path: row-major buffers,
unsafe hot loops, batched gradients, and Adam or SGD updates. The synth-driver
output includes core tract parameters plus expressive synthesis lanes: AM depth,
FM depth, LFO rate/depth, filter cutoff, filter resonance, and a full
mel-frequency spectral envelope vector for PAD-style wavetable/formant
coloration. Those lanes are not license to bypass tract anatomy; they are
renderable expression controls the later synth can choose to honor.
### 4. Morphology Model

544
src/AquaSynth.Core/PackedNeuralNetwork.cs Normal file
View file

@ -0,0 +1,544 @@
namespace AquaSynth.Dsl;
public sealed record PackedNeuralTrainingExample(
IReadOnlyList<float> Input,
IReadOnlyList<float> Target);
public sealed record PackedNeuralTrainingOptions(
int Epochs = 200,
float LearningRate = 0.01f,
int Seed = 1337,
PackedNeuralOptimizer Optimizer = PackedNeuralOptimizer.Adam,
int BatchSize = 32,
float Beta1 = 0.9f,
float Beta2 = 0.999f,
float Epsilon = 0.00000001f,
bool Shuffle = true);
public enum PackedNeuralOptimizer
{
Sgd,
Adam
}
public sealed record PackedNeuralTrainingStep(int Epoch, float Loss);
public sealed record PackedNeuralTrainingResult(
PackedNeuralNetwork Network,
IReadOnlyList<PackedNeuralTrainingStep> Steps);
public sealed class PackedNeuralNetwork
{
private readonly Layer[] layers;
private PackedNeuralNetwork(int inputSize, int outputSize, Layer[] layers)
{
InputSize = inputSize;
OutputSize = outputSize;
this.layers = layers;
HiddenLayerSizes = layers.Take(layers.Length - 1).Select(layer => layer.OutputSize).ToArray();
}
public int InputSize { get; }
public int OutputSize { get; }
public IReadOnlyList<int> HiddenLayerSizes { get; }
public static PackedNeuralNetwork Create(
int inputSize,
int outputSize,
IReadOnlyList<int>? hiddenLayerSizes = null,
int seed = 1337)
{
if (inputSize <= 0) throw new ArgumentOutOfRangeException(nameof(inputSize), "input size must be positive");
if (outputSize <= 0) throw new ArgumentOutOfRangeException(nameof(outputSize), "output size must be positive");
hiddenLayerSizes ??= [64, 64, 32];
var random = new Random(seed);
var sizes = new List<int> { inputSize };
sizes.AddRange(hiddenLayerSizes);
sizes.Add(outputSize);
var layers = new Layer[sizes.Count - 1];
for (var index = 0; index < layers.Length; index++)
{
var activation = index == layers.Length - 1
? Activation.Sigmoid
: Activation.Tanh;
layers[index] = Layer.Create(sizes[index], sizes[index + 1], activation, random);
}
return new PackedNeuralNetwork(inputSize, outputSize, layers);
}
public float[] Predict(IReadOnlyList<float> input)
{
if (input.Count != InputSize)
{
throw new ArgumentException($"input must have {InputSize} values", nameof(input));
}
var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
Copy(input, scratch.Activations[0]);
Forward(scratch);
return scratch.Output.ToArray();
}
public PackedNeuralTrainingResult Train(
IReadOnlyList<PackedNeuralTrainingExample> examples,
PackedNeuralTrainingOptions? options = null)
{
if (examples.Count == 0)
{
throw new ArgumentException("at least one training example is required", nameof(examples));
}
options ??= new PackedNeuralTrainingOptions();
Validate(options);
var steps = new List<PackedNeuralTrainingStep>();
var order = Enumerable.Range(0, examples.Count).ToArray();
var random = new Random(options.Seed);
var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
var gradients = LayerBuffers.Create(layers);
var adamState = options.Optimizer == PackedNeuralOptimizer.Adam
? OptimizerState.Create(layers)
: null;
var updateStep = 0;
for (var epoch = 0; epoch < options.Epochs; epoch++)
{
if (options.Shuffle)
{
Shuffle(order, random);
}
var loss = 0f;
for (var batchStart = 0; batchStart < order.Length; batchStart += options.BatchSize)
{
var batchEnd = Math.Min(batchStart + options.BatchSize, order.Length);
gradients.Clear();
for (var orderIndex = batchStart; orderIndex < batchEnd; orderIndex++)
{
var example = examples[order[orderIndex]];
CopyChecked(example.Input, scratch.Activations[0], "input");
CopyChecked(example.Target, scratch.Target, "target");
Forward(scratch);
loss += MeanSquaredError(scratch.Output, scratch.Target);
AccumulateGradients(scratch, gradients);
}
updateStep++;
var scale = 1f / (batchEnd - batchStart);
if (options.Optimizer == PackedNeuralOptimizer.Adam)
{
adamState!.ApplyAdam(layers, gradients, scale, options.LearningRate, options.Beta1, options.Beta2, options.Epsilon, updateStep);
}
else
{
ApplySgd(gradients, scale, options.LearningRate);
}
}
steps.Add(new PackedNeuralTrainingStep(epoch + 1, loss / examples.Count));
}
return new PackedNeuralTrainingResult(this, steps);
}
private static void Validate(PackedNeuralTrainingOptions options)
{
if (options.Epochs <= 0) throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive");
if (options.LearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(options), "learning rate must be positive");
if (options.BatchSize <= 0) throw new ArgumentOutOfRangeException(nameof(options), "batch size must be positive");
}
private void Forward(TrainingScratch scratch)
{
for (var index = 0; index < layers.Length; index++)
{
layers[index].Forward(scratch.Activations[index], scratch.Activations[index + 1]);
}
}
private void AccumulateGradients(TrainingScratch scratch, LayerBuffers gradients)
{
var lastLayerIndex = layers.Length - 1;
var outputDelta = scratch.Deltas[lastLayerIndex];
var output = scratch.Activations[^1];
for (var index = 0; index < outputDelta.Length; index++)
{
var actual = output[index];
outputDelta[index] = 2f * (actual - scratch.Target[index]) * layers[lastLayerIndex].Derivative(actual);
}
for (var layerIndex = lastLayerIndex - 1; layerIndex >= 0; layerIndex--)
{
layers[layerIndex].BackpropagateDelta(layers[layerIndex + 1], scratch.Deltas[layerIndex + 1], scratch.Activations[layerIndex + 1], scratch.Deltas[layerIndex]);
}
for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++)
{
gradients.Accumulate(layerIndex, scratch.Activations[layerIndex], scratch.Deltas[layerIndex]);
}
}
private void ApplySgd(LayerBuffers gradients, float scale, float learningRate)
{
for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++)
{
layers[layerIndex].ApplySgd(gradients.WeightBuffers[layerIndex], gradients.BiasBuffers[layerIndex], scale, learningRate);
}
}
private static void CopyChecked(IReadOnlyList<float> source, float[] target, string name)
{
if (source.Count != target.Length)
{
throw new ArgumentException($"{name} must have {target.Length} values");
}
Copy(source, target);
}
private static void Copy(IReadOnlyList<float> source, float[] target)
{
for (var index = 0; index < target.Length; index++)
{
target[index] = source[index];
}
}
private static void Shuffle(int[] values, Random random)
{
for (var index = values.Length - 1; index > 0; index--)
{
var swapIndex = random.Next(index + 1);
(values[index], values[swapIndex]) = (values[swapIndex], values[index]);
}
}
private static float MeanSquaredError(IReadOnlyList<float> actual, IReadOnlyList<float> target)
{
var sum = 0f;
for (var index = 0; index < actual.Count; index++)
{
var difference = actual[index] - target[index];
sum += difference * difference;
}
return sum / actual.Count;
}
private sealed class TrainingScratch
{
private TrainingScratch(float[][] activations, float[][] deltas, float[] target)
{
Activations = activations;
Deltas = deltas;
Target = target;
}
public float[][] Activations { get; }
public float[][] Deltas { get; }
public float[] Target { get; }
public float[] Output => Activations[^1];
public static TrainingScratch Create(IReadOnlyList<Layer> layers, int inputSize, int outputSize)
{
var activations = new float[layers.Count + 1][];
var deltas = new float[layers.Count][];
activations[0] = new float[inputSize];
for (var index = 0; index < layers.Count; index++)
{
activations[index + 1] = new float[layers[index].OutputSize];
deltas[index] = new float[layers[index].OutputSize];
}
return new TrainingScratch(activations, deltas, new float[outputSize]);
}
}
private sealed class LayerBuffers
{
private LayerBuffers(float[][] weightBuffers, float[][] biasBuffers)
{
WeightBuffers = weightBuffers;
BiasBuffers = biasBuffers;
}
public float[][] WeightBuffers { get; }
public float[][] BiasBuffers { get; }
public static LayerBuffers Create(IReadOnlyList<Layer> layers)
{
var weights = new float[layers.Count][];
var biases = new float[layers.Count][];
for (var index = 0; index < layers.Count; index++)
{
weights[index] = new float[layers[index].Weights.Length];
biases[index] = new float[layers[index].Biases.Length];
}
return new LayerBuffers(weights, biases);
}
public void Clear()
{
for (var index = 0; index < WeightBuffers.Length; index++)
{
Array.Clear(WeightBuffers[index]);
Array.Clear(BiasBuffers[index]);
}
}
public void Accumulate(int layerIndex, float[] input, float[] delta)
{
var weightGradients = WeightBuffers[layerIndex];
var biasGradients = BiasBuffers[layerIndex];
var inputSize = input.Length;
unsafe
{
fixed (float* inputBase = input)
fixed (float* deltaBase = delta)
fixed (float* weightBase = weightGradients)
fixed (float* biasBase = biasGradients)
{
for (var output = 0; output < delta.Length; output++)
{
var deltaValue = deltaBase[output];
var weightRow = weightBase + output * inputSize;
for (var column = 0; column < inputSize; column++)
{
weightRow[column] += deltaValue * inputBase[column];
}
biasBase[output] += deltaValue;
}
}
}
}
}
private sealed class Layer
{
private readonly Activation activation;
private Layer(int inputSize, int outputSize, float[] weights, float[] biases, Activation activation)
{
InputSize = inputSize;
OutputSize = outputSize;
Weights = weights;
Biases = biases;
this.activation = activation;
}
public int InputSize { get; }
public int OutputSize { get; }
public float[] Weights { get; }
public float[] Biases { get; }
public static Layer Create(int inputSize, int outputSize, Activation activation, Random random)
{
var scale = MathF.Sqrt(2f / Math.Max(1, inputSize));
var weights = new float[inputSize * outputSize];
for (var index = 0; index < weights.Length; index++)
{
weights[index] = ((float)random.NextDouble() * 2f - 1f) * scale;
}
return new Layer(inputSize, outputSize, weights, new float[outputSize], activation);
}
public void Forward(float[] input, float[] output)
{
unsafe
{
fixed (float* inputBase = input)
fixed (float* outputBase = output)
fixed (float* weightBase = Weights)
fixed (float* biasBase = Biases)
{
for (var row = 0; row < OutputSize; row++)
{
var sum = biasBase[row];
var weightRow = weightBase + row * InputSize;
for (var column = 0; column < InputSize; column++)
{
sum += weightRow[column] * inputBase[column];
}
outputBase[row] = Activate(sum);
}
}
}
}
public void BackpropagateDelta(Layer nextLayer, float[] nextDelta, float[] activationValues, float[] delta)
{
unsafe
{
fixed (float* nextDeltaBase = nextDelta)
fixed (float* activationBase = activationValues)
fixed (float* deltaBase = delta)
fixed (float* nextWeightBase = nextLayer.Weights)
{
for (var output = 0; output < OutputSize; output++)
{
var sum = 0f;
for (var nextOutput = 0; nextOutput < nextLayer.OutputSize; nextOutput++)
{
sum += nextWeightBase[nextOutput * nextLayer.InputSize + output] * nextDeltaBase[nextOutput];
}
deltaBase[output] = sum * Derivative(activationBase[output]);
}
}
}
}
public float Derivative(float activated) =>
activation switch
{
Activation.Sigmoid => activated * (1f - activated),
Activation.Tanh => 1f - activated * activated,
_ => 1
};
public void ApplySgd(float[] weightGradients, float[] biasGradients, float scale, float learningRate)
{
unsafe
{
fixed (float* weightBase = Weights)
fixed (float* biasBase = Biases)
fixed (float* weightGradientBase = weightGradients)
fixed (float* biasGradientBase = biasGradients)
{
for (var index = 0; index < Weights.Length; index++)
{
weightBase[index] -= learningRate * weightGradientBase[index] * scale;
}
for (var index = 0; index < Biases.Length; index++)
{
biasBase[index] -= learningRate * biasGradientBase[index] * scale;
}
}
}
}
private float Activate(float value) =>
activation switch
{
Activation.Sigmoid => 1f / (1f + MathF.Exp(-value)),
Activation.Tanh => MathF.Tanh(value),
_ => value
};
}
private enum Activation
{
Tanh,
Sigmoid
}
private sealed class OptimizerState
{
private readonly LayerBuffers firstMoment;
private readonly LayerBuffers secondMoment;
private OptimizerState(LayerBuffers firstMoment, LayerBuffers secondMoment)
{
this.firstMoment = firstMoment;
this.secondMoment = secondMoment;
}
public static OptimizerState Create(IReadOnlyList<Layer> layers) =>
new(LayerBuffers.Create(layers), LayerBuffers.Create(layers));
public void ApplyAdam(
IReadOnlyList<Layer> layers,
LayerBuffers gradients,
float scale,
float learningRate,
float beta1,
float beta2,
float epsilon,
int step)
{
var beta1Correction = 1f - MathF.Pow(beta1, step);
var beta2Correction = 1f - MathF.Pow(beta2, step);
for (var layerIndex = 0; layerIndex < layers.Count; layerIndex++)
{
ApplyAdam(
layers[layerIndex].Weights,
gradients.WeightBuffers[layerIndex],
firstMoment.WeightBuffers[layerIndex],
secondMoment.WeightBuffers[layerIndex],
scale,
learningRate,
beta1,
beta2,
epsilon,
beta1Correction,
beta2Correction);
ApplyAdam(
layers[layerIndex].Biases,
gradients.BiasBuffers[layerIndex],
firstMoment.BiasBuffers[layerIndex],
secondMoment.BiasBuffers[layerIndex],
scale,
learningRate,
beta1,
beta2,
epsilon,
beta1Correction,
beta2Correction);
}
}
private static void ApplyAdam(
float[] values,
float[] gradients,
float[] firstMoment,
float[] secondMoment,
float scale,
float learningRate,
float beta1,
float beta2,
float epsilon,
float beta1Correction,
float beta2Correction)
{
unsafe
{
fixed (float* valueBase = values)
fixed (float* gradientBase = gradients)
fixed (float* firstBase = firstMoment)
fixed (float* secondBase = secondMoment)
{
for (var index = 0; index < values.Length; index++)
{
var gradient = gradientBase[index] * scale;
firstBase[index] = beta1 * firstBase[index] + (1f - beta1) * gradient;
secondBase[index] = beta2 * secondBase[index] + (1f - beta2) * gradient * gradient;
var mHat = firstBase[index] / beta1Correction;
var vHat = secondBase[index] / beta2Correction;
valueBase[index] -= learningRate * mHat / (MathF.Sqrt(vHat) + epsilon);
}
}
}
}
}
}

112
src/AquaSynth.Core/UtteranceEmbeddingLearning.cs Normal file
View file

@ -0,0 +1,112 @@
namespace AquaSynth.Dsl;
public sealed record UtteranceEmbeddingInput(
IReadOnlyList<float> SpeechTextEmbedding,
IReadOnlyList<float> ProsodyAndEmphasisHints,
IReadOnlyList<float> CharacterStateVector)
{
public UtteranceEmbeddingFeatureVector ToFeatureVector()
{
var values = new float[SpeechTextEmbedding.Count + ProsodyAndEmphasisHints.Count + CharacterStateVector.Count];
var offset = Copy(SpeechTextEmbedding, values, 0);
offset = Copy(ProsodyAndEmphasisHints, values, offset);
Copy(CharacterStateVector, values, offset);
return new UtteranceEmbeddingFeatureVector(values);
}
private static int Copy(IReadOnlyList<float> source, float[] target, int offset)
{
for (var index = 0; index < source.Count; index++)
{
target[offset + index] = source[index];
}
return offset + source.Count;
}
}
public sealed record UtteranceEmbeddingFeatureVector(IReadOnlyList<float> Values);
public sealed record UtteranceEmbedding(IReadOnlyList<float> Values)
{
public VocalTractSemanticEmbedding ToSemanticEmbedding() =>
new(Values);
}
public sealed record UtteranceEmbeddingTrainingExample(
UtteranceEmbeddingFeatureVector Features,
UtteranceEmbedding Target);
public sealed record UtteranceEmbeddingTrainingResult(
UtteranceEmbeddingNeuralEncoder Encoder,
IReadOnlyList<PackedNeuralTrainingStep> Steps);
public sealed class UtteranceEmbeddingNeuralEncoder
{
private readonly PackedNeuralNetwork network;
private UtteranceEmbeddingNeuralEncoder(int inputSize, int embeddingSize, PackedNeuralNetwork network)
{
InputSize = inputSize;
EmbeddingSize = embeddingSize;
this.network = network;
}
public int InputSize { get; }
public int EmbeddingSize { get; }
public IReadOnlyList<int> HiddenLayerSizes => network.HiddenLayerSizes;
public static UtteranceEmbeddingNeuralEncoder Create(
int inputSize,
int embeddingSize,
IReadOnlyList<int>? hiddenLayerSizes = null,
int seed = 1337)
{
if (inputSize <= 0) throw new ArgumentOutOfRangeException(nameof(inputSize), "input size must be positive");
if (embeddingSize <= 0) throw new ArgumentOutOfRangeException(nameof(embeddingSize), "embedding size must be positive");
var network = PackedNeuralNetwork.Create(inputSize, embeddingSize, hiddenLayerSizes ?? [64, 64, 32], seed);
return new UtteranceEmbeddingNeuralEncoder(inputSize, embeddingSize, network);
}
public UtteranceEmbedding Encode(UtteranceEmbeddingInput input) =>
Encode(input.ToFeatureVector());
public UtteranceEmbedding Encode(UtteranceEmbeddingFeatureVector features)
{
ValidateSize(features.Values, InputSize, nameof(features));
return new UtteranceEmbedding(network.Predict(features.Values));
}
public UtteranceEmbeddingTrainingResult Train(
IReadOnlyList<UtteranceEmbeddingTrainingExample> examples,
PackedNeuralTrainingOptions? options = null)
{
if (examples.Count == 0)
{
throw new ArgumentException("at least one training example is required", nameof(examples));
}
var packed = new PackedNeuralTrainingExample[examples.Count];
for (var index = 0; index < examples.Count; index++)
{
var example = examples[index];
ValidateSize(example.Features.Values, InputSize, nameof(example.Features));
ValidateSize(example.Target.Values, EmbeddingSize, nameof(example.Target));
packed[index] = new PackedNeuralTrainingExample(example.Features.Values, example.Target.Values);
}
var result = network.Train(packed, options);
return new UtteranceEmbeddingTrainingResult(this, result.Steps);
}
private static void ValidateSize(IReadOnlyList<float> values, int expected, string name)
{
if (values.Count != expected)
{
throw new ArgumentException($"{name} must have {expected} values", name);
}
}
}

489
src/AquaSynth.Core/VocalTractLearning.cs
View file

@ -52,15 +52,13 @@ public sealed class VocalTractNeuralMapper
private const int PhoneticFeatureCount = 48;
private const int BaseOutputCount = 14;
private const float NeutralMelBand = 0.5f;
private readonly Layer[] layers;
private readonly int[] hiddenLayerSizes;
private readonly PackedNeuralNetwork network;
private VocalTractNeuralMapper(int semanticEmbeddingSize, int melBandCount, Layer[] layers)
private VocalTractNeuralMapper(int semanticEmbeddingSize, int melBandCount, PackedNeuralNetwork network)
{
SemanticEmbeddingSize = semanticEmbeddingSize;
MelBandCount = melBandCount;
this.layers = layers;
hiddenLayerSizes = layers.Take(layers.Length - 1).Select(layer => layer.OutputSize).ToArray();
this.network = network;
}
public int SemanticEmbeddingSize { get; }
@ -71,7 +69,7 @@ public sealed class VocalTractNeuralMapper
public int OutputSize => BaseOutputCount + MelBandCount;
public IReadOnlyList<int> HiddenLayerSizes => hiddenLayerSizes;
public IReadOnlyList<int> HiddenLayerSizes => network.HiddenLayerSizes;
public static VocalTractNeuralMapper Create(
int semanticEmbeddingSize,
@ -90,29 +88,18 @@ public sealed class VocalTractNeuralMapper
}
hiddenLayerSizes ??= [64, 64, 32];
var random = new Random(seed);
var sizes = new List<int> { PhoneticFeatureCount + 4 + semanticEmbeddingSize };
sizes.AddRange(hiddenLayerSizes);
sizes.Add(BaseOutputCount + melBandCount);
var network = PackedNeuralNetwork.Create(
PhoneticFeatureCount + 4 + semanticEmbeddingSize,
BaseOutputCount + melBandCount,
hiddenLayerSizes,
seed);
var layers = new Layer[sizes.Count - 1];
for (var index = 0; index < layers.Length; index++)
{
var activation = index == layers.Length - 1
? Activation.Sigmoid
: Activation.Tanh;
layers[index] = Layer.Create(sizes[index], sizes[index + 1], activation, random);
}
return new VocalTractNeuralMapper(semanticEmbeddingSize, melBandCount, layers);
return new VocalTractNeuralMapper(semanticEmbeddingSize, melBandCount, network);
}
public VocalTractControlTarget Predict(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding)
{
var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
EncodeInput(phoneticEvent, semanticEmbedding, scratch.Activations[0]);
Forward(scratch);
return TargetFromVector(scratch.Output);
return TargetFromVector(network.Predict(EncodeInput(phoneticEvent, semanticEmbedding)));
}
public VocalTractTrainingResult Train(
@ -125,56 +112,26 @@ public sealed class VocalTractNeuralMapper
}
options ??= new VocalTractTrainingOptions();
Validate(options);
var steps = new List<VocalTractTrainingStep>();
var order = Enumerable.Range(0, examples.Count).ToArray();
var random = new Random(options.Seed);
var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
var gradients = LayerBuffers.Create(layers);
var adamState = options.Optimizer == VocalTractOptimizer.Adam
? OptimizerState.Create(layers)
: null;
var updateStep = 0;
for (var epoch = 0; epoch < options.Epochs; epoch++)
var packedExamples = new PackedNeuralTrainingExample[examples.Count];
for (var index = 0; index < examples.Count; index++)
{
if (options.Shuffle)
{
Shuffle(order, random);
}
var loss = 0f;
for (var batchStart = 0; batchStart < order.Length; batchStart += options.BatchSize)
{
var batchEnd = Math.Min(batchStart + options.BatchSize, order.Length);
gradients.Clear();
for (var orderIndex = batchStart; orderIndex < batchEnd; orderIndex++)
{
var example = examples[order[orderIndex]];
EncodeInput(example.Event, example.SemanticEmbedding, scratch.Activations[0]);
FillTarget(example.Target, scratch.Target);
Forward(scratch);
loss += MeanSquaredError(scratch.Output, scratch.Target);
AccumulateGradients(scratch, gradients);
}
updateStep++;
var scale = 1f / (batchEnd - batchStart);
if (options.Optimizer == VocalTractOptimizer.Adam)
{
adamState!.ApplyAdam(layers, gradients, scale, options.LearningRate, options.Beta1, options.Beta2, options.Epsilon, updateStep);
}
else
{
ApplySgd(gradients, scale, options.LearningRate);
}
}
steps.Add(new VocalTractTrainingStep(epoch + 1, loss / examples.Count));
var example = examples[index];
packedExamples[index] = new PackedNeuralTrainingExample(
EncodeInput(example.Event, example.SemanticEmbedding),
example.Target.ToVector(MelBandCount, NeutralMelBand));
}
var result = network.Train(packedExamples, new PackedNeuralTrainingOptions(
options.Epochs,
options.LearningRate,
options.Seed,
options.Optimizer == VocalTractOptimizer.Adam ? PackedNeuralOptimizer.Adam : PackedNeuralOptimizer.Sgd,
options.BatchSize,
options.Beta1,
options.Beta2,
options.Epsilon,
options.Shuffle));
var steps = result.Steps.Select(step => new VocalTractTrainingStep(step.Epoch, step.Loss)).ToArray();
return new VocalTractTrainingResult(this, steps);
}
@ -185,62 +142,6 @@ public sealed class VocalTractNeuralMapper
return input;
}
private static void Validate(VocalTractTrainingOptions options)
{
if (options.Epochs <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive");
}
if (options.LearningRate <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), "learning rate must be positive");
}
if (options.BatchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), "batch size must be positive");
}
}
private void Forward(TrainingScratch scratch)
{
for (var index = 0; index < layers.Length; index++)
{
layers[index].Forward(scratch.Activations[index], scratch.Activations[index + 1]);
}
}
private void AccumulateGradients(TrainingScratch scratch, LayerBuffers gradients)
{
var lastLayerIndex = layers.Length - 1;
var outputDelta = scratch.Deltas[lastLayerIndex];
var output = scratch.Activations[^1];
for (var index = 0; index < outputDelta.Length; index++)
{
var actual = output[index];
outputDelta[index] = 2f * (actual - scratch.Target[index]) * layers[lastLayerIndex].Derivative(actual);
}
for (var layerIndex = lastLayerIndex - 1; layerIndex >= 0; layerIndex--)
{
layers[layerIndex].BackpropagateDelta(layers[layerIndex + 1], scratch.Deltas[layerIndex + 1], scratch.Activations[layerIndex + 1], scratch.Deltas[layerIndex]);
}
for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++)
{
gradients.Accumulate(layerIndex, scratch.Activations[layerIndex], scratch.Deltas[layerIndex]);
}
}
private void ApplySgd(LayerBuffers gradients, float scale, float learningRate)
{
for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++)
{
layers[layerIndex].ApplySgd(gradients.WeightBuffers[layerIndex], gradients.BiasBuffers[layerIndex], scale, learningRate);
}
}
private void EncodeInput(PhoneticEvent phoneticEvent, VocalTractSemanticEmbedding semanticEmbedding, float[] input)
{
if (semanticEmbedding.Values.Count != SemanticEmbeddingSize)
@ -270,24 +171,12 @@ public sealed class VocalTractNeuralMapper
}
}
private void FillTarget(VocalTractControlTarget target, float[] output) =>
target.ToVector(output, MelBandCount, NeutralMelBand);
private static int OneHot(float[] input, int offset, int value, int count)
{
input[offset + Math.Clamp(value, 0, count - 1)] = 1;
return offset + count;
}
private static void Shuffle(int[] values, Random random)
{
for (var index = values.Length - 1; index > 0; index--)
{
var swapIndex = random.Next(index + 1);
(values[index], values[swapIndex]) = (values[swapIndex], values[index]);
}
}
private static VocalTractControlTarget TargetFromVector(IReadOnlyList<float> values)
{
var mel = new float[Math.Max(0, values.Count - BaseOutputCount)];
@ -314,329 +203,7 @@ public sealed class VocalTractNeuralMapper
mel);
}
private static float MeanSquaredError(IReadOnlyList<float> actual, IReadOnlyList<float> target)
{
var sum = 0f;
for (var index = 0; index < actual.Count; index++)
{
var difference = actual[index] - target[index];
sum += difference * difference;
}
return sum / actual.Count;
}
private static float Clamp01(float value) => Math.Clamp(value, 0, 1);
private sealed class TrainingScratch
{
private TrainingScratch(float[][] activations, float[][] deltas, float[] target)
{
Activations = activations;
Deltas = deltas;
Target = target;
}
public float[][] Activations { get; }
public float[][] Deltas { get; }
public float[] Target { get; }
public float[] Output => Activations[^1];
public static TrainingScratch Create(IReadOnlyList<Layer> layers, int inputSize, int outputSize)
{
var activations = new float[layers.Count + 1][];
var deltas = new float[layers.Count][];
activations[0] = new float[inputSize];
for (var index = 0; index < layers.Count; index++)
{
activations[index + 1] = new float[layers[index].OutputSize];
deltas[index] = new float[layers[index].OutputSize];
}
return new TrainingScratch(activations, deltas, new float[outputSize]);
}
}
private sealed class LayerBuffers
{
private LayerBuffers(float[][] weightBuffers, float[][] biasBuffers)
{
WeightBuffers = weightBuffers;
BiasBuffers = biasBuffers;
}
public float[][] WeightBuffers { get; }
public float[][] BiasBuffers { get; }
public static LayerBuffers Create(IReadOnlyList<Layer> layers)
{
var weights = new float[layers.Count][];
var biases = new float[layers.Count][];
for (var index = 0; index < layers.Count; index++)
{
weights[index] = new float[layers[index].Weights.Length];
biases[index] = new float[layers[index].Biases.Length];
}
return new LayerBuffers(weights, biases);
}
public void Clear()
{
for (var index = 0; index < WeightBuffers.Length; index++)
{
Array.Clear(WeightBuffers[index]);
Array.Clear(BiasBuffers[index]);
}
}
public void Accumulate(int layerIndex, float[] input, float[] delta)
{
var weightGradients = WeightBuffers[layerIndex];
var biasGradients = BiasBuffers[layerIndex];
var inputSize = input.Length;
unsafe
{
fixed (float* inputBase = input)
fixed (float* deltaBase = delta)
fixed (float* weightBase = weightGradients)
fixed (float* biasBase = biasGradients)
{
for (var output = 0; output < delta.Length; output++)
{
var deltaValue = deltaBase[output];
var weightRow = weightBase + output * inputSize;
for (var column = 0; column < inputSize; column++)
{
weightRow[column] += deltaValue * inputBase[column];
}
biasBase[output] += deltaValue;
}
}
}
}
}
private sealed class Layer
{
private readonly Activation activation;
private Layer(int inputSize, int outputSize, float[] weights, float[] biases, Activation activation)
{
InputSize = inputSize;
OutputSize = outputSize;
Weights = weights;
Biases = biases;
this.activation = activation;
}
public int InputSize { get; }
public int OutputSize { get; }
public float[] Weights { get; }
public float[] Biases { get; }
public static Layer Create(int inputSize, int outputSize, Activation activation, Random random)
{
var scale = MathF.Sqrt(2f / Math.Max(1, inputSize));
var weights = new float[inputSize * outputSize];
for (var index = 0; index < weights.Length; index++)
{
weights[index] = ((float)random.NextDouble() * 2f - 1f) * scale;
}
return new Layer(inputSize, outputSize, weights, new float[outputSize], activation);
}
public void Forward(float[] input, float[] output)
{
unsafe
{
fixed (float* inputBase = input)
fixed (float* outputBase = output)
fixed (float* weightBase = Weights)
fixed (float* biasBase = Biases)
{
for (var row = 0; row < OutputSize; row++)
{
var sum = biasBase[row];
var weightRow = weightBase + row * InputSize;
for (var column = 0; column < InputSize; column++)
{
sum += weightRow[column] * inputBase[column];
}
outputBase[row] = Activate(sum);
}
}
}
}
public void BackpropagateDelta(Layer nextLayer, float[] nextDelta, float[] activationValues, float[] delta)
{
unsafe
{
fixed (float* nextDeltaBase = nextDelta)
fixed (float* activationBase = activationValues)
fixed (float* deltaBase = delta)
fixed (float* nextWeightBase = nextLayer.Weights)
{
for (var output = 0; output < OutputSize; output++)
{
var sum = 0f;
for (var nextOutput = 0; nextOutput < nextLayer.OutputSize; nextOutput++)
{
sum += nextWeightBase[nextOutput * nextLayer.InputSize + output] * nextDeltaBase[nextOutput];
}
deltaBase[output] = sum * Derivative(activationBase[output]);
}
}
}
}
public float Derivative(float activated) =>
activation switch
{
Activation.Sigmoid => activated * (1f - activated),
Activation.Tanh => 1f - activated * activated,
_ => 1
};
public void ApplySgd(float[] weightGradients, float[] biasGradients, float scale, float learningRate)
{
unsafe
{
fixed (float* weightBase = Weights)
fixed (float* biasBase = Biases)
fixed (float* weightGradientBase = weightGradients)
fixed (float* biasGradientBase = biasGradients)
{
for (var index = 0; index < Weights.Length; index++)
{
weightBase[index] -= learningRate * weightGradientBase[index] * scale;
}
for (var index = 0; index < Biases.Length; index++)
{
biasBase[index] -= learningRate * biasGradientBase[index] * scale;
}
}
}
}
private float Activate(float value) =>
activation switch
{
Activation.Sigmoid => 1f / (1f + MathF.Exp(-value)),
Activation.Tanh => MathF.Tanh(value),
_ => value
};
}
private enum Activation
{
Tanh,
Sigmoid
}
private sealed class OptimizerState
{
private readonly LayerBuffers firstMoment;
private readonly LayerBuffers secondMoment;
private OptimizerState(LayerBuffers firstMoment, LayerBuffers secondMoment)
{
this.firstMoment = firstMoment;
this.secondMoment = secondMoment;
}
public static OptimizerState Create(IReadOnlyList<Layer> layers) =>
new(LayerBuffers.Create(layers), LayerBuffers.Create(layers));
public void ApplyAdam(
IReadOnlyList<Layer> layers,
LayerBuffers gradients,
float scale,
float learningRate,
float beta1,
float beta2,
float epsilon,
int step)
{
var beta1Correction = 1f - MathF.Pow(beta1, step);
var beta2Correction = 1f - MathF.Pow(beta2, step);
for (var layerIndex = 0; layerIndex < layers.Count; layerIndex++)
{
ApplyAdam(
layers[layerIndex].Weights,
gradients.WeightBuffers[layerIndex],
firstMoment.WeightBuffers[layerIndex],
secondMoment.WeightBuffers[layerIndex],
scale,
learningRate,
beta1,
beta2,
epsilon,
beta1Correction,
beta2Correction);
ApplyAdam(
layers[layerIndex].Biases,
gradients.BiasBuffers[layerIndex],
firstMoment.BiasBuffers[layerIndex],
secondMoment.BiasBuffers[layerIndex],
scale,
learningRate,
beta1,
beta2,
epsilon,
beta1Correction,
beta2Correction);
}
}
private static void ApplyAdam(
float[] values,
float[] gradients,
float[] firstMoment,
float[] secondMoment,
float scale,
float learningRate,
float beta1,
float beta2,
float epsilon,
float beta1Correction,
float beta2Correction)
{
unsafe
{
fixed (float* valueBase = values)
fixed (float* gradientBase = gradients)
fixed (float* firstBase = firstMoment)
fixed (float* secondBase = secondMoment)
{
for (var index = 0; index < values.Length; index++)
{
var gradient = gradientBase[index] * scale;
firstBase[index] = beta1 * firstBase[index] + (1f - beta1) * gradient;
secondBase[index] = beta2 * secondBase[index] + (1f - beta2) * gradient * gradient;
var mHat = firstBase[index] / beta1Correction;
var vHat = secondBase[index] / beta2Correction;
valueBase[index] -= learningRate * mHat / (MathF.Sqrt(vHat) + epsilon);
}
}
}
}
}
}
public static class VocalTractControlTargetExtensions

1
state/evidence.jsonl
View file

@ -111,3 +111,4 @@
{"date":"2026-05-19","evidence":"Updated EspeakNgReferenceTests to record log_mel_frames, log_mel_bands, log_mel_mean, and log_mel_distance_matrix in report.md; updated the IPA vocal tract roadmap accordingly.","lesson":"The eSpeak NG speech workout should expose log-mel evidence immediately, not just basic audio features. Each fixture report now carries normalized log-mel band means plus a cross-fixture log-mel distance matrix so future AquaSynth IPA renders can plug into the same spectral comparison shape.","topic":"espeak-ng-logmel-workout"}
{"date":"2026-05-19","topic":"vocal-tract-learned-controller","lesson":"AquaSynth now has a small learned controller scaffold for the vocal-tract lane: phonetic features plus a semantic embedding vector feed a multilayer tanh/sigmoid MLP trained with SGD/backprop to predict tract controls plus AM/FM, LFO, filter, and mel spectral-envelope expression lanes. This is a trainable controller surface, not proof of eSpeak-level speech yet.","evidence":"Added VocalTractLearning.cs and VocalTractLearningTests.cs with feature encoding, semantic embeddings, 3-layer default network, loss-reduction tests, same-phone semantic steering, expressive modulator/filter control tests, and learned dark/bright mel-envelope steering for PAD-style spectral-table coloration."}
{"date":"2026-05-19","topic":"vocal-tract-optimizer-hot-path","lesson":"The first learned vocal-tract optimizer should stay native and inspectable until the render-loss boundary proves it needs a tensor framework. The trainer now uses packed row-major weight buffers, owned scratch activation/delta/target buffers, unsafe pointer loops, reusable gradient/moment buffers, batched gradients, deterministic shuffling, and Adam by default instead of allocation-heavy per-example jagged-array SGD.","evidence":"Rebuilt VocalTractLearning.cs hot path around flat float arrays and unsafe loops; enabled unsafe blocks in AquaSynth.Core; VocalTractLearningTests pass against the packed-buffer Adam trainer."}
{"date":"2026-05-19","topic":"utterance-embedding-learning-boundary","lesson":"AquaSynth should own gradient descent for Weksa speech handoff. Weksa supplies speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors; AquaSynth trains the tiny utterance-embedding encoder and passes that embedding into the synth-driver model.","evidence":"Extracted PackedNeuralNetwork as the shared packed-buffer Adam/SGD primitive, rewired VocalTractNeuralMapper through it, and added UtteranceEmbeddingInput/UtteranceEmbeddingNeuralEncoder plus tests for packing, training, and feeding the vocal-tract semantic embedding."}

7
state/map.yaml
View file

@ -32,6 +32,8 @@ modules:
model: src/AquaSynth.Core/Model.cs
parser: src/AquaSynth.Core/PatchScript.cs
vocal_tract_model: src/AquaSynth.Core/VocalTractModel.cs
packed_neural_network: src/AquaSynth.Core/PackedNeuralNetwork.cs
utterance_embedding_learning: src/AquaSynth.Core/UtteranceEmbeddingLearning.cs
vocal_tract_learning: src/AquaSynth.Core/VocalTractLearning.cs
faust_emitter: src/AquaSynth.Core/FaustExport.cs
faust_compiler: src/AquaSynth.Faust/FaustCompiler.cs
@ -43,6 +45,7 @@ modules:
ipa_vocal_tract_roadmap: docs/ipa-vocal-tract-roadmap.md
package_boundary_tests: tests/AquaSynth.Dsl.Tests/PackageBoundaryTests.cs
vocal_tract_model_tests: tests/AquaSynth.Dsl.Tests/VocalTractModelTests.cs
utterance_embedding_learning_tests: tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs
vocal_tract_learning_tests: tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs
sfxr: src/AquaSynth.Core/SfxrParams.cs
analysis: src/AquaSynth.Core/AudioAnalysis.cs
@ -66,7 +69,9 @@ boundaries:
- live patch compilation, native Faust factory lifetime, and offline sample-buffer render proof in AquaSynth.Faust
- IPA-to-phonetic-feature planning, articulatory gesture planning, and human/alien vocal-tract roadmap design
- phonetic intent, articulatory plan, morphology capability, and constraint-report contracts for vocal tract synthesis
- learned phonetic-plus-semantic embedding controller with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls
- packed-buffer native backprop/Adam as the shared AquaSynth gradient-descent primitive
- learned utterance embedding encoder from speech text embeddings, prosody/emphasis hints, and Ghostlight/Epiphany-shaped character state vectors
- learned phonetic-plus-utterance-embedding synth driver with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls
dev_only:
- public-domain DX7 SysEx fixtures
- pinned GPL ZynAddSubFX source for parity tests

2
state/memory.json
View file

@ -86,7 +86,7 @@
"rule": "Add DSL abstractions only when reference rebuilds prove the missing invariant. Do not consume references that cannot be backed by parity tests; parsing alone is inventory, not evidence of expressivity."
},
"ipa_vocal_tract_path": {
"status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. VocalTractNeuralMapper is the first learned controller: phonetic features plus semantic embedding vector feed a small multilayer tanh/sigmoid network trained with packed-buffer native C# backprop/Adam to predict tract, AM/FM, LFO, filter, and mel spectral-envelope controls.",
"status": "Roadmap drafted for an IPA-to-vocal-tract lane, and the first Core contracts now defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests now provide a first external speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing pressure, not anatomy truth. AquaSynth now owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls.",
"lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.",
"next": "Implement Stage 0/1 fixtures: IPA token spans, vowel feature bundles, human baseline vowel tract-area targets, and a tiny audible vowel render proof before expanding consonants."
},

106
tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs Normal file
View file

@ -0,0 +1,106 @@
using AquaSynth.Dsl;
namespace AquaSynth.Dsl.Tests;
public sealed class UtteranceEmbeddingLearningTests
{
[Fact]
public void UtteranceEmbeddingInputPacksTextProsodyAndCharacterState()
{
var input = new UtteranceEmbeddingInput(
SpeechTextEmbedding: [0.1f, 0.2f, 0.3f],
ProsodyAndEmphasisHints: [0.7f, 0.8f],
CharacterStateVector: [0.4f, 0.5f, 0.6f]);
var features = input.ToFeatureVector();
Assert.Equal([0.1f, 0.2f, 0.3f, 0.7f, 0.8f, 0.4f, 0.5f, 0.6f], features.Values);
}
[Fact]
public void UtteranceEmbeddingEncoderUsesPackedAquaSynthTraining()
{
var encoder = UtteranceEmbeddingNeuralEncoder.Create(inputSize: 10, embeddingSize: 16, hiddenLayerSizes: [48, 32, 24], seed: 61);
var examples = TrainingSet();
var before = AverageLoss(encoder, examples);
var result = encoder.Train(examples, new PackedNeuralTrainingOptions(Epochs: 320, LearningRate: 0.035f, Seed: 61, BatchSize: 2));
var after = AverageLoss(encoder, examples);
Assert.Equal([48, 32, 24], encoder.HiddenLayerSizes);
Assert.Equal(320, result.Steps.Count);
Assert.True(after < before * 0.35f, $"expected utterance-embedding loss to drop; before={before}, after={after}");
}
[Fact]
public void UtteranceEmbeddingCanFeedVocalTractSemanticEmbedding()
{
var vector = new UtteranceEmbedding(Enumerable.Range(0, 16).Select(index => index / 16f).ToArray());
var embedding = vector.ToSemanticEmbedding();
Assert.Equal(vector.Values, embedding.Values);
}
[Fact]
public void UtteranceEmbeddingEncoderSeparatesContrastingDeliveryTargets()
{
var encoder = UtteranceEmbeddingNeuralEncoder.Create(inputSize: 10, embeddingSize: 16, hiddenLayerSizes: [64, 48, 32], seed: 67);
var clippedThreat = new UtteranceEmbeddingFeatureVector([0.95f, 0.85f, 0.15f, 0.80f, 0.70f, 0.10f, 0.20f, 0.90f, 0.75f, 0.30f]);
var warmReassurance = new UtteranceEmbeddingFeatureVector([0.20f, 0.25f, 0.88f, 0.15f, 0.30f, 0.80f, 0.75f, 0.18f, 0.25f, 0.70f]);
var threatTarget = Vector(16, 0.86f, 0.72f, 0.18f);
var warmTarget = Vector(16, 0.22f, 0.48f, 0.88f);
var examples = new[]
{
new UtteranceEmbeddingTrainingExample(clippedThreat, threatTarget),
new UtteranceEmbeddingTrainingExample(warmReassurance, warmTarget)
};
encoder.Train(examples, new PackedNeuralTrainingOptions(Epochs: 420, LearningRate: 0.035f, Seed: 67, BatchSize: 2));
var threat = encoder.Encode(clippedThreat);
var warm = encoder.Encode(warmReassurance);
Assert.True(threat.Values[0] > warm.Values[0] + 0.25f);
Assert.True(warm.Values[^1] > threat.Values[^1] + 0.25f);
}
private static IReadOnlyList<UtteranceEmbeddingTrainingExample> TrainingSet() =>
[
new(new UtteranceEmbeddingFeatureVector([0.95f, 0.82f, 0.10f, 0.75f, 0.70f, 0.12f, 0.18f, 0.88f, 0.74f, 0.28f]), Vector(16, 0.86f, 0.70f, 0.20f)),
new(new UtteranceEmbeddingFeatureVector([0.20f, 0.22f, 0.88f, 0.18f, 0.28f, 0.82f, 0.72f, 0.18f, 0.26f, 0.74f]), Vector(16, 0.20f, 0.50f, 0.88f)),
new(new UtteranceEmbeddingFeatureVector([0.55f, 0.42f, 0.48f, 0.35f, 0.92f, 0.34f, 0.42f, 0.52f, 0.62f, 0.44f]), Vector(16, 0.48f, 0.82f, 0.50f))
];
private static UtteranceEmbedding Vector(int size, float left, float center, float right)
{
var values = new float[size];
for (var index = 0; index < size; index++)
{
var position = index / Math.Max(1f, size - 1f);
values[index] = position < 0.5f
? Lerp(left, center, position * 2f)
: Lerp(center, right, (position - 0.5f) * 2f);
}
return new UtteranceEmbedding(values);
}
private static float AverageLoss(UtteranceEmbeddingNeuralEncoder encoder, IReadOnlyList<UtteranceEmbeddingTrainingExample> examples)
{
var loss = 0f;
foreach (var example in examples)
{
var prediction = encoder.Encode(example.Features);
for (var index = 0; index < prediction.Values.Count; index++)
{
var difference = prediction.Values[index] - example.Target.Values[index];
loss += difference * difference;
}
}
return loss / examples.Count / encoder.EmbeddingSize;
}
private static float Lerp(float left, float right, float amount) =>
left + (right - left) * amount;
}