AquaSynth/tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs
2026-05-19 16:29:37 +01:00

230 lines
12 KiB
C#

using AquaSynth.Dsl;
namespace AquaSynth.Dsl.Tests;
public sealed class VocalTractLearningTests
{
[Fact]
public void NeuralMapperEncodesPhoneticInputAndSemanticEmbedding()
{
var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 12, hiddenLayerSizes: [32, 24, 16], seed: 7);
var input = mapper.EncodeInput(
new PhoneticEvent(
"phone-a",
"a",
new PhoneticFeatures(
PhoneticManner.Vowel,
Height: VowelHeight.Open,
Backness: VowelBackness.Front),
DurationSeconds: 0.2,
Prosody: new PhoneticProsody(Stress: 0.5f, PitchTarget: 0.25f, Intensity: 0.75f)),
new VocalTractSemanticEmbedding(Enumerable.Range(0, 12).Select(index => index / 12f).ToArray()));
Assert.Equal(64, input.Length);
Assert.Equal(32, mapper.MelBandCount);
Assert.Equal(46, mapper.OutputSize);
Assert.Equal([32, 24, 16], mapper.HiddenLayerSizes);
Assert.All(input, value => Assert.InRange(value, -1f, 1f));
}
[Fact]
public void NeuralMapperTrainingReducesLossForTinyVowelSet()
{
var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, hiddenLayerSizes: [48, 32, 24], seed: 11);
var examples = TinyTrainingSet();
var before = AverageLoss(mapper, examples);
var result = mapper.Train(examples, new VocalTractTrainingOptions(Epochs: 260, LearningRate: 0.05f, Seed: 11));
var after = AverageLoss(mapper, examples);
Assert.Equal(260, result.Steps.Count);
Assert.True(after < before * 0.35f, $"expected loss to drop hard; before={before}, after={after}");
Assert.True(result.Steps[^1].Loss < result.Steps[0].Loss, "training loss should decrease across epochs");
}
[Fact]
public void SemanticEmbeddingCanSteerSamePhoneToDifferentTractTargets()
{
var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, hiddenLayerSizes: [64, 48, 32], seed: 19);
var softContext = Embedding(16, 0.15f, 0.2f);
var hardContext = Embedding(16, 0.85f, -0.15f);
var eventA = new PhoneticEvent(
"phone-a",
"a",
new PhoneticFeatures(
PhoneticManner.Vowel,
Height: VowelHeight.Open,
Backness: VowelBackness.Front),
DurationSeconds: 0.2);
var examples = new[]
{
new VocalTractTrainingExample(eventA, softContext, new VocalTractControlTarget(0.78f, 0.35f, 0.90f, 0.05f, 0.0f, 0.42f, 0.0f, 0.35f, AmDepth: 0.12f, FmDepth: 0.08f, LfoRate: 0.18f, LfoDepth: 0.10f, FilterCutoff: 0.62f, FilterResonance: 0.08f)),
new VocalTractTrainingExample(eventA, hardContext, new VocalTractControlTarget(0.54f, 0.70f, 0.42f, 0.10f, 0.0f, 0.82f, 0.0f, 0.72f, AmDepth: 0.44f, FmDepth: 0.36f, LfoRate: 0.52f, LfoDepth: 0.41f, FilterCutoff: 0.38f, FilterResonance: 0.55f))
};
mapper.Train(examples, new VocalTractTrainingOptions(Epochs: 400, LearningRate: 0.04f, Seed: 19));
var soft = mapper.Predict(eventA, softContext);
var hard = mapper.Predict(eventA, hardContext);
Assert.True(soft.LipAperture > hard.LipAperture + 0.20f, $"soft aperture={soft.LipAperture}, hard aperture={hard.LipAperture}");
Assert.True(hard.GlottalTenseness > soft.GlottalTenseness + 0.20f, $"hard tenseness={hard.GlottalTenseness}, soft tenseness={soft.GlottalTenseness}");
Assert.True(hard.Pressure > soft.Pressure + 0.15f, $"hard pressure={hard.Pressure}, soft pressure={soft.Pressure}");
Assert.True(hard.FmDepth > soft.FmDepth + 0.15f, $"hard FM={hard.FmDepth}, soft FM={soft.FmDepth}");
Assert.True(hard.FilterResonance > soft.FilterResonance + 0.20f, $"hard resonance={hard.FilterResonance}, soft resonance={soft.FilterResonance}");
}
[Fact]
public void NeuralMapperCanLearnExpressiveModulatorAndFilterControls()
{
var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, hiddenLayerSizes: [64, 48, 32], seed: 31);
var fricative = new PhoneticEvent(
"phone-s",
"s",
new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless),
DurationSeconds: 0.12,
Prosody: new PhoneticProsody(Intensity: 0.9f));
var still = Embedding(16, -0.45f, 0.05f);
var animated = Embedding(16, 0.55f, 0.30f);
var examples = new[]
{
new VocalTractTrainingExample(fricative, still, new VocalTractControlTarget(0.42f, 0.82f, 0.18f, 0.0f, 0.0f, 0.04f, 0.92f, 0.64f, AmDepth: 0.04f, FmDepth: 0.02f, LfoRate: 0.08f, LfoDepth: 0.03f, FilterCutoff: 0.72f, FilterResonance: 0.25f)),
new VocalTractTrainingExample(fricative, animated, new VocalTractControlTarget(0.35f, 0.90f, 0.12f, 0.0f, 0.0f, 0.08f, 0.98f, 0.84f, AmDepth: 0.62f, FmDepth: 0.48f, LfoRate: 0.70f, LfoDepth: 0.58f, FilterCutoff: 0.42f, FilterResonance: 0.76f))
};
mapper.Train(examples, new VocalTractTrainingOptions(Epochs: 420, LearningRate: 0.04f, Seed: 31));
var stillPrediction = mapper.Predict(fricative, still);
var animatedPrediction = mapper.Predict(fricative, animated);
Assert.True(animatedPrediction.AmDepth > stillPrediction.AmDepth + 0.25f);
Assert.True(animatedPrediction.FmDepth > stillPrediction.FmDepth + 0.20f);
Assert.True(animatedPrediction.LfoDepth > stillPrediction.LfoDepth + 0.20f);
Assert.True(animatedPrediction.FilterResonance > stillPrediction.FilterResonance + 0.25f);
Assert.True(stillPrediction.FilterCutoff > animatedPrediction.FilterCutoff + 0.15f);
}
[Fact]
public void NeuralMapperCanLearnMelSpectralEnvelopeControls()
{
var mapper = VocalTractNeuralMapper.Create(semanticEmbeddingSize: 16, melBandCount: 12, hiddenLayerSizes: [72, 48, 32], seed: 43);
var vowel = new PhoneticEvent(
"phone-a",
"a",
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
DurationSeconds: 0.2,
Prosody: new PhoneticProsody(Intensity: 0.8f));
var darkContext = Embedding(16, -0.60f, 0.10f);
var brightContext = Embedding(16, 0.60f, 0.12f);
var darkEnvelope = MelEnvelope(12, low: 0.86f, mid: 0.52f, high: 0.18f);
var brightEnvelope = MelEnvelope(12, low: 0.22f, mid: 0.58f, high: 0.90f);
var examples = new[]
{
new VocalTractTrainingExample(vowel, darkContext, new VocalTractControlTarget(0.68f, 0.34f, 0.72f, 0.10f, 0.0f, 0.48f, 0.0f, 0.42f, MelSpectralEnvelope: darkEnvelope)),
new VocalTractTrainingExample(vowel, brightContext, new VocalTractControlTarget(0.58f, 0.52f, 0.56f, 0.02f, 0.0f, 0.54f, 0.0f, 0.46f, MelSpectralEnvelope: brightEnvelope))
};
mapper.Train(examples, new VocalTractTrainingOptions(Epochs: 520, LearningRate: 0.04f, Seed: 43));
var dark = mapper.Predict(vowel, darkContext);
var bright = mapper.Predict(vowel, brightContext);
Assert.NotNull(dark.MelSpectralEnvelope);
Assert.NotNull(bright.MelSpectralEnvelope);
Assert.Equal(12, dark.MelSpectralEnvelope.Count);
Assert.True(dark.MelSpectralEnvelope[1] > bright.MelSpectralEnvelope[1] + 0.25f);
Assert.True(bright.MelSpectralEnvelope[^2] > dark.MelSpectralEnvelope[^2] + 0.25f);
}
private static IReadOnlyList<VocalTractTrainingExample> TinyTrainingSet() =>
[
new(
new PhoneticEvent(
"a",
"a",
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
DurationSeconds: 0.20,
Prosody: new PhoneticProsody(Intensity: 0.8f)),
Embedding(16, 0.20f, 0.10f),
new VocalTractControlTarget(0.78f, 0.30f, 0.88f, 0.04f, 0.0f, 0.52f, 0.0f, 0.42f, AmDepth: 0.10f, FmDepth: 0.04f, LfoRate: 0.12f, LfoDepth: 0.06f, FilterCutoff: 0.62f, FilterResonance: 0.10f, MelSpectralEnvelope: MelEnvelope(32, 0.52f, 0.60f, 0.42f))),
new(
new PhoneticEvent(
"i",
"i",
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Close, Backness: VowelBackness.Front),
DurationSeconds: 0.18,
Prosody: new PhoneticProsody(Intensity: 0.7f)),
Embedding(16, -0.15f, 0.35f),
new VocalTractControlTarget(0.25f, 0.82f, 0.24f, 0.02f, 0.0f, 0.50f, 0.0f, 0.38f, AmDepth: 0.08f, FmDepth: 0.05f, LfoRate: 0.16f, LfoDepth: 0.05f, FilterCutoff: 0.80f, FilterResonance: 0.06f, MelSpectralEnvelope: MelEnvelope(32, 0.32f, 0.78f, 0.66f))),
new(
new PhoneticEvent(
"u",
"u",
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Close, Backness: VowelBackness.Back, Rounded: true),
DurationSeconds: 0.22,
Prosody: new PhoneticProsody(Intensity: 0.72f)),
Embedding(16, 0.40f, -0.25f),
new VocalTractControlTarget(0.18f, 0.26f, 0.22f, 0.88f, 0.0f, 0.48f, 0.0f, 0.36f, AmDepth: 0.18f, FmDepth: 0.12f, LfoRate: 0.22f, LfoDepth: 0.14f, FilterCutoff: 0.48f, FilterResonance: 0.18f, MelSpectralEnvelope: MelEnvelope(32, 0.72f, 0.46f, 0.22f))),
new(
new PhoneticEvent(
"s",
"s",
new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless),
DurationSeconds: 0.12,
Prosody: new PhoneticProsody(Intensity: 0.85f)),
Embedding(16, -0.30f, -0.20f),
new VocalTractControlTarget(0.42f, 0.86f, 0.18f, 0.0f, 0.0f, 0.05f, 0.92f, 0.70f, AmDepth: 0.45f, FmDepth: 0.35f, LfoRate: 0.55f, LfoDepth: 0.40f, FilterCutoff: 0.58f, FilterResonance: 0.62f, MelSpectralEnvelope: MelEnvelope(32, 0.18f, 0.56f, 0.92f)))
];
private static float AverageLoss(VocalTractNeuralMapper mapper, IReadOnlyList<VocalTractTrainingExample> examples)
{
var loss = 0f;
foreach (var example in examples)
{
var prediction = mapper.Predict(example.Event, example.SemanticEmbedding);
loss += Loss(mapper, prediction, example.Target);
}
return loss / examples.Count;
}
private static float Loss(VocalTractNeuralMapper mapper, VocalTractControlTarget prediction, VocalTractControlTarget target)
{
var predicted = prediction.ToVector(mapper.MelBandCount);
var expected = target.ToVector(mapper.MelBandCount);
var loss = 0f;
for (var index = 0; index < predicted.Length; index++)
{
var difference = predicted[index] - expected[index];
loss += difference * difference;
}
return loss / predicted.Length;
}
private static VocalTractSemanticEmbedding Embedding(int size, float offset, float slope)
{
var values = new float[size];
for (var index = 0; index < values.Length; index++)
{
values[index] = MathF.Tanh(offset + slope * index / Math.Max(1, size - 1));
}
return new VocalTractSemanticEmbedding(values);
}
private static float[] MelEnvelope(int size, float low, float mid, float high)
{
var values = new float[size];
for (var index = 0; index < values.Length; index++)
{
var position = index / Math.Max(1f, size - 1f);
values[index] = position < 0.5f
? Lerp(low, mid, position * 2f)
: Lerp(mid, high, (position - 0.5f) * 2f);
}
return values;
}
private static float Lerp(float left, float right, float amount) =>
left + (right - left) * amount;
}