GameCult/AquaSynth
Watch
9
0
Fork
You've already forked AquaSynth
 0
Code Issues Pull requests 4 Projects Releases Packages Wiki Activity Actions

Route speech loss controls into audible DSP

Browse source
This commit is contained in:
Meta Vens 2026-05-25 18:30:19 +01:00
commit bb94924f5c
3 changed files with 93 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

48
src/AquaSynth.Faust/FaustRenderedSpeechLossSurface.cs
View file

@ -71,12 +71,18 @@ public sealed class FaustRenderedSpeechLossSurface(FaustRenderedSpeechLossOption
public string SourceFor(VocalTractControlTarget target)
{
var mel = target.MelSpectralEnvelope ?? [];
var tract = TractResonances(target);
var source = new StringBuilder();
source.AppendLine("patch gain=0.85");
source.AppendLine(VoiceLine("saw", F(120 + target.GlottalTenseness * 110), F(0.08f + target.Pressure * 0.22f), target, target.Turbulence * 0.22f));
source.AppendLine(VoiceLine("saw", F(120 + target.GlottalTenseness * 110), F(0.08f + target.Pressure * 0.22f), target, target.Turbulence * 0.22f, tract));
if (target.Turbulence > 0.02f)
{
source.AppendLine(VoiceLine("noise", F(900 + target.Turbulence * 2600), F(target.Turbulence * 0.16f), target, 1));
source.AppendLine(VoiceLine("noise", F(900 + target.Turbulence * 2600 + target.TongueTip * 1200), F(target.Turbulence * 0.16f), target, 1, tract));
}
if (target.Velum > 0.02f)
{
source.AppendLine(VoiceLine("sine", F(240 + target.Velum * 520), F(target.Velum * target.Pressure * 0.07f), target, 0, tract));
}
for (var band = 0; band < options.MelBandCount; band++)
@ -84,7 +90,7 @@ public sealed class FaustRenderedSpeechLossSurface(FaustRenderedSpeechLossOption
var value = band < mel.Count ? Math.Clamp(mel[band], 0, 1) : 0.5f;
var hz = MelBandCenterHz(band);
var gain = value * value * 0.08f;
source.AppendLine(VoiceLine("sine", F(hz), F(gain), target, 0));
source.AppendLine(VoiceLine("sine", F(hz), F(gain), target, 0, tract));
}
return FaustEmitter.EmitScript(source.ToString(), new FaustExportOptions("aquasynth_speech_loss")).Source;
@ -171,8 +177,19 @@ public sealed class FaustRenderedSpeechLossSurface(FaustRenderedSpeechLossOption
mel);
}
private static string VoiceLine(string wave, string freq, string gain, VocalTractControlTarget target, float noise) =>
string.Create(CultureInfo.InvariantCulture, $"voice wave={wave} freq={freq} gain={gain} attack=0.004 sustain=0.16 decay=0.055 lpf={Math.Clamp(target.FilterCutoff, 0.04f, 1):0.######} lpf_q={Math.Clamp(0.2f + target.FilterResonance * 8, 0.2f, 12):0.######} noise={Math.Clamp(noise, 0, 1):0.######} fm=2.0 fm_index={Math.Clamp(target.FmDepth, 0, 1):0.######} tremolo={Math.Clamp(target.AmDepth, 0, 1):0.######} tremolo_hz={Math.Clamp(2 + target.LfoRate * 11, 0.1f, 20):0.######}");
private static TractResonanceSet TractResonances(VocalTractControlTarget target)
{
var lipDamping = 1.0f - Math.Clamp(target.LipRounding, 0, 1) * 0.32f;
return new(
Math.Clamp((280 + target.LipAperture * 760 - target.TongueBody * 360 - target.LipRounding * 160) * lipDamping, 90, 1400),
Math.Clamp(760 + target.TongueTip * 1700 + target.TongueBody * 520 - target.LipRounding * 300, 180, 3600),
Math.Clamp(1900 + target.TongueTip * 2300 + target.FilterCutoff * 2200 - target.LipRounding * 700, 500, 7200));
}
private static string VoiceLine(string wave, string freq, string gain, VocalTractControlTarget target, float noise, TractResonanceSet tract) =>
string.Create(CultureInfo.InvariantCulture, $"voice wave={wave} freq={freq} gain={gain} attack=0.004 sustain=0.16 decay=0.055 lpf={Math.Clamp(0.08f + target.FilterCutoff * 0.86f, 0.04f, 1):0.######} lpf_q={Math.Clamp(0.2f + target.FilterResonance * 8, 0.2f, 12):0.######} noise={Math.Clamp(noise, 0, 1):0.######} fm=2.0 fm_index={Math.Clamp(target.FmDepth, 0, 1):0.######} tremolo={Math.Clamp(target.AmDepth * (0.25f + target.LfoDepth * 0.75f), 0, 1):0.######} tremolo_hz={Math.Clamp(2 + target.LfoRate * 11, 0.1f, 20):0.######} formant_mix={Math.Clamp(0.2f + target.Pressure * 0.18f + target.Velum * 0.18f, 0, 0.7f):0.######} formants={tract.LowHz:0.######}:80:0.75,{tract.MidHz:0.######}:170:1,{tract.HighHz:0.######}:360:0.55");
private readonly record struct TractResonanceSet(float LowHz, float MidHz, float HighHz);
private static string F(float value) => value.ToString("0.######", CultureInfo.InvariantCulture);
private static float HzToMel(float hz) => 2595 * MathF.Log10(1 + hz / 700);
@ -293,9 +310,17 @@ public sealed class CompiledFaustRenderedSpeechLossSurface : IDisposable
source.AppendLine($"o{index} = hslider(\"{OutputPath(index)}\", 0.5, 0.0, 1.0, 0.0001) : si.smoo;");
}
source.AppendLine("fm = sine((120.0 + o5 * 110.0) * 2.0) * o9 * 0.18;");
source.AppendLine("glottis = sine((120.0 + o5 * 110.0) * (1.0 + fm)) * (0.08 + o7 * 0.22);");
source.AppendLine("breath = no.noise * o6 * 0.18;");
source.AppendLine("pitchHz = 120.0 + o5 * 110.0;");
source.AppendLine("fm = sine(pitchHz * (1.0 + o10 * 7.0)) * o9 * 0.18;");
source.AppendLine("glottis = sine(pitchHz * (1.0 + fm)) * (0.08 + o7 * 0.22);");
source.AppendLine("breath = no.noise * o6 * o7 * (0.08 + o1 * 0.14);");
source.AppendLine("lipDamping = 1.0 - o3 * 0.32;");
source.AppendLine("apertureGain = 0.35 + o2 * 0.95;");
source.AppendLine("tractLowHz = max(90.0, (280.0 + o2 * 760.0 - o0 * 360.0 - o3 * 160.0) * lipDamping);");
source.AppendLine("tractMidHz = max(180.0, 760.0 + o1 * 1700.0 + o0 * 520.0 - o3 * 300.0);");
source.AppendLine("tractHighHz = max(500.0, 1900.0 + o1 * 2300.0 + o12 * 2200.0 - o3 * 700.0);");
source.AppendLine("tractQ = 1.0 + o13 * 10.0;");
source.AppendLine("nasal = sine(240.0 + o4 * 520.0) * o4 * o7 * 0.08;");
for (var band = 0; band < options.MelBandCount; band++)
{
source.AppendLine($"mel{band} = sine({F(MelBandCenterHz(options, band))}) * o{14 + band} * o{14 + band} * 0.08;");
@ -304,9 +329,10 @@ public sealed class CompiledFaustRenderedSpeechLossSurface : IDisposable
var melMix = options.MelBandCount == 0
? "0.0"
: string.Join(" + ", Enumerable.Range(0, options.MelBandCount).Select(index => $"mel{index}"));
source.AppendLine($"raw = glottis + breath + {melMix};");
source.AppendLine("trem = 1.0 - o8 * (0.5 + 0.5 * sine(2.0 + o10 * 11.0));");
source.AppendLine("filtered = raw * trem : fi.lowpass(2, min(ma.SR * 0.45, max(20.0, o12 * 18000.0)));");
source.AppendLine($"raw = (glottis * apertureGain) + breath + nasal + {melMix};");
source.AppendLine("tract = raw * 0.35 + (raw : fi.resonbp(tractLowHz, tractQ, 1.0)) * (0.65 - o0 + o2) + (raw : fi.resonbp(tractMidHz, tractQ, 1.0)) * (o0 + o1) + (raw : fi.resonbp(tractHighHz, tractQ, 1.0)) * (o1 + o12);");
source.AppendLine("trem = 1.0 - o8 * (0.25 + o11 * 0.75) * (0.5 + 0.5 * sine(2.0 + o10 * 11.0));");
source.AppendLine("filtered = tract * trem : fi.lowpass(2, min(ma.SR * 0.45, max(80.0, 800.0 + o12 * 15000.0)));");
var keepControls = string.Join(" + ", Enumerable.Range(0, 14 + options.MelBandCount).Select(index => $"o{index}"));
source.AppendLine($"process = filtered * 0.85 + 0.000000001 * ({keepControls});");
return source.ToString();

1
state/evidence.jsonl
View file

@ -128,3 +128,4 @@
{"date":"2026-05-24","topic":"cultmesh-speech-worker-authority","lesson":"CultMesh should distribute speech render/scoring work, not optimizer authority. AquaSynth emits typed SpeechRenderRequest documents; CultMesh delivers compiled Faust renderers, scoring code, small model/encoder assemblies, and batch/artifact refs to compatible leased workers; workers return SpeechRenderResult documents with loss, gradients, artifacts, and timing receipts. The authority trainer or shard-primary applies gradients through SpeechBackpropagationPipeline and commits SpeechTrainingCheckpoint documents through normal CultCache/CultMesh shard authority.","evidence":"Recorded the authority map in docs/ipa-vocal-tract-roadmap.md, state/spine.yaml, state/map.yaml, state/memory.json, and state/scratch.md. The next implementation layer is transport, admission, worker leases, payload delivery, and result collection; federated averaging waits until the simple authority path is boring."}
{"date":"2026-05-24","topic":"cultmesh-speech-training-step","lesson":"AquaSynth can now run the first honest CultMesh-distributed speech training step: an authority trainer emits SpeechRenderRequest documents and a SpeechWorkerPayloadManifest, admits only CultMesh peers with valid speech-render-worker leases, records SpeechWorkerAdmissionReceipt documents for admitted/rejected peers, lets admitted workers produce SpeechRenderResult gradients, and applies those gradients centrally through SpeechBackpropagationPipeline. Workers still do not write checkpoints.","evidence":"Added GameCult.Mesh to AquaSynth.Core, added payload/admission/assignment/training-step types and store helpers in SpeechDistributedTraining.cs, added RunCultMeshTrainingStep, and added a test with two leased workers plus rejected unleased/read-replica peers. Focused verification: `dotnet test tests/AquaSynth.Dsl.Tests/AquaSynth.Dsl.Tests.csproj --filter \"SpeechDistributedTrainingTests\" -v minimal`: 3 passed. Remaining infrastructure gap: real remote artifact chunk transfer and process supervision belong to CultMesh, not an AquaSynth-local workaround."}
{"date":"2026-05-25","topic":"vocal-tract-control-playground","lesson":"The current AquaSynth vocal-tract lane is touchable as a voice-patch target surface before it is a Pink Trombone-class tract simulation. A useful playground should expose the real VocalTractControlTarget knobs directly: these are the controls the learned speech driver predicts from phonetic and utterance input, not learned controls themselves. Keep the tract-ish visualization honest that the WebAudio renderer is a lab proxy rather than full waveguide anatomy.","evidence":"Added tools/vocal-tract-playground with static HTML/CSS/JS, a Node static server, WebAudio source/filter synthesis, tract canvas visualization, presets, and sliders for the 14 base voice-patch target controls plus six mel bands. Verified with `node --check` on app.js/serve.mjs, curl against http://localhost:5125/, and browser interaction: Play toggled audio, Ee preset updated controls and canvas, console errors were empty."}
{"date":"2026-05-25","topic":"speech-loss-control-authority","lesson":"Exporting `/speech/output/N` controls is not proof that the speech training surface is usable. Every base VocalTractControlTarget output must route into audible DSP before gradient descent is trusted; epsilon keep-alive references are not control authority.","evidence":"The compiled Faust speech-loss surface previously used tongue body, tongue tip, lip aperture, lip rounding, and velum only in the final keep-controls epsilon. Routed tract controls into small resonant source-filter proxies, velum into a nasal tone, LFO depth into tremolo depth, and added tests that fail when base controls are only declared/kept alive. Focused verification: `dotnet test AquaSynth.sln --no-restore --filter FaustRenderedSpeechLossSurfaceTests` passed. Full `dotnet test AquaSynth.sln --no-restore` timed out twice at 120s and 300s without a result; orphaned dotnet workers were stopped."}

55
tests/AquaSynth.Dsl.Tests/FaustRenderedSpeechLossSurfaceTests.cs
View file

@ -5,6 +5,34 @@ namespace AquaSynth.Dsl.Tests;
public sealed class FaustRenderedSpeechLossSurfaceTests
{
[Fact]
public void CompiledSpeechLossSourceRoutesEveryBaseControlIntoAudibleDsp()
{
var source = CompiledFaustRenderedSpeechLossSurface.ControllableSource(new FaustRenderedSpeechLossOptions(MelBandCount: 4));
Assert.Contains("tractLowHz", source);
Assert.Contains("tractMidHz", source);
Assert.Contains("tractHighHz", source);
Assert.Contains("nasal =", source);
Assert.Contains("apertureGain", source);
Assert.Contains("o11 * 0.75", source);
for (var index = 0; index < 14; index++)
{
Assert.True(MeaningfulControlReferenceCount(source, $"o{index}") > 1, $"/speech/output/{index} is only declared/kept alive, not routed into audible DSP.");
}
}
[Fact]
public void ScriptSpeechLossSourceRoutesTractControlsIntoFormantProxy()
{
var surface = new FaustRenderedSpeechLossSurface(new FaustRenderedSpeechLossOptions(MelBandCount: 4));
var source = surface.SourceFor(Target([0.5f, 0.5f, 0.5f, 0.5f], turbulence: 0.20f, cutoff: 0.60f));
Assert.Contains("fi.resonbp", source);
Assert.Contains("_formant_", source);
Assert.Contains("lfo_sin", source);
}
[Fact]
public async Task FaustRenderedSpeechLossSurfaceReturnsFiniteDifferenceGradientWhenToolchainIsAvailable()
{
@ -92,4 +120,31 @@ public sealed class FaustRenderedSpeechLossSurfaceTests
FilterCutoff: cutoff,
FilterResonance: 0.20f,
MelSpectralEnvelope: mel);
private static int MeaningfulControlReferenceCount(string source, string controlName)
{
var processIndex = source.IndexOf("process =", StringComparison.Ordinal);
var audibleSource = processIndex < 0 ? source : source[..processIndex];
var count = 0;
var index = 0;
while ((index = audibleSource.IndexOf(controlName, index, StringComparison.Ordinal)) >= 0)
{
if (IsControlToken(audibleSource, index, controlName.Length))
{
count++;
}
index += controlName.Length;
}
return count;
}
private static bool IsControlToken(string source, int index, int length)
{
var before = index == 0 ? '\0' : source[index - 1];
var afterIndex = index + length;
var after = afterIndex >= source.Length ? '\0' : source[afterIndex];
return !char.IsLetterOrDigit(before) && !char.IsLetterOrDigit(after) && before != '_' && after != '_';
}
}