Adopt PanPhon realization channel

This commit is contained in:
Meta Vens 2026-05-20 22:47:37 +01:00
commit 21c1046f9d
11 changed files with 89 additions and 27 deletions

View file

@ -236,8 +236,8 @@ The first Weksa handoff target is `weksa.utterance_embedding_handoff.v0.1`.
AquaSynth should treat it as the training input contract: AquaSynth should treat it as the training input contract:
- 1024-float `bge-m3:latest` speech text embedding for semantic text evidence; - 1024-float `bge-m3:latest` speech text embedding for semantic text evidence;
- 256-float phonetic realization vector for IPA, phones, pronunciation, and - 256-float PanPhon-based phonetic realization vector for IPA, phones,
alien speech-shape evidence; pronunciation, and alien speech-shape evidence;
- 32-float deterministic prosody/emphasis hint vector; - 32-float deterministic prosody/emphasis hint vector;
- 64-float projected Ghostlight/Epiphany-shaped character-state vector; - 64-float projected Ghostlight/Epiphany-shaped character-state vector;
- 64-float AquaSynth-owned learned utterance embedding output. - 64-float AquaSynth-owned learned utterance embedding output.

View file

@ -121,10 +121,10 @@ beautiful falsehood.
Do not start serious gradient descent from toy embedding packets. The first Do not start serious gradient descent from toy embedding packets. The first
training contract coordinated with Weksa is training contract coordinated with Weksa is
`weksa.utterance_embedding_handoff.v0.1`: 1024 speech-text floats from `weksa.utterance_embedding_handoff.v0.1`: 1024 speech-text floats from
`bge-m3:latest`, 256 phonetic-realization floats for IPA/phones/alien speech `bge-m3:latest`, 256 PanPhon-based phonetic-realization floats for
shape, 32 deterministic prosody/emphasis floats, 64 projected character-state IPA/phones/alien speech shape, 32 deterministic prosody/emphasis floats, 64
floats, and a 64-float AquaSynth-owned utterance embedding output. Anything projected character-state floats, and a 64-float AquaSynth-owned utterance
smaller is a plumbing fixture, not a training corpus. embedding output. Anything smaller is a plumbing fixture, not a training corpus.
Keep semantic text and phonetic realization separate. English text embeddings Keep semantic text and phonetic realization separate. English text embeddings
may help preserve meaning and phrasing, but an IPA string is not English prose may help preserve meaning and phrasing, but an IPA string is not English prose

View file

@ -71,13 +71,14 @@ public sealed record SpeechTimingReceipt(
[MessagePackObject] [MessagePackObject]
public sealed record SpeechUtteranceEmbeddingInputSnapshot( public sealed record SpeechUtteranceEmbeddingInputSnapshot(
[property: Key(0)] float[] TextEmbedding, [property: Key(0)] float[] TextEmbedding,
[property: Key(1)] float[] Prosody, [property: Key(1)] float[] PhoneticRealization,
[property: Key(2)] float[] CharacterState) [property: Key(2)] float[] Prosody,
[property: Key(3)] float[] CharacterState)
{ {
public UtteranceEmbeddingInput ToInput() => new(TextEmbedding, Prosody, CharacterState); public UtteranceEmbeddingInput ToInput() => new(TextEmbedding, PhoneticRealization, Prosody, CharacterState);
public static SpeechUtteranceEmbeddingInputSnapshot From(UtteranceEmbeddingInput input) => public static SpeechUtteranceEmbeddingInputSnapshot From(UtteranceEmbeddingInput input) =>
new([.. input.SpeechTextEmbedding], [.. input.ProsodyAndEmphasisHints], [.. input.CharacterStateVector]); new([.. input.SpeechTextEmbedding], [.. input.PhoneticRealizationVector], [.. input.ProsodyAndEmphasisHints], [.. input.CharacterStateVector]);
} }
[MessagePackObject] [MessagePackObject]

View file

@ -2,13 +2,15 @@ namespace AquaSynth.Dsl;
public sealed record UtteranceEmbeddingInput( public sealed record UtteranceEmbeddingInput(
IReadOnlyList<float> SpeechTextEmbedding, IReadOnlyList<float> SpeechTextEmbedding,
IReadOnlyList<float> PhoneticRealizationVector,
IReadOnlyList<float> ProsodyAndEmphasisHints, IReadOnlyList<float> ProsodyAndEmphasisHints,
IReadOnlyList<float> CharacterStateVector) IReadOnlyList<float> CharacterStateVector)
{ {
public UtteranceEmbeddingFeatureVector ToFeatureVector() public UtteranceEmbeddingFeatureVector ToFeatureVector()
{ {
var values = new float[SpeechTextEmbedding.Count + ProsodyAndEmphasisHints.Count + CharacterStateVector.Count]; var values = new float[SpeechTextEmbedding.Count + PhoneticRealizationVector.Count + ProsodyAndEmphasisHints.Count + CharacterStateVector.Count];
var offset = Copy(SpeechTextEmbedding, values, 0); var offset = Copy(SpeechTextEmbedding, values, 0);
offset = Copy(PhoneticRealizationVector, values, offset);
offset = Copy(ProsodyAndEmphasisHints, values, offset); offset = Copy(ProsodyAndEmphasisHints, values, offset);
Copy(CharacterStateVector, values, offset); Copy(CharacterStateVector, values, offset);
return new UtteranceEmbeddingFeatureVector(values); return new UtteranceEmbeddingFeatureVector(values);
@ -25,6 +27,35 @@ public sealed record UtteranceEmbeddingInput(
} }
} }
public static class WeksaUtteranceEmbeddingHandoff
{
public const string SchemaVersion = "weksa.utterance_embedding_handoff.v0.1";
public const string SpeechTextEmbeddingModelId = "bge-m3:latest";
public const string PhoneticRealizationModelId = "weksa.panphon_realization.v0.1";
public const int SpeechTextEmbeddingSize = 1024;
public const int PhoneticRealizationVectorSize = 256;
public const int ProsodyAndEmphasisHintSize = 32;
public const int CharacterStateVectorSize = 64;
public const int UtteranceEmbeddingSize = 64;
public const int InputSize = SpeechTextEmbeddingSize + PhoneticRealizationVectorSize + ProsodyAndEmphasisHintSize + CharacterStateVectorSize;
public static void Validate(UtteranceEmbeddingInput input)
{
ValidateSize(input.SpeechTextEmbedding, SpeechTextEmbeddingSize, nameof(input.SpeechTextEmbedding));
ValidateSize(input.PhoneticRealizationVector, PhoneticRealizationVectorSize, nameof(input.PhoneticRealizationVector));
ValidateSize(input.ProsodyAndEmphasisHints, ProsodyAndEmphasisHintSize, nameof(input.ProsodyAndEmphasisHints));
ValidateSize(input.CharacterStateVector, CharacterStateVectorSize, nameof(input.CharacterStateVector));
}
private static void ValidateSize(IReadOnlyList<float> values, int expected, string name)
{
if (values.Count != expected)
{
throw new ArgumentException($"{SchemaVersion} requires {name} to have {expected} values", name);
}
}
}
public sealed record UtteranceEmbeddingFeatureVector(IReadOnlyList<float> Values); public sealed record UtteranceEmbeddingFeatureVector(IReadOnlyList<float> Values);
public sealed record UtteranceEmbedding(IReadOnlyList<float> Values) public sealed record UtteranceEmbedding(IReadOnlyList<float> Values)

View file

@ -122,4 +122,4 @@
{"date":"2026-05-20","topic":"utterance-parity-current-objective","lesson":"AquaSynth's active objective is now the utterance parity curriculum: structured utterance metadata lowers into a learned utterance embedding, then into learned synth automation for the vocal tract and associated envelopes/LFOs/filters/spectral lanes, then into rendered speech loss. Broad DX7/Zyn synth digestion is backburner work unless it directly improves this harness.","evidence":"Updated README, docs/audio-doctrine.md, docs/reference-synth-roadmap.md, docs/ipa-vocal-tract-roadmap.md, state/map.yaml, state/spine.yaml, and state/memory.json to make the speech parity harness the current center."} {"date":"2026-05-20","topic":"utterance-parity-current-objective","lesson":"AquaSynth's active objective is now the utterance parity curriculum: structured utterance metadata lowers into a learned utterance embedding, then into learned synth automation for the vocal tract and associated envelopes/LFOs/filters/spectral lanes, then into rendered speech loss. Broad DX7/Zyn synth digestion is backburner work unless it directly improves this harness.","evidence":"Updated README, docs/audio-doctrine.md, docs/reference-synth-roadmap.md, docs/ipa-vocal-tract-roadmap.md, state/map.yaml, state/spine.yaml, and state/memory.json to make the speech parity harness the current center."}
{"date":"2026-05-20","topic":"utterance-parity-prior-work","lesson":"Prior work supports AquaSynth's architecture when kept in its lane: Kelly-Lochbaum/Pink Trombone and Story area functions support tract/area authority; TADA/articulatory phonology supports overlapping gestures; eSpeak/Klatt-style formant synthesis is useful fast ground truth but not anatomy; DDSP/neural source-filter work supports learned controllers over interpretable DSP rather than black-box throat replacement.","evidence":"Added docs/utterance-parity-research.md with source ledger covering Pink Trombone, VocalTractLab, Story, TADA, eSpeak NG, PanPhon, PHOIBLE, DDSP, DDSP articulatory vocoders, neural source-filter models, and LF voice-source references."} {"date":"2026-05-20","topic":"utterance-parity-prior-work","lesson":"Prior work supports AquaSynth's architecture when kept in its lane: Kelly-Lochbaum/Pink Trombone and Story area functions support tract/area authority; TADA/articulatory phonology supports overlapping gestures; eSpeak/Klatt-style formant synthesis is useful fast ground truth but not anatomy; DDSP/neural source-filter work supports learned controllers over interpretable DSP rather than black-box throat replacement.","evidence":"Added docs/utterance-parity-research.md with source ledger covering Pink Trombone, VocalTractLab, Story, TADA, eSpeak NG, PanPhon, PHOIBLE, DDSP, DDSP articulatory vocoders, neural source-filter models, and LF voice-source references."}
{"date":"2026-05-20","topic":"speech-training-time-witness","lesson":"Speech training receipts need time and confidence as evidence, not only loss. Each curriculum step should record when the stage decided, observed latency, intended latency budget, confidence, and artifact/model identifiers; a good-sounding result from an unbounded clockless path is not trustworthy.","evidence":"Added SpeechTimingReceipt to SpeechRenderRequest, SpeechRenderResult, and SpeechTrainingCheckpoint; seeded timing/confidence receipts in the local distributed-training proof; updated audio doctrine, IPA roadmap, utterance parity research notes, spine, and memory."} {"date":"2026-05-20","topic":"speech-training-time-witness","lesson":"Speech training receipts need time and confidence as evidence, not only loss. Each curriculum step should record when the stage decided, observed latency, intended latency budget, confidence, and artifact/model identifiers; a good-sounding result from an unbounded clockless path is not trustworthy.","evidence":"Added SpeechTimingReceipt to SpeechRenderRequest, SpeechRenderResult, and SpeechTrainingCheckpoint; seeded timing/confidence receipts in the local distributed-training proof; updated audio doctrine, IPA roadmap, utterance parity research notes, spine, and memory."}
{"date":"2026-05-20","topic":"weksa-utterance-embedding-contract","lesson":"Do not start serious AquaSynth speech gradient descent from toy Weksa vectors, and do not treat semantic text embeddings as IPA embeddings. The coordinated Weksa handoff contract is `weksa.utterance_embedding_handoff.v0.1`: 1024 `bge-m3:latest` semantic speech-text floats, 256 phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth-owned utterance embedding output.","evidence":"Updated E:/Projects/weksa docs/schema/state to split speech text embedding from phonetic realization and marked the existing Nibu toy fixture as non-training; mirrored the training block in AquaSynth IPA roadmap, utterance parity research notes, spine, and memory."} {"date":"2026-05-20","topic":"weksa-utterance-embedding-contract","lesson":"Do not start serious AquaSynth speech gradient descent from toy Weksa vectors, and do not treat semantic text embeddings as IPA embeddings. The coordinated Weksa handoff contract is `weksa.utterance_embedding_handoff.v0.1`: 1024 `bge-m3:latest` semantic speech-text floats, 256 PanPhon-based phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth-owned utterance embedding output.","evidence":"Updated E:/Projects/weksa docs/schema/state to split speech text embedding from PanPhon-based phonetic realization and marked the existing Nibu toy fixture as non-training; mirrored the training block in AquaSynth IPA roadmap, utterance parity research notes, spine, and memory."}

View file

@ -92,7 +92,7 @@
"ipa_vocal_tract_path": { "ipa_vocal_tract_path": {
"status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets.", "status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets.",
"lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.", "lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.",
"next": "Do not start serious speech gradient descent from toy embedding packets. Mirror Weksa `weksa.utterance_embedding_handoff.v0.1` first: 1024 bge-m3 semantic speech-text floats, 256 phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Then tighten the parity harness: render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop." "next": "Do not start serious speech gradient descent from toy embedding packets. Mirror Weksa `weksa.utterance_embedding_handoff.v0.1` first: 1024 bge-m3 semantic speech-text floats, 256 PanPhon-based phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Then tighten the parity harness: render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop."
}, },
"dx7_reference_path": { "dx7_reference_path": {
"status": "DX7 SysEx parser extracts voice edit buffers, packed voices, packed 32-voice banks, six operators, envelopes, algorithm, feedback, oscillator sync, LFO, transpose, voice name, checksum validation, ReferencePatch features, algorithm topology from the reverse-engineered OPS algorithm ROM table, envelope-to-ADSR approximations, measured carrier output amplitude, isolated two-op modulation route scale, summed/cascaded route scale, calibrated feedback amount table, ratio detune lowering, voice transpose lowering, fixed-frequency operator lowering, ROM COM output compensation cut, operator key/rate scaling at the effective played note, topology-aware route lowering, staged operator envelopes with per-segment curves, raw and block-interpolated DX7 envelope tracing, applied-envelope curved lowering with optional sustain floor, two AquaSynth DSL rebuild attempts, first-class OperatorGraph support with graph-level vibrato, public-domain SysEx fixture coverage, dexed-py reference rendering, project-authored Dexed patch rendering, AquaSynth candidate rendering through Faust-generated C#, one thresholded rendered parity test for analog1.syx voice 13, a hard PRC SYNTH1 parity probe with listening WAV artifacts and log-mel <= 0.145, project-authored algorithm-8 summed-stack/cascade/summed-pair parity, rendered community parity for Mooger, Piano Bass, RES SYNTH1, and ANLGSYN 1 with a sustained high-band gate, a MELLOWSOLO pressure artifact test, and renderable delayed operator feedback.", "status": "DX7 SysEx parser extracts voice edit buffers, packed voices, packed 32-voice banks, six operators, envelopes, algorithm, feedback, oscillator sync, LFO, transpose, voice name, checksum validation, ReferencePatch features, algorithm topology from the reverse-engineered OPS algorithm ROM table, envelope-to-ADSR approximations, measured carrier output amplitude, isolated two-op modulation route scale, summed/cascaded route scale, calibrated feedback amount table, ratio detune lowering, voice transpose lowering, fixed-frequency operator lowering, ROM COM output compensation cut, operator key/rate scaling at the effective played note, topology-aware route lowering, staged operator envelopes with per-segment curves, raw and block-interpolated DX7 envelope tracing, applied-envelope curved lowering with optional sustain floor, two AquaSynth DSL rebuild attempts, first-class OperatorGraph support with graph-level vibrato, public-domain SysEx fixture coverage, dexed-py reference rendering, project-authored Dexed patch rendering, AquaSynth candidate rendering through Faust-generated C#, one thresholded rendered parity test for analog1.syx voice 13, a hard PRC SYNTH1 parity probe with listening WAV artifacts and log-mel <= 0.145, project-authored algorithm-8 summed-stack/cascade/summed-pair parity, rendered community parity for Mooger, Piano Bass, RES SYNTH1, and ANLGSYN 1 with a sustained high-band gate, a MELLOWSOLO pressure artifact test, and renderable delayed operator feedback.",

View file

@ -112,7 +112,7 @@ live_invariants:
- eSpeak NG is an early parity reference for timing, IPA coverage, and log-mel pressure, not an anatomy oracle or final target. - eSpeak NG is an early parity reference for timing, IPA coverage, and log-mel pressure, not an anatomy oracle or final target.
- Do not expand into prosody, emotional context, personality, or broad Weksa expressivity until the tiny IPA utterance harness can show useful loss movement. - Do not expand into prosody, emotional context, personality, or broad Weksa expressivity until the tiny IPA utterance harness can show useful loss movement.
- Speech training receipts must keep time as a witness: every request, render result, and checkpoint should record stage decision time, latency spent, latency budget, confidence, and artifact/model identifiers. - Speech training receipts must keep time as a witness: every request, render result, and checkpoint should record stage decision time, latency spent, latency budget, confidence, and artifact/model identifiers.
- Serious speech gradient descent is blocked until Weksa packets use `weksa.utterance_embedding_handoff.v0.1` or a deliberate successor: 1024 semantic speech-text floats, 256 phonetic-realization floats, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth utterance embedding output. - Serious speech gradient descent is blocked until Weksa packets use `weksa.utterance_embedding_handoff.v0.1` or a deliberate successor: 1024 semantic speech-text floats, 256 PanPhon-based phonetic-realization floats, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth utterance embedding output.
- Do not use semantic text embeddings as a proxy for IPA or alien phonetic strings. Phonetic/IPA realization is a separate input channel. - Do not use semantic text embeddings as a proxy for IPA or alien phonetic strings. Phonetic/IPA realization is a separate input channel.
reference_targets: reference_targets:

View file

@ -89,8 +89,19 @@ public sealed class EspeakNgGradientDescentTests
private static TrainingFixture Fixture(string id, string text, PhoneticFeatures features, IReadOnlyList<float> textEmbedding) private static TrainingFixture Fixture(string id, string text, PhoneticFeatures features, IReadOnlyList<float> textEmbedding)
{ {
var prosody = new[] { 0.75f, text.Length / 4f, features.Manner == PhoneticManner.Fricative ? 0.95f : 0.35f, features.Manner == PhoneticManner.Nasal ? 0.80f : 0.25f }; var prosody = new[] { 0.75f, text.Length / 4f, features.Manner == PhoneticManner.Fricative ? 0.95f : 0.35f, features.Manner == PhoneticManner.Nasal ? 0.80f : 0.25f };
var phoneticRealization = new[]
{
features.Manner == PhoneticManner.Vowel ? 1f : 0f,
features.Manner == PhoneticManner.Stop ? 1f : 0f,
features.Manner == PhoneticManner.Fricative ? 1f : 0f,
features.Manner == PhoneticManner.Nasal ? 1f : 0f,
features.Place == PhoneticPlace.Bilabial ? 1f : 0f,
features.Place == PhoneticPlace.Alveolar ? 1f : 0f,
features.Place == PhoneticPlace.Velar ? 1f : 0f,
features.Phonation == Phonation.Voiced ? 1f : 0f
};
var characterState = new[] { 0.65f, 0.35f, 0.55f, features.Place == PhoneticPlace.Bilabial ? 0.72f : 0.28f }; var characterState = new[] { 0.65f, 0.35f, 0.55f, features.Place == PhoneticPlace.Bilabial ? 0.72f : 0.28f };
var input = new UtteranceEmbeddingInput(textEmbedding, prosody, characterState); var input = new UtteranceEmbeddingInput(textEmbedding, phoneticRealization, prosody, characterState);
var phoneticEvent = new PhoneticEvent(id, text, features, DurationSeconds: 0.20, Prosody: new PhoneticProsody(Stress: 0.75f, Intensity: 0.8f)); var phoneticEvent = new PhoneticEvent(id, text, features, DurationSeconds: 0.20, Prosody: new PhoneticProsody(Stress: 0.75f, Intensity: 0.8f));
return new TrainingFixture(id, text, "en", phoneticEvent, input); return new TrainingFixture(id, text, "en", phoneticEvent, input);
} }

View file

@ -8,7 +8,7 @@ public sealed class SpeechBackpropagationPipelineTests
public void SpeechPipelineBackpropagatesSynthDriverLossIntoUtteranceEncoder() public void SpeechPipelineBackpropagatesSynthDriverLossIntoUtteranceEncoder()
{ {
var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
inputSize: 10, inputSize: 13,
embeddingSize: 16, embeddingSize: 16,
hiddenLayerSizes: [48, 32, 24], hiddenLayerSizes: [48, 32, 24],
seed: 211); seed: 211);
@ -37,7 +37,7 @@ public sealed class SpeechBackpropagationPipelineTests
public void SpeechPipelineAcceptsExternalRenderedLossGradient() public void SpeechPipelineAcceptsExternalRenderedLossGradient()
{ {
var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
inputSize: 10, inputSize: 13,
embeddingSize: 8, embeddingSize: 8,
hiddenLayerSizes: [24, 16], hiddenLayerSizes: [24, 16],
seed: 301); seed: 301);
@ -50,7 +50,7 @@ public sealed class SpeechBackpropagationPipelineTests
var example = Example( var example = Example(
"a", "a",
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
new UtteranceEmbeddingInput([0.9f, 0.1f, 0.2f, 0.7f], [0.8f, 0.3f, 0.2f], [0.6f, 0.2f, 0.4f]), new UtteranceEmbeddingInput([0.9f, 0.1f, 0.2f, 0.7f], [0.9f, 0.2f, 0.1f], [0.8f, 0.3f, 0.2f], [0.6f, 0.2f, 0.4f]),
SmallTarget([0.9f, 0.7f, 0.3f, 0.1f])); SmallTarget([0.9f, 0.7f, 0.3f, 0.1f]));
var before = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount); var before = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount);
var gradient = new float[pipeline.SynthDriver.OutputSize]; var gradient = new float[pipeline.SynthDriver.OutputSize];
@ -77,17 +77,17 @@ public sealed class SpeechBackpropagationPipelineTests
Example( Example(
"a", "a",
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]), new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.90f, 0.18f, 0.10f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]),
Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f, 0.34f, 0.28f, 0.22f, 0.18f, 0.16f, 0.14f])), Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f, 0.34f, 0.28f, 0.22f, 0.18f, 0.16f, 0.14f])),
Example( Example(
"sa", "sa",
new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless), new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless),
new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]), new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.22f, 0.86f, 0.76f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]),
Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f, 0.76f, 0.88f, 0.94f, 0.90f, 0.82f, 0.72f])), Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f, 0.76f, 0.88f, 0.94f, 0.90f, 0.82f, 0.72f])),
Example( Example(
"ma", "ma",
new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true), new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true),
new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]), new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.70f, 0.60f, 0.82f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]),
Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f, 0.40f, 0.36f, 0.32f, 0.30f, 0.28f, 0.26f])) Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f, 0.40f, 0.36f, 0.32f, 0.30f, 0.28f, 0.26f]))
]; ];

View file

@ -89,7 +89,7 @@ public sealed class SpeechDistributedTrainingTests
private static SpeechBackpropagationPipeline Pipeline() private static SpeechBackpropagationPipeline Pipeline()
{ {
var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
inputSize: 10, inputSize: 13,
embeddingSize: 8, embeddingSize: 8,
hiddenLayerSizes: [24, 16], hiddenLayerSizes: [24, 16],
seed: 701); seed: 701);
@ -106,17 +106,17 @@ public sealed class SpeechDistributedTrainingTests
Example( Example(
"a", "a",
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]), new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.90f, 0.18f, 0.10f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]),
Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f])), Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f])),
Example( Example(
"sa", "sa",
new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless), new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless),
new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]), new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.22f, 0.86f, 0.76f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]),
Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f])), Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f])),
Example( Example(
"ma", "ma",
new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true), new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true),
new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]), new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.70f, 0.60f, 0.82f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]),
Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f])) Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f]))
]; ];

View file

@ -5,16 +5,33 @@ namespace AquaSynth.Dsl.Tests;
public sealed class UtteranceEmbeddingLearningTests public sealed class UtteranceEmbeddingLearningTests
{ {
[Fact] [Fact]
public void UtteranceEmbeddingInputPacksTextProsodyAndCharacterState() public void UtteranceEmbeddingInputPacksTextPhoneticsProsodyAndCharacterState()
{ {
var input = new UtteranceEmbeddingInput( var input = new UtteranceEmbeddingInput(
SpeechTextEmbedding: [0.1f, 0.2f, 0.3f], SpeechTextEmbedding: [0.1f, 0.2f, 0.3f],
PhoneticRealizationVector: [0.9f, 0.6f],
ProsodyAndEmphasisHints: [0.7f, 0.8f], ProsodyAndEmphasisHints: [0.7f, 0.8f],
CharacterStateVector: [0.4f, 0.5f, 0.6f]); CharacterStateVector: [0.4f, 0.5f, 0.6f]);
var features = input.ToFeatureVector(); var features = input.ToFeatureVector();
Assert.Equal([0.1f, 0.2f, 0.3f, 0.7f, 0.8f, 0.4f, 0.5f, 0.6f], features.Values); Assert.Equal([0.1f, 0.2f, 0.3f, 0.9f, 0.6f, 0.7f, 0.8f, 0.4f, 0.5f, 0.6f], features.Values);
}
[Fact]
public void WeksaV01ContractRequiresSeparatePanphonRealizationChannel()
{
var input = new UtteranceEmbeddingInput(
SpeechTextEmbedding: Zeros(WeksaUtteranceEmbeddingHandoff.SpeechTextEmbeddingSize),
PhoneticRealizationVector: Zeros(WeksaUtteranceEmbeddingHandoff.PhoneticRealizationVectorSize),
ProsodyAndEmphasisHints: Zeros(WeksaUtteranceEmbeddingHandoff.ProsodyAndEmphasisHintSize),
CharacterStateVector: Zeros(WeksaUtteranceEmbeddingHandoff.CharacterStateVectorSize));
WeksaUtteranceEmbeddingHandoff.Validate(input);
Assert.Equal("weksa.utterance_embedding_handoff.v0.1", WeksaUtteranceEmbeddingHandoff.SchemaVersion);
Assert.Equal("weksa.panphon_realization.v0.1", WeksaUtteranceEmbeddingHandoff.PhoneticRealizationModelId);
Assert.Equal(1376, input.ToFeatureVector().Values.Count);
} }
[Fact] [Fact]
@ -103,4 +120,6 @@ public sealed class UtteranceEmbeddingLearningTests
private static float Lerp(float left, float right, float amount) => private static float Lerp(float left, float right, float amount) =>
left + (right - left) * amount; left + (right - left) * amount;
private static float[] Zeros(int count) => new float[count];
} }