diff --git a/docs/ipa-vocal-tract-roadmap.md b/docs/ipa-vocal-tract-roadmap.md index 5f4ce8b..3e9765e 100644 --- a/docs/ipa-vocal-tract-roadmap.md +++ b/docs/ipa-vocal-tract-roadmap.md @@ -236,8 +236,8 @@ The first Weksa handoff target is `weksa.utterance_embedding_handoff.v0.1`. AquaSynth should treat it as the training input contract: - 1024-float `bge-m3:latest` speech text embedding for semantic text evidence; -- 256-float phonetic realization vector for IPA, phones, pronunciation, and - alien speech-shape evidence; +- 256-float PanPhon-based phonetic realization vector for IPA, phones, + pronunciation, and alien speech-shape evidence; - 32-float deterministic prosody/emphasis hint vector; - 64-float projected Ghostlight/Epiphany-shaped character-state vector; - 64-float AquaSynth-owned learned utterance embedding output. diff --git a/docs/utterance-parity-research.md b/docs/utterance-parity-research.md index 8080c71..4fc3fcb 100644 --- a/docs/utterance-parity-research.md +++ b/docs/utterance-parity-research.md @@ -121,10 +121,10 @@ beautiful falsehood. Do not start serious gradient descent from toy embedding packets. The first training contract coordinated with Weksa is `weksa.utterance_embedding_handoff.v0.1`: 1024 speech-text floats from -`bge-m3:latest`, 256 phonetic-realization floats for IPA/phones/alien speech -shape, 32 deterministic prosody/emphasis floats, 64 projected character-state -floats, and a 64-float AquaSynth-owned utterance embedding output. Anything -smaller is a plumbing fixture, not a training corpus. +`bge-m3:latest`, 256 PanPhon-based phonetic-realization floats for +IPA/phones/alien speech shape, 32 deterministic prosody/emphasis floats, 64 +projected character-state floats, and a 64-float AquaSynth-owned utterance +embedding output. Anything smaller is a plumbing fixture, not a training corpus. Keep semantic text and phonetic realization separate. English text embeddings may help preserve meaning and phrasing, but an IPA string is not English prose diff --git a/src/AquaSynth.Core/SpeechDistributedTraining.cs b/src/AquaSynth.Core/SpeechDistributedTraining.cs index 6164945..3d71365 100644 --- a/src/AquaSynth.Core/SpeechDistributedTraining.cs +++ b/src/AquaSynth.Core/SpeechDistributedTraining.cs @@ -71,13 +71,14 @@ public sealed record SpeechTimingReceipt( [MessagePackObject] public sealed record SpeechUtteranceEmbeddingInputSnapshot( [property: Key(0)] float[] TextEmbedding, - [property: Key(1)] float[] Prosody, - [property: Key(2)] float[] CharacterState) + [property: Key(1)] float[] PhoneticRealization, + [property: Key(2)] float[] Prosody, + [property: Key(3)] float[] CharacterState) { - public UtteranceEmbeddingInput ToInput() => new(TextEmbedding, Prosody, CharacterState); + public UtteranceEmbeddingInput ToInput() => new(TextEmbedding, PhoneticRealization, Prosody, CharacterState); public static SpeechUtteranceEmbeddingInputSnapshot From(UtteranceEmbeddingInput input) => - new([.. input.SpeechTextEmbedding], [.. input.ProsodyAndEmphasisHints], [.. input.CharacterStateVector]); + new([.. input.SpeechTextEmbedding], [.. input.PhoneticRealizationVector], [.. input.ProsodyAndEmphasisHints], [.. input.CharacterStateVector]); } [MessagePackObject] diff --git a/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs b/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs index dbf40f7..06efb1f 100644 --- a/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs +++ b/src/AquaSynth.Core/UtteranceEmbeddingLearning.cs @@ -2,13 +2,15 @@ namespace AquaSynth.Dsl; public sealed record UtteranceEmbeddingInput( IReadOnlyList SpeechTextEmbedding, + IReadOnlyList PhoneticRealizationVector, IReadOnlyList ProsodyAndEmphasisHints, IReadOnlyList CharacterStateVector) { public UtteranceEmbeddingFeatureVector ToFeatureVector() { - var values = new float[SpeechTextEmbedding.Count + ProsodyAndEmphasisHints.Count + CharacterStateVector.Count]; + var values = new float[SpeechTextEmbedding.Count + PhoneticRealizationVector.Count + ProsodyAndEmphasisHints.Count + CharacterStateVector.Count]; var offset = Copy(SpeechTextEmbedding, values, 0); + offset = Copy(PhoneticRealizationVector, values, offset); offset = Copy(ProsodyAndEmphasisHints, values, offset); Copy(CharacterStateVector, values, offset); return new UtteranceEmbeddingFeatureVector(values); @@ -25,6 +27,35 @@ public sealed record UtteranceEmbeddingInput( } } +public static class WeksaUtteranceEmbeddingHandoff +{ + public const string SchemaVersion = "weksa.utterance_embedding_handoff.v0.1"; + public const string SpeechTextEmbeddingModelId = "bge-m3:latest"; + public const string PhoneticRealizationModelId = "weksa.panphon_realization.v0.1"; + public const int SpeechTextEmbeddingSize = 1024; + public const int PhoneticRealizationVectorSize = 256; + public const int ProsodyAndEmphasisHintSize = 32; + public const int CharacterStateVectorSize = 64; + public const int UtteranceEmbeddingSize = 64; + public const int InputSize = SpeechTextEmbeddingSize + PhoneticRealizationVectorSize + ProsodyAndEmphasisHintSize + CharacterStateVectorSize; + + public static void Validate(UtteranceEmbeddingInput input) + { + ValidateSize(input.SpeechTextEmbedding, SpeechTextEmbeddingSize, nameof(input.SpeechTextEmbedding)); + ValidateSize(input.PhoneticRealizationVector, PhoneticRealizationVectorSize, nameof(input.PhoneticRealizationVector)); + ValidateSize(input.ProsodyAndEmphasisHints, ProsodyAndEmphasisHintSize, nameof(input.ProsodyAndEmphasisHints)); + ValidateSize(input.CharacterStateVector, CharacterStateVectorSize, nameof(input.CharacterStateVector)); + } + + private static void ValidateSize(IReadOnlyList values, int expected, string name) + { + if (values.Count != expected) + { + throw new ArgumentException($"{SchemaVersion} requires {name} to have {expected} values", name); + } + } +} + public sealed record UtteranceEmbeddingFeatureVector(IReadOnlyList Values); public sealed record UtteranceEmbedding(IReadOnlyList Values) diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 84cffbf..391b0fa 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -122,4 +122,4 @@ {"date":"2026-05-20","topic":"utterance-parity-current-objective","lesson":"AquaSynth's active objective is now the utterance parity curriculum: structured utterance metadata lowers into a learned utterance embedding, then into learned synth automation for the vocal tract and associated envelopes/LFOs/filters/spectral lanes, then into rendered speech loss. Broad DX7/Zyn synth digestion is backburner work unless it directly improves this harness.","evidence":"Updated README, docs/audio-doctrine.md, docs/reference-synth-roadmap.md, docs/ipa-vocal-tract-roadmap.md, state/map.yaml, state/spine.yaml, and state/memory.json to make the speech parity harness the current center."} {"date":"2026-05-20","topic":"utterance-parity-prior-work","lesson":"Prior work supports AquaSynth's architecture when kept in its lane: Kelly-Lochbaum/Pink Trombone and Story area functions support tract/area authority; TADA/articulatory phonology supports overlapping gestures; eSpeak/Klatt-style formant synthesis is useful fast ground truth but not anatomy; DDSP/neural source-filter work supports learned controllers over interpretable DSP rather than black-box throat replacement.","evidence":"Added docs/utterance-parity-research.md with source ledger covering Pink Trombone, VocalTractLab, Story, TADA, eSpeak NG, PanPhon, PHOIBLE, DDSP, DDSP articulatory vocoders, neural source-filter models, and LF voice-source references."} {"date":"2026-05-20","topic":"speech-training-time-witness","lesson":"Speech training receipts need time and confidence as evidence, not only loss. Each curriculum step should record when the stage decided, observed latency, intended latency budget, confidence, and artifact/model identifiers; a good-sounding result from an unbounded clockless path is not trustworthy.","evidence":"Added SpeechTimingReceipt to SpeechRenderRequest, SpeechRenderResult, and SpeechTrainingCheckpoint; seeded timing/confidence receipts in the local distributed-training proof; updated audio doctrine, IPA roadmap, utterance parity research notes, spine, and memory."} -{"date":"2026-05-20","topic":"weksa-utterance-embedding-contract","lesson":"Do not start serious AquaSynth speech gradient descent from toy Weksa vectors, and do not treat semantic text embeddings as IPA embeddings. The coordinated Weksa handoff contract is `weksa.utterance_embedding_handoff.v0.1`: 1024 `bge-m3:latest` semantic speech-text floats, 256 phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth-owned utterance embedding output.","evidence":"Updated E:/Projects/weksa docs/schema/state to split speech text embedding from phonetic realization and marked the existing Nibu toy fixture as non-training; mirrored the training block in AquaSynth IPA roadmap, utterance parity research notes, spine, and memory."} +{"date":"2026-05-20","topic":"weksa-utterance-embedding-contract","lesson":"Do not start serious AquaSynth speech gradient descent from toy Weksa vectors, and do not treat semantic text embeddings as IPA embeddings. The coordinated Weksa handoff contract is `weksa.utterance_embedding_handoff.v0.1`: 1024 `bge-m3:latest` semantic speech-text floats, 256 PanPhon-based phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth-owned utterance embedding output.","evidence":"Updated E:/Projects/weksa docs/schema/state to split speech text embedding from PanPhon-based phonetic realization and marked the existing Nibu toy fixture as non-training; mirrored the training block in AquaSynth IPA roadmap, utterance parity research notes, spine, and memory."} diff --git a/state/memory.json b/state/memory.json index e200754..6ba29eb 100644 --- a/state/memory.json +++ b/state/memory.json @@ -92,7 +92,7 @@ "ipa_vocal_tract_path": { "status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets.", "lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.", - "next": "Do not start serious speech gradient descent from toy embedding packets. Mirror Weksa `weksa.utterance_embedding_handoff.v0.1` first: 1024 bge-m3 semantic speech-text floats, 256 phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Then tighten the parity harness: render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop." + "next": "Do not start serious speech gradient descent from toy embedding packets. Mirror Weksa `weksa.utterance_embedding_handoff.v0.1` first: 1024 bge-m3 semantic speech-text floats, 256 PanPhon-based phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Then tighten the parity harness: render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop." }, "dx7_reference_path": { "status": "DX7 SysEx parser extracts voice edit buffers, packed voices, packed 32-voice banks, six operators, envelopes, algorithm, feedback, oscillator sync, LFO, transpose, voice name, checksum validation, ReferencePatch features, algorithm topology from the reverse-engineered OPS algorithm ROM table, envelope-to-ADSR approximations, measured carrier output amplitude, isolated two-op modulation route scale, summed/cascaded route scale, calibrated feedback amount table, ratio detune lowering, voice transpose lowering, fixed-frequency operator lowering, ROM COM output compensation cut, operator key/rate scaling at the effective played note, topology-aware route lowering, staged operator envelopes with per-segment curves, raw and block-interpolated DX7 envelope tracing, applied-envelope curved lowering with optional sustain floor, two AquaSynth DSL rebuild attempts, first-class OperatorGraph support with graph-level vibrato, public-domain SysEx fixture coverage, dexed-py reference rendering, project-authored Dexed patch rendering, AquaSynth candidate rendering through Faust-generated C#, one thresholded rendered parity test for analog1.syx voice 13, a hard PRC SYNTH1 parity probe with listening WAV artifacts and log-mel <= 0.145, project-authored algorithm-8 summed-stack/cascade/summed-pair parity, rendered community parity for Mooger, Piano Bass, RES SYNTH1, and ANLGSYN 1 with a sustained high-band gate, a MELLOWSOLO pressure artifact test, and renderable delayed operator feedback.", diff --git a/state/spine.yaml b/state/spine.yaml index 1f03a51..0fb162d 100644 --- a/state/spine.yaml +++ b/state/spine.yaml @@ -112,7 +112,7 @@ live_invariants: - eSpeak NG is an early parity reference for timing, IPA coverage, and log-mel pressure, not an anatomy oracle or final target. - Do not expand into prosody, emotional context, personality, or broad Weksa expressivity until the tiny IPA utterance harness can show useful loss movement. - Speech training receipts must keep time as a witness: every request, render result, and checkpoint should record stage decision time, latency spent, latency budget, confidence, and artifact/model identifiers. - - Serious speech gradient descent is blocked until Weksa packets use `weksa.utterance_embedding_handoff.v0.1` or a deliberate successor: 1024 semantic speech-text floats, 256 phonetic-realization floats, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth utterance embedding output. + - Serious speech gradient descent is blocked until Weksa packets use `weksa.utterance_embedding_handoff.v0.1` or a deliberate successor: 1024 semantic speech-text floats, 256 PanPhon-based phonetic-realization floats, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth utterance embedding output. - Do not use semantic text embeddings as a proxy for IPA or alien phonetic strings. Phonetic/IPA realization is a separate input channel. reference_targets: diff --git a/tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs b/tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs index af0b057..b652264 100644 --- a/tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs +++ b/tests/AquaSynth.Dsl.Tests/EspeakNgGradientDescentTests.cs @@ -89,8 +89,19 @@ public sealed class EspeakNgGradientDescentTests private static TrainingFixture Fixture(string id, string text, PhoneticFeatures features, IReadOnlyList textEmbedding) { var prosody = new[] { 0.75f, text.Length / 4f, features.Manner == PhoneticManner.Fricative ? 0.95f : 0.35f, features.Manner == PhoneticManner.Nasal ? 0.80f : 0.25f }; + var phoneticRealization = new[] + { + features.Manner == PhoneticManner.Vowel ? 1f : 0f, + features.Manner == PhoneticManner.Stop ? 1f : 0f, + features.Manner == PhoneticManner.Fricative ? 1f : 0f, + features.Manner == PhoneticManner.Nasal ? 1f : 0f, + features.Place == PhoneticPlace.Bilabial ? 1f : 0f, + features.Place == PhoneticPlace.Alveolar ? 1f : 0f, + features.Place == PhoneticPlace.Velar ? 1f : 0f, + features.Phonation == Phonation.Voiced ? 1f : 0f + }; var characterState = new[] { 0.65f, 0.35f, 0.55f, features.Place == PhoneticPlace.Bilabial ? 0.72f : 0.28f }; - var input = new UtteranceEmbeddingInput(textEmbedding, prosody, characterState); + var input = new UtteranceEmbeddingInput(textEmbedding, phoneticRealization, prosody, characterState); var phoneticEvent = new PhoneticEvent(id, text, features, DurationSeconds: 0.20, Prosody: new PhoneticProsody(Stress: 0.75f, Intensity: 0.8f)); return new TrainingFixture(id, text, "en", phoneticEvent, input); } diff --git a/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs b/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs index 3f46bc0..906be67 100644 --- a/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs +++ b/tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs @@ -8,7 +8,7 @@ public sealed class SpeechBackpropagationPipelineTests public void SpeechPipelineBackpropagatesSynthDriverLossIntoUtteranceEncoder() { var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( - inputSize: 10, + inputSize: 13, embeddingSize: 16, hiddenLayerSizes: [48, 32, 24], seed: 211); @@ -37,7 +37,7 @@ public sealed class SpeechBackpropagationPipelineTests public void SpeechPipelineAcceptsExternalRenderedLossGradient() { var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( - inputSize: 10, + inputSize: 13, embeddingSize: 8, hiddenLayerSizes: [24, 16], seed: 301); @@ -50,7 +50,7 @@ public sealed class SpeechBackpropagationPipelineTests var example = Example( "a", new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), - new UtteranceEmbeddingInput([0.9f, 0.1f, 0.2f, 0.7f], [0.8f, 0.3f, 0.2f], [0.6f, 0.2f, 0.4f]), + new UtteranceEmbeddingInput([0.9f, 0.1f, 0.2f, 0.7f], [0.9f, 0.2f, 0.1f], [0.8f, 0.3f, 0.2f], [0.6f, 0.2f, 0.4f]), SmallTarget([0.9f, 0.7f, 0.3f, 0.1f])); var before = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount); var gradient = new float[pipeline.SynthDriver.OutputSize]; @@ -77,17 +77,17 @@ public sealed class SpeechBackpropagationPipelineTests Example( "a", new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), - new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]), + new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.90f, 0.18f, 0.10f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]), Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f, 0.34f, 0.28f, 0.22f, 0.18f, 0.16f, 0.14f])), Example( "sa", new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless), - new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]), + new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.22f, 0.86f, 0.76f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]), Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f, 0.76f, 0.88f, 0.94f, 0.90f, 0.82f, 0.72f])), Example( "ma", new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true), - new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]), + new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.70f, 0.60f, 0.82f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]), Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f, 0.40f, 0.36f, 0.32f, 0.30f, 0.28f, 0.26f])) ]; diff --git a/tests/AquaSynth.Dsl.Tests/SpeechDistributedTrainingTests.cs b/tests/AquaSynth.Dsl.Tests/SpeechDistributedTrainingTests.cs index 9af3255..0d76f7c 100644 --- a/tests/AquaSynth.Dsl.Tests/SpeechDistributedTrainingTests.cs +++ b/tests/AquaSynth.Dsl.Tests/SpeechDistributedTrainingTests.cs @@ -89,7 +89,7 @@ public sealed class SpeechDistributedTrainingTests private static SpeechBackpropagationPipeline Pipeline() { var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create( - inputSize: 10, + inputSize: 13, embeddingSize: 8, hiddenLayerSizes: [24, 16], seed: 701); @@ -106,17 +106,17 @@ public sealed class SpeechDistributedTrainingTests Example( "a", new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front), - new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]), + new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.90f, 0.18f, 0.10f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]), Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f])), Example( "sa", new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless), - new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]), + new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.22f, 0.86f, 0.76f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]), Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f])), Example( "ma", new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true), - new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]), + new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.70f, 0.60f, 0.82f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]), Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f])) ]; diff --git a/tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs b/tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs index e09b7f9..f3712c5 100644 --- a/tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs +++ b/tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs @@ -5,16 +5,33 @@ namespace AquaSynth.Dsl.Tests; public sealed class UtteranceEmbeddingLearningTests { [Fact] - public void UtteranceEmbeddingInputPacksTextProsodyAndCharacterState() + public void UtteranceEmbeddingInputPacksTextPhoneticsProsodyAndCharacterState() { var input = new UtteranceEmbeddingInput( SpeechTextEmbedding: [0.1f, 0.2f, 0.3f], + PhoneticRealizationVector: [0.9f, 0.6f], ProsodyAndEmphasisHints: [0.7f, 0.8f], CharacterStateVector: [0.4f, 0.5f, 0.6f]); var features = input.ToFeatureVector(); - Assert.Equal([0.1f, 0.2f, 0.3f, 0.7f, 0.8f, 0.4f, 0.5f, 0.6f], features.Values); + Assert.Equal([0.1f, 0.2f, 0.3f, 0.9f, 0.6f, 0.7f, 0.8f, 0.4f, 0.5f, 0.6f], features.Values); + } + + [Fact] + public void WeksaV01ContractRequiresSeparatePanphonRealizationChannel() + { + var input = new UtteranceEmbeddingInput( + SpeechTextEmbedding: Zeros(WeksaUtteranceEmbeddingHandoff.SpeechTextEmbeddingSize), + PhoneticRealizationVector: Zeros(WeksaUtteranceEmbeddingHandoff.PhoneticRealizationVectorSize), + ProsodyAndEmphasisHints: Zeros(WeksaUtteranceEmbeddingHandoff.ProsodyAndEmphasisHintSize), + CharacterStateVector: Zeros(WeksaUtteranceEmbeddingHandoff.CharacterStateVectorSize)); + + WeksaUtteranceEmbeddingHandoff.Validate(input); + + Assert.Equal("weksa.utterance_embedding_handoff.v0.1", WeksaUtteranceEmbeddingHandoff.SchemaVersion); + Assert.Equal("weksa.panphon_realization.v0.1", WeksaUtteranceEmbeddingHandoff.PhoneticRealizationModelId); + Assert.Equal(1376, input.ToFeatureVector().Values.Count); } [Fact] @@ -103,4 +120,6 @@ public sealed class UtteranceEmbeddingLearningTests private static float Lerp(float left, float right, float amount) => left + (right - left) * amount; + + private static float[] Zeros(int count) => new float[count]; }