Adopt PanPhon realization channel
This commit is contained in:
Meta Vens
parent
13c29fea12
commit
21c1046f9d
11 changed files with 89 additions and 27 deletions
|
|
@ -236,8 +236,8 @@ The first Weksa handoff target is `weksa.utterance_embedding_handoff.v0.1`.
|
|||
AquaSynth should treat it as the training input contract:
|
||||
|
||||
- 1024-float `bge-m3:latest` speech text embedding for semantic text evidence;
|
||||
- 256-float phonetic realization vector for IPA, phones, pronunciation, and
|
||||
alien speech-shape evidence;
|
||||
- 256-float PanPhon-based phonetic realization vector for IPA, phones,
|
||||
pronunciation, and alien speech-shape evidence;
|
||||
- 32-float deterministic prosody/emphasis hint vector;
|
||||
- 64-float projected Ghostlight/Epiphany-shaped character-state vector;
|
||||
- 64-float AquaSynth-owned learned utterance embedding output.
|
||||
|
|
|
|||
|
|
@ -121,10 +121,10 @@ beautiful falsehood.
|
|||
Do not start serious gradient descent from toy embedding packets. The first
|
||||
training contract coordinated with Weksa is
|
||||
`weksa.utterance_embedding_handoff.v0.1`: 1024 speech-text floats from
|
||||
`bge-m3:latest`, 256 phonetic-realization floats for IPA/phones/alien speech
|
||||
shape, 32 deterministic prosody/emphasis floats, 64 projected character-state
|
||||
floats, and a 64-float AquaSynth-owned utterance embedding output. Anything
|
||||
smaller is a plumbing fixture, not a training corpus.
|
||||
`bge-m3:latest`, 256 PanPhon-based phonetic-realization floats for
|
||||
IPA/phones/alien speech shape, 32 deterministic prosody/emphasis floats, 64
|
||||
projected character-state floats, and a 64-float AquaSynth-owned utterance
|
||||
embedding output. Anything smaller is a plumbing fixture, not a training corpus.
|
||||
|
||||
Keep semantic text and phonetic realization separate. English text embeddings
|
||||
may help preserve meaning and phrasing, but an IPA string is not English prose
|
||||
|
|
|
|||
|
|
@ -71,13 +71,14 @@ public sealed record SpeechTimingReceipt(
|
|||
[MessagePackObject]
|
||||
public sealed record SpeechUtteranceEmbeddingInputSnapshot(
|
||||
[property: Key(0)] float[] TextEmbedding,
|
||||
[property: Key(1)] float[] Prosody,
|
||||
[property: Key(2)] float[] CharacterState)
|
||||
[property: Key(1)] float[] PhoneticRealization,
|
||||
[property: Key(2)] float[] Prosody,
|
||||
[property: Key(3)] float[] CharacterState)
|
||||
{
|
||||
public UtteranceEmbeddingInput ToInput() => new(TextEmbedding, Prosody, CharacterState);
|
||||
public UtteranceEmbeddingInput ToInput() => new(TextEmbedding, PhoneticRealization, Prosody, CharacterState);
|
||||
|
||||
public static SpeechUtteranceEmbeddingInputSnapshot From(UtteranceEmbeddingInput input) =>
|
||||
new([.. input.SpeechTextEmbedding], [.. input.ProsodyAndEmphasisHints], [.. input.CharacterStateVector]);
|
||||
new([.. input.SpeechTextEmbedding], [.. input.PhoneticRealizationVector], [.. input.ProsodyAndEmphasisHints], [.. input.CharacterStateVector]);
|
||||
}
|
||||
|
||||
[MessagePackObject]
|
||||
|
|
|
|||
|
|
@ -2,13 +2,15 @@ namespace AquaSynth.Dsl;
|
|||
|
||||
public sealed record UtteranceEmbeddingInput(
|
||||
IReadOnlyList<float> SpeechTextEmbedding,
|
||||
IReadOnlyList<float> PhoneticRealizationVector,
|
||||
IReadOnlyList<float> ProsodyAndEmphasisHints,
|
||||
IReadOnlyList<float> CharacterStateVector)
|
||||
{
|
||||
public UtteranceEmbeddingFeatureVector ToFeatureVector()
|
||||
{
|
||||
var values = new float[SpeechTextEmbedding.Count + ProsodyAndEmphasisHints.Count + CharacterStateVector.Count];
|
||||
var values = new float[SpeechTextEmbedding.Count + PhoneticRealizationVector.Count + ProsodyAndEmphasisHints.Count + CharacterStateVector.Count];
|
||||
var offset = Copy(SpeechTextEmbedding, values, 0);
|
||||
offset = Copy(PhoneticRealizationVector, values, offset);
|
||||
offset = Copy(ProsodyAndEmphasisHints, values, offset);
|
||||
Copy(CharacterStateVector, values, offset);
|
||||
return new UtteranceEmbeddingFeatureVector(values);
|
||||
|
|
@ -25,6 +27,35 @@ public sealed record UtteranceEmbeddingInput(
|
|||
}
|
||||
}
|
||||
|
||||
public static class WeksaUtteranceEmbeddingHandoff
|
||||
{
|
||||
public const string SchemaVersion = "weksa.utterance_embedding_handoff.v0.1";
|
||||
public const string SpeechTextEmbeddingModelId = "bge-m3:latest";
|
||||
public const string PhoneticRealizationModelId = "weksa.panphon_realization.v0.1";
|
||||
public const int SpeechTextEmbeddingSize = 1024;
|
||||
public const int PhoneticRealizationVectorSize = 256;
|
||||
public const int ProsodyAndEmphasisHintSize = 32;
|
||||
public const int CharacterStateVectorSize = 64;
|
||||
public const int UtteranceEmbeddingSize = 64;
|
||||
public const int InputSize = SpeechTextEmbeddingSize + PhoneticRealizationVectorSize + ProsodyAndEmphasisHintSize + CharacterStateVectorSize;
|
||||
|
||||
public static void Validate(UtteranceEmbeddingInput input)
|
||||
{
|
||||
ValidateSize(input.SpeechTextEmbedding, SpeechTextEmbeddingSize, nameof(input.SpeechTextEmbedding));
|
||||
ValidateSize(input.PhoneticRealizationVector, PhoneticRealizationVectorSize, nameof(input.PhoneticRealizationVector));
|
||||
ValidateSize(input.ProsodyAndEmphasisHints, ProsodyAndEmphasisHintSize, nameof(input.ProsodyAndEmphasisHints));
|
||||
ValidateSize(input.CharacterStateVector, CharacterStateVectorSize, nameof(input.CharacterStateVector));
|
||||
}
|
||||
|
||||
private static void ValidateSize(IReadOnlyList<float> values, int expected, string name)
|
||||
{
|
||||
if (values.Count != expected)
|
||||
{
|
||||
throw new ArgumentException($"{SchemaVersion} requires {name} to have {expected} values", name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record UtteranceEmbeddingFeatureVector(IReadOnlyList<float> Values);
|
||||
|
||||
public sealed record UtteranceEmbedding(IReadOnlyList<float> Values)
|
||||
|
|
|
|||
|
|
@ -122,4 +122,4 @@
|
|||
{"date":"2026-05-20","topic":"utterance-parity-current-objective","lesson":"AquaSynth's active objective is now the utterance parity curriculum: structured utterance metadata lowers into a learned utterance embedding, then into learned synth automation for the vocal tract and associated envelopes/LFOs/filters/spectral lanes, then into rendered speech loss. Broad DX7/Zyn synth digestion is backburner work unless it directly improves this harness.","evidence":"Updated README, docs/audio-doctrine.md, docs/reference-synth-roadmap.md, docs/ipa-vocal-tract-roadmap.md, state/map.yaml, state/spine.yaml, and state/memory.json to make the speech parity harness the current center."}
|
||||
{"date":"2026-05-20","topic":"utterance-parity-prior-work","lesson":"Prior work supports AquaSynth's architecture when kept in its lane: Kelly-Lochbaum/Pink Trombone and Story area functions support tract/area authority; TADA/articulatory phonology supports overlapping gestures; eSpeak/Klatt-style formant synthesis is useful fast ground truth but not anatomy; DDSP/neural source-filter work supports learned controllers over interpretable DSP rather than black-box throat replacement.","evidence":"Added docs/utterance-parity-research.md with source ledger covering Pink Trombone, VocalTractLab, Story, TADA, eSpeak NG, PanPhon, PHOIBLE, DDSP, DDSP articulatory vocoders, neural source-filter models, and LF voice-source references."}
|
||||
{"date":"2026-05-20","topic":"speech-training-time-witness","lesson":"Speech training receipts need time and confidence as evidence, not only loss. Each curriculum step should record when the stage decided, observed latency, intended latency budget, confidence, and artifact/model identifiers; a good-sounding result from an unbounded clockless path is not trustworthy.","evidence":"Added SpeechTimingReceipt to SpeechRenderRequest, SpeechRenderResult, and SpeechTrainingCheckpoint; seeded timing/confidence receipts in the local distributed-training proof; updated audio doctrine, IPA roadmap, utterance parity research notes, spine, and memory."}
|
||||
{"date":"2026-05-20","topic":"weksa-utterance-embedding-contract","lesson":"Do not start serious AquaSynth speech gradient descent from toy Weksa vectors, and do not treat semantic text embeddings as IPA embeddings. The coordinated Weksa handoff contract is `weksa.utterance_embedding_handoff.v0.1`: 1024 `bge-m3:latest` semantic speech-text floats, 256 phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth-owned utterance embedding output.","evidence":"Updated E:/Projects/weksa docs/schema/state to split speech text embedding from phonetic realization and marked the existing Nibu toy fixture as non-training; mirrored the training block in AquaSynth IPA roadmap, utterance parity research notes, spine, and memory."}
|
||||
{"date":"2026-05-20","topic":"weksa-utterance-embedding-contract","lesson":"Do not start serious AquaSynth speech gradient descent from toy Weksa vectors, and do not treat semantic text embeddings as IPA embeddings. The coordinated Weksa handoff contract is `weksa.utterance_embedding_handoff.v0.1`: 1024 `bge-m3:latest` semantic speech-text floats, 256 PanPhon-based phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth-owned utterance embedding output.","evidence":"Updated E:/Projects/weksa docs/schema/state to split speech text embedding from PanPhon-based phonetic realization and marked the existing Nibu toy fixture as non-training; mirrored the training block in AquaSynth IPA roadmap, utterance parity research notes, spine, and memory."}
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@
|
|||
"ipa_vocal_tract_path": {
|
||||
"status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets.",
|
||||
"lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.",
|
||||
"next": "Do not start serious speech gradient descent from toy embedding packets. Mirror Weksa `weksa.utterance_embedding_handoff.v0.1` first: 1024 bge-m3 semantic speech-text floats, 256 phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Then tighten the parity harness: render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop."
|
||||
"next": "Do not start serious speech gradient descent from toy embedding packets. Mirror Weksa `weksa.utterance_embedding_handoff.v0.1` first: 1024 bge-m3 semantic speech-text floats, 256 PanPhon-based phonetic-realization floats for IPA/phones/alien speech shape, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Then tighten the parity harness: render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop."
|
||||
},
|
||||
"dx7_reference_path": {
|
||||
"status": "DX7 SysEx parser extracts voice edit buffers, packed voices, packed 32-voice banks, six operators, envelopes, algorithm, feedback, oscillator sync, LFO, transpose, voice name, checksum validation, ReferencePatch features, algorithm topology from the reverse-engineered OPS algorithm ROM table, envelope-to-ADSR approximations, measured carrier output amplitude, isolated two-op modulation route scale, summed/cascaded route scale, calibrated feedback amount table, ratio detune lowering, voice transpose lowering, fixed-frequency operator lowering, ROM COM output compensation cut, operator key/rate scaling at the effective played note, topology-aware route lowering, staged operator envelopes with per-segment curves, raw and block-interpolated DX7 envelope tracing, applied-envelope curved lowering with optional sustain floor, two AquaSynth DSL rebuild attempts, first-class OperatorGraph support with graph-level vibrato, public-domain SysEx fixture coverage, dexed-py reference rendering, project-authored Dexed patch rendering, AquaSynth candidate rendering through Faust-generated C#, one thresholded rendered parity test for analog1.syx voice 13, a hard PRC SYNTH1 parity probe with listening WAV artifacts and log-mel <= 0.145, project-authored algorithm-8 summed-stack/cascade/summed-pair parity, rendered community parity for Mooger, Piano Bass, RES SYNTH1, and ANLGSYN 1 with a sustained high-band gate, a MELLOWSOLO pressure artifact test, and renderable delayed operator feedback.",
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ live_invariants:
|
|||
- eSpeak NG is an early parity reference for timing, IPA coverage, and log-mel pressure, not an anatomy oracle or final target.
|
||||
- Do not expand into prosody, emotional context, personality, or broad Weksa expressivity until the tiny IPA utterance harness can show useful loss movement.
|
||||
- Speech training receipts must keep time as a witness: every request, render result, and checkpoint should record stage decision time, latency spent, latency budget, confidence, and artifact/model identifiers.
|
||||
- Serious speech gradient descent is blocked until Weksa packets use `weksa.utterance_embedding_handoff.v0.1` or a deliberate successor: 1024 semantic speech-text floats, 256 phonetic-realization floats, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth utterance embedding output.
|
||||
- Serious speech gradient descent is blocked until Weksa packets use `weksa.utterance_embedding_handoff.v0.1` or a deliberate successor: 1024 semantic speech-text floats, 256 PanPhon-based phonetic-realization floats, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth utterance embedding output.
|
||||
- Do not use semantic text embeddings as a proxy for IPA or alien phonetic strings. Phonetic/IPA realization is a separate input channel.
|
||||
|
||||
reference_targets:
|
||||
|
|
|
|||
|
|
@ -89,8 +89,19 @@ public sealed class EspeakNgGradientDescentTests
|
|||
private static TrainingFixture Fixture(string id, string text, PhoneticFeatures features, IReadOnlyList<float> textEmbedding)
|
||||
{
|
||||
var prosody = new[] { 0.75f, text.Length / 4f, features.Manner == PhoneticManner.Fricative ? 0.95f : 0.35f, features.Manner == PhoneticManner.Nasal ? 0.80f : 0.25f };
|
||||
var phoneticRealization = new[]
|
||||
{
|
||||
features.Manner == PhoneticManner.Vowel ? 1f : 0f,
|
||||
features.Manner == PhoneticManner.Stop ? 1f : 0f,
|
||||
features.Manner == PhoneticManner.Fricative ? 1f : 0f,
|
||||
features.Manner == PhoneticManner.Nasal ? 1f : 0f,
|
||||
features.Place == PhoneticPlace.Bilabial ? 1f : 0f,
|
||||
features.Place == PhoneticPlace.Alveolar ? 1f : 0f,
|
||||
features.Place == PhoneticPlace.Velar ? 1f : 0f,
|
||||
features.Phonation == Phonation.Voiced ? 1f : 0f
|
||||
};
|
||||
var characterState = new[] { 0.65f, 0.35f, 0.55f, features.Place == PhoneticPlace.Bilabial ? 0.72f : 0.28f };
|
||||
var input = new UtteranceEmbeddingInput(textEmbedding, prosody, characterState);
|
||||
var input = new UtteranceEmbeddingInput(textEmbedding, phoneticRealization, prosody, characterState);
|
||||
var phoneticEvent = new PhoneticEvent(id, text, features, DurationSeconds: 0.20, Prosody: new PhoneticProsody(Stress: 0.75f, Intensity: 0.8f));
|
||||
return new TrainingFixture(id, text, "en", phoneticEvent, input);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ public sealed class SpeechBackpropagationPipelineTests
|
|||
public void SpeechPipelineBackpropagatesSynthDriverLossIntoUtteranceEncoder()
|
||||
{
|
||||
var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
|
||||
inputSize: 10,
|
||||
inputSize: 13,
|
||||
embeddingSize: 16,
|
||||
hiddenLayerSizes: [48, 32, 24],
|
||||
seed: 211);
|
||||
|
|
@ -37,7 +37,7 @@ public sealed class SpeechBackpropagationPipelineTests
|
|||
public void SpeechPipelineAcceptsExternalRenderedLossGradient()
|
||||
{
|
||||
var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
|
||||
inputSize: 10,
|
||||
inputSize: 13,
|
||||
embeddingSize: 8,
|
||||
hiddenLayerSizes: [24, 16],
|
||||
seed: 301);
|
||||
|
|
@ -50,7 +50,7 @@ public sealed class SpeechBackpropagationPipelineTests
|
|||
var example = Example(
|
||||
"a",
|
||||
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
|
||||
new UtteranceEmbeddingInput([0.9f, 0.1f, 0.2f, 0.7f], [0.8f, 0.3f, 0.2f], [0.6f, 0.2f, 0.4f]),
|
||||
new UtteranceEmbeddingInput([0.9f, 0.1f, 0.2f, 0.7f], [0.9f, 0.2f, 0.1f], [0.8f, 0.3f, 0.2f], [0.6f, 0.2f, 0.4f]),
|
||||
SmallTarget([0.9f, 0.7f, 0.3f, 0.1f]));
|
||||
var before = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount);
|
||||
var gradient = new float[pipeline.SynthDriver.OutputSize];
|
||||
|
|
@ -77,17 +77,17 @@ public sealed class SpeechBackpropagationPipelineTests
|
|||
Example(
|
||||
"a",
|
||||
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
|
||||
new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]),
|
||||
new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.90f, 0.18f, 0.10f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]),
|
||||
Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f, 0.34f, 0.28f, 0.22f, 0.18f, 0.16f, 0.14f])),
|
||||
Example(
|
||||
"sa",
|
||||
new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless),
|
||||
new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]),
|
||||
new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.22f, 0.86f, 0.76f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]),
|
||||
Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f, 0.76f, 0.88f, 0.94f, 0.90f, 0.82f, 0.72f])),
|
||||
Example(
|
||||
"ma",
|
||||
new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true),
|
||||
new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]),
|
||||
new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.70f, 0.60f, 0.82f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]),
|
||||
Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f, 0.40f, 0.36f, 0.32f, 0.30f, 0.28f, 0.26f]))
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ public sealed class SpeechDistributedTrainingTests
|
|||
private static SpeechBackpropagationPipeline Pipeline()
|
||||
{
|
||||
var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
|
||||
inputSize: 10,
|
||||
inputSize: 13,
|
||||
embeddingSize: 8,
|
||||
hiddenLayerSizes: [24, 16],
|
||||
seed: 701);
|
||||
|
|
@ -106,17 +106,17 @@ public sealed class SpeechDistributedTrainingTests
|
|||
Example(
|
||||
"a",
|
||||
new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
|
||||
new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]),
|
||||
new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.90f, 0.18f, 0.10f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]),
|
||||
Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f])),
|
||||
Example(
|
||||
"sa",
|
||||
new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless),
|
||||
new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]),
|
||||
new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.22f, 0.86f, 0.76f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]),
|
||||
Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f])),
|
||||
Example(
|
||||
"ma",
|
||||
new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true),
|
||||
new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]),
|
||||
new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.70f, 0.60f, 0.82f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]),
|
||||
Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f]))
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -5,16 +5,33 @@ namespace AquaSynth.Dsl.Tests;
|
|||
public sealed class UtteranceEmbeddingLearningTests
|
||||
{
|
||||
[Fact]
|
||||
public void UtteranceEmbeddingInputPacksTextProsodyAndCharacterState()
|
||||
public void UtteranceEmbeddingInputPacksTextPhoneticsProsodyAndCharacterState()
|
||||
{
|
||||
var input = new UtteranceEmbeddingInput(
|
||||
SpeechTextEmbedding: [0.1f, 0.2f, 0.3f],
|
||||
PhoneticRealizationVector: [0.9f, 0.6f],
|
||||
ProsodyAndEmphasisHints: [0.7f, 0.8f],
|
||||
CharacterStateVector: [0.4f, 0.5f, 0.6f]);
|
||||
|
||||
var features = input.ToFeatureVector();
|
||||
|
||||
Assert.Equal([0.1f, 0.2f, 0.3f, 0.7f, 0.8f, 0.4f, 0.5f, 0.6f], features.Values);
|
||||
Assert.Equal([0.1f, 0.2f, 0.3f, 0.9f, 0.6f, 0.7f, 0.8f, 0.4f, 0.5f, 0.6f], features.Values);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WeksaV01ContractRequiresSeparatePanphonRealizationChannel()
|
||||
{
|
||||
var input = new UtteranceEmbeddingInput(
|
||||
SpeechTextEmbedding: Zeros(WeksaUtteranceEmbeddingHandoff.SpeechTextEmbeddingSize),
|
||||
PhoneticRealizationVector: Zeros(WeksaUtteranceEmbeddingHandoff.PhoneticRealizationVectorSize),
|
||||
ProsodyAndEmphasisHints: Zeros(WeksaUtteranceEmbeddingHandoff.ProsodyAndEmphasisHintSize),
|
||||
CharacterStateVector: Zeros(WeksaUtteranceEmbeddingHandoff.CharacterStateVectorSize));
|
||||
|
||||
WeksaUtteranceEmbeddingHandoff.Validate(input);
|
||||
|
||||
Assert.Equal("weksa.utterance_embedding_handoff.v0.1", WeksaUtteranceEmbeddingHandoff.SchemaVersion);
|
||||
Assert.Equal("weksa.panphon_realization.v0.1", WeksaUtteranceEmbeddingHandoff.PhoneticRealizationModelId);
|
||||
Assert.Equal(1376, input.ToFeatureVector().Values.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
|
@ -103,4 +120,6 @@ public sealed class UtteranceEmbeddingLearningTests
|
|||
|
||||
private static float Lerp(float left, float right, float amount) =>
|
||||
left + (right - left) * amount;
|
||||
|
||||
private static float[] Zeros(int count) => new float[count];
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue