Add distributed speech training cache contracts

src/AquaSynth.Core/SpeechDistributedTraining.cs

@@ -0,0 +1,308 @@
+using GameCult.Caching;
+using GameCult.Caching.MessagePack;
+using MessagePack;
+
+namespace AquaSynth.Dsl;
+
+[CultDocument("aquasynth.speech_render_request", "aquasynth.speech_render_request.v1")]
+[MessagePackObject]
+public sealed record SpeechRenderRequest(
+    [property: Key(0)]
+    [property: CultName]
+    string RequestId,
+    [property: Key(1)] string BatchId,
+    [property: Key(2)] string CreatedAtUtc,
+    [property: Key(3)] string CurriculumTier,
+    [property: Key(4)] string TargetReferenceId,
+    [property: Key(5)] string MorphologyId,
+    [property: Key(6)] string RendererProfileId,
+    [property: Key(7)] string ScoringRecipeId,
+    [property: Key(8)] SpeechUtteranceEmbeddingInputSnapshot UtteranceInput,
+    [property: Key(9)] SpeechPhoneticEventSnapshot PhoneticEvent,
+    [property: Key(10)] float[] CandidateControlVector,
+    [property: Key(11)] float[] ReferenceControlVector);
+
+[CultDocument("aquasynth.speech_render_result", "aquasynth.speech_render_result.v1")]
+[MessagePackObject]
+public sealed record SpeechRenderResult(
+    [property: Key(0)]
+    [property: CultName]
+    string ResultId,
+    [property: Key(1)]
+    [property: CultIndex("request")]
+    string RequestId,
+    [property: Key(2)] string WorkerId,
+    [property: Key(3)] string RendererToolchain,
+    [property: Key(4)] string CompletedAtUtc,
+    [property: Key(5)] SpeechRenderStatus Status,
+    [property: Key(6)] float Loss,
+    [property: Key(7)] float[] OutputGradient,
+    [property: Key(8)] SpeechScoreMetric[] Metrics,
+    [property: Key(9)] SpeechRenderArtifact[] Artifacts,
+    [property: Key(10)] string FailureCode = "",
+    [property: Key(11)] string FailureMessage = "");
+
+[CultDocument("aquasynth.speech_training_checkpoint", "aquasynth.speech_training_checkpoint.v1")]
+[MessagePackObject]
+public sealed record SpeechTrainingCheckpoint(
+    [property: Key(0)]
+    [property: CultName]
+    string CheckpointId,
+    [property: Key(1)] string BatchId,
+    [property: Key(2)] string CreatedAtUtc,
+    [property: Key(3)] int AppliedResultCount,
+    [property: Key(4)] float MeanAppliedLoss,
+    [property: Key(5)] string[] AppliedResultIds,
+    [property: Key(6)] string Notes);
+
+[MessagePackObject]
+public sealed record SpeechUtteranceEmbeddingInputSnapshot(
+    [property: Key(0)] float[] TextEmbedding,
+    [property: Key(1)] float[] Prosody,
+    [property: Key(2)] float[] CharacterState)
+{
+    public UtteranceEmbeddingInput ToInput() => new(TextEmbedding, Prosody, CharacterState);
+
+    public static SpeechUtteranceEmbeddingInputSnapshot From(UtteranceEmbeddingInput input) =>
+        new([.. input.SpeechTextEmbedding], [.. input.ProsodyAndEmphasisHints], [.. input.CharacterStateVector]);
+}
+
+[MessagePackObject]
+public sealed record SpeechPhoneticEventSnapshot(
+    [property: Key(0)] string Id,
+    [property: Key(1)] string Ipa,
+    [property: Key(2)] PhoneticFeatures Features,
+    [property: Key(3)] double StartSeconds,
+    [property: Key(4)] double DurationSeconds,
+    [property: Key(5)] PhoneticProsody Prosody)
+{
+    public PhoneticEvent ToEvent() => new(Id, Ipa, Features, StartSeconds, DurationSeconds, Prosody);
+
+    public static SpeechPhoneticEventSnapshot From(PhoneticEvent phoneticEvent) =>
+        new(
+            phoneticEvent.Id,
+            phoneticEvent.Ipa,
+            phoneticEvent.Features,
+            phoneticEvent.StartSeconds,
+            phoneticEvent.DurationSeconds,
+            phoneticEvent.Prosody);
+}
+
+[MessagePackObject]
+public sealed record SpeechScoreMetric(
+    [property: Key(0)] string Name,
+    [property: Key(1)] float Value,
+    [property: Key(2)] float Weight);
+
+[MessagePackObject]
+public sealed record SpeechRenderArtifact(
+    [property: Key(0)] string Kind,
+    [property: Key(1)] string Uri,
+    [property: Key(2)] string ContentHash);
+
+public enum SpeechRenderStatus
+{
+    Succeeded,
+    Failed
+}
+
+public sealed record SpeechDistributedTrainingOptions(
+    string CurriculumTier = "tiny-local-proof",
+    string TargetReferenceId = "local/reference-control-vector",
+    string MorphologyId = "neutral-human-ish",
+    string RendererProfileId = "compiled-faust-worker",
+    string ScoringRecipeId = "control-vector-mse-proof",
+    float UtteranceLearningRate = 0.04f,
+    float SynthDriverLearningRate = 0.04f);
+
+public sealed record SpeechDistributedTrainingApplyResult(
+    SpeechTrainingCheckpoint Checkpoint,
+    IReadOnlyList<PackedNeuralBackpropagation> Backpropagations);
+
+public static class SpeechDistributedTrainingCultCacheStore
+{
+    public static Task UpsertRequestsAsync(string filePath, IEnumerable<SpeechRenderRequest> requests) =>
+        UpsertAsync(filePath, requests, request => $"speech-render-request:{request.RequestId}");
+
+    public static Task UpsertResultsAsync(string filePath, IEnumerable<SpeechRenderResult> results) =>
+        UpsertAsync(filePath, results, result => $"speech-render-result:{result.ResultId}");
+
+    public static Task UpsertCheckpointsAsync(string filePath, IEnumerable<SpeechTrainingCheckpoint> checkpoints) =>
+        UpsertAsync(filePath, checkpoints, checkpoint => $"speech-training-checkpoint:{checkpoint.CheckpointId}");
+
+    public static Task<IReadOnlyList<SpeechRenderRequest>> ReadRequestsAsync(string filePath) =>
+        ReadAllAsync<SpeechRenderRequest>(filePath, request => request.RequestId);
+
+    public static Task<IReadOnlyList<SpeechRenderResult>> ReadResultsAsync(string filePath) =>
+        ReadAllAsync<SpeechRenderResult>(filePath, result => result.ResultId);
+
+    public static Task<IReadOnlyList<SpeechTrainingCheckpoint>> ReadCheckpointsAsync(string filePath) =>
+        ReadAllAsync<SpeechTrainingCheckpoint>(filePath, checkpoint => checkpoint.CheckpointId);
+
+    private static async Task UpsertAsync<T>(
+        string filePath,
+        IEnumerable<T> documents,
+        Func<T, string> key)
+        where T : class
+    {
+        using var cache = await OpenForMutationAsync(filePath).ConfigureAwait(false);
+        foreach (var document in documents)
+        {
+            await cache.UpsertAsync(document, new CultRecordHandle<T>(new CultRecordKey(key(document)))).ConfigureAwait(false);
+        }
+
+        await cache.FlushAsync().ConfigureAwait(false);
+    }
+
+    private static async Task<IReadOnlyList<T>> ReadAllAsync<T>(string filePath, Func<T, string> orderKey)
+        where T : class
+    {
+        if (!File.Exists(filePath))
+        {
+            return [];
+        }
+
+        using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false);
+        return cache.GetAll<T>().OrderBy(orderKey, StringComparer.Ordinal).ToArray();
+    }
+
+    private static async Task<CultCache> OpenForMutationAsync(string filePath)
+    {
+        var options = new CultCacheOpenOptions { PullOnOpen = File.Exists(filePath) };
+        return await CultCacheMessagePack.OpenAsync(filePath, options).ConfigureAwait(false);
+    }
+}
+
+public static class SpeechDistributedTrainingCoordinator
+{
+    public static IReadOnlyList<SpeechRenderRequest> CreateRenderRequests(
+        string batchId,
+        SpeechBackpropagationPipeline pipeline,
+        IReadOnlyList<SpeechBackpropagationTrainingExample> examples,
+        SpeechDistributedTrainingOptions? options = null)
+    {
+        options ??= new SpeechDistributedTrainingOptions();
+        var createdAt = DateTimeOffset.UtcNow.ToString("O");
+        var requests = new SpeechRenderRequest[examples.Count];
+        for (var index = 0; index < examples.Count; index++)
+        {
+            var example = examples[index];
+            requests[index] = new SpeechRenderRequest(
+                $"{batchId}:{index:0000}",
+                batchId,
+                createdAt,
+                options.CurriculumTier,
+                options.TargetReferenceId,
+                options.MorphologyId,
+                options.RendererProfileId,
+                options.ScoringRecipeId,
+                SpeechUtteranceEmbeddingInputSnapshot.From(example.UtteranceInput),
+                SpeechPhoneticEventSnapshot.From(example.Event),
+                pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount),
+                example.Target.ToVector(pipeline.SynthDriver.MelBandCount));
+        }
+
+        return requests;
+    }
+
+    public static IReadOnlyList<SpeechRenderResult> ScoreControlVectorRequests(
+        IReadOnlyList<SpeechRenderRequest> requests,
+        string workerId,
+        string rendererToolchain = "local-control-vector-proof")
+    {
+        var completedAt = DateTimeOffset.UtcNow.ToString("O");
+        var results = new SpeechRenderResult[requests.Count];
+        for (var index = 0; index < requests.Count; index++)
+        {
+            var request = requests[index];
+            if (request.CandidateControlVector.Length != request.ReferenceControlVector.Length)
+            {
+                results[index] = new SpeechRenderResult(
+                    $"{request.RequestId}:failed",
+                    request.RequestId,
+                    workerId,
+                    rendererToolchain,
+                    completedAt,
+                    SpeechRenderStatus.Failed,
+                    0,
+                    [],
+                    [],
+                    [],
+                    "control_vector_size_mismatch",
+                    "Candidate and reference control vectors must have the same length.");
+                continue;
+            }
+
+            var gradient = new float[request.CandidateControlVector.Length];
+            var loss = 0f;
+            for (var valueIndex = 0; valueIndex < gradient.Length; valueIndex++)
+            {
+                var difference = request.CandidateControlVector[valueIndex] - request.ReferenceControlVector[valueIndex];
+                gradient[valueIndex] = 2f * difference / gradient.Length;
+                loss += difference * difference;
+            }
+
+            loss /= gradient.Length;
+            results[index] = new SpeechRenderResult(
+                $"{request.RequestId}:score",
+                request.RequestId,
+                workerId,
+                rendererToolchain,
+                completedAt,
+                SpeechRenderStatus.Succeeded,
+                loss,
+                gradient,
+                [new SpeechScoreMetric("control_mse", loss, 1)],
+                [new SpeechRenderArtifact("score-report", $"cultcache://speech-render-result/{request.RequestId}", "")]);
+        }
+
+        return results;
+    }
+
+    public static SpeechDistributedTrainingApplyResult ApplyResults(
+        SpeechBackpropagationPipeline pipeline,
+        IReadOnlyList<SpeechRenderRequest> requests,
+        IReadOnlyList<SpeechRenderResult> results,
+        string checkpointId,
+        SpeechDistributedTrainingOptions? options = null)
+    {
+        options ??= new SpeechDistributedTrainingOptions();
+        var requestById = requests.ToDictionary(request => request.RequestId, StringComparer.Ordinal);
+        var applied = new List<PackedNeuralBackpropagation>();
+        var appliedIds = new List<string>();
+        var loss = 0f;
+
+        foreach (var result in results.Where(result => result.Status == SpeechRenderStatus.Succeeded))
+        {
+            if (!requestById.TryGetValue(result.RequestId, out var request))
+            {
+                continue;
+            }
+
+            applied.Add(pipeline.TrainSingleFromSynthOutputGradient(
+                request.UtteranceInput.ToInput(),
+                request.PhoneticEvent.ToEvent(),
+                result.OutputGradient,
+                options.UtteranceLearningRate,
+                options.SynthDriverLearningRate,
+                result.Loss));
+            appliedIds.Add(result.ResultId);
+            loss += result.Loss;
+        }
+
+        if (applied.Count == 0)
+        {
+            throw new InvalidOperationException("no successful speech render results matched the request batch");
+        }
+
+        var checkpoint = new SpeechTrainingCheckpoint(
+            checkpointId,
+            requests.FirstOrDefault()?.BatchId ?? "",
+            DateTimeOffset.UtcNow.ToString("O"),
+            applied.Count,
+            loss / applied.Count,
+            [.. appliedIds],
+            "Applied remote-render score gradients to the utterance encoder and synth driver.");
+        return new SpeechDistributedTrainingApplyResult(checkpoint, applied);
+    }
+}

src/AquaSynth.Core/VocalTractModel.cs

@@ -1,3 +1,5 @@
+using MessagePack;
+
 namespace AquaSynth.Dsl;
 
 public enum PhoneticManner
@@ -68,22 +70,38 @@ public enum VowelBackness
     Back
 }
 
+[MessagePackObject]
 public sealed record PhoneticFeatures(
+    [property: Key(0)]
     PhoneticManner Manner,
+    [property: Key(1)]
     PhoneticPlace Place = PhoneticPlace.None,
+    [property: Key(2)]
     Phonation Phonation = Phonation.Voiced,
+    [property: Key(3)]
     AirstreamMechanism Airstream = AirstreamMechanism.PulmonicEgressive,
+    [property: Key(4)]
     VowelHeight Height = VowelHeight.None,
+    [property: Key(5)]
     VowelBackness Backness = VowelBackness.None,
+    [property: Key(6)]
     bool Rounded = false,
+    [property: Key(7)]
     bool Long = false,
+    [property: Key(8)]
     bool Nasalized = false,
+    [property: Key(9)]
     bool Lateral = false);
 
+[MessagePackObject]
 public sealed record PhoneticProsody(
+    [property: Key(0)]
     float Stress = 0,
+    [property: Key(1)]
     float PitchTarget = 0,
+    [property: Key(2)]
     float Intensity = 1,
+    [property: Key(3)]
     string Register = "");
 
 public sealed record PhoneticEvent(

state/evidence.jsonl

@@ -118,3 +118,4 @@
 {"date":"2026-05-19","topic":"compiled-faust-speech-loss-surface","lesson":"The usable speech loss surface should compile topology once and vary every neural mapper output through exported Faust controls. Batch rendering through a native compiled factory makes finite-difference and curriculum learning plausible; recompiling per candidate is only a proof scaffold.","evidence":"Added native Faust UI control discovery/set support, AquaSynthCompiledPatch.Render(controlValues), and CompiledFaustRenderedSpeechLossSurface with speech/output/N knobs for all mapper outputs. The opt-in proof renders multiple candidates through one compiled patch and caught the need to clamp dynamic filter cutoff below Nyquist."}
 {"date":"2026-05-19","topic":"cultcache-sound-claim-cards","lesson":"Serious AquaSynth sound claims should be distributed as typed data beside the patch database. Patch and speech claim cards must name the reference/source, intent, tier, proof artifact, latest listening receipt surface, metrics, and known lies; stock sketches stay in the database but should not masquerade as parity without receipts.","evidence":"Added CultCachePatchDocument types, patches/aquasynth-patch-cultcache.json, and tests that load the typed document, check patch database count against library.yaml, and require every ReferenceRebuildCatalog entry to have a claim card."}
 {"date":"2026-05-19","topic":"cultcache-cc-artifact-boundary","lesson":"AquaSynth must not impersonate CultCache persistence. The sound-claim fixture should be a real .cc CultCache store produced and consumed through CultLib; AquaSynth owns typed sound-claim meaning, while CultLib owns serialization, schema cataloging, record keys, and MessagePack backing-store mechanics.","evidence":"Deleted the local CultCacheBinaryCodec, replaced the JSON cache artifact with patches/aquasynth-patch-cultcache.cc, added a GameCult.Caching.MessagePack project reference, and updated CultCachePatchDocumentTests to load the fixture through CultCachePatchDocumentStore.ReadAsync."}
+{"date":"2026-05-19","topic":"distributed-speech-training-cache-contract","lesson":"The speech-learning machine should distribute expensive render evidence instead of renderer authority. AquaSynth emits typed speech render requests, Faust workers write typed score results, and the trainer consumes synced gradients to update the utterance encoder and synth driver; CultCache/CultMesh carry the documents between machines.","evidence":"Added SpeechDistributedTraining.cs with SpeechRenderRequest, SpeechRenderResult, SpeechTrainingCheckpoint, CultCache store helpers, and a coordinator proof. SpeechDistributedTrainingTests round-trip request/result/checkpoint .cc stores and prove synced score gradients reduce local pipeline loss."}

state/map.yaml

@@ -35,6 +35,7 @@ modules:
   packed_neural_network: src/AquaSynth.Core/PackedNeuralNetwork.cs
   utterance_embedding_learning: src/AquaSynth.Core/UtteranceEmbeddingLearning.cs
   speech_backpropagation_pipeline: src/AquaSynth.Core/SpeechBackpropagationPipeline.cs
+  speech_distributed_training: src/AquaSynth.Core/SpeechDistributedTraining.cs
   vocal_tract_learning: src/AquaSynth.Core/VocalTractLearning.cs
   faust_emitter: src/AquaSynth.Core/FaustExport.cs
   faust_compiler: src/AquaSynth.Faust/FaustCompiler.cs
@@ -52,6 +53,7 @@ modules:
   vocal_tract_model_tests: tests/AquaSynth.Dsl.Tests/VocalTractModelTests.cs
   utterance_embedding_learning_tests: tests/AquaSynth.Dsl.Tests/UtteranceEmbeddingLearningTests.cs
   speech_backpropagation_pipeline_tests: tests/AquaSynth.Dsl.Tests/SpeechBackpropagationPipelineTests.cs
+  speech_distributed_training_tests: tests/AquaSynth.Dsl.Tests/SpeechDistributedTrainingTests.cs
   vocal_tract_learning_tests: tests/AquaSynth.Dsl.Tests/VocalTractLearningTests.cs
   sfxr: src/AquaSynth.Core/SfxrParams.cs
   analysis: src/AquaSynth.Core/AudioAnalysis.cs
@@ -80,6 +82,7 @@ boundaries:
     - learned phonetic-plus-utterance-embedding synth driver with packed-buffer native backprop/Adam for tract, AM/FM, LFO, filter, and mel spectral-envelope controls
     - chained utterance-embedding-to-synth-driver backpropagation that returns synth input gradients and updates the utterance encoder from downstream speech loss
     - external-gradient speech backpropagation entry point for rendered loss surfaces
+    - distributed speech render request/result/checkpoint CultCache contracts for multi-machine Faust rendering and local backpropagation
     - optional eSpeak NG generated ground-truth fixture for supervised utterance-embedding and synth-driver gradient descent
     - Faust-rendered speech loss surface in AquaSynth.Faust, using rendered audio analysis and finite-difference controller-output gradients
     - compiled native Faust speech-loss surface with exported /speech/output/N controls for batch candidate rendering without recompiling DSP

state/scratch.md

@@ -2,6 +2,24 @@
 
 Current slice:
 
+- Added the first distributed speech-training contract:
+  `SpeechRenderRequest`, `SpeechRenderResult`, and
+  `SpeechTrainingCheckpoint` are typed CultCache documents in
+  `src/AquaSynth.Core/SpeechDistributedTraining.cs`.
+- `SpeechDistributedTrainingCoordinator` now creates render requests from the
+  current pipeline, accepts worker-produced score gradients, and applies those
+  gradients through `SpeechBackpropagationPipeline.TrainSingleFromSynthOutputGradient`.
+  The current worker proof scores control-vector MSE locally; the ownership
+  shape is the same as remote compiled-Faust scoring.
+- `SpeechDistributedTrainingCultCacheStore` reads and writes request, result,
+  and checkpoint `.cc` stores through CultLib. No JSON queue, no bespoke
+  serializer, no trainer pretending to own the renderer.
+- Verification:
+  `dotnet test tests\AquaSynth.Dsl.Tests\AquaSynth.Dsl.Tests.csproj --filter "SpeechDistributedTrainingTests" -v minimal`:
+  2 passed.
+
+Previous slice:
+
 - Added a typed CultCache sound-claim document for CultMesh distribution:
   `patches/aquasynth-patch-cultcache.cc`.
 - Cut the JSON artifact and the local hand-written MessagePack codec. AquaSynth

tests/AquaSynth.Dsl.Tests/SpeechDistributedTrainingTests.cs

@@ -0,0 +1,174 @@
+using AquaSynth.Dsl;
+
+namespace AquaSynth.Dsl.Tests;
+
+public sealed class SpeechDistributedTrainingTests
+{
+    [Fact]
+    public async Task SpeechRenderRequestsAndResultsRoundTripThroughCultCache()
+    {
+        var directory = TempDirectory();
+        try
+        {
+            var requestStore = Path.Combine(directory, "speech-requests.cc");
+            var resultStore = Path.Combine(directory, "speech-results.cc");
+            var pipeline = Pipeline();
+            var examples = TrainingSet();
+
+            var requests = SpeechDistributedTrainingCoordinator.CreateRenderRequests("batch-a", pipeline, examples);
+            await SpeechDistributedTrainingCultCacheStore.UpsertRequestsAsync(requestStore, requests);
+            var loadedRequests = await SpeechDistributedTrainingCultCacheStore.ReadRequestsAsync(requestStore);
+            var results = SpeechDistributedTrainingCoordinator.ScoreControlVectorRequests(loadedRequests, "worker-manycore-01");
+            await SpeechDistributedTrainingCultCacheStore.UpsertResultsAsync(resultStore, results);
+            var loadedResults = await SpeechDistributedTrainingCultCacheStore.ReadResultsAsync(resultStore);
+
+            Assert.Equal(examples.Count, loadedRequests.Count);
+            Assert.Equal(examples.Count, loadedResults.Count);
+            Assert.All(loadedResults, result => Assert.Equal(SpeechRenderStatus.Succeeded, result.Status));
+            Assert.All(loadedResults, result => Assert.Equal(pipeline.SynthDriver.OutputSize, result.OutputGradient.Length));
+            Assert.Contains(loadedRequests, request => request.RendererProfileId == "compiled-faust-worker");
+        }
+        finally
+        {
+            Directory.Delete(directory, recursive: true);
+        }
+    }
+
+    [Fact]
+    public async Task SyncedSpeechScoresDriveBackpropagationAndCheckpoint()
+    {
+        var directory = TempDirectory();
+        try
+        {
+            var requestStore = Path.Combine(directory, "speech-requests.cc");
+            var resultStore = Path.Combine(directory, "speech-results.cc");
+            var checkpointStore = Path.Combine(directory, "speech-checkpoints.cc");
+            var pipeline = Pipeline();
+            var examples = TrainingSet();
+            var before = AverageLoss(pipeline, examples);
+
+            var requests = SpeechDistributedTrainingCoordinator.CreateRenderRequests("batch-train", pipeline, examples);
+            await SpeechDistributedTrainingCultCacheStore.UpsertRequestsAsync(requestStore, requests);
+
+            var workerRequests = await SpeechDistributedTrainingCultCacheStore.ReadRequestsAsync(requestStore);
+            var workerResults = SpeechDistributedTrainingCoordinator.ScoreControlVectorRequests(workerRequests, "worker-manycore-01");
+            await SpeechDistributedTrainingCultCacheStore.UpsertResultsAsync(resultStore, workerResults);
+
+            var trainerRequests = await SpeechDistributedTrainingCultCacheStore.ReadRequestsAsync(requestStore);
+            var trainerResults = await SpeechDistributedTrainingCultCacheStore.ReadResultsAsync(resultStore);
+            var applied = SpeechDistributedTrainingCoordinator.ApplyResults(
+                pipeline,
+                trainerRequests,
+                trainerResults,
+                "checkpoint-0001",
+                new SpeechDistributedTrainingOptions(
+                    UtteranceLearningRate: 0.18f,
+                    SynthDriverLearningRate: 0.18f));
+            await SpeechDistributedTrainingCultCacheStore.UpsertCheckpointsAsync(checkpointStore, [applied.Checkpoint]);
+
+            var after = AverageLoss(pipeline, examples);
+            var checkpoints = await SpeechDistributedTrainingCultCacheStore.ReadCheckpointsAsync(checkpointStore);
+
+            Assert.True(after < before, $"expected synced score gradients to reduce loss; before={before}, after={after}");
+            Assert.Equal(examples.Count, applied.Checkpoint.AppliedResultCount);
+            Assert.Single(checkpoints);
+            Assert.Equal("checkpoint-0001", checkpoints[0].CheckpointId);
+            Assert.All(applied.Backpropagations, backprop => Assert.True(backprop.InputGradient.Length > 0));
+        }
+        finally
+        {
+            Directory.Delete(directory, recursive: true);
+        }
+    }
+
+    private static SpeechBackpropagationPipeline Pipeline()
+    {
+        var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(
+            inputSize: 10,
+            embeddingSize: 8,
+            hiddenLayerSizes: [24, 16],
+            seed: 701);
+        var synthDriver = VocalTractNeuralMapper.Create(
+            semanticEmbeddingSize: 8,
+            melBandCount: 6,
+            hiddenLayerSizes: [32, 24],
+            seed: 709);
+        return new SpeechBackpropagationPipeline(utteranceEncoder, synthDriver);
+    }
+
+    private static IReadOnlyList<SpeechBackpropagationTrainingExample> TrainingSet() =>
+    [
+        Example(
+            "a",
+            new PhoneticFeatures(PhoneticManner.Vowel, Height: VowelHeight.Open, Backness: VowelBackness.Front),
+            new UtteranceEmbeddingInput([0.92f, 0.20f, 0.35f, 0.70f], [0.88f, 0.32f, 0.20f], [0.65f, 0.20f, 0.44f]),
+            Target([0.82f, 0.74f, 0.66f, 0.58f, 0.50f, 0.42f])),
+        Example(
+            "sa",
+            new PhoneticFeatures(PhoneticManner.Fricative, PhoneticPlace.Alveolar, Phonation.Voiceless),
+            new UtteranceEmbeddingInput([0.35f, 0.42f, 0.90f, 0.44f], [0.70f, 0.92f, 0.75f], [0.28f, 0.60f, 0.82f]),
+            Target([0.12f, 0.16f, 0.22f, 0.34f, 0.48f, 0.62f])),
+        Example(
+            "ma",
+            new PhoneticFeatures(PhoneticManner.Nasal, PhoneticPlace.Bilabial, Phonation.Voiced, Nasalized: true),
+            new UtteranceEmbeddingInput([0.72f, 0.52f, 0.24f, 0.64f], [0.62f, 0.40f, 0.32f], [0.74f, 0.80f, 0.36f]),
+            Target([0.68f, 0.72f, 0.70f, 0.62f, 0.54f, 0.46f]))
+    ];
+
+    private static SpeechBackpropagationTrainingExample Example(
+        string ipa,
+        PhoneticFeatures features,
+        UtteranceEmbeddingInput input,
+        VocalTractControlTarget target) =>
+        new(input, new PhoneticEvent($"phone-{ipa}", ipa, features, DurationSeconds: 0.2), target);
+
+    private static VocalTractControlTarget Target(IReadOnlyList<float> mel) =>
+        new(
+            TongueBody: mel[3],
+            TongueTip: mel[4],
+            LipAperture: mel[0],
+            LipRounding: 0.08f,
+            Velum: mel[1],
+            GlottalTenseness: mel[2],
+            Turbulence: mel[5],
+            Pressure: mel[3],
+            AmDepth: mel[1],
+            FmDepth: mel[2],
+            LfoRate: 0.22f,
+            LfoDepth: mel[4],
+            FilterCutoff: mel[5],
+            FilterResonance: 0.18f,
+            MelSpectralEnvelope: mel);
+
+    private static float AverageLoss(SpeechBackpropagationPipeline pipeline, IReadOnlyList<SpeechBackpropagationTrainingExample> examples)
+    {
+        var loss = 0f;
+        foreach (var example in examples)
+        {
+            loss += MeanSquaredError(
+                pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount),
+                example.Target.ToVector(pipeline.SynthDriver.MelBandCount));
+        }
+
+        return loss / examples.Count;
+    }
+
+    private static float MeanSquaredError(IReadOnlyList<float> actual, IReadOnlyList<float> target)
+    {
+        var loss = 0f;
+        for (var index = 0; index < actual.Count; index++)
+        {
+            var difference = actual[index] - target[index];
+            loss += difference * difference;
+        }
+
+        return loss / actual.Count;
+    }
+
+    private static string TempDirectory()
+    {
+        var path = Path.Combine(Path.GetTempPath(), $"aquasynth-speech-distributed-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(path);
+        return path;
+    }
+}