Add CultMesh speech training step

docs/ipa-vocal-tract-roadmap.md

@@ -547,11 +547,15 @@ The clean path is:
 6. New checkpoints are committed through normal CultCache/CultMesh shard
    authority.
 
-The missing layer is therefore payload transport, runtime admission, worker
-leases, and result collection. The first target is to distribute compiled Faust
-render/scoring jobs, collect gradients/results, and apply them centrally or by
-one shard-primary trainer. Federated averaging can be explored later only after
-the simple authority path is boring.
+The first implemented layer in AquaSynth now covers payload manifests, CultMesh
+peer/lease admission, worker assignment, result collection, and central
+application through `SpeechBackpropagationPipeline`. This is enough to run a
+distributed training step against leased CultMesh workers that return
+`SpeechRenderResult` gradients. Real network artifact chunk transfer and remote
+process supervision still belong to CultMesh infrastructure; AquaSynth's
+authority boundary is ready for that transport instead of inventing its own.
+Federated averaging can be explored later only after the simple authority path
+is boring.
 
 ## Optional eSpeak NG Workout
 

src/AquaSynth.Core/AquaSynth.Core.csproj

@@ -23,6 +23,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\..\CultLib\src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj" />
+    <ProjectReference Include="..\..\..\CultLib\src\GameCult.Mesh\GameCult.Mesh.csproj" />
   </ItemGroup>
 
 </Project>

src/AquaSynth.Core/SpeechDistributedTraining.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics;
 using GameCult.Caching;
 using GameCult.Caching.MessagePack;
+using GameCult.Mesh;
 using MessagePack;
 
 namespace AquaSynth.Dsl;
@@ -120,12 +121,76 @@ public enum SpeechRenderStatus
     Failed
 }
 
+[CultDocument("aquasynth.speech_worker_payload_manifest", "aquasynth.speech_worker_payload_manifest.v1")]
+[MessagePackObject]
+public sealed record SpeechWorkerPayloadManifest(
+    [property: Key(0)]
+    [property: CultName]
+    string PayloadId,
+    [property: Key(1)] string BatchId,
+    [property: Key(2)] string CreatedAtUtc,
+    [property: Key(3)] string RendererProfileId,
+    [property: Key(4)] string ScoringRecipeId,
+    [property: Key(5)] string RequiredRuntimeKind,
+    [property: Key(6)] string WorkerRole,
+    [property: Key(7)] SpeechWorkerPayloadArtifact[] Artifacts,
+    [property: Key(8)] string Notes);
+
+[MessagePackObject]
+public sealed record SpeechWorkerPayloadArtifact(
+    [property: Key(0)] string Kind,
+    [property: Key(1)] string Uri,
+    [property: Key(2)] string ContentHash,
+    [property: Key(3)] string MediaType);
+
+[CultDocument("aquasynth.speech_worker_admission_receipt", "aquasynth.speech_worker_admission_receipt.v1")]
+[MessagePackObject]
+public sealed record SpeechWorkerAdmissionReceipt(
+    [property: Key(0)]
+    [property: CultName]
+    string ReceiptId,
+    [property: Key(1)]
+    [property: CultIndex("payload")]
+    string PayloadId,
+    [property: Key(2)] string PeerId,
+    [property: Key(3)] string LeaseId,
+    [property: Key(4)] string DecidedAtUtc,
+    [property: Key(5)] SpeechWorkerAdmissionStatus Status,
+    [property: Key(6)] string[] AssignedRequestIds,
+    [property: Key(7)] string Reason,
+    [property: Key(8)] SpeechTimingReceipt[] TimingReceipts);
+
+public enum SpeechWorkerAdmissionStatus
+{
+    Admitted,
+    Rejected
+}
+
+public sealed record SpeechCultMeshWorkerAssignment(
+    CultMeshPeerCard Peer,
+    CultMeshAuthorityLease Lease,
+    SpeechWorkerAdmissionReceipt Receipt,
+    IReadOnlyList<SpeechRenderRequest> Requests);
+
+public sealed record SpeechCultMeshTrainingStepResult(
+    SpeechWorkerPayloadManifest PayloadManifest,
+    IReadOnlyList<SpeechWorkerAdmissionReceipt> AdmissionReceipts,
+    IReadOnlyList<SpeechRenderResult> WorkerResults,
+    SpeechDistributedTrainingApplyResult Applied);
+
+public static class SpeechCultMeshRoles
+{
+    public const string RenderWorker = "speech-render-worker";
+}
+
 public sealed record SpeechDistributedTrainingOptions(
     string CurriculumTier = "tiny-local-proof",
     string TargetReferenceId = "local/reference-control-vector",
     string MorphologyId = "neutral-human-ish",
     string RendererProfileId = "compiled-faust-worker",
     string ScoringRecipeId = "control-vector-mse-proof",
+    string RequiredRuntimeKind = "dotnet-csharp",
+    string WorkerRole = SpeechCultMeshRoles.RenderWorker,
     float UtteranceLearningRate = 0.04f,
     float SynthDriverLearningRate = 0.04f);
 
@@ -144,6 +209,12 @@ public static class SpeechDistributedTrainingCultCacheStore
     public static Task UpsertCheckpointsAsync(string filePath, IEnumerable<SpeechTrainingCheckpoint> checkpoints) =>
         UpsertAsync(filePath, checkpoints, checkpoint => $"speech-training-checkpoint:{checkpoint.CheckpointId}");
 
+    public static Task UpsertPayloadManifestsAsync(string filePath, IEnumerable<SpeechWorkerPayloadManifest> payloads) =>
+        UpsertAsync(filePath, payloads, payload => $"speech-worker-payload:{payload.PayloadId}");
+
+    public static Task UpsertAdmissionReceiptsAsync(string filePath, IEnumerable<SpeechWorkerAdmissionReceipt> receipts) =>
+        UpsertAsync(filePath, receipts, receipt => $"speech-worker-admission:{receipt.ReceiptId}");
+
     public static Task<IReadOnlyList<SpeechRenderRequest>> ReadRequestsAsync(string filePath) =>
         ReadAllAsync<SpeechRenderRequest>(filePath, request => request.RequestId);
 
@@ -153,6 +224,12 @@ public static class SpeechDistributedTrainingCultCacheStore
     public static Task<IReadOnlyList<SpeechTrainingCheckpoint>> ReadCheckpointsAsync(string filePath) =>
         ReadAllAsync<SpeechTrainingCheckpoint>(filePath, checkpoint => checkpoint.CheckpointId);
 
+    public static Task<IReadOnlyList<SpeechWorkerPayloadManifest>> ReadPayloadManifestsAsync(string filePath) =>
+        ReadAllAsync<SpeechWorkerPayloadManifest>(filePath, payload => payload.PayloadId);
+
+    public static Task<IReadOnlyList<SpeechWorkerAdmissionReceipt>> ReadAdmissionReceiptsAsync(string filePath) =>
+        ReadAllAsync<SpeechWorkerAdmissionReceipt>(filePath, receipt => receipt.ReceiptId);
+
     private static async Task UpsertAsync<T>(
         string filePath,
         IEnumerable<T> documents,
@@ -306,6 +383,202 @@ public static class SpeechDistributedTrainingCoordinator
         return results;
     }
 
+    public static SpeechWorkerPayloadManifest CreateCultMeshPayloadManifest(
+        string payloadId,
+        IReadOnlyList<SpeechRenderRequest> requests,
+        IReadOnlyList<SpeechWorkerPayloadArtifact>? artifacts = null,
+        SpeechDistributedTrainingOptions? options = null)
+    {
+        if (requests.Count == 0)
+        {
+            throw new ArgumentException("A CultMesh speech payload needs at least one render request.", nameof(requests));
+        }
+
+        options ??= new SpeechDistributedTrainingOptions();
+        return new SpeechWorkerPayloadManifest(
+            payloadId,
+            requests[0].BatchId,
+            DateTimeOffset.UtcNow.ToString("O"),
+            options.RendererProfileId,
+            options.ScoringRecipeId,
+            options.RequiredRuntimeKind,
+            options.WorkerRole,
+            [.. artifacts ?? DefaultPayloadArtifacts(options)],
+            "Payload manifest authorizes transport/admission only; workers return gradients and receipts, not checkpoints.");
+    }
+
+    public static IReadOnlyList<SpeechCultMeshWorkerAssignment> AdmitCultMeshWorkers(
+        SpeechWorkerPayloadManifest payload,
+        IReadOnlyList<SpeechRenderRequest> requests,
+        IReadOnlyList<CultMeshPeerCard> peers,
+        CultMeshAuthorityLeaseCatalog leases,
+        string? shardId = null,
+        DateTimeOffset? at = null)
+    {
+        if (requests.Count == 0)
+        {
+            return [];
+        }
+
+        var admittedPeers = new List<(CultMeshPeerCard Peer, CultMeshAuthorityLease Lease)>();
+        foreach (var peer in peers.OrderBy(peer => peer.PeerId, StringComparer.Ordinal))
+        {
+            if (!peer.HasRole(payload.WorkerRole))
+            {
+                continue;
+            }
+
+            var leaseId = peer.AuthorityLeaseId;
+            if (string.IsNullOrWhiteSpace(leaseId))
+            {
+                continue;
+            }
+
+            var lease = leases.Get(leaseId);
+            if (lease is null || !leases.IsAuthorized(peer, payload.WorkerRole, shardId, at))
+            {
+                continue;
+            }
+
+            admittedPeers.Add((peer, lease));
+        }
+
+        if (admittedPeers.Count == 0)
+        {
+            throw new InvalidOperationException("no CultMesh speech render workers were admitted for the payload");
+        }
+
+        var requestBuckets = admittedPeers
+            .Select(_ => new List<SpeechRenderRequest>())
+            .ToArray();
+        for (var index = 0; index < requests.Count; index++)
+        {
+            requestBuckets[index % requestBuckets.Length].Add(requests[index]);
+        }
+
+        var decidedAt = DateTimeOffset.UtcNow.ToString("O");
+        var assignments = new List<SpeechCultMeshWorkerAssignment>();
+        for (var index = 0; index < admittedPeers.Count; index++)
+        {
+            var bucket = requestBuckets[index];
+            if (bucket.Count == 0)
+            {
+                continue;
+            }
+
+            var (peer, lease) = admittedPeers[index];
+            var requestIds = bucket.Select(request => request.RequestId).ToArray();
+            var receipt = new SpeechWorkerAdmissionReceipt(
+                $"{payload.PayloadId}:{peer.PeerId}:admitted",
+                payload.PayloadId,
+                peer.PeerId,
+                lease.LeaseId,
+                decidedAt,
+                SpeechWorkerAdmissionStatus.Admitted,
+                requestIds,
+                "Peer advertised the speech render role and held a valid CultMesh worker lease.",
+                [
+                    new SpeechTimingReceipt(
+                        "cultmesh-worker-admission",
+                        decidedAt,
+                        0,
+                        0,
+                        1,
+                        $"Admitted {requestIds.Length} request(s) for score-only worker execution.")
+                ]);
+            assignments.Add(new SpeechCultMeshWorkerAssignment(peer, lease, receipt, bucket));
+        }
+
+        return assignments;
+    }
+
+    public static IReadOnlyList<SpeechWorkerAdmissionReceipt> RejectCultMeshWorkers(
+        SpeechWorkerPayloadManifest payload,
+        IReadOnlyList<CultMeshPeerCard> peers,
+        CultMeshAuthorityLeaseCatalog leases,
+        string? shardId = null,
+        DateTimeOffset? at = null)
+    {
+        var decidedAt = DateTimeOffset.UtcNow.ToString("O");
+        var rejected = new List<SpeechWorkerAdmissionReceipt>();
+        foreach (var peer in peers.OrderBy(peer => peer.PeerId, StringComparer.Ordinal))
+        {
+            var reason = RejectionReason(payload, peer, leases, shardId, at);
+            if (reason.Length == 0)
+            {
+                continue;
+            }
+
+            rejected.Add(new SpeechWorkerAdmissionReceipt(
+                $"{payload.PayloadId}:{peer.PeerId}:rejected",
+                payload.PayloadId,
+                peer.PeerId,
+                peer.AuthorityLeaseId ?? "",
+                decidedAt,
+                SpeechWorkerAdmissionStatus.Rejected,
+                [],
+                reason,
+                [
+                    new SpeechTimingReceipt(
+                        "cultmesh-worker-admission",
+                        decidedAt,
+                        0,
+                        0,
+                        0,
+                        reason)
+                ]));
+        }
+
+        return rejected;
+    }
+
+    public static IReadOnlyList<SpeechRenderResult> RunCultMeshRenderWorkers(
+        IReadOnlyList<SpeechCultMeshWorkerAssignment> assignments,
+        SpeechWorkerPayloadManifest payload)
+    {
+        var results = new List<SpeechRenderResult>();
+        foreach (var assignment in assignments)
+        {
+            if (assignment.Receipt.Status != SpeechWorkerAdmissionStatus.Admitted)
+            {
+                continue;
+            }
+
+            results.AddRange(ScoreControlVectorRequests(
+                assignment.Requests,
+                assignment.Peer.PeerId,
+                $"{payload.RendererProfileId}+{payload.ScoringRecipeId}+cultmesh-lease:{assignment.Lease.LeaseId}"));
+        }
+
+        return results;
+    }
+
+    public static SpeechCultMeshTrainingStepResult RunCultMeshTrainingStep(
+        string batchId,
+        string payloadId,
+        string checkpointId,
+        SpeechBackpropagationPipeline pipeline,
+        IReadOnlyList<SpeechBackpropagationTrainingExample> examples,
+        IReadOnlyList<CultMeshPeerCard> peers,
+        CultMeshAuthorityLeaseCatalog leases,
+        SpeechDistributedTrainingOptions? options = null,
+        string? shardId = null,
+        DateTimeOffset? at = null)
+    {
+        options ??= new SpeechDistributedTrainingOptions();
+        var requests = CreateRenderRequests(batchId, pipeline, examples, options);
+        var payload = CreateCultMeshPayloadManifest(payloadId, requests, options: options);
+        var assignments = AdmitCultMeshWorkers(payload, requests, peers, leases, shardId, at);
+        var rejected = RejectCultMeshWorkers(payload, peers, leases, shardId, at);
+        var results = RunCultMeshRenderWorkers(assignments, payload);
+        var applied = ApplyResults(pipeline, requests, results, checkpointId, options);
+        return new SpeechCultMeshTrainingStepResult(
+            payload,
+            [.. assignments.Select(assignment => assignment.Receipt), .. rejected],
+            results,
+            applied);
+    }
+
     public static SpeechDistributedTrainingApplyResult ApplyResults(
         SpeechBackpropagationPipeline pipeline,
         IReadOnlyList<SpeechRenderRequest> requests,
@@ -388,4 +661,42 @@ public static class SpeechDistributedTrainingCoordinator
 
         return Math.Clamp(1f / (1f + loss), 0f, 1f);
     }
+
+    private static IReadOnlyList<SpeechWorkerPayloadArtifact> DefaultPayloadArtifacts(SpeechDistributedTrainingOptions options) =>
+    [
+        new("compiled-faust-renderer", $"cultcache://speech-renderer/{options.RendererProfileId}", "", "application/wasm-or-native-faust"),
+        new("scoring-code", $"cultcache://speech-scoring/{options.ScoringRecipeId}", "", "application/dotnet-assembly"),
+        new("model-encoder-assembly", "cultcache://speech-model/aquasynth-core", "", "application/dotnet-assembly")
+    ];
+
+    private static string RejectionReason(
+        SpeechWorkerPayloadManifest payload,
+        CultMeshPeerCard peer,
+        CultMeshAuthorityLeaseCatalog leases,
+        string? shardId,
+        DateTimeOffset? at)
+    {
+        if (!peer.HasRole(payload.WorkerRole))
+        {
+            return $"Peer does not advertise required worker role `{payload.WorkerRole}`.";
+        }
+
+        if (string.IsNullOrWhiteSpace(peer.AuthorityLeaseId))
+        {
+            return "Peer has no CultMesh authority lease id for worker admission.";
+        }
+
+        var lease = leases.Get(peer.AuthorityLeaseId!);
+        if (lease is null)
+        {
+            return "Peer references a worker lease that is not present in the local lease catalog.";
+        }
+
+        if (!leases.IsAuthorized(peer, payload.WorkerRole, shardId, at))
+        {
+            return "Peer worker lease does not authorize the requested role, shard, or time window.";
+        }
+
+        return "";
+    }
 }

state/evidence.jsonl

@@ -126,3 +126,4 @@
 {"date":"2026-05-20","topic":"panphon-sequence-encoder-organ","lesson":"The 256D phonetic realization surface is now an AquaSynth-owned learned organ, not fixed phone slots. Weksa supplies variable-length PanPhon feature sequences; AquaSynth encodes them through a trainable phonetic sequence encoder before the utterance embedding encoder consumes the 256D result.","evidence":"Added PhoneticSequenceEncoding.cs with PanPhon feature frames, deterministic sequence sketching, PhoneticSequenceNeuralEncoder, training records, and tests proving the encoder learns compact PanPhon realization embeddings."}
 {"date":"2026-05-20","topic":"weksa-v01-training-artifact","lesson":"The Weksa blocker has moved from schema definition to artifact consumption. Weksa now emits a machine-readable `weksa.utterance_embedding_handoff.v0.1` tiny PanPhon seed batch for `a`, `pa`, `ta`, `ka`, `sa`, and `ma`; AquaSynth's next serious training pass should load that artifact rather than local toy vectors.","evidence":"Added Weksa emitter `E:/Projects/weksa/tools/emit_tiny_panphon_handoffs.py` and generated `E:/Projects/weksa/examples/speech-training/tiny-panphon-v0.1/batch.json` with fixed 1024/32/64 input widths, inline PanPhon 22-feature frames, and pending AquaSynth-owned 256/64 learned outputs."}
 {"date":"2026-05-24","topic":"cultmesh-speech-worker-authority","lesson":"CultMesh should distribute speech render/scoring work, not optimizer authority. AquaSynth emits typed SpeechRenderRequest documents; CultMesh delivers compiled Faust renderers, scoring code, small model/encoder assemblies, and batch/artifact refs to compatible leased workers; workers return SpeechRenderResult documents with loss, gradients, artifacts, and timing receipts. The authority trainer or shard-primary applies gradients through SpeechBackpropagationPipeline and commits SpeechTrainingCheckpoint documents through normal CultCache/CultMesh shard authority.","evidence":"Recorded the authority map in docs/ipa-vocal-tract-roadmap.md, state/spine.yaml, state/map.yaml, state/memory.json, and state/scratch.md. The next implementation layer is transport, admission, worker leases, payload delivery, and result collection; federated averaging waits until the simple authority path is boring."}
+{"date":"2026-05-24","topic":"cultmesh-speech-training-step","lesson":"AquaSynth can now run the first honest CultMesh-distributed speech training step: an authority trainer emits SpeechRenderRequest documents and a SpeechWorkerPayloadManifest, admits only CultMesh peers with valid speech-render-worker leases, records SpeechWorkerAdmissionReceipt documents for admitted/rejected peers, lets admitted workers produce SpeechRenderResult gradients, and applies those gradients centrally through SpeechBackpropagationPipeline. Workers still do not write checkpoints.","evidence":"Added GameCult.Mesh to AquaSynth.Core, added payload/admission/assignment/training-step types and store helpers in SpeechDistributedTraining.cs, added RunCultMeshTrainingStep, and added a test with two leased workers plus rejected unleased/read-replica peers. Focused verification: `dotnet test tests/AquaSynth.Dsl.Tests/AquaSynth.Dsl.Tests.csproj --filter \"SpeechDistributedTrainingTests\" -v minimal`: 3 passed. Remaining infrastructure gap: real remote artifact chunk transfer and process supervision belong to CultMesh, not an AquaSynth-local workaround."}

state/map.yaml

@@ -87,7 +87,7 @@ boundaries:
     - chained utterance-embedding-to-synth-driver backpropagation that returns synth input gradients and updates the utterance encoder from downstream speech loss
     - external-gradient speech backpropagation entry point for rendered loss surfaces
     - distributed speech render request/result/checkpoint CultCache contracts for multi-machine Faust rendering and local backpropagation
-    - CultMesh speech render/scoring worker orchestration as transport/admission/lease/result collection; workers return gradients and receipts, while one authority trainer owns optimizer updates and checkpoint commits
+    - CultMesh speech render/scoring worker orchestration with payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application; workers return gradients and receipts, while one authority trainer owns optimizer updates and checkpoint commits
     - optional eSpeak NG generated ground-truth fixture for supervised utterance-embedding and synth-driver gradient descent
     - Faust-rendered speech loss surface in AquaSynth.Faust, using rendered audio analysis and finite-difference controller-output gradients
     - compiled native Faust speech-loss surface with exported /speech/output/N controls for batch candidate rendering without recompiling DSP

state/memory.json

@@ -57,7 +57,7 @@
       "analysis/comparison metrics",
       "Faust-generated C# candidate rendering",
       "speech timing/confidence receipts for render requests, results, and checkpoints",
-      "CultMesh speech render/scoring orchestration as payload transport, runtime admission, worker leasing, and result collection while the authority trainer owns optimizer steps and checkpoint commits",
+      "CultMesh speech render/scoring orchestration with payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application while the authority trainer owns optimizer steps and checkpoint commits",
       "script scoring",
       "preset catalogs",
       "Faust emission and compiler wrapper"
@@ -91,9 +91,9 @@
     "rule": "For speech, add schema fields and learned-control lanes only when the parity harness has a loss surface that can exercise them. For reference synths, add DSL abstractions only when rebuilds prove the missing invariant. Parsing alone is inventory, not evidence of expressivity."
   },
   "ipa_vocal_tract_path": {
-    "status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets. CultMesh should distribute compiled Faust render/scoring payloads and collect worker gradients/results, but workers do not own model updates; the authority trainer or shard-primary applies gradients and commits SpeechTrainingCheckpoint documents through normal shard authority.",
+    "status": "This is now the active project center. Roadmap and research notes align the curriculum around structured utterance metadata -> learned utterance embedding -> learned synth automation -> vocal tract/envelope/LFO/filter/spectral controls -> rendered audio loss. The first Core contracts still defend the boundary: PhoneticIntent is the Weksa/language-owned handoff, ArticulatoryPlan is the AquaSynth realization layer, VocalTractMorphology declares anatomy capabilities, ArticulatoryConstraintReport is the host-visible veto surface, and VocalTractPlanResult is the tiny end-to-end proof artifact for host branching. IPA is an authoring input only: tokenization and feature mapping lower into language phonology profiles, timed articulatory gestures, morphology-specific tract/excitation curves, and Faust DSP. The staged support plan starts with tokenization and vowels, then core pulmonic consonants, expanded human IPA, airstream/phonation, and Weksa/alien morphologies. Optional eSpeak NG tests provide the first generated speech workout when ESPEAK_NG or espeak-ng is installed; eSpeak is intelligibility/timing/log-mel pressure, not anatomy truth. AquaSynth owns the learned utterance embedding encoder and synth driver: speech text embeddings plus prosody/emphasis hints plus Ghostlight/Epiphany-shaped character state vectors compress into utterance embeddings, and phonetic features plus that embedding feed a packed-buffer native C# backprop/Adam mapper that predicts tract, AM/FM, LFO, filter, and mel spectral-envelope controls. SpeechBackpropagationPipeline chains the two models: synth-driver target loss returns gradients with respect to its input, slices the utterance-embedding gradient, and updates the utterance encoder from downstream speech loss. The generated ground-truth training fixture renders eSpeak syllables, extracts log-mel evidence, and trains that chained pipeline toward eSpeak-seeded mel/control targets. CultMesh speech training now has payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application; workers do not own model updates. Real remote artifact chunk transfer and process supervision remain CultMesh infrastructure surfaces, not AquaSynth workaround territory.",
     "lesson": "Do not build a complete IPA parser before the tract can say basic syllables. The durable machine is tokenizer -> feature mapper -> phonology profile -> gesture planner -> morphology -> tract plan -> Faust renderer, with dependency-injected seams and mocks at each boundary. Weksa should use IPA-like phonetic input without letting human IPA semantics own alien anatomy. Anatomy has veto power: if a morphology lacks bilabial closure or another required capability, AquaSynth reports that failure instead of smearing over it.",
-    "next": "Start the next speech gradient descent pass from Weksa's emitted v0.1 seed batch at `E:\\Projects\\weksa\\examples\\speech-training\\tiny-panphon-v0.1\\batch.json`: 1024 bge-m3 semantic speech-text floats, variable-length PanPhon sequence evidence, 256 AquaSynth-owned phonetic-realization embedding floats from `aquasynth.panphon_sequence_encoder.v0.1`, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Next tighten the parity harness: load that batch into AquaSynth training records, render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop. For distributed work, make the first CultMesh target boring: payload delivery, runtime admission, worker leases, result collection, and central or shard-primary gradient application before any federated averaging experiment."
+    "next": "Start the next speech gradient descent pass from Weksa's emitted v0.1 seed batch at `E:\\Projects\\weksa\\examples\\speech-training\\tiny-panphon-v0.1\\batch.json`: 1024 bge-m3 semantic speech-text floats, variable-length PanPhon sequence evidence, 256 AquaSynth-owned phonetic-realization embedding floats from `aquasynth.panphon_sequence_encoder.v0.1`, 32 prosody/emphasis floats, 64 projected character-state floats, and 64 AquaSynth utterance-embedding output floats. Keep semantic text embeddings and phonetic realization separate. Next tighten the parity harness: load that batch into AquaSynth training records, render tiny utterance fixtures, sweep controller outputs through compiled Faust controls, score log-mel/timing/envelope loss, record per-stage decision time/latency/budget/confidence receipts, backpropagate into the utterance encoder plus synth driver, and connect Stage 0/1 IPA token spans, vowel feature bundles, and human baseline tract-area targets to that loop. For distributed work, use RunCultMeshTrainingStep as the boring authority path: admitted leased workers return gradients/results, then the trainer applies centrally. Do not attempt federated averaging until this path is exercised against real Faust-rendered speech jobs."
   },
   "dx7_reference_path": {
     "status": "DX7 SysEx parser extracts voice edit buffers, packed voices, packed 32-voice banks, six operators, envelopes, algorithm, feedback, oscillator sync, LFO, transpose, voice name, checksum validation, ReferencePatch features, algorithm topology from the reverse-engineered OPS algorithm ROM table, envelope-to-ADSR approximations, measured carrier output amplitude, isolated two-op modulation route scale, summed/cascaded route scale, calibrated feedback amount table, ratio detune lowering, voice transpose lowering, fixed-frequency operator lowering, ROM COM output compensation cut, operator key/rate scaling at the effective played note, topology-aware route lowering, staged operator envelopes with per-segment curves, raw and block-interpolated DX7 envelope tracing, applied-envelope curved lowering with optional sustain floor, two AquaSynth DSL rebuild attempts, first-class OperatorGraph support with graph-level vibrato, public-domain SysEx fixture coverage, dexed-py reference rendering, project-authored Dexed patch rendering, AquaSynth candidate rendering through Faust-generated C#, one thresholded rendered parity test for analog1.syx voice 13, a hard PRC SYNTH1 parity probe with listening WAV artifacts and log-mel <= 0.145, project-authored algorithm-8 summed-stack/cascade/summed-pair parity, rendered community parity for Mooger, Piano Bass, RES SYNTH1, and ANLGSYN 1 with a sustained high-band gate, a MELLOWSOLO pressure artifact test, and renderable delayed operator feedback.",

state/scratch.md

@@ -52,6 +52,25 @@ Current slice:
   `dotnet test tests\AquaSynth.Dsl.Tests\AquaSynth.Dsl.Tests.csproj --filter "SpeechDistributedTrainingTests" -v minimal`:
   2 passed.
 
+- Implemented the first CultMesh-distributed speech training step:
+  Core now references `GameCult.Mesh`, defines `SpeechWorkerPayloadManifest`,
+  `SpeechWorkerPayloadArtifact`, `SpeechWorkerAdmissionReceipt`,
+  `SpeechCultMeshWorkerAssignment`, `SpeechCultMeshTrainingStepResult`, and
+  `SpeechCultMeshRoles.RenderWorker`, and persists payload/admission documents
+  through `SpeechDistributedTrainingCultCacheStore`.
+- `SpeechDistributedTrainingCoordinator.RunCultMeshTrainingStep` now creates
+  requests, creates a worker payload manifest, admits only peers with valid
+  CultMesh worker leases, records rejected peers, runs score-only worker
+  assignments, collects `SpeechRenderResult` gradients, and applies them
+  centrally through `ApplyResults`.
+- This is distributed training via CultMesh lease/admission and typed CultCache
+  documents. It is not yet real remote process supervision or artifact chunk
+  transfer; those remain CultMesh infrastructure surfaces. No worker API owns
+  checkpoint mutation.
+- Verification:
+  `dotnet test tests\AquaSynth.Dsl.Tests\AquaSynth.Dsl.Tests.csproj --filter "SpeechDistributedTrainingTests" -v minimal`:
+  3 passed.
+
 Previous slice:
 
 - Added a typed CultCache sound-claim document for CultMesh distribution:

state/spine.yaml

@@ -112,7 +112,7 @@ live_invariants:
   - eSpeak NG is an early parity reference for timing, IPA coverage, and log-mel pressure, not an anatomy oracle or final target.
   - Do not expand into prosody, emotional context, personality, or broad Weksa expressivity until the tiny IPA utterance harness can show useful loss movement.
   - Speech training receipts must keep time as a witness: every request, render result, and checkpoint should record stage decision time, latency spent, latency budget, confidence, and artifact/model identifiers.
-  - CultMesh is the speech render/scoring transport and worker-orchestration layer, not distributed checkpoint authority. Workers may render, score, return gradients, artifacts, and timing receipts under a lease; the authority trainer or shard-primary applies gradients through `SpeechBackpropagationPipeline` and commits checkpoints through normal CultCache/CultMesh shard authority.
+  - CultMesh is the speech render/scoring transport and worker-orchestration layer, not distributed checkpoint authority. AquaSynth now has payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application. Workers may render, score, return gradients, artifacts, and timing receipts under a lease; the authority trainer or shard-primary applies gradients through `SpeechBackpropagationPipeline` and commits checkpoints through normal CultCache/CultMesh shard authority.
   - Serious speech gradient descent may now start from Weksa's first emitted v0.1 seed batch at `E:\Projects\weksa\examples\speech-training\tiny-panphon-v0.1\batch.json`: 1024 semantic speech-text floats, variable-length PanPhon sequence evidence, 256 AquaSynth-owned phonetic realization embedding floats, 32 prosody/emphasis floats, 64 projected character-state floats, and a 64-float AquaSynth utterance embedding output.
   - Do not use semantic text embeddings as a proxy for IPA or alien phonetic strings. Phonetic/IPA realization starts as PanPhon sequence evidence and is compressed by AquaSynth's phonetic sequence encoder.
 
@@ -123,13 +123,13 @@ reference_targets:
     sources:
       - "docs/ipa-vocal-tract-roadmap.md"
       - "docs/utterance-parity-research.md"
-    status: "Boundary contracts, packed neural networks, utterance encoder, phonetic sequence encoder, synth-driver mapper, chained backprop, optional eSpeak rendered fixtures, Faust finite-difference loss surface, compiled Faust speech-loss controls, distributed render request/result/checkpoint contracts, timing/confidence receipts, sound-claim cards, and a Weksa-emitted v0.1 tiny PanPhon batch exist. CultMesh should now be used for transport, runtime admission, worker leases, payload delivery, and result collection, while the authority trainer owns optimizer application and checkpoint commits. The active need is a loader/training pass that consumes Weksa's batch artifact and tests many controller weights as rendered speech output against simple ground truth."
+    status: "Boundary contracts, packed neural networks, utterance encoder, phonetic sequence encoder, synth-driver mapper, chained backprop, optional eSpeak rendered fixtures, Faust finite-difference loss surface, compiled Faust speech-loss controls, distributed render request/result/checkpoint contracts, timing/confidence receipts, sound-claim cards, and a Weksa-emitted v0.1 tiny PanPhon batch exist. CultMesh speech training now has payload manifests, peer/lease admission receipts, worker assignment, result collection, and central checkpoint application; real network artifact transfer/process supervision remains CultMesh infrastructure. The active need is a loader/training pass that consumes Weksa's batch artifact and tests many controller weights as rendered speech output against simple ground truth."
     first_slice:
       - "Keep the utterance schema small and testable: segments, spans, duration/stress/boundary hints, speaker/morphology identity, pitch/loudness/rate/breath controls, and named context vectors."
       - "Load Weksa's tiny PanPhon v0.1 batch artifact into AquaSynth training records before the next serious gradient descent run."
       - "Use eSpeak NG fixtures for generated ground truth while remembering that formant parity is not anatomy parity."
       - "Connect Stage 0/1 IPA and vowel tract-area fixtures to the compiled Faust loss surface before expanding consonants."
-      - "Use CultMesh first for compiled Faust render/scoring jobs: ship payloads, admit compatible workers under leases, collect `SpeechRenderResult` gradients/receipts, and apply centrally or by one shard-primary trainer."
+      - "Use the CultMesh speech training step for compiled Faust render/scoring jobs: admit compatible workers under leases, collect `SpeechRenderResult` gradients/receipts, and apply centrally or by one shard-primary trainer. Do not let worker payload transport become optimizer authority."
   dx7_dexed:
     priority: 1
     purpose: "Structured FM/operator graph pressure test."

tests/AquaSynth.Dsl.Tests/SpeechDistributedTrainingTests.cs

@@ -1,4 +1,5 @@
 using AquaSynth.Dsl;
+using GameCult.Mesh;
 
 namespace AquaSynth.Dsl.Tests;
 
@@ -86,6 +87,119 @@ public sealed class SpeechDistributedTrainingTests
         }
     }
 
+    [Fact]
+    public async Task CultMeshLeasedWorkersScoreButAuthorityTrainerAppliesCheckpoint()
+    {
+        var directory = TempDirectory();
+        try
+        {
+            var payloadStore = Path.Combine(directory, "speech-payloads.cc");
+            var admissionStore = Path.Combine(directory, "speech-admissions.cc");
+            var resultStore = Path.Combine(directory, "speech-results.cc");
+            var checkpointStore = Path.Combine(directory, "speech-checkpoints.cc");
+            var pipeline = Pipeline();
+            var examples = TrainingSet();
+            var before = AverageLoss(pipeline, examples);
+            var now = DateTimeOffset.Parse("2026-05-24T12:00:00.0000000Z");
+            var leases = CultMesh.CreateAuthorityLeaseCatalog();
+            leases.Upsert(new CultMeshAuthorityLease(
+                "lease:worker-a",
+                "aquasynth-speech",
+                "worker-a",
+                [SpeechCultMeshRoles.RenderWorker],
+                ["speech-train"],
+                "trainer-primary",
+                now.AddMinutes(-1),
+                now.AddMinutes(10)));
+            leases.Upsert(new CultMeshAuthorityLease(
+                "lease:worker-b",
+                "aquasynth-speech",
+                "worker-b",
+                [SpeechCultMeshRoles.RenderWorker],
+                ["speech-train"],
+                "trainer-primary",
+                now.AddMinutes(-1),
+                now.AddMinutes(10)));
+
+            var peers = new[]
+            {
+                new CultMeshPeerCard(
+                    "worker-a",
+                    "aquasynth-speech",
+                    ["cultmesh://worker-a"],
+                    roles: [SpeechCultMeshRoles.RenderWorker],
+                    shardIds: ["speech-train"],
+                    authorityLeaseId: "lease:worker-a"),
+                new CultMeshPeerCard(
+                    "worker-b",
+                    "aquasynth-speech",
+                    ["cultmesh://worker-b"],
+                    roles: [SpeechCultMeshRoles.RenderWorker],
+                    shardIds: ["speech-train"],
+                    authorityLeaseId: "lease:worker-b"),
+                new CultMeshPeerCard(
+                    "unleased-worker",
+                    "aquasynth-speech",
+                    ["cultmesh://unleased"],
+                    roles: [SpeechCultMeshRoles.RenderWorker],
+                    shardIds: ["speech-train"]),
+                new CultMeshPeerCard(
+                    "read-replica",
+                    "aquasynth-speech",
+                    ["cultmesh://read-replica"],
+                    roles: [CultMeshPeerRoles.ReadReplica],
+                    shardIds: ["speech-train"])
+            };
+
+            var run = SpeechDistributedTrainingCoordinator.RunCultMeshTrainingStep(
+                "batch-cultmesh",
+                "payload-cultmesh-0001",
+                "checkpoint-cultmesh-0001",
+                pipeline,
+                examples,
+                peers,
+                leases,
+                new SpeechDistributedTrainingOptions(
+                    CurriculumTier: "tiny-cultmesh-proof",
+                    UtteranceLearningRate: 0.18f,
+                    SynthDriverLearningRate: 0.18f),
+                "speech-train",
+                now);
+
+            await SpeechDistributedTrainingCultCacheStore.UpsertPayloadManifestsAsync(payloadStore, [run.PayloadManifest]);
+            await SpeechDistributedTrainingCultCacheStore.UpsertAdmissionReceiptsAsync(admissionStore, run.AdmissionReceipts);
+            await SpeechDistributedTrainingCultCacheStore.UpsertResultsAsync(resultStore, run.WorkerResults);
+            await SpeechDistributedTrainingCultCacheStore.UpsertCheckpointsAsync(checkpointStore, [run.Applied.Checkpoint]);
+
+            var payloads = await SpeechDistributedTrainingCultCacheStore.ReadPayloadManifestsAsync(payloadStore);
+            var admissions = await SpeechDistributedTrainingCultCacheStore.ReadAdmissionReceiptsAsync(admissionStore);
+            var results = await SpeechDistributedTrainingCultCacheStore.ReadResultsAsync(resultStore);
+            var checkpoints = await SpeechDistributedTrainingCultCacheStore.ReadCheckpointsAsync(checkpointStore);
+            var after = AverageLoss(pipeline, examples);
+
+            Assert.True(after < before, $"expected CultMesh worker gradients to reduce loss; before={before}, after={after}");
+            Assert.Single(payloads);
+            Assert.Equal(SpeechCultMeshRoles.RenderWorker, payloads[0].WorkerRole);
+            Assert.Equal(2, admissions.Count(receipt => receipt.Status == SpeechWorkerAdmissionStatus.Admitted));
+            Assert.Equal(2, admissions.Count(receipt => receipt.Status == SpeechWorkerAdmissionStatus.Rejected));
+            Assert.All(admissions.Where(receipt => receipt.Status == SpeechWorkerAdmissionStatus.Admitted), receipt => Assert.NotEmpty(receipt.AssignedRequestIds));
+            Assert.All(results, result => Assert.Equal(SpeechRenderStatus.Succeeded, result.Status));
+            Assert.Equal(examples.Count, results.Count);
+            Assert.Equal(examples.Count, run.Applied.Checkpoint.AppliedResultCount);
+            Assert.Single(checkpoints);
+            Assert.Equal("checkpoint-cultmesh-0001", checkpoints[0].CheckpointId);
+            Assert.DoesNotContain(checkpoints[0].AppliedResultIds, id => id.Contains("unleased-worker", StringComparison.Ordinal));
+            Assert.Contains(results, result => result.WorkerId == "worker-a");
+            Assert.Contains(results, result => result.WorkerId == "worker-b");
+            Assert.DoesNotContain(results, result => result.WorkerId == "unleased-worker");
+            Assert.Contains(checkpoints[0].TimingReceipts, receipt => receipt.StageId == "apply-render-gradients");
+        }
+        finally
+        {
+            Directory.Delete(directory, recursive: true);
+        }
+    }
+
     private static SpeechBackpropagationPipeline Pipeline()
     {
         var utteranceEncoder = UtteranceEmbeddingNeuralEncoder.Create(