AquaSynth/src/AquaSynth.Core/SpeechDistributedTraining.cs
2026-05-24 20:02:51 +01:00

702 lines
27 KiB
C#

using System.Diagnostics;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using GameCult.Mesh;
using MessagePack;
namespace AquaSynth.Dsl;
[CultDocument("aquasynth.speech_render_request", "aquasynth.speech_render_request.v1")]
[MessagePackObject]
public sealed record SpeechRenderRequest(
[property: Key(0)]
[property: CultName]
string RequestId,
[property: Key(1)] string BatchId,
[property: Key(2)] string CreatedAtUtc,
[property: Key(3)] string CurriculumTier,
[property: Key(4)] string TargetReferenceId,
[property: Key(5)] string MorphologyId,
[property: Key(6)] string RendererProfileId,
[property: Key(7)] string ScoringRecipeId,
[property: Key(8)] SpeechUtteranceEmbeddingInputSnapshot UtteranceInput,
[property: Key(9)] SpeechPhoneticEventSnapshot PhoneticEvent,
[property: Key(10)] float[] CandidateControlVector,
[property: Key(11)] float[] ReferenceControlVector,
[property: Key(12)] SpeechTimingReceipt[] TimingReceipts);
[CultDocument("aquasynth.speech_render_result", "aquasynth.speech_render_result.v1")]
[MessagePackObject]
public sealed record SpeechRenderResult(
[property: Key(0)]
[property: CultName]
string ResultId,
[property: Key(1)]
[property: CultIndex("request")]
string RequestId,
[property: Key(2)] string WorkerId,
[property: Key(3)] string RendererToolchain,
[property: Key(4)] string CompletedAtUtc,
[property: Key(5)] SpeechRenderStatus Status,
[property: Key(6)] float Loss,
[property: Key(7)] float[] OutputGradient,
[property: Key(8)] SpeechScoreMetric[] Metrics,
[property: Key(9)] SpeechRenderArtifact[] Artifacts,
[property: Key(10)] SpeechTimingReceipt[] TimingReceipts,
[property: Key(11)] string FailureCode = "",
[property: Key(12)] string FailureMessage = "");
[CultDocument("aquasynth.speech_training_checkpoint", "aquasynth.speech_training_checkpoint.v1")]
[MessagePackObject]
public sealed record SpeechTrainingCheckpoint(
[property: Key(0)]
[property: CultName]
string CheckpointId,
[property: Key(1)] string BatchId,
[property: Key(2)] string CreatedAtUtc,
[property: Key(3)] int AppliedResultCount,
[property: Key(4)] float MeanAppliedLoss,
[property: Key(5)] string[] AppliedResultIds,
[property: Key(6)] string Notes,
[property: Key(7)] SpeechTimingReceipt[] TimingReceipts);
[MessagePackObject]
public sealed record SpeechTimingReceipt(
[property: Key(0)] string StageId,
[property: Key(1)] string DecidedAtUtc,
[property: Key(2)] double LatencyMilliseconds,
[property: Key(3)] double BudgetMilliseconds,
[property: Key(4)] float Confidence,
[property: Key(5)] string Notes);
[MessagePackObject]
public sealed record SpeechUtteranceEmbeddingInputSnapshot(
[property: Key(0)] float[] TextEmbedding,
[property: Key(1)] float[] PhoneticRealizationEmbedding,
[property: Key(2)] float[] Prosody,
[property: Key(3)] float[] CharacterState)
{
public UtteranceEmbeddingInput ToInput() => new(TextEmbedding, PhoneticRealizationEmbedding, Prosody, CharacterState);
public static SpeechUtteranceEmbeddingInputSnapshot From(UtteranceEmbeddingInput input) =>
new([.. input.SpeechTextEmbedding], [.. input.PhoneticRealizationEmbedding], [.. input.ProsodyAndEmphasisHints], [.. input.CharacterStateVector]);
}
[MessagePackObject]
public sealed record SpeechPhoneticEventSnapshot(
[property: Key(0)] string Id,
[property: Key(1)] string Ipa,
[property: Key(2)] PhoneticFeatures Features,
[property: Key(3)] double StartSeconds,
[property: Key(4)] double DurationSeconds,
[property: Key(5)] PhoneticProsody Prosody)
{
public PhoneticEvent ToEvent() => new(Id, Ipa, Features, StartSeconds, DurationSeconds, Prosody);
public static SpeechPhoneticEventSnapshot From(PhoneticEvent phoneticEvent) =>
new(
phoneticEvent.Id,
phoneticEvent.Ipa,
phoneticEvent.Features,
phoneticEvent.StartSeconds,
phoneticEvent.DurationSeconds,
phoneticEvent.Prosody);
}
[MessagePackObject]
public sealed record SpeechScoreMetric(
[property: Key(0)] string Name,
[property: Key(1)] float Value,
[property: Key(2)] float Weight);
[MessagePackObject]
public sealed record SpeechRenderArtifact(
[property: Key(0)] string Kind,
[property: Key(1)] string Uri,
[property: Key(2)] string ContentHash);
public enum SpeechRenderStatus
{
Succeeded,
Failed
}
[CultDocument("aquasynth.speech_worker_payload_manifest", "aquasynth.speech_worker_payload_manifest.v1")]
[MessagePackObject]
public sealed record SpeechWorkerPayloadManifest(
[property: Key(0)]
[property: CultName]
string PayloadId,
[property: Key(1)] string BatchId,
[property: Key(2)] string CreatedAtUtc,
[property: Key(3)] string RendererProfileId,
[property: Key(4)] string ScoringRecipeId,
[property: Key(5)] string RequiredRuntimeKind,
[property: Key(6)] string WorkerRole,
[property: Key(7)] SpeechWorkerPayloadArtifact[] Artifacts,
[property: Key(8)] string Notes);
[MessagePackObject]
public sealed record SpeechWorkerPayloadArtifact(
[property: Key(0)] string Kind,
[property: Key(1)] string Uri,
[property: Key(2)] string ContentHash,
[property: Key(3)] string MediaType);
[CultDocument("aquasynth.speech_worker_admission_receipt", "aquasynth.speech_worker_admission_receipt.v1")]
[MessagePackObject]
public sealed record SpeechWorkerAdmissionReceipt(
[property: Key(0)]
[property: CultName]
string ReceiptId,
[property: Key(1)]
[property: CultIndex("payload")]
string PayloadId,
[property: Key(2)] string PeerId,
[property: Key(3)] string LeaseId,
[property: Key(4)] string DecidedAtUtc,
[property: Key(5)] SpeechWorkerAdmissionStatus Status,
[property: Key(6)] string[] AssignedRequestIds,
[property: Key(7)] string Reason,
[property: Key(8)] SpeechTimingReceipt[] TimingReceipts);
public enum SpeechWorkerAdmissionStatus
{
Admitted,
Rejected
}
public sealed record SpeechCultMeshWorkerAssignment(
CultMeshPeerCard Peer,
CultMeshAuthorityLease Lease,
SpeechWorkerAdmissionReceipt Receipt,
IReadOnlyList<SpeechRenderRequest> Requests);
public sealed record SpeechCultMeshTrainingStepResult(
SpeechWorkerPayloadManifest PayloadManifest,
IReadOnlyList<SpeechWorkerAdmissionReceipt> AdmissionReceipts,
IReadOnlyList<SpeechRenderResult> WorkerResults,
SpeechDistributedTrainingApplyResult Applied);
public static class SpeechCultMeshRoles
{
public const string RenderWorker = "speech-render-worker";
}
public sealed record SpeechDistributedTrainingOptions(
string CurriculumTier = "tiny-local-proof",
string TargetReferenceId = "local/reference-control-vector",
string MorphologyId = "neutral-human-ish",
string RendererProfileId = "compiled-faust-worker",
string ScoringRecipeId = "control-vector-mse-proof",
string RequiredRuntimeKind = "dotnet-csharp",
string WorkerRole = SpeechCultMeshRoles.RenderWorker,
float UtteranceLearningRate = 0.04f,
float SynthDriverLearningRate = 0.04f);
public sealed record SpeechDistributedTrainingApplyResult(
SpeechTrainingCheckpoint Checkpoint,
IReadOnlyList<PackedNeuralBackpropagation> Backpropagations);
public static class SpeechDistributedTrainingCultCacheStore
{
public static Task UpsertRequestsAsync(string filePath, IEnumerable<SpeechRenderRequest> requests) =>
UpsertAsync(filePath, requests, request => $"speech-render-request:{request.RequestId}");
public static Task UpsertResultsAsync(string filePath, IEnumerable<SpeechRenderResult> results) =>
UpsertAsync(filePath, results, result => $"speech-render-result:{result.ResultId}");
public static Task UpsertCheckpointsAsync(string filePath, IEnumerable<SpeechTrainingCheckpoint> checkpoints) =>
UpsertAsync(filePath, checkpoints, checkpoint => $"speech-training-checkpoint:{checkpoint.CheckpointId}");
public static Task UpsertPayloadManifestsAsync(string filePath, IEnumerable<SpeechWorkerPayloadManifest> payloads) =>
UpsertAsync(filePath, payloads, payload => $"speech-worker-payload:{payload.PayloadId}");
public static Task UpsertAdmissionReceiptsAsync(string filePath, IEnumerable<SpeechWorkerAdmissionReceipt> receipts) =>
UpsertAsync(filePath, receipts, receipt => $"speech-worker-admission:{receipt.ReceiptId}");
public static Task<IReadOnlyList<SpeechRenderRequest>> ReadRequestsAsync(string filePath) =>
ReadAllAsync<SpeechRenderRequest>(filePath, request => request.RequestId);
public static Task<IReadOnlyList<SpeechRenderResult>> ReadResultsAsync(string filePath) =>
ReadAllAsync<SpeechRenderResult>(filePath, result => result.ResultId);
public static Task<IReadOnlyList<SpeechTrainingCheckpoint>> ReadCheckpointsAsync(string filePath) =>
ReadAllAsync<SpeechTrainingCheckpoint>(filePath, checkpoint => checkpoint.CheckpointId);
public static Task<IReadOnlyList<SpeechWorkerPayloadManifest>> ReadPayloadManifestsAsync(string filePath) =>
ReadAllAsync<SpeechWorkerPayloadManifest>(filePath, payload => payload.PayloadId);
public static Task<IReadOnlyList<SpeechWorkerAdmissionReceipt>> ReadAdmissionReceiptsAsync(string filePath) =>
ReadAllAsync<SpeechWorkerAdmissionReceipt>(filePath, receipt => receipt.ReceiptId);
private static async Task UpsertAsync<T>(
string filePath,
IEnumerable<T> documents,
Func<T, string> key)
where T : class
{
using var cache = await OpenForMutationAsync(filePath).ConfigureAwait(false);
foreach (var document in documents)
{
await cache.UpsertAsync(document, new CultRecordHandle<T>(new CultRecordKey(key(document)))).ConfigureAwait(false);
}
await cache.FlushAsync().ConfigureAwait(false);
}
private static async Task<IReadOnlyList<T>> ReadAllAsync<T>(string filePath, Func<T, string> orderKey)
where T : class
{
if (!File.Exists(filePath))
{
return [];
}
using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false);
return cache.GetAll<T>().OrderBy(orderKey, StringComparer.Ordinal).ToArray();
}
private static async Task<CultCache> OpenForMutationAsync(string filePath)
{
var options = new CultCacheOpenOptions { PullOnOpen = File.Exists(filePath) };
return await CultCacheMessagePack.OpenAsync(filePath, options).ConfigureAwait(false);
}
}
public static class SpeechDistributedTrainingCoordinator
{
public static IReadOnlyList<SpeechRenderRequest> CreateRenderRequests(
string batchId,
SpeechBackpropagationPipeline pipeline,
IReadOnlyList<SpeechBackpropagationTrainingExample> examples,
SpeechDistributedTrainingOptions? options = null)
{
options ??= new SpeechDistributedTrainingOptions();
var createdAt = DateTimeOffset.UtcNow;
var createdAtText = createdAt.ToString("O");
var requests = new SpeechRenderRequest[examples.Count];
for (var index = 0; index < examples.Count; index++)
{
var example = examples[index];
var candidate = pipeline.Predict(example).ToVector(pipeline.SynthDriver.MelBandCount);
requests[index] = new SpeechRenderRequest(
$"{batchId}:{index:0000}",
batchId,
createdAtText,
options.CurriculumTier,
options.TargetReferenceId,
options.MorphologyId,
options.RendererProfileId,
options.ScoringRecipeId,
SpeechUtteranceEmbeddingInputSnapshot.From(example.UtteranceInput),
SpeechPhoneticEventSnapshot.From(example.Event),
candidate,
example.Target.ToVector(pipeline.SynthDriver.MelBandCount),
[
new SpeechTimingReceipt(
"utterance-to-candidate-controls",
createdAtText,
0,
0,
CandidateConfidence(candidate),
"Local request creation captured the current utterance encoder and synth-driver decision.")
]);
}
return requests;
}
public static IReadOnlyList<SpeechRenderResult> ScoreControlVectorRequests(
IReadOnlyList<SpeechRenderRequest> requests,
string workerId,
string rendererToolchain = "local-control-vector-proof")
{
var completedAt = DateTimeOffset.UtcNow.ToString("O");
var results = new SpeechRenderResult[requests.Count];
for (var index = 0; index < requests.Count; index++)
{
var startedAt = DateTimeOffset.UtcNow;
var startedStamp = Stopwatch.GetTimestamp();
var request = requests[index];
if (request.CandidateControlVector.Length != request.ReferenceControlVector.Length)
{
var latency = Stopwatch.GetElapsedTime(startedStamp).TotalMilliseconds;
results[index] = new SpeechRenderResult(
$"{request.RequestId}:failed",
request.RequestId,
workerId,
rendererToolchain,
completedAt,
SpeechRenderStatus.Failed,
0,
[],
[],
[],
[
new SpeechTimingReceipt(
"score-render-result",
startedAt.ToString("O"),
latency,
0,
0,
"Rejected before scoring because the candidate/reference control vector sizes differ.")
],
"control_vector_size_mismatch",
"Candidate and reference control vectors must have the same length.");
continue;
}
var gradient = new float[request.CandidateControlVector.Length];
var loss = 0f;
for (var valueIndex = 0; valueIndex < gradient.Length; valueIndex++)
{
var difference = request.CandidateControlVector[valueIndex] - request.ReferenceControlVector[valueIndex];
gradient[valueIndex] = 2f * difference / gradient.Length;
loss += difference * difference;
}
loss /= gradient.Length;
var scoreLatency = Stopwatch.GetElapsedTime(startedStamp).TotalMilliseconds;
results[index] = new SpeechRenderResult(
$"{request.RequestId}:score",
request.RequestId,
workerId,
rendererToolchain,
completedAt,
SpeechRenderStatus.Succeeded,
loss,
gradient,
[new SpeechScoreMetric("control_mse", loss, 1)],
[new SpeechRenderArtifact("score-report", $"cultcache://speech-render-result/{request.RequestId}", "")],
[
new SpeechTimingReceipt(
"score-render-result",
startedAt.ToString("O"),
scoreLatency,
0,
LossConfidence(loss),
"Local control-vector proof scored candidate output against reference output.")
]);
}
return results;
}
public static SpeechWorkerPayloadManifest CreateCultMeshPayloadManifest(
string payloadId,
IReadOnlyList<SpeechRenderRequest> requests,
IReadOnlyList<SpeechWorkerPayloadArtifact>? artifacts = null,
SpeechDistributedTrainingOptions? options = null)
{
if (requests.Count == 0)
{
throw new ArgumentException("A CultMesh speech payload needs at least one render request.", nameof(requests));
}
options ??= new SpeechDistributedTrainingOptions();
return new SpeechWorkerPayloadManifest(
payloadId,
requests[0].BatchId,
DateTimeOffset.UtcNow.ToString("O"),
options.RendererProfileId,
options.ScoringRecipeId,
options.RequiredRuntimeKind,
options.WorkerRole,
[.. artifacts ?? DefaultPayloadArtifacts(options)],
"Payload manifest authorizes transport/admission only; workers return gradients and receipts, not checkpoints.");
}
public static IReadOnlyList<SpeechCultMeshWorkerAssignment> AdmitCultMeshWorkers(
SpeechWorkerPayloadManifest payload,
IReadOnlyList<SpeechRenderRequest> requests,
IReadOnlyList<CultMeshPeerCard> peers,
CultMeshAuthorityLeaseCatalog leases,
string? shardId = null,
DateTimeOffset? at = null)
{
if (requests.Count == 0)
{
return [];
}
var admittedPeers = new List<(CultMeshPeerCard Peer, CultMeshAuthorityLease Lease)>();
foreach (var peer in peers.OrderBy(peer => peer.PeerId, StringComparer.Ordinal))
{
if (!peer.HasRole(payload.WorkerRole))
{
continue;
}
var leaseId = peer.AuthorityLeaseId;
if (string.IsNullOrWhiteSpace(leaseId))
{
continue;
}
var lease = leases.Get(leaseId);
if (lease is null || !leases.IsAuthorized(peer, payload.WorkerRole, shardId, at))
{
continue;
}
admittedPeers.Add((peer, lease));
}
if (admittedPeers.Count == 0)
{
throw new InvalidOperationException("no CultMesh speech render workers were admitted for the payload");
}
var requestBuckets = admittedPeers
.Select(_ => new List<SpeechRenderRequest>())
.ToArray();
for (var index = 0; index < requests.Count; index++)
{
requestBuckets[index % requestBuckets.Length].Add(requests[index]);
}
var decidedAt = DateTimeOffset.UtcNow.ToString("O");
var assignments = new List<SpeechCultMeshWorkerAssignment>();
for (var index = 0; index < admittedPeers.Count; index++)
{
var bucket = requestBuckets[index];
if (bucket.Count == 0)
{
continue;
}
var (peer, lease) = admittedPeers[index];
var requestIds = bucket.Select(request => request.RequestId).ToArray();
var receipt = new SpeechWorkerAdmissionReceipt(
$"{payload.PayloadId}:{peer.PeerId}:admitted",
payload.PayloadId,
peer.PeerId,
lease.LeaseId,
decidedAt,
SpeechWorkerAdmissionStatus.Admitted,
requestIds,
"Peer advertised the speech render role and held a valid CultMesh worker lease.",
[
new SpeechTimingReceipt(
"cultmesh-worker-admission",
decidedAt,
0,
0,
1,
$"Admitted {requestIds.Length} request(s) for score-only worker execution.")
]);
assignments.Add(new SpeechCultMeshWorkerAssignment(peer, lease, receipt, bucket));
}
return assignments;
}
public static IReadOnlyList<SpeechWorkerAdmissionReceipt> RejectCultMeshWorkers(
SpeechWorkerPayloadManifest payload,
IReadOnlyList<CultMeshPeerCard> peers,
CultMeshAuthorityLeaseCatalog leases,
string? shardId = null,
DateTimeOffset? at = null)
{
var decidedAt = DateTimeOffset.UtcNow.ToString("O");
var rejected = new List<SpeechWorkerAdmissionReceipt>();
foreach (var peer in peers.OrderBy(peer => peer.PeerId, StringComparer.Ordinal))
{
var reason = RejectionReason(payload, peer, leases, shardId, at);
if (reason.Length == 0)
{
continue;
}
rejected.Add(new SpeechWorkerAdmissionReceipt(
$"{payload.PayloadId}:{peer.PeerId}:rejected",
payload.PayloadId,
peer.PeerId,
peer.AuthorityLeaseId ?? "",
decidedAt,
SpeechWorkerAdmissionStatus.Rejected,
[],
reason,
[
new SpeechTimingReceipt(
"cultmesh-worker-admission",
decidedAt,
0,
0,
0,
reason)
]));
}
return rejected;
}
public static IReadOnlyList<SpeechRenderResult> RunCultMeshRenderWorkers(
IReadOnlyList<SpeechCultMeshWorkerAssignment> assignments,
SpeechWorkerPayloadManifest payload)
{
var results = new List<SpeechRenderResult>();
foreach (var assignment in assignments)
{
if (assignment.Receipt.Status != SpeechWorkerAdmissionStatus.Admitted)
{
continue;
}
results.AddRange(ScoreControlVectorRequests(
assignment.Requests,
assignment.Peer.PeerId,
$"{payload.RendererProfileId}+{payload.ScoringRecipeId}+cultmesh-lease:{assignment.Lease.LeaseId}"));
}
return results;
}
public static SpeechCultMeshTrainingStepResult RunCultMeshTrainingStep(
string batchId,
string payloadId,
string checkpointId,
SpeechBackpropagationPipeline pipeline,
IReadOnlyList<SpeechBackpropagationTrainingExample> examples,
IReadOnlyList<CultMeshPeerCard> peers,
CultMeshAuthorityLeaseCatalog leases,
SpeechDistributedTrainingOptions? options = null,
string? shardId = null,
DateTimeOffset? at = null)
{
options ??= new SpeechDistributedTrainingOptions();
var requests = CreateRenderRequests(batchId, pipeline, examples, options);
var payload = CreateCultMeshPayloadManifest(payloadId, requests, options: options);
var assignments = AdmitCultMeshWorkers(payload, requests, peers, leases, shardId, at);
var rejected = RejectCultMeshWorkers(payload, peers, leases, shardId, at);
var results = RunCultMeshRenderWorkers(assignments, payload);
var applied = ApplyResults(pipeline, requests, results, checkpointId, options);
return new SpeechCultMeshTrainingStepResult(
payload,
[.. assignments.Select(assignment => assignment.Receipt), .. rejected],
results,
applied);
}
public static SpeechDistributedTrainingApplyResult ApplyResults(
SpeechBackpropagationPipeline pipeline,
IReadOnlyList<SpeechRenderRequest> requests,
IReadOnlyList<SpeechRenderResult> results,
string checkpointId,
SpeechDistributedTrainingOptions? options = null)
{
options ??= new SpeechDistributedTrainingOptions();
var requestById = requests.ToDictionary(request => request.RequestId, StringComparer.Ordinal);
var applied = new List<PackedNeuralBackpropagation>();
var appliedIds = new List<string>();
var loss = 0f;
foreach (var result in results.Where(result => result.Status == SpeechRenderStatus.Succeeded))
{
if (!requestById.TryGetValue(result.RequestId, out var request))
{
continue;
}
applied.Add(pipeline.TrainSingleFromSynthOutputGradient(
request.UtteranceInput.ToInput(),
request.PhoneticEvent.ToEvent(),
result.OutputGradient,
options.UtteranceLearningRate,
options.SynthDriverLearningRate,
result.Loss));
appliedIds.Add(result.ResultId);
loss += result.Loss;
}
if (applied.Count == 0)
{
throw new InvalidOperationException("no successful speech render results matched the request batch");
}
var checkpointCreatedAt = DateTimeOffset.UtcNow.ToString("O");
var checkpoint = new SpeechTrainingCheckpoint(
checkpointId,
requests.FirstOrDefault()?.BatchId ?? "",
checkpointCreatedAt,
applied.Count,
loss / applied.Count,
[.. appliedIds],
"Applied remote-render score gradients to the utterance encoder and synth driver.",
[
new SpeechTimingReceipt(
"apply-render-gradients",
checkpointCreatedAt,
0,
0,
LossConfidence(loss / applied.Count),
"Checkpoint records the training step that consumed successful render results.")
]);
return new SpeechDistributedTrainingApplyResult(checkpoint, applied);
}
private static float CandidateConfidence(IReadOnlyList<float> candidate)
{
if (candidate.Count == 0)
{
return 0;
}
var sum = 0f;
for (var index = 0; index < candidate.Count; index++)
{
sum += Math.Abs(candidate[index] - 0.5f) * 2f;
}
return Math.Clamp(sum / candidate.Count, 0f, 1f);
}
private static float LossConfidence(float loss)
{
if (!float.IsFinite(loss))
{
return 0;
}
return Math.Clamp(1f / (1f + loss), 0f, 1f);
}
private static IReadOnlyList<SpeechWorkerPayloadArtifact> DefaultPayloadArtifacts(SpeechDistributedTrainingOptions options) =>
[
new("compiled-faust-renderer", $"cultcache://speech-renderer/{options.RendererProfileId}", "", "application/wasm-or-native-faust"),
new("scoring-code", $"cultcache://speech-scoring/{options.ScoringRecipeId}", "", "application/dotnet-assembly"),
new("model-encoder-assembly", "cultcache://speech-model/aquasynth-core", "", "application/dotnet-assembly")
];
private static string RejectionReason(
SpeechWorkerPayloadManifest payload,
CultMeshPeerCard peer,
CultMeshAuthorityLeaseCatalog leases,
string? shardId,
DateTimeOffset? at)
{
if (!peer.HasRole(payload.WorkerRole))
{
return $"Peer does not advertise required worker role `{payload.WorkerRole}`.";
}
if (string.IsNullOrWhiteSpace(peer.AuthorityLeaseId))
{
return "Peer has no CultMesh authority lease id for worker admission.";
}
var lease = leases.Get(peer.AuthorityLeaseId!);
if (lease is null)
{
return "Peer references a worker lease that is not present in the local lease catalog.";
}
if (!leases.IsAuthorized(peer, payload.WorkerRole, shardId, at))
{
return "Peer worker lease does not authorize the requested role, shard, or time window.";
}
return "";
}
}