Add CultNet shard write forwarding policy

This commit is contained in:
Meta Vens 2026-05-19 17:40:19 +01:00
commit 1f527ae99b
8 changed files with 218 additions and 29 deletions

View file

@ -161,16 +161,16 @@ Shard catalog exchange now exists:
- per-shard id, owner runtime id, epoch, schema ids, key prefix/range
- stale epoch rejection on writes
- routing error that tells clients where the primary lives when known
- injectable non-primary write forwarding policy
## Next Cut
Build write forwarding:
Build a concrete schema-v0 write forwarder:
- non-primary nodes forward raw put/delete messages to primary endpoints when
policy allows
- dial the selected primary endpoint from the shard catalog
- forwarded writes preserve shard id and epoch
- stale epochs still fail
- clients can opt into direct retry using routing hints
- stale epochs still fail before forwarding
- failed forwarding returns endpoint-aware routing errors
That is the next foundation needed before membership, Raft-style replication,
or CRDT policy work can be honest.

View file

@ -44,12 +44,14 @@ Implemented:
primary endpoints, replica endpoints, read replica endpoints, and region
- epoch-aware raw put/delete messages
- routing hints on shard authority errors
- injectable non-primary write forwarding policy
Not implemented yet:
- membership/failure detection
- leader election or automatic shard failover
- replicated log
- concrete endpoint dialer for forwarded writes
- declared CRDT merge policies
## Live Invariants
@ -66,14 +68,13 @@ Not implemented yet:
## Next Coherent Slice
Add write forwarding:
Add a concrete endpoint dialer for write forwarding:
- non-primary nodes can forward raw put/delete messages to the advertised
primary endpoint when policy allows it
- forwarded writes must preserve the original shard id and epoch
- stale-epoch writes still fail instead of being "helpfully" retried against
unknown authority
- clients that opt out of forwarding receive routing hints and retry directly
- implement `ICultNetShardWriteForwarder` over the schema-v0 client transport
- choose the best primary endpoint from the shard descriptor
- preserve shard id and epoch on forwarded messages
- surface forwarding failures as routing errors with endpoint context
Forwarding is the next useful layer now that shard catalogs exist. Membership,
leader election, and replicated logs should wait until routing is boring.
Forwarding now has the policy seam. The next useful layer is the actual network
dialer. Membership, leader election, and replicated logs should wait until
routing is boring.

View file

@ -237,15 +237,20 @@ namespace GameCult.Networking
/// <summary>
/// Creates a shard authority exception.
/// </summary>
public CultNetShardAuthorityException(CultNetShardDescriptor shard, string message) : base(message)
public CultNetShardAuthorityException(CultNetShardDescriptor shard, string message, string reason = "not_primary") : base(message)
{
Shard = shard;
Reason = reason;
}
/// <summary>
/// Gets the shard that rejected the write.
/// </summary>
public CultNetShardDescriptor Shard { get; }
/// <summary>
/// Gets the machine-readable authority rejection reason.
/// </summary>
public string Reason { get; }
}
/// <summary>
@ -316,6 +321,15 @@ namespace GameCult.Networking
};
}
/// <summary>
/// Resolves the shard that governs the supplied schema and key.
/// </summary>
public CultNetShardDescriptor ResolveShard(string schemaId, CultRecordKey key)
{
ThrowIfDisposed();
return ResolveShardInternal(schemaId, key);
}
/// <summary>
/// Gets a document by key.
/// </summary>
@ -334,7 +348,7 @@ namespace GameCult.Networking
if (document == null) throw new ArgumentNullException(nameof(document));
var descriptor = _cache.Registry.GetRequired<T>();
var shard = ResolveShard(descriptor.SchemaId, key);
var shard = ResolveShardInternal(descriptor.SchemaId, key);
EnsurePrimary(shard, descriptor.SchemaId, key);
var previous = _cache.Get<T>(key);
@ -356,7 +370,7 @@ namespace GameCult.Networking
{
ThrowIfDisposed();
var descriptor = _cache.Registry.GetRequired<T>();
var shard = ResolveShard(descriptor.SchemaId, key);
var shard = ResolveShardInternal(descriptor.SchemaId, key);
EnsurePrimary(shard, descriptor.SchemaId, key);
var previous = _cache.Get<T>(key);
@ -388,7 +402,7 @@ namespace GameCult.Networking
}
var key = new CultRecordKey(message.Document.RecordKey);
var shard = ResolveShard(message.Document.SchemaId, key);
var shard = ResolveShardInternal(message.Document.SchemaId, key);
EnsurePrimary(shard, message.Document.SchemaId, key, message.ShardEpoch);
var descriptor = _cache.Registry.GetRequiredBySchemaId(message.Document.SchemaId);
var previous = _cache.Get(key);
@ -413,7 +427,7 @@ namespace GameCult.Networking
if (message == null) throw new ArgumentNullException(nameof(message));
var key = new CultRecordKey(message.RecordKey);
var shard = ResolveShard(message.SchemaId, key);
var shard = ResolveShardInternal(message.SchemaId, key);
EnsurePrimary(shard, message.SchemaId, key, message.ShardEpoch);
var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId);
var previous = _cache.Get(key);
@ -526,7 +540,7 @@ namespace GameCult.Networking
var key = current != null
? GetTrackedKey(current, documentType)
: new CultRecordKey(string.Empty);
var shard = ResolveShard(descriptor.SchemaId, key);
var shard = ResolveShardInternal(descriptor.SchemaId, key);
var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType);
var change = Activator.CreateInstance(
changeType,
@ -591,7 +605,7 @@ namespace GameCult.Networking
}
}
private CultNetShardDescriptor ResolveShard(string schemaId, CultRecordKey key)
private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key)
{
return _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? _shards[0];
}
@ -602,7 +616,8 @@ namespace GameCult.Networking
{
throw new CultNetShardAuthorityException(
shard,
$"Shard '{shard.ShardId}' is at epoch {shard.Epoch}, not requested epoch {expectedEpoch.Value}.");
$"Shard '{shard.ShardId}' is at epoch {shard.Epoch}, not requested epoch {expectedEpoch.Value}.",
"stale_epoch");
}
if (shard.IsPrimary)
@ -612,7 +627,8 @@ namespace GameCult.Networking
throw new CultNetShardAuthorityException(
shard,
$"Shard '{shard.ShardId}' owned by '{shard.OwnerRuntimeId}' does not accept local writes for schema '{schemaId}' key '{key.Value}'.");
$"Shard '{shard.ShardId}' owned by '{shard.OwnerRuntimeId}' does not accept local writes for schema '{schemaId}' key '{key.Value}'.",
"not_primary");
}
private static bool MatchesCatalogFilter(

View file

@ -16,6 +16,7 @@ namespace GameCult.Networking
{
private readonly Server _server;
private readonly CultNetDatabase _database;
private readonly CultNetDatabaseServerOptions _options;
private readonly Func<CultNetSnapshotRequestMessage, NetPeer, Task> _snapshotHandler;
private readonly Func<CultNetDocumentPutRawMessage, NetPeer, Task> _putHandler;
private readonly Func<CultNetDocumentDeleteMessage, NetPeer, Task> _deleteHandler;
@ -28,10 +29,14 @@ namespace GameCult.Networking
/// <summary>
/// Creates and attaches a database message bridge to a server.
/// </summary>
public CultNetDatabaseServer(Server server, CultNetDatabase database)
public CultNetDatabaseServer(
Server server,
CultNetDatabase database,
CultNetDatabaseServerOptions? options = null)
{
_server = server ?? throw new ArgumentNullException(nameof(server));
_database = database ?? throw new ArgumentNullException(nameof(database));
_options = options ?? new CultNetDatabaseServerOptions();
_snapshotHandler = HandleSnapshotRequestAsync;
_putHandler = HandlePutAsync;
_deleteHandler = HandleDeleteAsync;
@ -133,7 +138,10 @@ namespace GameCult.Networking
}
catch (CultNetShardAuthorityException ex)
{
peer.SendCultNet(CreateRoutingError(ex));
if (!await TryForwardPutAsync(ex, message).ConfigureAwait(false))
{
peer.SendCultNet(CreateRoutingError(ex));
}
}
catch (Exception ex)
{
@ -149,7 +157,10 @@ namespace GameCult.Networking
}
catch (CultNetShardAuthorityException ex)
{
peer.SendCultNet(CreateRoutingError(ex));
if (!await TryForwardDeleteAsync(ex, message).ConfigureAwait(false))
{
peer.SendCultNet(CreateRoutingError(ex));
}
}
catch (Exception ex)
{
@ -294,10 +305,50 @@ namespace GameCult.Networking
Error = exception.Message,
RoutingHint = new CultNetShardRoutingHint
{
Reason = exception.Shard.IsPrimary ? "stale_epoch" : "not_primary",
Reason = exception.Reason,
Shard = CultNetDatabase.ToMessage(exception.Shard)
}
};
}
/// <summary>
/// Attempts to forward a rejected raw put to the shard owner.
/// </summary>
public async Task<bool> TryForwardPutAsync(
CultNetShardAuthorityException exception,
CultNetDocumentPutRawMessage message)
{
if (!_options.ForwardNonPrimaryWrites ||
_options.WriteForwarder == null ||
exception.Reason != "not_primary")
{
return false;
}
message.ShardId ??= exception.Shard.ShardId;
message.ShardEpoch ??= exception.Shard.Epoch;
await _options.WriteForwarder.ForwardPutAsync(exception.Shard, message).ConfigureAwait(false);
return true;
}
/// <summary>
/// Attempts to forward a rejected raw delete to the shard owner.
/// </summary>
public async Task<bool> TryForwardDeleteAsync(
CultNetShardAuthorityException exception,
CultNetDocumentDeleteMessage message)
{
if (!_options.ForwardNonPrimaryWrites ||
_options.WriteForwarder == null ||
exception.Reason != "not_primary")
{
return false;
}
message.ShardId ??= exception.Shard.ShardId;
message.ShardEpoch ??= exception.Shard.Epoch;
await _options.WriteForwarder.ForwardDeleteAsync(exception.Shard, message).ConfigureAwait(false);
return true;
}
}
}

View file

@ -31,6 +31,11 @@ namespace GameCult.Networking
/// </summary>
public CultNetDatabaseOptions? DatabaseOptions { get; set; }
/// <summary>
/// Gets or sets the database server bridge options used for shard routing and forwarding.
/// </summary>
public CultNetDatabaseServerOptions? DatabaseServerOptions { get; set; }
/// <summary>
/// Gets or sets an optional callback used to customize the server before start.
/// </summary>
@ -130,7 +135,7 @@ namespace GameCult.Networking
var server = new Server(cache, options.Security ?? ServerSecurityOptions.Development());
var databaseOptions = options.DatabaseOptions ?? new CultNetDatabaseOptions();
var database = new CultNetDatabase(cache, databaseOptions);
var databaseServer = new CultNetDatabaseServer(server, database);
var databaseServer = new CultNetDatabaseServer(server, database, options.DatabaseServerOptions);
options.ConfigureServer?.Invoke(server);
var host = new CultNetHost(cache, store, server, database, databaseServer);

View file

@ -0,0 +1,36 @@
using System.Threading.Tasks;
namespace GameCult.Networking
{
/// <summary>
/// Forwards shard writes to an authoritative owner.
/// </summary>
public interface ICultNetShardWriteForwarder
{
/// <summary>
/// Forwards a raw document put to the shard owner.
/// </summary>
Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message);
/// <summary>
/// Forwards a raw document delete to the shard owner.
/// </summary>
Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message);
}
/// <summary>
/// Options controlling shard write forwarding from a database server bridge.
/// </summary>
public sealed class CultNetDatabaseServerOptions
{
/// <summary>
/// Gets or sets whether non-primary writes may be forwarded to shard owners.
/// </summary>
public bool ForwardNonPrimaryWrites { get; set; }
/// <summary>
/// Gets or sets the forwarder used when non-primary write forwarding is enabled.
/// </summary>
public ICultNetShardWriteForwarder? WriteForwarder { get; set; }
}
}

View file

@ -133,7 +133,9 @@ The current first slice exposes this as `CultNetDatabase` and wires hosted
servers through `CultNetDatabaseServer`. It handles primary-shard writes, typed
R3 watch streams, raw document puts/deletes, raw snapshot responses,
schema/key-filtered live subscriptions, shard catalog exchange, and stale-epoch
write rejection. Write forwarding is the next missing organ.
write rejection. Non-primary write forwarding is available behind an injectable
`ICultNetShardWriteForwarder`; a concrete schema-v0 endpoint dialer is the next
missing organ.
The first coherent shard policy is primary ownership. Each shard has one
authoritative writer at a time. Clients may connect to any node; a non-owner

View file

@ -545,6 +545,64 @@ namespace GameCult.Networking.Tests
Assert.That(response.Shards[0].ReadReplicaEndpoints, Is.EqualTo(["cultnet://edge-a:3075"]));
}
[Test]
public async Task CultNetDatabaseServer_Forwards_NonPrimaryWrites_WhenConfigured()
{
var cache = new CultCache();
var registry = new CultNetDocumentRegistry(cache.Registry)
.Register(CultNetDocumentBinding.ForDocument<PlayerData>(
cache.Registry,
payloadSerializer: SerializePlayerDataPayload,
payloadDeserializer: DeserializePlayerDataPayload));
var schemaId = cache.Registry.GetRequired<PlayerData>().SchemaId;
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
DocumentRegistry = registry,
Shards =
[
new CultNetShardDescriptor(
"players-remote",
"runtime-owner",
epoch: 9,
isPrimary: false,
schemaIds: [schemaId],
keyPrefix: "player:",
primaryEndpoints: ["cultnet://runtime-owner:3075"])
]
});
var forwarder = new CapturingShardWriteForwarder();
using var server = new Server(cache, DevelopmentServerSecurity);
using var databaseServer = new CultNetDatabaseServer(
server,
database,
new CultNetDatabaseServerOptions
{
ForwardNonPrimaryWrites = true,
WriteForwarder = forwarder
});
var key = new CultRecordKey("player:forward");
var message = registry.CreateRawDocumentPutMessage(
"put-forward",
new CultRecordHandle<PlayerData>(key),
new PlayerData
{
PlayerId = Guid.NewGuid(),
Email = "forward@example.test",
PasswordHash = "hash",
Username = "Forward"
});
var exception = Assert.ThrowsAsync<CultNetShardAuthorityException>(
async () => await database.ApplyPutAsync(message));
var forwarded = await databaseServer.TryForwardPutAsync(exception!, message);
Assert.That(forwarded, Is.True);
Assert.That(forwarder.PutCount, Is.EqualTo(1));
Assert.That(forwarder.LastPutShard!.ShardId, Is.EqualTo("players-remote"));
Assert.That(forwarder.LastPutMessage!.ShardId, Is.EqualTo("players-remote"));
Assert.That(forwarder.LastPutMessage.ShardEpoch, Is.EqualTo(9));
}
[Test]
public void CultNetDatabaseServer_Creates_Filtered_SubscriptionChange()
{
@ -754,6 +812,26 @@ namespace GameCult.Networking.Tests
}
}
private sealed class CapturingShardWriteForwarder : ICultNetShardWriteForwarder
{
public int PutCount { get; private set; }
public CultNetShardDescriptor? LastPutShard { get; private set; }
public CultNetDocumentPutRawMessage? LastPutMessage { get; private set; }
public Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message)
{
PutCount++;
LastPutShard = shard;
LastPutMessage = message;
return Task.CompletedTask;
}
public Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message)
{
return Task.CompletedTask;
}
}
[MessagePack.MessagePackObject]
public sealed class PlayerDataPayload
{