Add CultNet shard write forwarding policy
This commit is contained in:
parent
4af500a87d
commit
1f527ae99b
8 changed files with 218 additions and 29 deletions
|
|
@ -161,16 +161,16 @@ Shard catalog exchange now exists:
|
|||
- per-shard id, owner runtime id, epoch, schema ids, key prefix/range
|
||||
- stale epoch rejection on writes
|
||||
- routing error that tells clients where the primary lives when known
|
||||
- injectable non-primary write forwarding policy
|
||||
|
||||
## Next Cut
|
||||
|
||||
Build write forwarding:
|
||||
Build a concrete schema-v0 write forwarder:
|
||||
|
||||
- non-primary nodes forward raw put/delete messages to primary endpoints when
|
||||
policy allows
|
||||
- dial the selected primary endpoint from the shard catalog
|
||||
- forwarded writes preserve shard id and epoch
|
||||
- stale epochs still fail
|
||||
- clients can opt into direct retry using routing hints
|
||||
- stale epochs still fail before forwarding
|
||||
- failed forwarding returns endpoint-aware routing errors
|
||||
|
||||
That is the next foundation needed before membership, Raft-style replication,
|
||||
or CRDT policy work can be honest.
|
||||
|
|
|
|||
|
|
@ -44,12 +44,14 @@ Implemented:
|
|||
primary endpoints, replica endpoints, read replica endpoints, and region
|
||||
- epoch-aware raw put/delete messages
|
||||
- routing hints on shard authority errors
|
||||
- injectable non-primary write forwarding policy
|
||||
|
||||
Not implemented yet:
|
||||
|
||||
- membership/failure detection
|
||||
- leader election or automatic shard failover
|
||||
- replicated log
|
||||
- concrete endpoint dialer for forwarded writes
|
||||
- declared CRDT merge policies
|
||||
|
||||
## Live Invariants
|
||||
|
|
@ -66,14 +68,13 @@ Not implemented yet:
|
|||
|
||||
## Next Coherent Slice
|
||||
|
||||
Add write forwarding:
|
||||
Add a concrete endpoint dialer for write forwarding:
|
||||
|
||||
- non-primary nodes can forward raw put/delete messages to the advertised
|
||||
primary endpoint when policy allows it
|
||||
- forwarded writes must preserve the original shard id and epoch
|
||||
- stale-epoch writes still fail instead of being "helpfully" retried against
|
||||
unknown authority
|
||||
- clients that opt out of forwarding receive routing hints and retry directly
|
||||
- implement `ICultNetShardWriteForwarder` over the schema-v0 client transport
|
||||
- choose the best primary endpoint from the shard descriptor
|
||||
- preserve shard id and epoch on forwarded messages
|
||||
- surface forwarding failures as routing errors with endpoint context
|
||||
|
||||
Forwarding is the next useful layer now that shard catalogs exist. Membership,
|
||||
leader election, and replicated logs should wait until routing is boring.
|
||||
Forwarding now has the policy seam. The next useful layer is the actual network
|
||||
dialer. Membership, leader election, and replicated logs should wait until
|
||||
routing is boring.
|
||||
|
|
|
|||
|
|
@ -237,15 +237,20 @@ namespace GameCult.Networking
|
|||
/// <summary>
|
||||
/// Creates a shard authority exception.
|
||||
/// </summary>
|
||||
public CultNetShardAuthorityException(CultNetShardDescriptor shard, string message) : base(message)
|
||||
public CultNetShardAuthorityException(CultNetShardDescriptor shard, string message, string reason = "not_primary") : base(message)
|
||||
{
|
||||
Shard = shard;
|
||||
Reason = reason;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the shard that rejected the write.
|
||||
/// </summary>
|
||||
public CultNetShardDescriptor Shard { get; }
|
||||
/// <summary>
|
||||
/// Gets the machine-readable authority rejection reason.
|
||||
/// </summary>
|
||||
public string Reason { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -316,6 +321,15 @@ namespace GameCult.Networking
|
|||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the shard that governs the supplied schema and key.
|
||||
/// </summary>
|
||||
public CultNetShardDescriptor ResolveShard(string schemaId, CultRecordKey key)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
return ResolveShardInternal(schemaId, key);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a document by key.
|
||||
/// </summary>
|
||||
|
|
@ -334,7 +348,7 @@ namespace GameCult.Networking
|
|||
if (document == null) throw new ArgumentNullException(nameof(document));
|
||||
|
||||
var descriptor = _cache.Registry.GetRequired<T>();
|
||||
var shard = ResolveShard(descriptor.SchemaId, key);
|
||||
var shard = ResolveShardInternal(descriptor.SchemaId, key);
|
||||
EnsurePrimary(shard, descriptor.SchemaId, key);
|
||||
|
||||
var previous = _cache.Get<T>(key);
|
||||
|
|
@ -356,7 +370,7 @@ namespace GameCult.Networking
|
|||
{
|
||||
ThrowIfDisposed();
|
||||
var descriptor = _cache.Registry.GetRequired<T>();
|
||||
var shard = ResolveShard(descriptor.SchemaId, key);
|
||||
var shard = ResolveShardInternal(descriptor.SchemaId, key);
|
||||
EnsurePrimary(shard, descriptor.SchemaId, key);
|
||||
|
||||
var previous = _cache.Get<T>(key);
|
||||
|
|
@ -388,7 +402,7 @@ namespace GameCult.Networking
|
|||
}
|
||||
|
||||
var key = new CultRecordKey(message.Document.RecordKey);
|
||||
var shard = ResolveShard(message.Document.SchemaId, key);
|
||||
var shard = ResolveShardInternal(message.Document.SchemaId, key);
|
||||
EnsurePrimary(shard, message.Document.SchemaId, key, message.ShardEpoch);
|
||||
var descriptor = _cache.Registry.GetRequiredBySchemaId(message.Document.SchemaId);
|
||||
var previous = _cache.Get(key);
|
||||
|
|
@ -413,7 +427,7 @@ namespace GameCult.Networking
|
|||
if (message == null) throw new ArgumentNullException(nameof(message));
|
||||
|
||||
var key = new CultRecordKey(message.RecordKey);
|
||||
var shard = ResolveShard(message.SchemaId, key);
|
||||
var shard = ResolveShardInternal(message.SchemaId, key);
|
||||
EnsurePrimary(shard, message.SchemaId, key, message.ShardEpoch);
|
||||
var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId);
|
||||
var previous = _cache.Get(key);
|
||||
|
|
@ -526,7 +540,7 @@ namespace GameCult.Networking
|
|||
var key = current != null
|
||||
? GetTrackedKey(current, documentType)
|
||||
: new CultRecordKey(string.Empty);
|
||||
var shard = ResolveShard(descriptor.SchemaId, key);
|
||||
var shard = ResolveShardInternal(descriptor.SchemaId, key);
|
||||
var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType);
|
||||
var change = Activator.CreateInstance(
|
||||
changeType,
|
||||
|
|
@ -591,7 +605,7 @@ namespace GameCult.Networking
|
|||
}
|
||||
}
|
||||
|
||||
private CultNetShardDescriptor ResolveShard(string schemaId, CultRecordKey key)
|
||||
private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key)
|
||||
{
|
||||
return _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? _shards[0];
|
||||
}
|
||||
|
|
@ -602,7 +616,8 @@ namespace GameCult.Networking
|
|||
{
|
||||
throw new CultNetShardAuthorityException(
|
||||
shard,
|
||||
$"Shard '{shard.ShardId}' is at epoch {shard.Epoch}, not requested epoch {expectedEpoch.Value}.");
|
||||
$"Shard '{shard.ShardId}' is at epoch {shard.Epoch}, not requested epoch {expectedEpoch.Value}.",
|
||||
"stale_epoch");
|
||||
}
|
||||
|
||||
if (shard.IsPrimary)
|
||||
|
|
@ -612,7 +627,8 @@ namespace GameCult.Networking
|
|||
|
||||
throw new CultNetShardAuthorityException(
|
||||
shard,
|
||||
$"Shard '{shard.ShardId}' owned by '{shard.OwnerRuntimeId}' does not accept local writes for schema '{schemaId}' key '{key.Value}'.");
|
||||
$"Shard '{shard.ShardId}' owned by '{shard.OwnerRuntimeId}' does not accept local writes for schema '{schemaId}' key '{key.Value}'.",
|
||||
"not_primary");
|
||||
}
|
||||
|
||||
private static bool MatchesCatalogFilter(
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ namespace GameCult.Networking
|
|||
{
|
||||
private readonly Server _server;
|
||||
private readonly CultNetDatabase _database;
|
||||
private readonly CultNetDatabaseServerOptions _options;
|
||||
private readonly Func<CultNetSnapshotRequestMessage, NetPeer, Task> _snapshotHandler;
|
||||
private readonly Func<CultNetDocumentPutRawMessage, NetPeer, Task> _putHandler;
|
||||
private readonly Func<CultNetDocumentDeleteMessage, NetPeer, Task> _deleteHandler;
|
||||
|
|
@ -28,10 +29,14 @@ namespace GameCult.Networking
|
|||
/// <summary>
|
||||
/// Creates and attaches a database message bridge to a server.
|
||||
/// </summary>
|
||||
public CultNetDatabaseServer(Server server, CultNetDatabase database)
|
||||
public CultNetDatabaseServer(
|
||||
Server server,
|
||||
CultNetDatabase database,
|
||||
CultNetDatabaseServerOptions? options = null)
|
||||
{
|
||||
_server = server ?? throw new ArgumentNullException(nameof(server));
|
||||
_database = database ?? throw new ArgumentNullException(nameof(database));
|
||||
_options = options ?? new CultNetDatabaseServerOptions();
|
||||
_snapshotHandler = HandleSnapshotRequestAsync;
|
||||
_putHandler = HandlePutAsync;
|
||||
_deleteHandler = HandleDeleteAsync;
|
||||
|
|
@ -133,7 +138,10 @@ namespace GameCult.Networking
|
|||
}
|
||||
catch (CultNetShardAuthorityException ex)
|
||||
{
|
||||
peer.SendCultNet(CreateRoutingError(ex));
|
||||
if (!await TryForwardPutAsync(ex, message).ConfigureAwait(false))
|
||||
{
|
||||
peer.SendCultNet(CreateRoutingError(ex));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
|
@ -149,7 +157,10 @@ namespace GameCult.Networking
|
|||
}
|
||||
catch (CultNetShardAuthorityException ex)
|
||||
{
|
||||
peer.SendCultNet(CreateRoutingError(ex));
|
||||
if (!await TryForwardDeleteAsync(ex, message).ConfigureAwait(false))
|
||||
{
|
||||
peer.SendCultNet(CreateRoutingError(ex));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
|
@ -294,10 +305,50 @@ namespace GameCult.Networking
|
|||
Error = exception.Message,
|
||||
RoutingHint = new CultNetShardRoutingHint
|
||||
{
|
||||
Reason = exception.Shard.IsPrimary ? "stale_epoch" : "not_primary",
|
||||
Reason = exception.Reason,
|
||||
Shard = CultNetDatabase.ToMessage(exception.Shard)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to forward a rejected raw put to the shard owner.
|
||||
/// </summary>
|
||||
public async Task<bool> TryForwardPutAsync(
|
||||
CultNetShardAuthorityException exception,
|
||||
CultNetDocumentPutRawMessage message)
|
||||
{
|
||||
if (!_options.ForwardNonPrimaryWrites ||
|
||||
_options.WriteForwarder == null ||
|
||||
exception.Reason != "not_primary")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
message.ShardId ??= exception.Shard.ShardId;
|
||||
message.ShardEpoch ??= exception.Shard.Epoch;
|
||||
await _options.WriteForwarder.ForwardPutAsync(exception.Shard, message).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to forward a rejected raw delete to the shard owner.
|
||||
/// </summary>
|
||||
public async Task<bool> TryForwardDeleteAsync(
|
||||
CultNetShardAuthorityException exception,
|
||||
CultNetDocumentDeleteMessage message)
|
||||
{
|
||||
if (!_options.ForwardNonPrimaryWrites ||
|
||||
_options.WriteForwarder == null ||
|
||||
exception.Reason != "not_primary")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
message.ShardId ??= exception.Shard.ShardId;
|
||||
message.ShardEpoch ??= exception.Shard.Epoch;
|
||||
await _options.WriteForwarder.ForwardDeleteAsync(exception.Shard, message).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,11 @@ namespace GameCult.Networking
|
|||
/// </summary>
|
||||
public CultNetDatabaseOptions? DatabaseOptions { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the database server bridge options used for shard routing and forwarding.
|
||||
/// </summary>
|
||||
public CultNetDatabaseServerOptions? DatabaseServerOptions { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets an optional callback used to customize the server before start.
|
||||
/// </summary>
|
||||
|
|
@ -130,7 +135,7 @@ namespace GameCult.Networking
|
|||
var server = new Server(cache, options.Security ?? ServerSecurityOptions.Development());
|
||||
var databaseOptions = options.DatabaseOptions ?? new CultNetDatabaseOptions();
|
||||
var database = new CultNetDatabase(cache, databaseOptions);
|
||||
var databaseServer = new CultNetDatabaseServer(server, database);
|
||||
var databaseServer = new CultNetDatabaseServer(server, database, options.DatabaseServerOptions);
|
||||
options.ConfigureServer?.Invoke(server);
|
||||
|
||||
var host = new CultNetHost(cache, store, server, database, databaseServer);
|
||||
|
|
|
|||
36
src/GameCult.Networking/CultNetShardWriteForwarding.cs
Normal file
36
src/GameCult.Networking/CultNetShardWriteForwarding.cs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
using System.Threading.Tasks;
|
||||
|
||||
namespace GameCult.Networking
|
||||
{
|
||||
/// <summary>
|
||||
/// Forwards shard writes to an authoritative owner.
|
||||
/// </summary>
|
||||
public interface ICultNetShardWriteForwarder
|
||||
{
|
||||
/// <summary>
|
||||
/// Forwards a raw document put to the shard owner.
|
||||
/// </summary>
|
||||
Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message);
|
||||
|
||||
/// <summary>
|
||||
/// Forwards a raw document delete to the shard owner.
|
||||
/// </summary>
|
||||
Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options controlling shard write forwarding from a database server bridge.
|
||||
/// </summary>
|
||||
public sealed class CultNetDatabaseServerOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets whether non-primary writes may be forwarded to shard owners.
|
||||
/// </summary>
|
||||
public bool ForwardNonPrimaryWrites { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the forwarder used when non-primary write forwarding is enabled.
|
||||
/// </summary>
|
||||
public ICultNetShardWriteForwarder? WriteForwarder { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -133,7 +133,9 @@ The current first slice exposes this as `CultNetDatabase` and wires hosted
|
|||
servers through `CultNetDatabaseServer`. It handles primary-shard writes, typed
|
||||
R3 watch streams, raw document puts/deletes, raw snapshot responses,
|
||||
schema/key-filtered live subscriptions, shard catalog exchange, and stale-epoch
|
||||
write rejection. Write forwarding is the next missing organ.
|
||||
write rejection. Non-primary write forwarding is available behind an injectable
|
||||
`ICultNetShardWriteForwarder`; a concrete schema-v0 endpoint dialer is the next
|
||||
missing organ.
|
||||
|
||||
The first coherent shard policy is primary ownership. Each shard has one
|
||||
authoritative writer at a time. Clients may connect to any node; a non-owner
|
||||
|
|
|
|||
|
|
@ -545,6 +545,64 @@ namespace GameCult.Networking.Tests
|
|||
Assert.That(response.Shards[0].ReadReplicaEndpoints, Is.EqualTo(["cultnet://edge-a:3075"]));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CultNetDatabaseServer_Forwards_NonPrimaryWrites_WhenConfigured()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var registry = new CultNetDocumentRegistry(cache.Registry)
|
||||
.Register(CultNetDocumentBinding.ForDocument<PlayerData>(
|
||||
cache.Registry,
|
||||
payloadSerializer: SerializePlayerDataPayload,
|
||||
payloadDeserializer: DeserializePlayerDataPayload));
|
||||
var schemaId = cache.Registry.GetRequired<PlayerData>().SchemaId;
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
DocumentRegistry = registry,
|
||||
Shards =
|
||||
[
|
||||
new CultNetShardDescriptor(
|
||||
"players-remote",
|
||||
"runtime-owner",
|
||||
epoch: 9,
|
||||
isPrimary: false,
|
||||
schemaIds: [schemaId],
|
||||
keyPrefix: "player:",
|
||||
primaryEndpoints: ["cultnet://runtime-owner:3075"])
|
||||
]
|
||||
});
|
||||
var forwarder = new CapturingShardWriteForwarder();
|
||||
using var server = new Server(cache, DevelopmentServerSecurity);
|
||||
using var databaseServer = new CultNetDatabaseServer(
|
||||
server,
|
||||
database,
|
||||
new CultNetDatabaseServerOptions
|
||||
{
|
||||
ForwardNonPrimaryWrites = true,
|
||||
WriteForwarder = forwarder
|
||||
});
|
||||
var key = new CultRecordKey("player:forward");
|
||||
var message = registry.CreateRawDocumentPutMessage(
|
||||
"put-forward",
|
||||
new CultRecordHandle<PlayerData>(key),
|
||||
new PlayerData
|
||||
{
|
||||
PlayerId = Guid.NewGuid(),
|
||||
Email = "forward@example.test",
|
||||
PasswordHash = "hash",
|
||||
Username = "Forward"
|
||||
});
|
||||
var exception = Assert.ThrowsAsync<CultNetShardAuthorityException>(
|
||||
async () => await database.ApplyPutAsync(message));
|
||||
|
||||
var forwarded = await databaseServer.TryForwardPutAsync(exception!, message);
|
||||
|
||||
Assert.That(forwarded, Is.True);
|
||||
Assert.That(forwarder.PutCount, Is.EqualTo(1));
|
||||
Assert.That(forwarder.LastPutShard!.ShardId, Is.EqualTo("players-remote"));
|
||||
Assert.That(forwarder.LastPutMessage!.ShardId, Is.EqualTo("players-remote"));
|
||||
Assert.That(forwarder.LastPutMessage.ShardEpoch, Is.EqualTo(9));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CultNetDatabaseServer_Creates_Filtered_SubscriptionChange()
|
||||
{
|
||||
|
|
@ -754,6 +812,26 @@ namespace GameCult.Networking.Tests
|
|||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingShardWriteForwarder : ICultNetShardWriteForwarder
|
||||
{
|
||||
public int PutCount { get; private set; }
|
||||
public CultNetShardDescriptor? LastPutShard { get; private set; }
|
||||
public CultNetDocumentPutRawMessage? LastPutMessage { get; private set; }
|
||||
|
||||
public Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message)
|
||||
{
|
||||
PutCount++;
|
||||
LastPutShard = shard;
|
||||
LastPutMessage = message;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
[MessagePack.MessagePackObject]
|
||||
public sealed class PlayerDataPayload
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue