Persist CultNet replica cursors
This commit is contained in:
parent
d778c36c27
commit
ff0436b529
6 changed files with 240 additions and 13 deletions
|
|
@ -169,14 +169,16 @@ Shard catalog exchange now exists:
|
|||
- replica-side application of shard log responses with epoch checks, gap
|
||||
rejection, and idempotent replay
|
||||
- background shard log replicator plus schema-v0 fetcher for primary endpoints
|
||||
- restart-safe replica cursor store with a local MessagePack file
|
||||
implementation
|
||||
|
||||
## Next Cut
|
||||
|
||||
Build durable replica catch-up state:
|
||||
Build durable authoritative mutation logs:
|
||||
|
||||
- persist per-shard logs and last-applied replica sequence
|
||||
- persist per-shard primary logs
|
||||
- define the snapshot fallback boundary for compacted history
|
||||
- store replica cursors across restarts
|
||||
- return explicit resync-required responses when requested history is gone
|
||||
|
||||
That is the next foundation needed before membership, Raft-style failover, or
|
||||
CRDT policy work can be honest.
|
||||
|
|
|
|||
|
|
@ -57,13 +57,15 @@ Implemented:
|
|||
- shard log replicator that pulls non-primary shards from catalog-advertised
|
||||
primary endpoints
|
||||
- schema-v0 shard log fetcher for `cultnet://host:port` endpoints
|
||||
- restart-safe replica cursor store, including a local MessagePack file-backed
|
||||
implementation
|
||||
|
||||
Not implemented yet:
|
||||
|
||||
- membership/failure detection
|
||||
- leader election or automatic shard failover
|
||||
- durable log persistence and compaction
|
||||
- durable replica cursor storage across restarts
|
||||
- snapshot fallback for compacted log history
|
||||
- declared CRDT merge policies
|
||||
|
||||
## Live Invariants
|
||||
|
|
@ -80,13 +82,14 @@ Not implemented yet:
|
|||
|
||||
## Next Coherent Slice
|
||||
|
||||
Add durable log storage and restart-safe replica cursors:
|
||||
Add durable mutation-log storage and snapshot fallback:
|
||||
|
||||
- persist per-shard mutation logs instead of keeping them only in memory
|
||||
- define the compaction boundary that turns old log reads into snapshot reads
|
||||
- store last applied sequence per shard across restarts
|
||||
- return `ResyncRequired` when a replica asks for compacted history
|
||||
|
||||
The log is now wire-readable, replicas can apply it explicitly, and a pull loop
|
||||
can drive non-primary shards from primary endpoints. The next useful layer is
|
||||
making that state survive restart. Membership and leader election should wait
|
||||
until basic replica catch-up is boring.
|
||||
The log is wire-readable, replicas can apply it explicitly, a pull loop can
|
||||
drive non-primary shards from primary endpoints, and replica cursors can survive
|
||||
restart. The next useful layer is making authoritative mutation logs durable and
|
||||
compaction-aware. Membership and leader election should wait until basic replica
|
||||
catch-up is boring.
|
||||
|
|
|
|||
|
|
@ -400,6 +400,14 @@ namespace GameCult.Networking
|
|||
: 0;
|
||||
}
|
||||
|
||||
internal void SetAppliedShardSequence(string shardId, long sequence)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
|
||||
if (sequence < 0) throw new ArgumentOutOfRangeException(nameof(sequence), "Sequence must be non-negative.");
|
||||
_appliedShardSequences[shardId] = sequence;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a shard catalog response for optional schema/key filters.
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MessagePack;
|
||||
|
||||
namespace GameCult.Networking
|
||||
{
|
||||
|
|
@ -21,6 +23,46 @@ namespace GameCult.Networking
|
|||
int? limit = null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Persisted replica cursor for one shard.
|
||||
/// </summary>
|
||||
[MessagePackObject]
|
||||
public sealed class CultNetShardReplicaCursor
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the shard id.
|
||||
/// </summary>
|
||||
[Key("shardId")] public string ShardId { get; set; } = string.Empty;
|
||||
/// <summary>
|
||||
/// Gets or sets the shard epoch.
|
||||
/// </summary>
|
||||
[Key("shardEpoch")] public long ShardEpoch { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the last applied shard-log sequence.
|
||||
/// </summary>
|
||||
[Key("lastAppliedSequence")] public long LastAppliedSequence { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the cursor update timestamp.
|
||||
/// </summary>
|
||||
[Key("updatedAt")] public string UpdatedAt { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stores restart-safe replica cursors.
|
||||
/// </summary>
|
||||
public interface ICultNetShardReplicaCursorStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads the cursor for a shard, if one exists.
|
||||
/// </summary>
|
||||
Task<CultNetShardReplicaCursor?> ReadAsync(string shardId);
|
||||
|
||||
/// <summary>
|
||||
/// Writes the cursor for a shard.
|
||||
/// </summary>
|
||||
Task WriteAsync(CultNetShardReplicaCursor cursor);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options for background shard-log replication.
|
||||
/// </summary>
|
||||
|
|
@ -45,6 +87,11 @@ namespace GameCult.Networking
|
|||
/// Gets or sets a callback for background pull failures.
|
||||
/// </summary>
|
||||
public Action<Exception>? OnError { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets optional restart-safe cursor storage.
|
||||
/// </summary>
|
||||
public ICultNetShardReplicaCursorStore? CursorStore { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -129,11 +176,13 @@ namespace GameCult.Networking
|
|||
throw new InvalidOperationException($"Shard '{shard.ShardId}' is primary on this node and does not need replica pulling.");
|
||||
}
|
||||
|
||||
var afterSequence = _database.GetAppliedShardSequence(shard.ShardId);
|
||||
var afterSequence = await GetAfterSequenceAsync(shard).ConfigureAwait(false);
|
||||
var response = await _options.Fetcher
|
||||
.FetchAsync(shard, afterSequence, _options.BatchSize)
|
||||
.ConfigureAwait(false);
|
||||
return await _database.ApplyShardLogResponseAsync(response).ConfigureAwait(false);
|
||||
var appliedSequence = await _database.ApplyShardLogResponseAsync(response).ConfigureAwait(false);
|
||||
await WriteCursorAsync(shard, appliedSequence).ConfigureAwait(false);
|
||||
return appliedSequence;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
|
@ -175,6 +224,40 @@ namespace GameCult.Networking
|
|||
}
|
||||
}
|
||||
|
||||
private async Task<long> GetAfterSequenceAsync(CultNetShardDescriptor shard)
|
||||
{
|
||||
var afterSequence = _database.GetAppliedShardSequence(shard.ShardId);
|
||||
if (afterSequence > 0 || _options.CursorStore == null)
|
||||
{
|
||||
return afterSequence;
|
||||
}
|
||||
|
||||
var cursor = await _options.CursorStore.ReadAsync(shard.ShardId).ConfigureAwait(false);
|
||||
if (cursor == null || cursor.ShardEpoch != shard.Epoch)
|
||||
{
|
||||
return afterSequence;
|
||||
}
|
||||
|
||||
_database.SetAppliedShardSequence(shard.ShardId, cursor.LastAppliedSequence);
|
||||
return cursor.LastAppliedSequence;
|
||||
}
|
||||
|
||||
private async Task WriteCursorAsync(CultNetShardDescriptor shard, long appliedSequence)
|
||||
{
|
||||
if (_options.CursorStore == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _options.CursorStore.WriteAsync(new CultNetShardReplicaCursor
|
||||
{
|
||||
ShardId = shard.ShardId,
|
||||
ShardEpoch = shard.Epoch,
|
||||
LastAppliedSequence = appliedSequence,
|
||||
UpdatedAt = DateTimeOffset.UtcNow.ToString("O")
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private void ThrowIfDisposed()
|
||||
{
|
||||
if (_disposed)
|
||||
|
|
@ -184,6 +267,77 @@ namespace GameCult.Networking
|
|||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stores replica cursors in one local MessagePack file.
|
||||
/// </summary>
|
||||
public sealed class CultNetFileShardReplicaCursorStore : ICultNetShardReplicaCursorStore
|
||||
{
|
||||
private readonly string _filePath;
|
||||
private readonly object _gate = new();
|
||||
|
||||
/// <summary>
|
||||
/// Creates a file-backed cursor store.
|
||||
/// </summary>
|
||||
public CultNetFileShardReplicaCursorStore(string filePath)
|
||||
{
|
||||
_filePath = string.IsNullOrWhiteSpace(filePath)
|
||||
? throw new ArgumentException("Value must be non-empty.", nameof(filePath))
|
||||
: filePath;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<CultNetShardReplicaCursor?> ReadAsync(string shardId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
|
||||
lock (_gate)
|
||||
{
|
||||
CultNetShardReplicaCursor? cursor = ReadAll().FirstOrDefault(candidate =>
|
||||
string.Equals(candidate.ShardId, shardId, StringComparison.Ordinal));
|
||||
return Task.FromResult<CultNetShardReplicaCursor?>(cursor);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task WriteAsync(CultNetShardReplicaCursor cursor)
|
||||
{
|
||||
if (cursor == null) throw new ArgumentNullException(nameof(cursor));
|
||||
if (string.IsNullOrWhiteSpace(cursor.ShardId))
|
||||
{
|
||||
throw new ArgumentException("Cursor requires a shardId.", nameof(cursor));
|
||||
}
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
var cursors = ReadAll()
|
||||
.Where(existing => !string.Equals(existing.ShardId, cursor.ShardId, StringComparison.Ordinal))
|
||||
.Concat(new[] { cursor })
|
||||
.OrderBy(existing => existing.ShardId, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
var directory = Path.GetDirectoryName(Path.GetFullPath(_filePath));
|
||||
if (!string.IsNullOrWhiteSpace(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
File.WriteAllBytes(_filePath, MessagePackSerializer.Serialize(cursors));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private CultNetShardReplicaCursor[] ReadAll()
|
||||
{
|
||||
if (!File.Exists(_filePath))
|
||||
{
|
||||
return Array.Empty<CultNetShardReplicaCursor>();
|
||||
}
|
||||
|
||||
var bytes = File.ReadAllBytes(_filePath);
|
||||
return bytes.Length == 0
|
||||
? Array.Empty<CultNetShardReplicaCursor>()
|
||||
: MessagePackSerializer.Deserialize<CultNetShardReplicaCursor[]>(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options for the schema-v0 client-based shard-log fetcher.
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -143,7 +143,10 @@ catch-up. Replicas apply those committed entries through
|
|||
sequence gaps, and treats replayed entries as already applied.
|
||||
`CultNetShardReplicator` can then pull non-primary shard logs from advertised
|
||||
primary endpoints, using `CultNetSchemaShardLogFetcher` for the schema-v0
|
||||
transport path.
|
||||
transport path. Replica cursors can be stored through
|
||||
`ICultNetShardReplicaCursorStore`; `CultNetFileShardReplicaCursorStore` keeps
|
||||
those cursors in one local MessagePack file so replicas can resume after a
|
||||
restart.
|
||||
|
||||
The first coherent shard policy is primary ownership. Each shard has one
|
||||
authoritative writer at a time. Clients may connect to any node; a non-owner
|
||||
|
|
|
|||
|
|
@ -1037,6 +1037,63 @@ namespace GameCult.Networking.Tests
|
|||
Assert.That(cache.Get<PlayerData>(key)!.Username, Is.EqualTo("Pull"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CultNetShardReplicator_Resumes_From_FileCursorStore()
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), $"cultnet-cursors-{Guid.NewGuid():N}.mpack");
|
||||
try
|
||||
{
|
||||
var store = new CultNetFileShardReplicaCursorStore(path);
|
||||
await store.WriteAsync(new CultNetShardReplicaCursor
|
||||
{
|
||||
ShardId = "players-cursor",
|
||||
ShardEpoch = 3,
|
||||
LastAppliedSequence = 5,
|
||||
UpdatedAt = "2026-05-19T12:00:00.0000000Z"
|
||||
});
|
||||
var cache = new CultCache();
|
||||
var schemaId = cache.Registry.GetRequired<PlayerData>().SchemaId;
|
||||
var shard = new CultNetShardDescriptor(
|
||||
"players-cursor",
|
||||
"runtime-a",
|
||||
epoch: 3,
|
||||
isPrimary: false,
|
||||
schemaIds: [schemaId],
|
||||
keyPrefix: "player:",
|
||||
primaryEndpoints: ["cultnet://primary.example.test:3075"]);
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [shard]
|
||||
});
|
||||
var fetcher = new CapturingShardLogFetcher(new CultNetShardLogResponseMessage
|
||||
{
|
||||
ShardId = "players-cursor",
|
||||
ShardEpoch = 3
|
||||
});
|
||||
using var replicator = new CultNetShardReplicator(database, new CultNetShardReplicatorOptions
|
||||
{
|
||||
Fetcher = fetcher,
|
||||
CursorStore = store
|
||||
});
|
||||
|
||||
var sequence = await replicator.PullOnceAsync("players-cursor");
|
||||
var cursor = await store.ReadAsync("players-cursor");
|
||||
|
||||
Assert.That(fetcher.LastAfterSequence, Is.EqualTo(5));
|
||||
Assert.That(sequence, Is.EqualTo(5));
|
||||
Assert.That(database.GetAppliedShardSequence("players-cursor"), Is.EqualTo(5));
|
||||
Assert.That(cursor, Is.Not.Null);
|
||||
Assert.That(cursor!.LastAppliedSequence, Is.EqualTo(5));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CultNetDatabaseServer_Creates_Filtered_SubscriptionChange()
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue