Persist CultNet replica cursors

This commit is contained in:
Meta Vens 2026-05-19 18:19:09 +01:00
commit ff0436b529
6 changed files with 240 additions and 13 deletions

View file

@ -169,14 +169,16 @@ Shard catalog exchange now exists:
- replica-side application of shard log responses with epoch checks, gap
rejection, and idempotent replay
- background shard log replicator plus schema-v0 fetcher for primary endpoints
- restart-safe replica cursor store with a local MessagePack file
implementation
## Next Cut
Build durable replica catch-up state:
Build durable authoritative mutation logs:
- persist per-shard logs and last-applied replica sequence
- persist per-shard primary logs
- define the snapshot fallback boundary for compacted history
- store replica cursors across restarts
- return explicit resync-required responses when requested history is gone
That is the next foundation needed before membership, Raft-style failover, or
CRDT policy work can be honest.

View file

@ -57,13 +57,15 @@ Implemented:
- shard log replicator that pulls non-primary shards from catalog-advertised
primary endpoints
- schema-v0 shard log fetcher for `cultnet://host:port` endpoints
- restart-safe replica cursor store, including a local MessagePack file-backed
implementation
Not implemented yet:
- membership/failure detection
- leader election or automatic shard failover
- durable log persistence and compaction
- durable replica cursor storage across restarts
- snapshot fallback for compacted log history
- declared CRDT merge policies
## Live Invariants
@ -80,13 +82,14 @@ Not implemented yet:
## Next Coherent Slice
Add durable log storage and restart-safe replica cursors:
Add durable mutation-log storage and snapshot fallback:
- persist per-shard mutation logs instead of keeping them only in memory
- define the compaction boundary that turns old log reads into snapshot reads
- store last applied sequence per shard across restarts
- return `ResyncRequired` when a replica asks for compacted history
The log is now wire-readable, replicas can apply it explicitly, and a pull loop
can drive non-primary shards from primary endpoints. The next useful layer is
making that state survive restart. Membership and leader election should wait
until basic replica catch-up is boring.
The log is wire-readable, replicas can apply it explicitly, a pull loop can
drive non-primary shards from primary endpoints, and replica cursors can survive
restart. The next useful layer is making authoritative mutation logs durable and
compaction-aware. Membership and leader election should wait until basic replica
catch-up is boring.

View file

@ -400,6 +400,14 @@ namespace GameCult.Networking
: 0;
}
internal void SetAppliedShardSequence(string shardId, long sequence)
{
ThrowIfDisposed();
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
if (sequence < 0) throw new ArgumentOutOfRangeException(nameof(sequence), "Sequence must be non-negative.");
_appliedShardSequences[shardId] = sequence;
}
/// <summary>
/// Creates a shard catalog response for optional schema/key filters.
/// </summary>

View file

@ -1,9 +1,11 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MessagePack;
namespace GameCult.Networking
{
@ -21,6 +23,46 @@ namespace GameCult.Networking
int? limit = null);
}
/// <summary>
/// Persisted replica cursor for one shard.
/// </summary>
[MessagePackObject]
public sealed class CultNetShardReplicaCursor
{
/// <summary>
/// Gets or sets the shard id.
/// </summary>
[Key("shardId")] public string ShardId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the shard epoch.
/// </summary>
[Key("shardEpoch")] public long ShardEpoch { get; set; }
/// <summary>
/// Gets or sets the last applied shard-log sequence.
/// </summary>
[Key("lastAppliedSequence")] public long LastAppliedSequence { get; set; }
/// <summary>
/// Gets or sets the cursor update timestamp.
/// </summary>
[Key("updatedAt")] public string UpdatedAt { get; set; } = string.Empty;
}
/// <summary>
/// Stores restart-safe replica cursors.
/// </summary>
public interface ICultNetShardReplicaCursorStore
{
/// <summary>
/// Reads the cursor for a shard, if one exists.
/// </summary>
Task<CultNetShardReplicaCursor?> ReadAsync(string shardId);
/// <summary>
/// Writes the cursor for a shard.
/// </summary>
Task WriteAsync(CultNetShardReplicaCursor cursor);
}
/// <summary>
/// Options for background shard-log replication.
/// </summary>
@ -45,6 +87,11 @@ namespace GameCult.Networking
/// Gets or sets a callback for background pull failures.
/// </summary>
public Action<Exception>? OnError { get; set; }
/// <summary>
/// Gets or sets optional restart-safe cursor storage.
/// </summary>
public ICultNetShardReplicaCursorStore? CursorStore { get; set; }
}
/// <summary>
@ -129,11 +176,13 @@ namespace GameCult.Networking
throw new InvalidOperationException($"Shard '{shard.ShardId}' is primary on this node and does not need replica pulling.");
}
var afterSequence = _database.GetAppliedShardSequence(shard.ShardId);
var afterSequence = await GetAfterSequenceAsync(shard).ConfigureAwait(false);
var response = await _options.Fetcher
.FetchAsync(shard, afterSequence, _options.BatchSize)
.ConfigureAwait(false);
return await _database.ApplyShardLogResponseAsync(response).ConfigureAwait(false);
var appliedSequence = await _database.ApplyShardLogResponseAsync(response).ConfigureAwait(false);
await WriteCursorAsync(shard, appliedSequence).ConfigureAwait(false);
return appliedSequence;
}
/// <inheritdoc />
@ -175,6 +224,40 @@ namespace GameCult.Networking
}
}
private async Task<long> GetAfterSequenceAsync(CultNetShardDescriptor shard)
{
var afterSequence = _database.GetAppliedShardSequence(shard.ShardId);
if (afterSequence > 0 || _options.CursorStore == null)
{
return afterSequence;
}
var cursor = await _options.CursorStore.ReadAsync(shard.ShardId).ConfigureAwait(false);
if (cursor == null || cursor.ShardEpoch != shard.Epoch)
{
return afterSequence;
}
_database.SetAppliedShardSequence(shard.ShardId, cursor.LastAppliedSequence);
return cursor.LastAppliedSequence;
}
private async Task WriteCursorAsync(CultNetShardDescriptor shard, long appliedSequence)
{
if (_options.CursorStore == null)
{
return;
}
await _options.CursorStore.WriteAsync(new CultNetShardReplicaCursor
{
ShardId = shard.ShardId,
ShardEpoch = shard.Epoch,
LastAppliedSequence = appliedSequence,
UpdatedAt = DateTimeOffset.UtcNow.ToString("O")
}).ConfigureAwait(false);
}
private void ThrowIfDisposed()
{
if (_disposed)
@ -184,6 +267,77 @@ namespace GameCult.Networking
}
}
/// <summary>
/// Stores replica cursors in one local MessagePack file.
/// </summary>
public sealed class CultNetFileShardReplicaCursorStore : ICultNetShardReplicaCursorStore
{
private readonly string _filePath;
private readonly object _gate = new();
/// <summary>
/// Creates a file-backed cursor store.
/// </summary>
public CultNetFileShardReplicaCursorStore(string filePath)
{
_filePath = string.IsNullOrWhiteSpace(filePath)
? throw new ArgumentException("Value must be non-empty.", nameof(filePath))
: filePath;
}
/// <inheritdoc />
public Task<CultNetShardReplicaCursor?> ReadAsync(string shardId)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
lock (_gate)
{
CultNetShardReplicaCursor? cursor = ReadAll().FirstOrDefault(candidate =>
string.Equals(candidate.ShardId, shardId, StringComparison.Ordinal));
return Task.FromResult<CultNetShardReplicaCursor?>(cursor);
}
}
/// <inheritdoc />
public Task WriteAsync(CultNetShardReplicaCursor cursor)
{
if (cursor == null) throw new ArgumentNullException(nameof(cursor));
if (string.IsNullOrWhiteSpace(cursor.ShardId))
{
throw new ArgumentException("Cursor requires a shardId.", nameof(cursor));
}
lock (_gate)
{
var cursors = ReadAll()
.Where(existing => !string.Equals(existing.ShardId, cursor.ShardId, StringComparison.Ordinal))
.Concat(new[] { cursor })
.OrderBy(existing => existing.ShardId, StringComparer.Ordinal)
.ToArray();
var directory = Path.GetDirectoryName(Path.GetFullPath(_filePath));
if (!string.IsNullOrWhiteSpace(directory))
{
Directory.CreateDirectory(directory);
}
File.WriteAllBytes(_filePath, MessagePackSerializer.Serialize(cursors));
return Task.CompletedTask;
}
}
private CultNetShardReplicaCursor[] ReadAll()
{
if (!File.Exists(_filePath))
{
return Array.Empty<CultNetShardReplicaCursor>();
}
var bytes = File.ReadAllBytes(_filePath);
return bytes.Length == 0
? Array.Empty<CultNetShardReplicaCursor>()
: MessagePackSerializer.Deserialize<CultNetShardReplicaCursor[]>(bytes);
}
}
/// <summary>
/// Options for the schema-v0 client-based shard-log fetcher.
/// </summary>

View file

@ -143,7 +143,10 @@ catch-up. Replicas apply those committed entries through
sequence gaps, and treats replayed entries as already applied.
`CultNetShardReplicator` can then pull non-primary shard logs from advertised
primary endpoints, using `CultNetSchemaShardLogFetcher` for the schema-v0
transport path.
transport path. Replica cursors can be stored through
`ICultNetShardReplicaCursorStore`; `CultNetFileShardReplicaCursorStore` keeps
those cursors in one local MessagePack file so replicas can resume after a
restart.
The first coherent shard policy is primary ownership. Each shard has one
authoritative writer at a time. Clients may connect to any node; a non-owner

View file

@ -1037,6 +1037,63 @@ namespace GameCult.Networking.Tests
Assert.That(cache.Get<PlayerData>(key)!.Username, Is.EqualTo("Pull"));
}
[Test]
public async Task CultNetShardReplicator_Resumes_From_FileCursorStore()
{
var path = Path.Combine(Path.GetTempPath(), $"cultnet-cursors-{Guid.NewGuid():N}.mpack");
try
{
var store = new CultNetFileShardReplicaCursorStore(path);
await store.WriteAsync(new CultNetShardReplicaCursor
{
ShardId = "players-cursor",
ShardEpoch = 3,
LastAppliedSequence = 5,
UpdatedAt = "2026-05-19T12:00:00.0000000Z"
});
var cache = new CultCache();
var schemaId = cache.Registry.GetRequired<PlayerData>().SchemaId;
var shard = new CultNetShardDescriptor(
"players-cursor",
"runtime-a",
epoch: 3,
isPrimary: false,
schemaIds: [schemaId],
keyPrefix: "player:",
primaryEndpoints: ["cultnet://primary.example.test:3075"]);
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
Shards = [shard]
});
var fetcher = new CapturingShardLogFetcher(new CultNetShardLogResponseMessage
{
ShardId = "players-cursor",
ShardEpoch = 3
});
using var replicator = new CultNetShardReplicator(database, new CultNetShardReplicatorOptions
{
Fetcher = fetcher,
CursorStore = store
});
var sequence = await replicator.PullOnceAsync("players-cursor");
var cursor = await store.ReadAsync("players-cursor");
Assert.That(fetcher.LastAfterSequence, Is.EqualTo(5));
Assert.That(sequence, Is.EqualTo(5));
Assert.That(database.GetAppliedShardSequence("players-cursor"), Is.EqualTo(5));
Assert.That(cursor, Is.Not.Null);
Assert.That(cursor!.LastAppliedSequence, Is.EqualTo(5));
}
finally
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
[Test]
public void CultNetDatabaseServer_Creates_Filtered_SubscriptionChange()
{