From ff0436b5291a2c9731eed3dc766c8afc01c91a05 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 19 May 2026 18:19:09 +0100 Subject: [PATCH] Persist CultNet replica cursors --- .../cultnet-distributed-database/summary.md | 8 +- .../Contracts/cultnet-distributed-database.md | 17 +- src/GameCult.Networking/CultNetDatabase.cs | 8 + .../CultNetShardReplication.cs | 158 +++++++++++++++++- src/GameCult.Networking/README.md | 5 +- .../NetworkingTests.cs | 57 +++++++ 6 files changed, 240 insertions(+), 13 deletions(-) diff --git a/research/cultnet-distributed-database/summary.md b/research/cultnet-distributed-database/summary.md index 698f9a92..9a8f4eb3 100644 --- a/research/cultnet-distributed-database/summary.md +++ b/research/cultnet-distributed-database/summary.md @@ -169,14 +169,16 @@ Shard catalog exchange now exists: - replica-side application of shard log responses with epoch checks, gap rejection, and idempotent replay - background shard log replicator plus schema-v0 fetcher for primary endpoints +- restart-safe replica cursor store with a local MessagePack file + implementation ## Next Cut -Build durable replica catch-up state: +Build durable authoritative mutation logs: -- persist per-shard logs and last-applied replica sequence +- persist per-shard primary logs - define the snapshot fallback boundary for compacted history -- store replica cursors across restarts +- return explicit resync-required responses when requested history is gone That is the next foundation needed before membership, Raft-style failover, or CRDT policy work can be honest. diff --git a/src/GameCult.Networking/Contracts/cultnet-distributed-database.md b/src/GameCult.Networking/Contracts/cultnet-distributed-database.md index eb159b96..b696b522 100644 --- a/src/GameCult.Networking/Contracts/cultnet-distributed-database.md +++ b/src/GameCult.Networking/Contracts/cultnet-distributed-database.md @@ -57,13 +57,15 @@ Implemented: - shard log replicator that pulls non-primary shards from catalog-advertised primary endpoints - schema-v0 shard log fetcher for `cultnet://host:port` endpoints +- restart-safe replica cursor store, including a local MessagePack file-backed + implementation Not implemented yet: - membership/failure detection - leader election or automatic shard failover - durable log persistence and compaction -- durable replica cursor storage across restarts +- snapshot fallback for compacted log history - declared CRDT merge policies ## Live Invariants @@ -80,13 +82,14 @@ Not implemented yet: ## Next Coherent Slice -Add durable log storage and restart-safe replica cursors: +Add durable mutation-log storage and snapshot fallback: - persist per-shard mutation logs instead of keeping them only in memory - define the compaction boundary that turns old log reads into snapshot reads -- store last applied sequence per shard across restarts +- return `ResyncRequired` when a replica asks for compacted history -The log is now wire-readable, replicas can apply it explicitly, and a pull loop -can drive non-primary shards from primary endpoints. The next useful layer is -making that state survive restart. Membership and leader election should wait -until basic replica catch-up is boring. +The log is wire-readable, replicas can apply it explicitly, a pull loop can +drive non-primary shards from primary endpoints, and replica cursors can survive +restart. The next useful layer is making authoritative mutation logs durable and +compaction-aware. Membership and leader election should wait until basic replica +catch-up is boring. diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 6e8631e2..0672ea8c 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -400,6 +400,14 @@ namespace GameCult.Networking : 0; } + internal void SetAppliedShardSequence(string shardId, long sequence) + { + ThrowIfDisposed(); + if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); + if (sequence < 0) throw new ArgumentOutOfRangeException(nameof(sequence), "Sequence must be non-negative."); + _appliedShardSequences[shardId] = sequence; + } + /// /// Creates a shard catalog response for optional schema/key filters. /// diff --git a/src/GameCult.Networking/CultNetShardReplication.cs b/src/GameCult.Networking/CultNetShardReplication.cs index e03e01cf..14e614a7 100644 --- a/src/GameCult.Networking/CultNetShardReplication.cs +++ b/src/GameCult.Networking/CultNetShardReplication.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using MessagePack; namespace GameCult.Networking { @@ -21,6 +23,46 @@ namespace GameCult.Networking int? limit = null); } + /// + /// Persisted replica cursor for one shard. + /// + [MessagePackObject] + public sealed class CultNetShardReplicaCursor + { + /// + /// Gets or sets the shard id. + /// + [Key("shardId")] public string ShardId { get; set; } = string.Empty; + /// + /// Gets or sets the shard epoch. + /// + [Key("shardEpoch")] public long ShardEpoch { get; set; } + /// + /// Gets or sets the last applied shard-log sequence. + /// + [Key("lastAppliedSequence")] public long LastAppliedSequence { get; set; } + /// + /// Gets or sets the cursor update timestamp. + /// + [Key("updatedAt")] public string UpdatedAt { get; set; } = string.Empty; + } + + /// + /// Stores restart-safe replica cursors. + /// + public interface ICultNetShardReplicaCursorStore + { + /// + /// Reads the cursor for a shard, if one exists. + /// + Task ReadAsync(string shardId); + + /// + /// Writes the cursor for a shard. + /// + Task WriteAsync(CultNetShardReplicaCursor cursor); + } + /// /// Options for background shard-log replication. /// @@ -45,6 +87,11 @@ namespace GameCult.Networking /// Gets or sets a callback for background pull failures. /// public Action? OnError { get; set; } + + /// + /// Gets or sets optional restart-safe cursor storage. + /// + public ICultNetShardReplicaCursorStore? CursorStore { get; set; } } /// @@ -129,11 +176,13 @@ namespace GameCult.Networking throw new InvalidOperationException($"Shard '{shard.ShardId}' is primary on this node and does not need replica pulling."); } - var afterSequence = _database.GetAppliedShardSequence(shard.ShardId); + var afterSequence = await GetAfterSequenceAsync(shard).ConfigureAwait(false); var response = await _options.Fetcher .FetchAsync(shard, afterSequence, _options.BatchSize) .ConfigureAwait(false); - return await _database.ApplyShardLogResponseAsync(response).ConfigureAwait(false); + var appliedSequence = await _database.ApplyShardLogResponseAsync(response).ConfigureAwait(false); + await WriteCursorAsync(shard, appliedSequence).ConfigureAwait(false); + return appliedSequence; } /// @@ -175,6 +224,40 @@ namespace GameCult.Networking } } + private async Task GetAfterSequenceAsync(CultNetShardDescriptor shard) + { + var afterSequence = _database.GetAppliedShardSequence(shard.ShardId); + if (afterSequence > 0 || _options.CursorStore == null) + { + return afterSequence; + } + + var cursor = await _options.CursorStore.ReadAsync(shard.ShardId).ConfigureAwait(false); + if (cursor == null || cursor.ShardEpoch != shard.Epoch) + { + return afterSequence; + } + + _database.SetAppliedShardSequence(shard.ShardId, cursor.LastAppliedSequence); + return cursor.LastAppliedSequence; + } + + private async Task WriteCursorAsync(CultNetShardDescriptor shard, long appliedSequence) + { + if (_options.CursorStore == null) + { + return; + } + + await _options.CursorStore.WriteAsync(new CultNetShardReplicaCursor + { + ShardId = shard.ShardId, + ShardEpoch = shard.Epoch, + LastAppliedSequence = appliedSequence, + UpdatedAt = DateTimeOffset.UtcNow.ToString("O") + }).ConfigureAwait(false); + } + private void ThrowIfDisposed() { if (_disposed) @@ -184,6 +267,77 @@ namespace GameCult.Networking } } + /// + /// Stores replica cursors in one local MessagePack file. + /// + public sealed class CultNetFileShardReplicaCursorStore : ICultNetShardReplicaCursorStore + { + private readonly string _filePath; + private readonly object _gate = new(); + + /// + /// Creates a file-backed cursor store. + /// + public CultNetFileShardReplicaCursorStore(string filePath) + { + _filePath = string.IsNullOrWhiteSpace(filePath) + ? throw new ArgumentException("Value must be non-empty.", nameof(filePath)) + : filePath; + } + + /// + public Task ReadAsync(string shardId) + { + if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); + lock (_gate) + { + CultNetShardReplicaCursor? cursor = ReadAll().FirstOrDefault(candidate => + string.Equals(candidate.ShardId, shardId, StringComparison.Ordinal)); + return Task.FromResult(cursor); + } + } + + /// + public Task WriteAsync(CultNetShardReplicaCursor cursor) + { + if (cursor == null) throw new ArgumentNullException(nameof(cursor)); + if (string.IsNullOrWhiteSpace(cursor.ShardId)) + { + throw new ArgumentException("Cursor requires a shardId.", nameof(cursor)); + } + + lock (_gate) + { + var cursors = ReadAll() + .Where(existing => !string.Equals(existing.ShardId, cursor.ShardId, StringComparison.Ordinal)) + .Concat(new[] { cursor }) + .OrderBy(existing => existing.ShardId, StringComparer.Ordinal) + .ToArray(); + var directory = Path.GetDirectoryName(Path.GetFullPath(_filePath)); + if (!string.IsNullOrWhiteSpace(directory)) + { + Directory.CreateDirectory(directory); + } + + File.WriteAllBytes(_filePath, MessagePackSerializer.Serialize(cursors)); + return Task.CompletedTask; + } + } + + private CultNetShardReplicaCursor[] ReadAll() + { + if (!File.Exists(_filePath)) + { + return Array.Empty(); + } + + var bytes = File.ReadAllBytes(_filePath); + return bytes.Length == 0 + ? Array.Empty() + : MessagePackSerializer.Deserialize(bytes); + } + } + /// /// Options for the schema-v0 client-based shard-log fetcher. /// diff --git a/src/GameCult.Networking/README.md b/src/GameCult.Networking/README.md index 030623c1..c092c8ca 100644 --- a/src/GameCult.Networking/README.md +++ b/src/GameCult.Networking/README.md @@ -143,7 +143,10 @@ catch-up. Replicas apply those committed entries through sequence gaps, and treats replayed entries as already applied. `CultNetShardReplicator` can then pull non-primary shard logs from advertised primary endpoints, using `CultNetSchemaShardLogFetcher` for the schema-v0 -transport path. +transport path. Replica cursors can be stored through +`ICultNetShardReplicaCursorStore`; `CultNetFileShardReplicaCursorStore` keeps +those cursors in one local MessagePack file so replicas can resume after a +restart. The first coherent shard policy is primary ownership. Each shard has one authoritative writer at a time. Clients may connect to any node; a non-owner diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 6eea0179..dfba1bc5 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1037,6 +1037,63 @@ namespace GameCult.Networking.Tests Assert.That(cache.Get(key)!.Username, Is.EqualTo("Pull")); } + [Test] + public async Task CultNetShardReplicator_Resumes_From_FileCursorStore() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-cursors-{Guid.NewGuid():N}.mpack"); + try + { + var store = new CultNetFileShardReplicaCursorStore(path); + await store.WriteAsync(new CultNetShardReplicaCursor + { + ShardId = "players-cursor", + ShardEpoch = 3, + LastAppliedSequence = 5, + UpdatedAt = "2026-05-19T12:00:00.0000000Z" + }); + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var shard = new CultNetShardDescriptor( + "players-cursor", + "runtime-a", + epoch: 3, + isPrimary: false, + schemaIds: [schemaId], + keyPrefix: "player:", + primaryEndpoints: ["cultnet://primary.example.test:3075"]); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + Shards = [shard] + }); + var fetcher = new CapturingShardLogFetcher(new CultNetShardLogResponseMessage + { + ShardId = "players-cursor", + ShardEpoch = 3 + }); + using var replicator = new CultNetShardReplicator(database, new CultNetShardReplicatorOptions + { + Fetcher = fetcher, + CursorStore = store + }); + + var sequence = await replicator.PullOnceAsync("players-cursor"); + var cursor = await store.ReadAsync("players-cursor"); + + Assert.That(fetcher.LastAfterSequence, Is.EqualTo(5)); + Assert.That(sequence, Is.EqualTo(5)); + Assert.That(database.GetAppliedShardSequence("players-cursor"), Is.EqualTo(5)); + Assert.That(cursor, Is.Not.Null); + Assert.That(cursor!.LastAppliedSequence, Is.EqualTo(5)); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } + [Test] public void CultNetDatabaseServer_Creates_Filtered_SubscriptionChange() {